1#!/usr/bin/env python3
2
3import sys
4import re
5import tempfile
6import os
7import logging
8import argparse
9import errno
10import contextlib
11from os import path
12from hashlib import md5
13from io import StringIO
14
15from prettify_cp2k import normalizeFortranFile
16from prettify_cp2k import replacer
17
18sys.path.insert(0, path.join(path.dirname(path.abspath(__file__)), "fprettify"))
19from fprettify import reformat_ffile, fparse_utils, log_exception
20
21
22TO_UPCASE_RE = re.compile(
23    r"""
24(?P<toUpcase>
25    \.(?:and|eq|eqv|false|ge|gt|le|lt|ne|neqv|not|or|true)\.
26    |
27    (?<![\w%#])  # do not match stmts/intrinsics midword or called as type-bound procedures
28    (?<!%\ )  # do not match stmts/intrinsics when being called as type-bound procedure with a space in between
29    (?<!subroutine\ )(?<!function\ )(?<!::\ )  # do not match stmts/intrinsics when used as procedure names
30    (?:
31        (?: # statements:
32            a(?:llocat(?:able|e)|ssign(?:|ment))
33            |c(?:a(?:ll|se)|haracter|lose|o(?:m(?:mon|plex)|nt(?:ains|inue))|ycle)
34            |d(?:ata|eallocate|imension|o(?:|uble))
35            |e(?:lse(?:|if|where)|n(?:d(?:|do|file|if)|try)|quivalence|x(?:it|ternal))
36            |f(?:or(?:all|mat)|unction)
37            |goto
38            |i(?:f|mplicit|n(?:clude|quire|t(?:e(?:ger|nt|rface)|rinsic)))
39            |logical
40            |module
41            |n(?:amelist|one|ullify)
42            |o(?:nly|p(?:en|erator|tional))
43            |p(?:a(?:rameter|use)|ointer|r(?:ecision|i(?:nt|vate)|o(?:cedure|gram))|ublic)
44            |re(?:a[dl]|cursive|sult|turn|wind)
45            |s(?:ave|e(?:lect|quence)|top|ubroutine)
46            |t(?:arget|hen|ype)
47            |use
48            |w(?:h(?:ere|ile)|rite)
49        )
50        | (?: # intrinsic functions:
51            a(?:bs|c(?:har|os)|djust[lr]|i(?:mag|nt)|ll(?:|ocated)|n(?:int|y)|s(?:in|sociated)|tan2?)
52            |b(?:it_size|test)
53            |c(?:eiling|har|mplx|o(?:njg|sh?|unt)|shift)
54            |d(?:ate_and_time|ble|i(?:gits|m)|ot_product|prod)
55            |e(?:oshift|psilon|xp(?:|onent))
56            |f(?:loor|raction)
57            |huge
58            |i(?:a(?:char|nd)|b(?:clr|its|set)|char|eor|n(?:dex|t)|or|shftc?)
59            |kind
60            |l(?:bound|en(?:|_trim)|g[et]|l[et]|og(?:|10|ical))
61            |m(?:a(?:tmul|x(?:|exponent|loc|val))|erge|in(?:|exponent|loc|val)|od(?:|ulo)|vbits)
62            |n(?:earest|int|ot)
63            |p(?:ack|r(?:e(?:cision|sent)|oduct))
64            |r(?:a(?:dix|n(?:dom_(?:number|seed)|ge))|e(?:peat|shape)|rspacing)
65            |s(?:ca(?:le|n)|e(?:lected_(?:int_kind|real_kind)|t_exponent)|hape|i(?:gn|nh?|ze)|p(?:acing|read)|qrt|um|ystem_clock)
66            |t(?:anh?|iny|r(?:ans(?:fer|pose)|im))
67            |u(?:bound|npack)
68            |verify
69        ) (?=\ *\()
70    )
71    (?![\w%])
72)
73""",
74    flags=re.IGNORECASE | re.VERBOSE,
75)
76
77TO_UPCASE_OMP_RE = re.compile(
78    r"""
79(?<![\w%#])
80(?P<toUpcase>
81    (?:
82        atomic|barrier|c(?:apture|ritical)|do|end|flush|if|master|num_threads|ordered|parallel|read
83        |s(?:ection(?:|s)|ingle)|t(?:ask(?:|wait|yield)|hreadprivate)|update|w(?:orkshare|rite)|!\$omp
84    )
85    | (?:
86        a|co(?:llapse|py(?:in|private))|default|fi(?:nal|rstprivate)|i(?:and|eor|or)|lastprivate
87        |m(?:ax|ergeable|in)|n(?:one|owait)|ordered|private|reduction|shared|untied|\.(?:and|eqv|neqv|or)\.
88    )
89    | omp_(?:dynamic|max_active_levels|n(?:ested|um_threads)|proc_bind|s(?:tacksize|chedule)|thread_limit|wait_policy)
90)
91(?![\w%])
92""",
93    flags=re.IGNORECASE | re.VERBOSE,
94)
95
96LINE_PARTS_RE = re.compile(
97    r"""
98    (?P<commands>[^\"'!]*)
99    (?P<comment>!.*)?
100    (?P<string>
101        (?P<qchar>[\"'])
102        .*?
103        (?P=qchar))?
104""",
105    re.VERBOSE,
106)
107
108
109@contextlib.contextmanager
110def rewind(fhandle):
111    try:
112        yield fhandle
113    finally:
114        fhandle.seek(0)
115
116
117def upcaseStringKeywords(line):
118    """Upcases the fortran keywords, operators and intrinsic routines
119    in line"""
120    res = ""
121    start = 0
122    while start < len(line):
123        m = LINE_PARTS_RE.match(line[start:])
124        if not m:
125            raise SyntaxError("Syntax error, open string")
126        res = res + TO_UPCASE_RE.sub(
127            lambda match: match.group("toUpcase").upper(), m.group("commands")
128        )
129        if m.group("comment"):
130            res = res + m.group("comment")
131        if m.group("string"):
132            res = res + m.group("string")
133        start = start + m.end()
134    return res
135
136
137def upcaseKeywords(infile, outfile, upcase_omp):
138    """Writes infile to outfile with all the fortran keywords upcased"""
139
140    for line in infile:
141        line = upcaseStringKeywords(line)
142
143        if upcase_omp and normalizeFortranFile.OMP_DIR_RE.match(line):
144            line = TO_UPCASE_OMP_RE.sub(
145                lambda match: match.group("toUpcase").upper(), line
146            )
147
148        outfile.write(line)
149
150
151def prettifyFile(
152    infile,
153    filename,
154    normalize_use,
155    decl_linelength,
156    decl_offset,
157    reformat,
158    indent,
159    whitespace,
160    upcase_keywords,
161    upcase_omp,
162    replace,
163):
164    """prettifyes the fortran source in infile into a temporary file that is
165    returned. It can be the same as infile.
166    if normalize_use normalizes the use statements (defaults to true)
167    if upcase_keywords upcases the keywords (defaults to true)
168    if replace does the replacements contained in replacer.py (defaults
169    to false)
170
171    does not close the input file"""
172    max_pretty_iter = 5
173
174    logger = logging.getLogger("prettify-logger")
175
176    if is_fypp(infile):
177        logger.warning(
178            "fypp directives not fully supported, running only fprettify",
179            extra={"ffilename": filename},
180        )
181        replace = False
182        normalize_use = False
183        upcase_keywords = False
184
185    # create a temporary file first as a copy of the input file
186    inbuf = StringIO(infile.read())
187
188    hash_prev = md5(inbuf.getvalue().encode("utf8"))
189
190    for _ in range(max_pretty_iter):
191        if replace:
192            outbuf = StringIO()
193            replacer.replaceWords(inbuf, outbuf)
194            outbuf.seek(0)
195            inbuf.close()
196            inbuf = outbuf
197
198        if reformat:  # reformat needs to be done first
199            outbuf = StringIO()
200            try:
201                reformat_ffile(
202                    inbuf,
203                    outbuf,
204                    indent_size=indent,
205                    whitespace=whitespace,
206                    orig_filename=filename,
207                )
208            except fparse_utils.FprettifyParseException as e:
209                log_exception(
210                    e, "fprettify could not parse file, file is not prettified"
211                )
212                outbuf.close()
213                inbuf.seek(0)
214            else:
215                outbuf.seek(0)
216                inbuf.close()
217                inbuf = outbuf
218
219        normalize_use_succeeded = True
220
221        if normalize_use:
222            outbuf = StringIO()
223            try:
224                normalizeFortranFile.rewriteFortranFile(
225                    inbuf,
226                    outbuf,
227                    indent,
228                    decl_linelength,
229                    decl_offset,
230                    orig_filename=filename,
231                )
232            except normalizeFortranFile.InputStreamError as exc:
233                logger.exception(
234                    "normalizeFortranFile could not parse file, file is not normalized",
235                    extra={"ffilename": filename},
236                )
237                outbuf.close()
238                inbuf.seek(0)
239                normalize_use_succeeded = False
240            else:
241                outbuf.seek(0)
242                inbuf.close()
243                inbuf = outbuf
244
245        if upcase_keywords and normalize_use_succeeded:
246            outbuf = StringIO()
247            upcaseKeywords(inbuf, outbuf, upcase_omp)
248            outbuf.seek(0)
249            inbuf.close()
250            inbuf = outbuf
251
252        hash_new = md5(inbuf.getvalue().encode("utf8"))
253
254        if hash_prev.digest() == hash_new.digest():
255            return inbuf
256
257        hash_prev = hash_new
258
259    else:
260        raise RuntimeError(
261            "Prettify did not converge in {} steps.".format(max_pretty_iter)
262        )
263
264
265def prettifyInplace(filename, backupdir=None, stdout=False, **kwargs):
266    """Same as prettify, but inplace, replaces only if needed
267    :return: True if file was modified, False if otherwise. If stdin/stdout, always False.
268    """
269
270    if filename == "stdin":
271        infile = tempfile.TemporaryFile(mode="r+")
272        infile.write(sys.stdin.read())
273        infile.seek(0)
274    else:
275        infile = open(filename, "r")
276
277    outfile = prettifyFile(infile=infile, filename=filename, **kwargs)
278
279    if stdout:
280        outfile.seek(0)
281        sys.stdout.write(outfile.read())
282        outfile.close()
283        return False
284
285    if infile == outfile:
286        infile.close()
287        return False
288
289    infile.seek(0)
290    outfile.seek(0)
291    changed = True
292    for line1, line2 in zip(infile, outfile):
293        if line1 != line2:
294            break
295    else:
296        changed = False
297
298    if changed:
299        if backupdir:
300            bkName = path.join(backupdir, path.basename(filename))
301
302            with open(bkName, "w") as fhandle:
303                infile.seek(0)
304                fhandle.write(infile.read())
305
306        infile.close()  # close it here since we're going to overwrite it
307
308        with open(filename, "w") as fhandle:
309            outfile.seek(0)
310            fhandle.write(outfile.read())
311
312    else:
313        infile.close()
314
315    outfile.close()
316
317    return changed
318
319
320def is_fypp(infile):
321    FYPP_SYMBOLS = r"(#|\$|@)"
322    FYPP_LINE = r"^\s*" + FYPP_SYMBOLS + r":"
323    FYPP_INLINE = r"(" + FYPP_SYMBOLS + r"{|}" + FYPP_SYMBOLS + r")"
324    FYPP_RE = re.compile(r"(" + FYPP_LINE + r"|" + FYPP_INLINE + r")")
325
326    infile.seek(0)
327
328    with rewind(infile) as fhandle:
329        for line in fhandle.readlines():
330            if FYPP_RE.search(line):
331                return True
332
333        return False
334
335
336# based on https://stackoverflow.com/a/31347222
337def argparse_add_bool_arg(parser, name, default, helptxt):
338    dname = name.replace("-", "_")
339    group = parser.add_mutually_exclusive_group(required=False)
340    group.add_argument(
341        "--{}".format(name), dest=dname, action="store_true", help=helptxt
342    )
343    group.add_argument("--no-{}".format(name), dest=dname, action="store_false")
344    parser.set_defaults(**{dname: default})
345
346
347# from https://stackoverflow.com/a/600612
348def mkdir_p(p):
349    try:
350        os.makedirs(p)
351    except OSError as exc:  # Python >2.5
352        if exc.errno == errno.EEXIST and path.isdir(p):
353            pass
354        else:
355            raise
356
357
358def abspath(p):
359    return path.abspath(path.expanduser(p))
360
361
362def main(argv):
363    parser = argparse.ArgumentParser(
364        description="Auto-format F90 source files",
365        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
366        epilog="""\
367            If no files are given, stdin is used.
368            Note: for editor integration, use options --no-normalize-use --no-report-errors""",
369    )
370
371    parser.add_argument("--indent", type=int, default=3)
372    parser.add_argument("--whitespace", type=int, default=2, choices=range(0, 5))
373    parser.add_argument("--decl-linelength", type=int, default=100)
374    parser.add_argument("--decl-offset", type=int, default=50)
375    parser.add_argument("--backup-dir", type=abspath, default=abspath("preprettify"))
376
377    argparse_add_bool_arg(parser, "upcase", True, "Upcasing fortran keywords."),
378    argparse_add_bool_arg(
379        parser,
380        "normalize-use",
381        True,
382        """\
383        Sorting and alignment of variable declarations and USE statements, removal of unused list entries.
384        The line length of declarations is controlled by --decl-linelength=n, the offset of the variable list
385        is controlled by --decl-offset=n.""",
386    )
387    argparse_add_bool_arg(parser, "omp-upcase", True, "Upcasing OMP directives.")
388    argparse_add_bool_arg(
389        parser,
390        "reformat",
391        True,
392        """\
393        Auto-indentation, auto-alignment and whitespace formatting.
394        Amount of whitespace controlled by --whitespace = 0, 1, 2.
395        For indenting with a relative width of n columns specify --indent=n.
396        For manual formatting of specific lines:
397        - disable auto-alignment by starting line continuation with an ampersand '&'.
398        - completely disable reformatting by adding a comment '!&'.
399        For manual formatting of a code block, use:
400        - start a manually formatted block with a '!&<' comment and close it with a '!&>' comment.""",
401    )
402    argparse_add_bool_arg(
403        parser,
404        "replace",
405        True,
406        "If requested the replacements performed by the replacer.py script are also performed. Note: these replacements are specific to CP2K.",
407    )
408    argparse_add_bool_arg(parser, "stdout", False, "write output to stdout")
409    argparse_add_bool_arg(
410        parser,
411        "do-backup",
412        False,
413        "store backups of original files in backup-dir (--backup-dir option)",
414    )
415    argparse_add_bool_arg(
416        parser, "quiet", False, "don't show summary or list of reformatted files"
417    )
418    argparse_add_bool_arg(parser, "report-errors", True, "report warnings and errors")
419    argparse_add_bool_arg(parser, "debug", False, "increase log level to debug")
420
421    parser.add_argument("files", metavar="file", type=str, nargs="*", default=["stdin"])
422
423    args = parser.parse_args(argv)
424
425    if args.do_backup and not (args.stdout or args.files == ["stdin"]):
426        mkdir_p(args.backup_dir)
427
428    level = logging.CRITICAL
429
430    if args.report_errors:
431        if args.debug:
432            level = logging.DEBUG
433        else:
434            level = logging.ERROR
435
436    # the fprettify logger provides filename and line number in case of errors
437    shandler = logging.StreamHandler()
438    shandler.setLevel(level)
439    shandler.setFormatter(
440        logging.Formatter("%(levelname)s %(ffilename)s:%(fline)s: %(message)s")
441    )
442
443    fprettify_logger = logging.getLogger("fprettify-logger")
444    fprettify_logger.setLevel(level)
445    fprettify_logger.addHandler(shandler)
446
447    # the prettify_cp2k loggers only provide the filename in their messages
448    shandler = logging.StreamHandler()
449    shandler.setLevel(level)
450    shandler.setFormatter(logging.Formatter("%(levelname)s %(ffilename)s: %(message)s"))
451
452    logger = logging.getLogger("prettify-logger")
453    logger.setLevel(level)
454    logger.addHandler(shandler)
455
456    failure = 0
457    total_prettified = 0
458
459    for filename in args.files:
460        if not path.isfile(filename) and not filename == "stdin":
461            logger.error(f"file '{filename}' does not exist")
462            failure += 1
463            continue
464
465        try:
466            changed = prettifyInplace(
467                filename,
468                backupdir=args.backup_dir if args.do_backup else None,
469                stdout=args.stdout or filename == "stdin",
470                normalize_use=args.normalize_use,
471                decl_linelength=args.decl_linelength,
472                decl_offset=args.decl_offset,
473                reformat=args.reformat,
474                indent=args.indent,
475                whitespace=args.whitespace,
476                upcase_keywords=args.upcase,
477                upcase_omp=args.omp_upcase,
478                replace=args.replace,
479            )
480
481            if changed and not args.quiet:
482                print(f"prettified {filename}")
483                total_prettified += 1
484
485        except:
486            logger.exception("processing file failed", extra={"ffilename": filename})
487            failure += 1
488
489    if not (args.stdout or filename == "stdin" or args.quiet):
490        print(
491            f"{total_prettified} file(s) prettified, {len(args.files) - total_prettified} file(s) left unchanged."
492        )
493
494    return failure > 0
495
496
497if __name__ == "__main__":
498    sys.exit(main(sys.argv[1:]))
499