1#!/usr/bin/env python
2
3from __future__ import print_function
4
5import sys
6import re
7import tempfile
8import os
9from os import path
10import logging
11import argparse
12import errno
13import traceback
14
15try:
16    from hashlib import md5
17except ImportError:
18    from md5 import new as md5
19
20try:
21    from cStringIO import StringIO
22except ImportError:
23    from io import StringIO
24
25from prettify_cp2k import normalizeFortranFile
26from prettify_cp2k import replacer
27
28sys.path.insert(0, path.join(path.dirname(path.abspath(__file__)), "fprettify"))
29from fprettify import reformat_ffile, fparse_utils, log_exception
30
31
32TO_UPCASE_RE = re.compile(
33    r"""
34(?P<toUpcase>
35    \.(?:and|eq|eqv|false|ge|gt|le|lt|ne|neqv|not|or|true)\.
36    |
37    (?<![\w%#])  # do not match stmts/intrinsics midword or called as type-bound procedures
38    (?<!%\ )  # do not match stmts/intrinsics when being called as type-bound procedure with a space in between
39    (?<!subroutine\ )(?<!function\ )(?<!::\ )  # do not match stmts/intrinsics when used as procedure names
40    (?:
41        (?: # statements:
42            a(?:llocat(?:able|e)|ssign(?:|ment))
43            |c(?:a(?:ll|se)|haracter|lose|o(?:m(?:mon|plex)|nt(?:ains|inue))|ycle)
44            |d(?:ata|eallocate|imension|o(?:|uble))
45            |e(?:lse(?:|if|where)|n(?:d(?:|do|file|if)|try)|quivalence|x(?:it|ternal))
46            |f(?:or(?:all|mat)|unction)
47            |goto
48            |i(?:f|mplicit|n(?:clude|quire|t(?:e(?:ger|nt|rface)|rinsic)))
49            |logical
50            |module
51            |n(?:amelist|one|ullify)
52            |o(?:nly|p(?:en|erator|tional))
53            |p(?:a(?:rameter|use)|ointer|r(?:ecision|i(?:nt|vate)|o(?:cedure|gram))|ublic)
54            |re(?:a[dl]|cursive|sult|turn|wind)
55            |s(?:ave|e(?:lect|quence)|top|ubroutine)
56            |t(?:arget|hen|ype)
57            |use
58            |w(?:h(?:ere|ile)|rite)
59        )
60        | (?: # intrinsic functions:
61            a(?:bs|c(?:har|os)|djust[lr]|i(?:mag|nt)|ll(?:|ocated)|n(?:int|y)|s(?:in|sociated)|tan2?)
62            |b(?:it_size|test)
63            |c(?:eiling|har|mplx|o(?:njg|sh?|unt)|shift)
64            |d(?:ate_and_time|ble|i(?:gits|m)|ot_product|prod)
65            |e(?:oshift|psilon|xp(?:|onent))
66            |f(?:loor|raction)
67            |huge
68            |i(?:a(?:char|nd)|b(?:clr|its|set)|char|eor|n(?:dex|t)|or|shftc?)
69            |kind
70            |l(?:bound|en(?:|_trim)|g[et]|l[et]|og(?:|10|ical))
71            |m(?:a(?:tmul|x(?:|exponent|loc|val))|erge|in(?:|exponent|loc|val)|od(?:|ulo)|vbits)
72            |n(?:earest|int|ot)
73            |p(?:ack|r(?:e(?:cision|sent)|oduct))
74            |r(?:a(?:dix|n(?:dom_(?:number|seed)|ge))|e(?:peat|shape)|rspacing)
75            |s(?:ca(?:le|n)|e(?:lected_(?:int_kind|real_kind)|t_exponent)|hape|i(?:gn|nh?|ze)|p(?:acing|read)|qrt|um|ystem_clock)
76            |t(?:anh?|iny|r(?:ans(?:fer|pose)|im))
77            |u(?:bound|npack)
78            |verify
79        ) (?=\ *\()
80    )
81    (?![\w%])
82)
83""",
84    flags=re.IGNORECASE | re.VERBOSE,
85)
86
87TO_UPCASE_OMP_RE = re.compile(
88    r"""
89(?<![\w%#])
90(?P<toUpcase>
91    (?:
92        atomic|barrier|c(?:apture|ritical)|do|end|flush|if|master|num_threads|ordered|parallel|read
93        |s(?:ection(?:|s)|ingle)|t(?:ask(?:|wait|yield)|hreadprivate)|update|w(?:orkshare|rite)|!\$omp
94    )
95    | (?:
96        a|co(?:llapse|py(?:in|private))|default|fi(?:nal|rstprivate)|i(?:and|eor|or)|lastprivate
97        |m(?:ax|ergeable|in)|n(?:one|owait)|ordered|private|reduction|shared|untied|\.(?:and|eqv|neqv|or)\.
98    )
99    | omp_(?:dynamic|max_active_levels|n(?:ested|um_threads)|proc_bind|s(?:tacksize|chedule)|thread_limit|wait_policy)
100)
101(?![\w%])
102""",
103    flags=re.IGNORECASE | re.VERBOSE,
104)
105
106LINE_PARTS_RE = re.compile(
107    r"""
108    (?P<commands>[^\"'!]*)
109    (?P<comment>!.*)?
110    (?P<string>
111        (?P<qchar>[\"'])
112        .*?
113        (?P=qchar))?
114""",
115    re.VERBOSE,
116)
117
118
119def upcaseStringKeywords(line):
120    """Upcases the fortran keywords, operators and intrinsic routines
121    in line"""
122    res = ""
123    start = 0
124    while start < len(line):
125        m = LINE_PARTS_RE.match(line[start:])
126        if not m:
127            raise SyntaxError("Syntax error, open string")
128        res = res + TO_UPCASE_RE.sub(
129            lambda match: match.group("toUpcase").upper(), m.group("commands")
130        )
131        if m.group("comment"):
132            res = res + m.group("comment")
133        if m.group("string"):
134            res = res + m.group("string")
135        start = start + m.end()
136    return res
137
138
139def upcaseKeywords(infile, outfile, upcase_omp):
140    """Writes infile to outfile with all the fortran keywords upcased"""
141
142    for line in infile:
143        line = upcaseStringKeywords(line)
144
145        if upcase_omp and normalizeFortranFile.OMP_DIR_RE.match(line):
146            line = TO_UPCASE_OMP_RE.sub(
147                lambda match: match.group("toUpcase").upper(), line
148            )
149
150        outfile.write(line)
151
152
153def prettifyFile(
154    infile,
155    filename,
156    normalize_use,
157    decl_linelength,
158    decl_offset,
159    reformat,
160    indent,
161    whitespace,
162    upcase_keywords,
163    upcase_omp,
164    replace,
165):
166    """prettifyes the fortran source in infile into a temporary file that is
167    returned. It can be the same as infile.
168    if normalize_use normalizes the use statements (defaults to true)
169    if upcase_keywords upcases the keywords (defaults to true)
170    if replace does the replacements contained in replacer.py (defaults
171    to false)
172
173    does not close the input file"""
174    max_pretty_iter = 5
175
176    logger = logging.getLogger("fprettify-logger")
177
178    if is_fypp(infile):
179        logger.warning(
180            "fypp directives not fully supported, running only fprettify",
181            extra={"ffilename": filename, "fline": 0},
182        )
183        replace = False
184        normalize_use = False
185        upcase_keywords = False
186
187    # create a temporary file first as a copy of the input file
188    inbuf = StringIO(infile.read())
189
190    hash_prev = md5(inbuf.getvalue().encode("utf8"))
191
192    for _ in range(max_pretty_iter):
193        try:
194            if replace:
195                outbuf = StringIO()
196                replacer.replaceWords(inbuf, outbuf)
197                outbuf.seek(0)
198                inbuf.close()
199                inbuf = outbuf
200
201            if reformat:  # reformat needs to be done first
202                outbuf = StringIO()
203                try:
204                    reformat_ffile(
205                        inbuf,
206                        outbuf,
207                        indent_size=indent,
208                        whitespace=whitespace,
209                        orig_filename=filename,
210                    )
211                except fparse_utils.FprettifyParseException as e:
212                    log_exception(
213                        e, "fprettify could not parse file, file is not prettified"
214                    )
215                    outbuf.close()
216                    inbuf.seek(0)
217                else:
218                    outbuf.seek(0)
219                    inbuf.close()
220                    inbuf = outbuf
221
222            normalize_use_succeeded = True
223
224            if normalize_use:
225                outbuf = StringIO()
226                try:
227                    normalizeFortranFile.rewriteFortranFile(
228                        inbuf,
229                        outbuf,
230                        indent,
231                        decl_linelength,
232                        decl_offset,
233                        orig_filename=filename,
234                    )
235                except normalizeFortranFile.InputStreamError as exc:
236                    logger.error(
237                        "normalizeFortranFile could not parse file, file is not normalized",
238                        extra={"ffilename": filename, "fline": 0},
239                    )
240                    outbuf.close()
241                    inbuf.seek(0)
242                    normalize_use_succeeded = False
243                else:
244                    outbuf.seek(0)
245                    inbuf.close()
246                    inbuf = outbuf
247
248            if upcase_keywords and normalize_use_succeeded:
249                outbuf = StringIO()
250                upcaseKeywords(inbuf, outbuf, upcase_omp)
251                outbuf.seek(0)
252                inbuf.close()
253                inbuf = outbuf
254
255            hash_new = md5(inbuf.getvalue().encode("utf8"))
256
257            if hash_prev.digest() == hash_new.digest():
258                return inbuf
259
260            hash_prev = hash_new
261
262        except:
263            logger.critical(
264                "error processing file", extra={"ffilename": filename, "fline": 0}
265            )
266            raise
267
268    else:
269        raise RuntimeError(
270            "Prettify did not converge in {} steps.".format(max_pretty_iter)
271        )
272
273
274def prettifyInplace(filename, backupdir=None, stdout=False, **kwargs):
275    """Same as prettify, but inplace, replaces only if needed"""
276
277    if filename == "stdin":
278        infile = tempfile.TemporaryFile(mode="r+")
279        infile.write(sys.stdin.read())
280        infile.seek(0)
281    else:
282        infile = open(filename, "r")
283
284    outfile = prettifyFile(infile=infile, filename=filename, **kwargs)
285
286    if stdout:
287        outfile.seek(0)
288        sys.stdout.write(outfile.read())
289        outfile.close()
290        return
291
292    if infile == outfile:
293        infile.close()
294        return
295
296    infile.seek(0)
297    outfile.seek(0)
298    changed = True
299    for line1, line2 in zip(infile, outfile):
300        if line1 != line2:
301            break
302    else:
303        changed = False
304
305    if changed:
306        if backupdir:
307            bkName = path.join(backupdir, path.basename(filename))
308
309            with open(bkName, "w") as fhandle:
310                infile.seek(0)
311                fhandle.write(infile.read())
312
313        infile.close()  # close it here since we're going to overwrite it
314
315        with open(filename, "w") as fhandle:
316            outfile.seek(0)
317            fhandle.write(outfile.read())
318
319    else:
320        infile.close()
321
322    outfile.close()
323
324
325def is_fypp(infile):
326    FYPP_SYMBOLS = r"(#|\$|@)"
327    FYPP_LINE = r"^\s*" + FYPP_SYMBOLS + r":"
328    FYPP_INLINE = r"(" + FYPP_SYMBOLS + r"{|}" + FYPP_SYMBOLS + r")"
329    FYPP_RE = re.compile(r"(" + FYPP_LINE + r"|" + FYPP_INLINE + r")")
330
331    infile.seek(0)
332    for line in infile.readlines():
333        if FYPP_RE.search(line):
334            return True
335
336    infile.seek(0)
337    return False
338
339
340# based on https://stackoverflow.com/a/31347222
341def argparse_add_bool_arg(parser, name, default, helptxt):
342    dname = name.replace("-", "_")
343    group = parser.add_mutually_exclusive_group(required=False)
344    group.add_argument(
345        "--{}".format(name), dest=dname, action="store_true", help=helptxt
346    )
347    group.add_argument("--no-{}".format(name), dest=dname, action="store_false")
348    parser.set_defaults(**{dname: default})
349
350
351# from https://stackoverflow.com/a/600612
352def mkdir_p(p):
353    try:
354        os.makedirs(p)
355    except OSError as exc:  # Python >2.5
356        if exc.errno == errno.EEXIST and path.isdir(p):
357            pass
358        else:
359            raise
360
361
362# from https://stackoverflow.com/a/14981125
363def eprint(*args, **kwargs):
364    print(*args, file=sys.stderr, **kwargs)
365
366
367def abspath(p):
368    return path.abspath(path.expanduser(p))
369
370
371def main(argv):
372    parser = argparse.ArgumentParser(
373        description="Auto-format F90 source files",
374        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
375        epilog="""\
376            If no files are given, stdin is used.
377            Note: for editor integration, use options --no-normalize-use --no-report-errors""",
378    )
379
380    parser.add_argument("--indent", type=int, default=3)
381    parser.add_argument("--whitespace", type=int, default=2, choices=range(0, 5))
382    parser.add_argument("--decl-linelength", type=int, default=100)
383    parser.add_argument("--decl-offset", type=int, default=50)
384    parser.add_argument("--backup-dir", type=abspath, default=abspath("preprettify"))
385
386    argparse_add_bool_arg(parser, "upcase", True, "Upcasing fortran keywords."),
387    argparse_add_bool_arg(
388        parser,
389        "normalize-use",
390        True,
391        """\
392        Sorting and alignment of variable declarations and USE statements, removal of unused list entries.
393        The line length of declarations is controlled by --decl-linelength=n, the offset of the variable list
394        is controlled by --decl-offset=n.""",
395    )
396    argparse_add_bool_arg(parser, "omp-upcase", True, "Upcasing OMP directives.")
397    argparse_add_bool_arg(
398        parser,
399        "reformat",
400        True,
401        """\
402        Auto-indentation, auto-alignment and whitespace formatting.
403        Amount of whitespace controlled by --whitespace = 0, 1, 2.
404        For indenting with a relative width of n columns specify --indent=n.
405        For manual formatting of specific lines:
406        - disable auto-alignment by starting line continuation with an ampersand '&'.
407        - completely disable reformatting by adding a comment '!&'.
408        For manual formatting of a code block, use:
409        - start a manually formatted block with a '!&<' comment and close it with a '!&>' comment.""",
410    )
411    argparse_add_bool_arg(
412        parser,
413        "replace",
414        True,
415        "If requested the replacements performed by the replacer.py script are also performed. Note: these replacements are specific to CP2K.",
416    )
417    argparse_add_bool_arg(parser, "stdout", False, "write output to stdout")
418    argparse_add_bool_arg(
419        parser,
420        "do-backup",
421        False,
422        "store backups of original files in backup-dir (--backup-dir option)",
423    )
424    argparse_add_bool_arg(parser, "report-errors", True, "report warnings and errors")
425    argparse_add_bool_arg(parser, "debug", False, "increase log level to debug")
426
427    parser.add_argument("files", metavar="file", type=str, nargs="*", default=["stdin"])
428
429    args = parser.parse_args(argv)
430
431    if args.do_backup and not (args.stdout or args.files == ["stdin"]):
432        mkdir_p(args.backup_dir)
433
434    failure = 0
435
436    for filename in args.files:
437        if not path.isfile(filename) and not filename == "stdin":
438            eprint("file '{}' does not exist!".format(filename))
439            failure += 1
440            continue
441
442        level = logging.CRITICAL
443
444        if args.report_errors:
445            if args.debug:
446                level = logging.DEBUG
447            else:
448                level = logging.INFO
449
450        logger = logging.getLogger("fprettify-logger")
451        logger.setLevel(level)
452        sh = logging.StreamHandler()
453        sh.setLevel(level)
454        formatter = logging.Formatter(
455            "%(levelname)s %(ffilename)s:%(fline)s: %(message)s"
456        )
457        sh.setFormatter(formatter)
458        logger.addHandler(sh)
459
460        try:
461            prettifyInplace(
462                filename,
463                backupdir=args.backup_dir if args.do_backup else None,
464                stdout=args.stdout or filename == "stdin",
465                normalize_use=args.normalize_use,
466                decl_linelength=args.decl_linelength,
467                decl_offset=args.decl_offset,
468                reformat=args.reformat,
469                indent=args.indent,
470                whitespace=args.whitespace,
471                upcase_keywords=args.upcase,
472                upcase_omp=args.omp_upcase,
473                replace=args.replace,
474            )
475        except:
476            eprint("-" * 60)
477            traceback.print_exc(file=sys.stderr)
478            eprint("-" * 60)
479            eprint("Processing file '{}'".format(filename))
480            failure += 1
481
482    return failure > 0
483
484
485if __name__ == "__main__":
486    sys.exit(main(sys.argv[1:]))
487