1#!/usr/bin/env python 2 3from __future__ import print_function 4 5import sys 6import re 7import tempfile 8import os 9from os import path 10import logging 11import argparse 12import errno 13import traceback 14 15try: 16 from hashlib import md5 17except ImportError: 18 from md5 import new as md5 19 20try: 21 from cStringIO import StringIO 22except ImportError: 23 from io import StringIO 24 25from prettify_cp2k import normalizeFortranFile 26from prettify_cp2k import replacer 27 28sys.path.insert(0, path.join(path.dirname(path.abspath(__file__)), "fprettify")) 29from fprettify import reformat_ffile, fparse_utils, log_exception 30 31 32TO_UPCASE_RE = re.compile( 33 r""" 34(?P<toUpcase> 35 \.(?:and|eq|eqv|false|ge|gt|le|lt|ne|neqv|not|or|true)\. 36 | 37 (?<![\w%#]) # do not match stmts/intrinsics midword or called as type-bound procedures 38 (?<!%\ ) # do not match stmts/intrinsics when being called as type-bound procedure with a space in between 39 (?<!subroutine\ )(?<!function\ )(?<!::\ ) # do not match stmts/intrinsics when used as procedure names 40 (?: 41 (?: # statements: 42 a(?:llocat(?:able|e)|ssign(?:|ment)) 43 |c(?:a(?:ll|se)|haracter|lose|o(?:m(?:mon|plex)|nt(?:ains|inue))|ycle) 44 |d(?:ata|eallocate|imension|o(?:|uble)) 45 |e(?:lse(?:|if|where)|n(?:d(?:|do|file|if)|try)|quivalence|x(?:it|ternal)) 46 |f(?:or(?:all|mat)|unction) 47 |goto 48 |i(?:f|mplicit|n(?:clude|quire|t(?:e(?:ger|nt|rface)|rinsic))) 49 |logical 50 |module 51 |n(?:amelist|one|ullify) 52 |o(?:nly|p(?:en|erator|tional)) 53 |p(?:a(?:rameter|use)|ointer|r(?:ecision|i(?:nt|vate)|o(?:cedure|gram))|ublic) 54 |re(?:a[dl]|cursive|sult|turn|wind) 55 |s(?:ave|e(?:lect|quence)|top|ubroutine) 56 |t(?:arget|hen|ype) 57 |use 58 |w(?:h(?:ere|ile)|rite) 59 ) 60 | (?: # intrinsic functions: 61 a(?:bs|c(?:har|os)|djust[lr]|i(?:mag|nt)|ll(?:|ocated)|n(?:int|y)|s(?:in|sociated)|tan2?) 62 |b(?:it_size|test) 63 |c(?:eiling|har|mplx|o(?:njg|sh?|unt)|shift) 64 |d(?:ate_and_time|ble|i(?:gits|m)|ot_product|prod) 65 |e(?:oshift|psilon|xp(?:|onent)) 66 |f(?:loor|raction) 67 |huge 68 |i(?:a(?:char|nd)|b(?:clr|its|set)|char|eor|n(?:dex|t)|or|shftc?) 69 |kind 70 |l(?:bound|en(?:|_trim)|g[et]|l[et]|og(?:|10|ical)) 71 |m(?:a(?:tmul|x(?:|exponent|loc|val))|erge|in(?:|exponent|loc|val)|od(?:|ulo)|vbits) 72 |n(?:earest|int|ot) 73 |p(?:ack|r(?:e(?:cision|sent)|oduct)) 74 |r(?:a(?:dix|n(?:dom_(?:number|seed)|ge))|e(?:peat|shape)|rspacing) 75 |s(?:ca(?:le|n)|e(?:lected_(?:int_kind|real_kind)|t_exponent)|hape|i(?:gn|nh?|ze)|p(?:acing|read)|qrt|um|ystem_clock) 76 |t(?:anh?|iny|r(?:ans(?:fer|pose)|im)) 77 |u(?:bound|npack) 78 |verify 79 ) (?=\ *\() 80 ) 81 (?![\w%]) 82) 83""", 84 flags=re.IGNORECASE | re.VERBOSE, 85) 86 87TO_UPCASE_OMP_RE = re.compile( 88 r""" 89(?<![\w%#]) 90(?P<toUpcase> 91 (?: 92 atomic|barrier|c(?:apture|ritical)|do|end|flush|if|master|num_threads|ordered|parallel|read 93 |s(?:ection(?:|s)|ingle)|t(?:ask(?:|wait|yield)|hreadprivate)|update|w(?:orkshare|rite)|!\$omp 94 ) 95 | (?: 96 a|co(?:llapse|py(?:in|private))|default|fi(?:nal|rstprivate)|i(?:and|eor|or)|lastprivate 97 |m(?:ax|ergeable|in)|n(?:one|owait)|ordered|private|reduction|shared|untied|\.(?:and|eqv|neqv|or)\. 98 ) 99 | omp_(?:dynamic|max_active_levels|n(?:ested|um_threads)|proc_bind|s(?:tacksize|chedule)|thread_limit|wait_policy) 100) 101(?![\w%]) 102""", 103 flags=re.IGNORECASE | re.VERBOSE, 104) 105 106LINE_PARTS_RE = re.compile( 107 r""" 108 (?P<commands>[^\"'!]*) 109 (?P<comment>!.*)? 110 (?P<string> 111 (?P<qchar>[\"']) 112 .*? 113 (?P=qchar))? 114""", 115 re.VERBOSE, 116) 117 118 119def upcaseStringKeywords(line): 120 """Upcases the fortran keywords, operators and intrinsic routines 121 in line""" 122 res = "" 123 start = 0 124 while start < len(line): 125 m = LINE_PARTS_RE.match(line[start:]) 126 if not m: 127 raise SyntaxError("Syntax error, open string") 128 res = res + TO_UPCASE_RE.sub( 129 lambda match: match.group("toUpcase").upper(), m.group("commands") 130 ) 131 if m.group("comment"): 132 res = res + m.group("comment") 133 if m.group("string"): 134 res = res + m.group("string") 135 start = start + m.end() 136 return res 137 138 139def upcaseKeywords(infile, outfile, upcase_omp): 140 """Writes infile to outfile with all the fortran keywords upcased""" 141 142 for line in infile: 143 line = upcaseStringKeywords(line) 144 145 if upcase_omp and normalizeFortranFile.OMP_DIR_RE.match(line): 146 line = TO_UPCASE_OMP_RE.sub( 147 lambda match: match.group("toUpcase").upper(), line 148 ) 149 150 outfile.write(line) 151 152 153def prettifyFile( 154 infile, 155 filename, 156 normalize_use, 157 decl_linelength, 158 decl_offset, 159 reformat, 160 indent, 161 whitespace, 162 upcase_keywords, 163 upcase_omp, 164 replace, 165): 166 """prettifyes the fortran source in infile into a temporary file that is 167 returned. It can be the same as infile. 168 if normalize_use normalizes the use statements (defaults to true) 169 if upcase_keywords upcases the keywords (defaults to true) 170 if replace does the replacements contained in replacer.py (defaults 171 to false) 172 173 does not close the input file""" 174 max_pretty_iter = 5 175 176 logger = logging.getLogger("fprettify-logger") 177 178 if is_fypp(infile): 179 logger.warning( 180 "fypp directives not fully supported, running only fprettify", 181 extra={"ffilename": filename, "fline": 0}, 182 ) 183 replace = False 184 normalize_use = False 185 upcase_keywords = False 186 187 # create a temporary file first as a copy of the input file 188 inbuf = StringIO(infile.read()) 189 190 hash_prev = md5(inbuf.getvalue().encode("utf8")) 191 192 for _ in range(max_pretty_iter): 193 try: 194 if replace: 195 outbuf = StringIO() 196 replacer.replaceWords(inbuf, outbuf) 197 outbuf.seek(0) 198 inbuf.close() 199 inbuf = outbuf 200 201 if reformat: # reformat needs to be done first 202 outbuf = StringIO() 203 try: 204 reformat_ffile( 205 inbuf, 206 outbuf, 207 indent_size=indent, 208 whitespace=whitespace, 209 orig_filename=filename, 210 ) 211 except fparse_utils.FprettifyParseException as e: 212 log_exception( 213 e, "fprettify could not parse file, file is not prettified" 214 ) 215 outbuf.close() 216 inbuf.seek(0) 217 else: 218 outbuf.seek(0) 219 inbuf.close() 220 inbuf = outbuf 221 222 normalize_use_succeeded = True 223 224 if normalize_use: 225 outbuf = StringIO() 226 try: 227 normalizeFortranFile.rewriteFortranFile( 228 inbuf, 229 outbuf, 230 indent, 231 decl_linelength, 232 decl_offset, 233 orig_filename=filename, 234 ) 235 except normalizeFortranFile.InputStreamError as exc: 236 logger.error( 237 "normalizeFortranFile could not parse file, file is not normalized", 238 extra={"ffilename": filename, "fline": 0}, 239 ) 240 outbuf.close() 241 inbuf.seek(0) 242 normalize_use_succeeded = False 243 else: 244 outbuf.seek(0) 245 inbuf.close() 246 inbuf = outbuf 247 248 if upcase_keywords and normalize_use_succeeded: 249 outbuf = StringIO() 250 upcaseKeywords(inbuf, outbuf, upcase_omp) 251 outbuf.seek(0) 252 inbuf.close() 253 inbuf = outbuf 254 255 hash_new = md5(inbuf.getvalue().encode("utf8")) 256 257 if hash_prev.digest() == hash_new.digest(): 258 return inbuf 259 260 hash_prev = hash_new 261 262 except: 263 logger.critical( 264 "error processing file", extra={"ffilename": filename, "fline": 0} 265 ) 266 raise 267 268 else: 269 raise RuntimeError( 270 "Prettify did not converge in {} steps.".format(max_pretty_iter) 271 ) 272 273 274def prettifyInplace(filename, backupdir=None, stdout=False, **kwargs): 275 """Same as prettify, but inplace, replaces only if needed""" 276 277 if filename == "stdin": 278 infile = tempfile.TemporaryFile(mode="r+") 279 infile.write(sys.stdin.read()) 280 infile.seek(0) 281 else: 282 infile = open(filename, "r") 283 284 outfile = prettifyFile(infile=infile, filename=filename, **kwargs) 285 286 if stdout: 287 outfile.seek(0) 288 sys.stdout.write(outfile.read()) 289 outfile.close() 290 return 291 292 if infile == outfile: 293 infile.close() 294 return 295 296 infile.seek(0) 297 outfile.seek(0) 298 changed = True 299 for line1, line2 in zip(infile, outfile): 300 if line1 != line2: 301 break 302 else: 303 changed = False 304 305 if changed: 306 if backupdir: 307 bkName = path.join(backupdir, path.basename(filename)) 308 309 with open(bkName, "w") as fhandle: 310 infile.seek(0) 311 fhandle.write(infile.read()) 312 313 infile.close() # close it here since we're going to overwrite it 314 315 with open(filename, "w") as fhandle: 316 outfile.seek(0) 317 fhandle.write(outfile.read()) 318 319 else: 320 infile.close() 321 322 outfile.close() 323 324 325def is_fypp(infile): 326 FYPP_SYMBOLS = r"(#|\$|@)" 327 FYPP_LINE = r"^\s*" + FYPP_SYMBOLS + r":" 328 FYPP_INLINE = r"(" + FYPP_SYMBOLS + r"{|}" + FYPP_SYMBOLS + r")" 329 FYPP_RE = re.compile(r"(" + FYPP_LINE + r"|" + FYPP_INLINE + r")") 330 331 infile.seek(0) 332 for line in infile.readlines(): 333 if FYPP_RE.search(line): 334 return True 335 336 infile.seek(0) 337 return False 338 339 340# based on https://stackoverflow.com/a/31347222 341def argparse_add_bool_arg(parser, name, default, helptxt): 342 dname = name.replace("-", "_") 343 group = parser.add_mutually_exclusive_group(required=False) 344 group.add_argument( 345 "--{}".format(name), dest=dname, action="store_true", help=helptxt 346 ) 347 group.add_argument("--no-{}".format(name), dest=dname, action="store_false") 348 parser.set_defaults(**{dname: default}) 349 350 351# from https://stackoverflow.com/a/600612 352def mkdir_p(p): 353 try: 354 os.makedirs(p) 355 except OSError as exc: # Python >2.5 356 if exc.errno == errno.EEXIST and path.isdir(p): 357 pass 358 else: 359 raise 360 361 362# from https://stackoverflow.com/a/14981125 363def eprint(*args, **kwargs): 364 print(*args, file=sys.stderr, **kwargs) 365 366 367def abspath(p): 368 return path.abspath(path.expanduser(p)) 369 370 371def main(argv): 372 parser = argparse.ArgumentParser( 373 description="Auto-format F90 source files", 374 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 375 epilog="""\ 376 If no files are given, stdin is used. 377 Note: for editor integration, use options --no-normalize-use --no-report-errors""", 378 ) 379 380 parser.add_argument("--indent", type=int, default=3) 381 parser.add_argument("--whitespace", type=int, default=2, choices=range(0, 5)) 382 parser.add_argument("--decl-linelength", type=int, default=100) 383 parser.add_argument("--decl-offset", type=int, default=50) 384 parser.add_argument("--backup-dir", type=abspath, default=abspath("preprettify")) 385 386 argparse_add_bool_arg(parser, "upcase", True, "Upcasing fortran keywords."), 387 argparse_add_bool_arg( 388 parser, 389 "normalize-use", 390 True, 391 """\ 392 Sorting and alignment of variable declarations and USE statements, removal of unused list entries. 393 The line length of declarations is controlled by --decl-linelength=n, the offset of the variable list 394 is controlled by --decl-offset=n.""", 395 ) 396 argparse_add_bool_arg(parser, "omp-upcase", True, "Upcasing OMP directives.") 397 argparse_add_bool_arg( 398 parser, 399 "reformat", 400 True, 401 """\ 402 Auto-indentation, auto-alignment and whitespace formatting. 403 Amount of whitespace controlled by --whitespace = 0, 1, 2. 404 For indenting with a relative width of n columns specify --indent=n. 405 For manual formatting of specific lines: 406 - disable auto-alignment by starting line continuation with an ampersand '&'. 407 - completely disable reformatting by adding a comment '!&'. 408 For manual formatting of a code block, use: 409 - start a manually formatted block with a '!&<' comment and close it with a '!&>' comment.""", 410 ) 411 argparse_add_bool_arg( 412 parser, 413 "replace", 414 True, 415 "If requested the replacements performed by the replacer.py script are also performed. Note: these replacements are specific to CP2K.", 416 ) 417 argparse_add_bool_arg(parser, "stdout", False, "write output to stdout") 418 argparse_add_bool_arg( 419 parser, 420 "do-backup", 421 False, 422 "store backups of original files in backup-dir (--backup-dir option)", 423 ) 424 argparse_add_bool_arg(parser, "report-errors", True, "report warnings and errors") 425 argparse_add_bool_arg(parser, "debug", False, "increase log level to debug") 426 427 parser.add_argument("files", metavar="file", type=str, nargs="*", default=["stdin"]) 428 429 args = parser.parse_args(argv) 430 431 if args.do_backup and not (args.stdout or args.files == ["stdin"]): 432 mkdir_p(args.backup_dir) 433 434 failure = 0 435 436 for filename in args.files: 437 if not path.isfile(filename) and not filename == "stdin": 438 eprint("file '{}' does not exist!".format(filename)) 439 failure += 1 440 continue 441 442 level = logging.CRITICAL 443 444 if args.report_errors: 445 if args.debug: 446 level = logging.DEBUG 447 else: 448 level = logging.INFO 449 450 logger = logging.getLogger("fprettify-logger") 451 logger.setLevel(level) 452 sh = logging.StreamHandler() 453 sh.setLevel(level) 454 formatter = logging.Formatter( 455 "%(levelname)s %(ffilename)s:%(fline)s: %(message)s" 456 ) 457 sh.setFormatter(formatter) 458 logger.addHandler(sh) 459 460 try: 461 prettifyInplace( 462 filename, 463 backupdir=args.backup_dir if args.do_backup else None, 464 stdout=args.stdout or filename == "stdin", 465 normalize_use=args.normalize_use, 466 decl_linelength=args.decl_linelength, 467 decl_offset=args.decl_offset, 468 reformat=args.reformat, 469 indent=args.indent, 470 whitespace=args.whitespace, 471 upcase_keywords=args.upcase, 472 upcase_omp=args.omp_upcase, 473 replace=args.replace, 474 ) 475 except: 476 eprint("-" * 60) 477 traceback.print_exc(file=sys.stderr) 478 eprint("-" * 60) 479 eprint("Processing file '{}'".format(filename)) 480 failure += 1 481 482 return failure > 0 483 484 485if __name__ == "__main__": 486 sys.exit(main(sys.argv[1:])) 487