1#!/usr/bin/env python3 2 3import sys 4import re 5import tempfile 6import os 7import logging 8import argparse 9import errno 10import contextlib 11from os import path 12from hashlib import md5 13from io import StringIO 14 15from prettify_cp2k import normalizeFortranFile 16from prettify_cp2k import replacer 17 18sys.path.insert(0, path.join(path.dirname(path.abspath(__file__)), "fprettify")) 19from fprettify import reformat_ffile, fparse_utils, log_exception 20 21 22TO_UPCASE_RE = re.compile( 23 r""" 24(?P<toUpcase> 25 \.(?:and|eq|eqv|false|ge|gt|le|lt|ne|neqv|not|or|true)\. 26 | 27 (?<![\w%#]) # do not match stmts/intrinsics midword or called as type-bound procedures 28 (?<!%\ ) # do not match stmts/intrinsics when being called as type-bound procedure with a space in between 29 (?<!subroutine\ )(?<!function\ )(?<!::\ ) # do not match stmts/intrinsics when used as procedure names 30 (?: 31 (?: # statements: 32 a(?:llocat(?:able|e)|ssign(?:|ment)) 33 |c(?:a(?:ll|se)|haracter|lose|o(?:m(?:mon|plex)|nt(?:ains|inue))|ycle) 34 |d(?:ata|eallocate|imension|o(?:|uble)) 35 |e(?:lse(?:|if|where)|n(?:d(?:|do|file|if)|try)|quivalence|x(?:it|ternal)) 36 |f(?:or(?:all|mat)|unction) 37 |goto 38 |i(?:f|mplicit|n(?:clude|quire|t(?:e(?:ger|nt|rface)|rinsic))) 39 |logical 40 |module 41 |n(?:amelist|one|ullify) 42 |o(?:nly|p(?:en|erator|tional)) 43 |p(?:a(?:rameter|use)|ointer|r(?:ecision|i(?:nt|vate)|o(?:cedure|gram))|ublic) 44 |re(?:a[dl]|cursive|sult|turn|wind) 45 |s(?:ave|e(?:lect|quence)|top|ubroutine) 46 |t(?:arget|hen|ype) 47 |use 48 |w(?:h(?:ere|ile)|rite) 49 ) 50 | (?: # intrinsic functions: 51 a(?:bs|c(?:har|os)|djust[lr]|i(?:mag|nt)|ll(?:|ocated)|n(?:int|y)|s(?:in|sociated)|tan2?) 52 |b(?:it_size|test) 53 |c(?:eiling|har|mplx|o(?:njg|sh?|unt)|shift) 54 |d(?:ate_and_time|ble|i(?:gits|m)|ot_product|prod) 55 |e(?:oshift|psilon|xp(?:|onent)) 56 |f(?:loor|raction) 57 |huge 58 |i(?:a(?:char|nd)|b(?:clr|its|set)|char|eor|n(?:dex|t)|or|shftc?) 59 |kind 60 |l(?:bound|en(?:|_trim)|g[et]|l[et]|og(?:|10|ical)) 61 |m(?:a(?:tmul|x(?:|exponent|loc|val))|erge|in(?:|exponent|loc|val)|od(?:|ulo)|vbits) 62 |n(?:earest|int|ot) 63 |p(?:ack|r(?:e(?:cision|sent)|oduct)) 64 |r(?:a(?:dix|n(?:dom_(?:number|seed)|ge))|e(?:peat|shape)|rspacing) 65 |s(?:ca(?:le|n)|e(?:lected_(?:int_kind|real_kind)|t_exponent)|hape|i(?:gn|nh?|ze)|p(?:acing|read)|qrt|um|ystem_clock) 66 |t(?:anh?|iny|r(?:ans(?:fer|pose)|im)) 67 |u(?:bound|npack) 68 |verify 69 ) (?=\ *\() 70 ) 71 (?![\w%]) 72) 73""", 74 flags=re.IGNORECASE | re.VERBOSE, 75) 76 77TO_UPCASE_OMP_RE = re.compile( 78 r""" 79(?<![\w%#]) 80(?P<toUpcase> 81 (?: 82 atomic|barrier|c(?:apture|ritical)|do|end|flush|if|master|num_threads|ordered|parallel|read 83 |s(?:ection(?:|s)|ingle)|t(?:ask(?:|wait|yield)|hreadprivate)|update|w(?:orkshare|rite)|!\$omp 84 ) 85 | (?: 86 a|co(?:llapse|py(?:in|private))|default|fi(?:nal|rstprivate)|i(?:and|eor|or)|lastprivate 87 |m(?:ax|ergeable|in)|n(?:one|owait)|ordered|private|reduction|shared|untied|\.(?:and|eqv|neqv|or)\. 88 ) 89 | omp_(?:dynamic|max_active_levels|n(?:ested|um_threads)|proc_bind|s(?:tacksize|chedule)|thread_limit|wait_policy) 90) 91(?![\w%]) 92""", 93 flags=re.IGNORECASE | re.VERBOSE, 94) 95 96LINE_PARTS_RE = re.compile( 97 r""" 98 (?P<commands>[^\"'!]*) 99 (?P<comment>!.*)? 100 (?P<string> 101 (?P<qchar>[\"']) 102 .*? 103 (?P=qchar))? 104""", 105 re.VERBOSE, 106) 107 108 109@contextlib.contextmanager 110def rewind(fhandle): 111 try: 112 yield fhandle 113 finally: 114 fhandle.seek(0) 115 116 117def upcaseStringKeywords(line): 118 """Upcases the fortran keywords, operators and intrinsic routines 119 in line""" 120 res = "" 121 start = 0 122 while start < len(line): 123 m = LINE_PARTS_RE.match(line[start:]) 124 if not m: 125 raise SyntaxError("Syntax error, open string") 126 res = res + TO_UPCASE_RE.sub( 127 lambda match: match.group("toUpcase").upper(), m.group("commands") 128 ) 129 if m.group("comment"): 130 res = res + m.group("comment") 131 if m.group("string"): 132 res = res + m.group("string") 133 start = start + m.end() 134 return res 135 136 137def upcaseKeywords(infile, outfile, upcase_omp): 138 """Writes infile to outfile with all the fortran keywords upcased""" 139 140 for line in infile: 141 line = upcaseStringKeywords(line) 142 143 if upcase_omp and normalizeFortranFile.OMP_DIR_RE.match(line): 144 line = TO_UPCASE_OMP_RE.sub( 145 lambda match: match.group("toUpcase").upper(), line 146 ) 147 148 outfile.write(line) 149 150 151def prettifyFile( 152 infile, 153 filename, 154 normalize_use, 155 decl_linelength, 156 decl_offset, 157 reformat, 158 indent, 159 whitespace, 160 upcase_keywords, 161 upcase_omp, 162 replace, 163): 164 """prettifyes the fortran source in infile into a temporary file that is 165 returned. It can be the same as infile. 166 if normalize_use normalizes the use statements (defaults to true) 167 if upcase_keywords upcases the keywords (defaults to true) 168 if replace does the replacements contained in replacer.py (defaults 169 to false) 170 171 does not close the input file""" 172 max_pretty_iter = 5 173 174 logger = logging.getLogger("prettify-logger") 175 176 if is_fypp(infile): 177 logger.warning( 178 "fypp directives not fully supported, running only fprettify", 179 extra={"ffilename": filename}, 180 ) 181 replace = False 182 normalize_use = False 183 upcase_keywords = False 184 185 # create a temporary file first as a copy of the input file 186 inbuf = StringIO(infile.read()) 187 188 hash_prev = md5(inbuf.getvalue().encode("utf8")) 189 190 for _ in range(max_pretty_iter): 191 if replace: 192 outbuf = StringIO() 193 replacer.replaceWords(inbuf, outbuf) 194 outbuf.seek(0) 195 inbuf.close() 196 inbuf = outbuf 197 198 if reformat: # reformat needs to be done first 199 outbuf = StringIO() 200 try: 201 reformat_ffile( 202 inbuf, 203 outbuf, 204 indent_size=indent, 205 whitespace=whitespace, 206 orig_filename=filename, 207 ) 208 except fparse_utils.FprettifyParseException as e: 209 log_exception( 210 e, "fprettify could not parse file, file is not prettified" 211 ) 212 outbuf.close() 213 inbuf.seek(0) 214 else: 215 outbuf.seek(0) 216 inbuf.close() 217 inbuf = outbuf 218 219 normalize_use_succeeded = True 220 221 if normalize_use: 222 outbuf = StringIO() 223 try: 224 normalizeFortranFile.rewriteFortranFile( 225 inbuf, 226 outbuf, 227 indent, 228 decl_linelength, 229 decl_offset, 230 orig_filename=filename, 231 ) 232 except normalizeFortranFile.InputStreamError as exc: 233 logger.exception( 234 "normalizeFortranFile could not parse file, file is not normalized", 235 extra={"ffilename": filename}, 236 ) 237 outbuf.close() 238 inbuf.seek(0) 239 normalize_use_succeeded = False 240 else: 241 outbuf.seek(0) 242 inbuf.close() 243 inbuf = outbuf 244 245 if upcase_keywords and normalize_use_succeeded: 246 outbuf = StringIO() 247 upcaseKeywords(inbuf, outbuf, upcase_omp) 248 outbuf.seek(0) 249 inbuf.close() 250 inbuf = outbuf 251 252 hash_new = md5(inbuf.getvalue().encode("utf8")) 253 254 if hash_prev.digest() == hash_new.digest(): 255 return inbuf 256 257 hash_prev = hash_new 258 259 else: 260 raise RuntimeError( 261 "Prettify did not converge in {} steps.".format(max_pretty_iter) 262 ) 263 264 265def prettifyInplace(filename, backupdir=None, stdout=False, **kwargs): 266 """Same as prettify, but inplace, replaces only if needed 267 :return: True if file was modified, False if otherwise. If stdin/stdout, always False. 268 """ 269 270 if filename == "stdin": 271 infile = tempfile.TemporaryFile(mode="r+") 272 infile.write(sys.stdin.read()) 273 infile.seek(0) 274 else: 275 infile = open(filename, "r") 276 277 outfile = prettifyFile(infile=infile, filename=filename, **kwargs) 278 279 if stdout: 280 outfile.seek(0) 281 sys.stdout.write(outfile.read()) 282 outfile.close() 283 return False 284 285 if infile == outfile: 286 infile.close() 287 return False 288 289 infile.seek(0) 290 outfile.seek(0) 291 changed = True 292 for line1, line2 in zip(infile, outfile): 293 if line1 != line2: 294 break 295 else: 296 changed = False 297 298 if changed: 299 if backupdir: 300 bkName = path.join(backupdir, path.basename(filename)) 301 302 with open(bkName, "w") as fhandle: 303 infile.seek(0) 304 fhandle.write(infile.read()) 305 306 infile.close() # close it here since we're going to overwrite it 307 308 with open(filename, "w") as fhandle: 309 outfile.seek(0) 310 fhandle.write(outfile.read()) 311 312 else: 313 infile.close() 314 315 outfile.close() 316 317 return changed 318 319 320def is_fypp(infile): 321 FYPP_SYMBOLS = r"(#|\$|@)" 322 FYPP_LINE = r"^\s*" + FYPP_SYMBOLS + r":" 323 FYPP_INLINE = r"(" + FYPP_SYMBOLS + r"{|}" + FYPP_SYMBOLS + r")" 324 FYPP_RE = re.compile(r"(" + FYPP_LINE + r"|" + FYPP_INLINE + r")") 325 326 infile.seek(0) 327 328 with rewind(infile) as fhandle: 329 for line in fhandle.readlines(): 330 if FYPP_RE.search(line): 331 return True 332 333 return False 334 335 336# based on https://stackoverflow.com/a/31347222 337def argparse_add_bool_arg(parser, name, default, helptxt): 338 dname = name.replace("-", "_") 339 group = parser.add_mutually_exclusive_group(required=False) 340 group.add_argument( 341 "--{}".format(name), dest=dname, action="store_true", help=helptxt 342 ) 343 group.add_argument("--no-{}".format(name), dest=dname, action="store_false") 344 parser.set_defaults(**{dname: default}) 345 346 347# from https://stackoverflow.com/a/600612 348def mkdir_p(p): 349 try: 350 os.makedirs(p) 351 except OSError as exc: # Python >2.5 352 if exc.errno == errno.EEXIST and path.isdir(p): 353 pass 354 else: 355 raise 356 357 358def abspath(p): 359 return path.abspath(path.expanduser(p)) 360 361 362def main(argv): 363 parser = argparse.ArgumentParser( 364 description="Auto-format F90 source files", 365 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 366 epilog="""\ 367 If no files are given, stdin is used. 368 Note: for editor integration, use options --no-normalize-use --no-report-errors""", 369 ) 370 371 parser.add_argument("--indent", type=int, default=3) 372 parser.add_argument("--whitespace", type=int, default=2, choices=range(0, 5)) 373 parser.add_argument("--decl-linelength", type=int, default=100) 374 parser.add_argument("--decl-offset", type=int, default=50) 375 parser.add_argument("--backup-dir", type=abspath, default=abspath("preprettify")) 376 377 argparse_add_bool_arg(parser, "upcase", True, "Upcasing fortran keywords."), 378 argparse_add_bool_arg( 379 parser, 380 "normalize-use", 381 True, 382 """\ 383 Sorting and alignment of variable declarations and USE statements, removal of unused list entries. 384 The line length of declarations is controlled by --decl-linelength=n, the offset of the variable list 385 is controlled by --decl-offset=n.""", 386 ) 387 argparse_add_bool_arg(parser, "omp-upcase", True, "Upcasing OMP directives.") 388 argparse_add_bool_arg( 389 parser, 390 "reformat", 391 True, 392 """\ 393 Auto-indentation, auto-alignment and whitespace formatting. 394 Amount of whitespace controlled by --whitespace = 0, 1, 2. 395 For indenting with a relative width of n columns specify --indent=n. 396 For manual formatting of specific lines: 397 - disable auto-alignment by starting line continuation with an ampersand '&'. 398 - completely disable reformatting by adding a comment '!&'. 399 For manual formatting of a code block, use: 400 - start a manually formatted block with a '!&<' comment and close it with a '!&>' comment.""", 401 ) 402 argparse_add_bool_arg( 403 parser, 404 "replace", 405 True, 406 "If requested the replacements performed by the replacer.py script are also performed. Note: these replacements are specific to CP2K.", 407 ) 408 argparse_add_bool_arg(parser, "stdout", False, "write output to stdout") 409 argparse_add_bool_arg( 410 parser, 411 "do-backup", 412 False, 413 "store backups of original files in backup-dir (--backup-dir option)", 414 ) 415 argparse_add_bool_arg( 416 parser, "quiet", False, "don't show summary or list of reformatted files" 417 ) 418 argparse_add_bool_arg(parser, "report-errors", True, "report warnings and errors") 419 argparse_add_bool_arg(parser, "debug", False, "increase log level to debug") 420 421 parser.add_argument("files", metavar="file", type=str, nargs="*", default=["stdin"]) 422 423 args = parser.parse_args(argv) 424 425 if args.do_backup and not (args.stdout or args.files == ["stdin"]): 426 mkdir_p(args.backup_dir) 427 428 level = logging.CRITICAL 429 430 if args.report_errors: 431 if args.debug: 432 level = logging.DEBUG 433 else: 434 level = logging.ERROR 435 436 # the fprettify logger provides filename and line number in case of errors 437 shandler = logging.StreamHandler() 438 shandler.setLevel(level) 439 shandler.setFormatter( 440 logging.Formatter("%(levelname)s %(ffilename)s:%(fline)s: %(message)s") 441 ) 442 443 fprettify_logger = logging.getLogger("fprettify-logger") 444 fprettify_logger.setLevel(level) 445 fprettify_logger.addHandler(shandler) 446 447 # the prettify_cp2k loggers only provide the filename in their messages 448 shandler = logging.StreamHandler() 449 shandler.setLevel(level) 450 shandler.setFormatter(logging.Formatter("%(levelname)s %(ffilename)s: %(message)s")) 451 452 logger = logging.getLogger("prettify-logger") 453 logger.setLevel(level) 454 logger.addHandler(shandler) 455 456 failure = 0 457 total_prettified = 0 458 459 for filename in args.files: 460 if not path.isfile(filename) and not filename == "stdin": 461 logger.error(f"file '{filename}' does not exist") 462 failure += 1 463 continue 464 465 try: 466 changed = prettifyInplace( 467 filename, 468 backupdir=args.backup_dir if args.do_backup else None, 469 stdout=args.stdout or filename == "stdin", 470 normalize_use=args.normalize_use, 471 decl_linelength=args.decl_linelength, 472 decl_offset=args.decl_offset, 473 reformat=args.reformat, 474 indent=args.indent, 475 whitespace=args.whitespace, 476 upcase_keywords=args.upcase, 477 upcase_omp=args.omp_upcase, 478 replace=args.replace, 479 ) 480 481 if changed and not args.quiet: 482 print(f"prettified {filename}") 483 total_prettified += 1 484 485 except: 486 logger.exception("processing file failed", extra={"ffilename": filename}) 487 failure += 1 488 489 if not (args.stdout or filename == "stdin" or args.quiet): 490 print( 491 f"{total_prettified} file(s) prettified, {len(args.files) - total_prettified} file(s) left unchanged." 492 ) 493 494 return failure > 0 495 496 497if __name__ == "__main__": 498 sys.exit(main(sys.argv[1:])) 499