1#!/usr/local/bin/python3.8 2# -*- coding: utf-8 -*- 3from __future__ import division, print_function, unicode_literals, absolute_import 4import os 5import re 6import sys 7import copy 8from io import open 9import argparse 10import functools 11import random 12import pprint 13from collections import defaultdict 14import json 15import math 16import keyword 17 18sys.dont_write_bytecode = True 19 20py3 = False 21pyv = sys.version_info 22if pyv >= (3,): 23 py3 = True 24 xrange = range 25 unicode = str 26 27__version__ = '20200619.0' 28 29__all__ = ['pyprepro','Immutable','Mutable','ImmutableValDict','dprepro','convert_dakota'] 30 31DESCRIPTION="""\ 32%(prog)s -- python-based input deck pre-processor and template engine. 33 34version: __version__ 35 36""".replace('__version__',__version__) 37 38EPILOG = """\ 39Fallback Flags 40----------------- 41Will also accept `--(left/right)-delimiter` as an alias to the 42respective parts of `--inline`. 43 44Include Ordering 45---------------- 46All include files are read and set as Immutable immediately. They are read 47in the following order: include files, json-include, python-include. 48Therefore, if a variable is, for example, set in the include file and the 49python-include, the original value will hold. 50 51Sources: 52-------- 53Built from BottlePy's SimpleTemplateEngine[1] with changes to better match 54the behavior of APREPRO[2] and DPREPRO[3] and more tuned to simulation 55input files 56 57[1]: https://bottlepy.org/docs/dev/stpl.html 58[2]: https://github.com/gsjaardema/seacas 59[3]: https://dakota.sandia.gov/ 60""" 61 62DEBUGCLI = False 63 64########################################################################### 65############################# Global Settings ############################# 66########################################################################### 67# These may be changed from within the main function. Globals 68# are *not* an ideal approach, but it makes it easier to combine 69# different Bottle code w/o turning it into its own class 70DEFAULT_FMT = '%0.10g' 71DEFAULT_FMT0 = DEFAULT_FMT # Store the original in case `setfmt` is called 72 73BLOCK_START = '{%' 74BLOCK_CLOSE = '%}' 75LINE_START = '%' 76INLINE_START = '{' 77INLINE_END = '}' 78 79CLI_MODE = False # Reset in the if __name__ == '__main__' 80########################################################################### 81############################## Main Functions ############################# 82########################################################################### 83def pyprepro(tpl,include_files=None, 84 json_include=None, 85 python_include=None, 86 dakota_include=None, 87 env=None,immutable_env=None,fmt='%0.10g', 88 code='%',code_block='{% %}',inline='{ }', 89 warn=True,output=None): 90 """ 91 Main pyprepro function. 92 93 Arguments: 94 ---------- 95 tpl: 96 Either a string representing the template or a filename. Will 97 automatically read if a filename. 98 99 Options: 100 -------- 101 include_files: (filename string or list of filenames) 102 Files to be read *first* and take precendance (i.e. set as immutable). 103 No output from these are printed! See include order below 104 105 json_include: (filename string or list of filenames) 106 JSON files of variables to be read that take precendance 107 (i.e. set as immutable). See include order below 108 109 python_include: (filename string or list of filenames) 110 python files to be evaluated as pure python where indentation 111 matters and blocks should *not* have an `end` statement (contrary 112 to the modified python language in a template code block). All 113 variables take precedence (i.e. set as immutable). See include 114 order below. 115 116 dakota_include (filename string or list of filenames) 117 Input files that are formatted as Dakota paramater files. 118 All variables are immutable. See include 119 order below. 120 121 env: (dictionary) 122 A dictionary of additional settings. If passed as an ImmutableValDict, 123 immutability of the params will be maintained 124 125 immutable_env: (dictionary) 126 Like `env` but will automatically set every element as immutable. Just 127 for convenience 128 129 fmt: 130 String formatting code for numerical output. Can be overidden inline 131 with (for example) `{ "%5.2e" % var }`, Can specify with '%' or '{}' 132 notation 133 134 code: ['%'] 135 Specify the string that, when it is the first non-whitespace character 136 on a line, signifies a code line 137 138 code_block: ['{% %}'] 139 Specify the open and closing strings to delineate a code block. 140 Note tha the inner-most character must *not* be any of "{}[]()" 141 142 inline: ['{ }'] 143 Specify the open and closing strings to specify an inline expression. 144 Use a space to separate the start and the end. 145 146 warn [True] 147 Whether to allow warnings on changed param names (likely from 148 Dakota) 149 150 output [None] 151 Convenience option to write the output string. 152 153 Returns: 154 --------- 155 resulting string from the template 156 157 Include Order: 158 -------------- 159 All methods of including parameters set them as Immutable before 160 reading the final template file. However, they also have a specific 161 ordering aa follows: include_files, json_include,python_include 162 Therefore, if a parameter is set in the include_file and also set in a 163 json_include, the original value will hold! 164 165 """ 166 # (re)set some globals from this function call. 167 global DEFAULT_FMT,DEFAULT_FMT0 168 global LINE_START,INLINE_START,INLINE_END,BLOCK_START,BLOCK_CLOSE 169 170 DEFAULT_FMT = DEFAULT_FMT0 = fmt 171 LINE_START = code 172 INLINE_START,INLINE_END = inline.split() 173 BLOCK_START,BLOCK_CLOSE = code_block.split() 174 175 _check_block_syntax() 176 177 if include_files is None: 178 include_files = [] 179 if json_include is None: 180 json_include = [] 181 if python_include is None: 182 python_include = [] 183 if dakota_include is None: 184 dakota_include = [] 185 186 if isinstance(include_files,(str,unicode)): 187 include_files = [include_files] 188 if isinstance(json_include,(str,unicode)): 189 json_include = [json_include] 190 if isinstance(python_include,(str,unicode)): 191 python_include = [python_include] 192 if isinstance(dakota_include,(str,unicode)): 193 dakota_include = [dakota_include] 194 195 # The broken_bottle code is designed (modified) such that when an 196 # environment is passed in, that environment is modified and not copied 197 # Alternatively, if none is passed in, you can use `return_env` to 198 # get the output. 199 200 if env is None: 201 env = ImmutableValDict() 202 elif not isinstance(env,ImmutableValDict): 203 # Make sure env is ImmutableValDict 204 # IMPORTANT: pass the incoming env as an arg and not kw 205 # to ensure immutability is maintained. 206 env = ImmutableValDict(env) 207 208 if immutable_env is not None: 209 for key,val in immutable_env.items(): 210 env[key] = Immutable(val) 211 212 # Parse all include files. Do not send in the environment since we will 213 # reserve that for later. 214 for include in include_files: 215 _,subenv = _template(include,return_env=True) 216 217 # remove the initial variables (even though they will be the same for all 218 for init_var in INIT_VARS: # init_vars is a set. May init_vars 219 del subenv[init_var] 220 221 # Update the main but set as immutable 222 for key,val in subenv.items(): 223 env[key] = Immutable(val) 224 225 for json_file in json_include: 226 with open(json_file,'rb') as F: 227 subenv = json.loads(_touni(F.read())) 228 for key,val in subenv.items(): 229 env[key] = Immutable(val) 230 231 for python_file in python_include: 232 subenv = dict() 233 with open(python_file,'rb') as F: 234 exec_(F.read(),subenv) 235 for key,val in subenv.items(): 236 env[key] = Immutable(val) 237 238 for dakota_file in dakota_include: 239 subenv = convert_dakota(dakota_file) 240 for key,val in subenv.items(): 241 param0 = key 242 param = _fix_param_name(param0,warn=warn) 243 if param0 != param and warn: 244 txt = """ Or, may be accessed via "DakotaParams['{0}']"\n""".format(param0) 245 if sys.version_info < (2,7): 246 txt = txt.encode('utf8') 247 sys.stderr.write(txt) 248 env[param] = Immutable(val) 249 env['DakotaParams'] = subenv 250 251 # perform the final evaluation. Note that we do *NOT* pass `**env` since that 252 # would create a copy. 253 txtout = _template(tpl,env=env) 254 255 if output: 256 with open(output,'wt') as out: 257 out.write(txtout) 258 259 return txtout 260 261def _parse_cli(argv,dprepro=False): 262 """ 263 Handle command line input. 264 265 266 Inputs: 267 argv: The command line argumnets. Ex: sys.argv[1:] 268 269 Options: 270 dprepo [False] 271 If True, will expect a *single* include file as the first 272 positional argument. Otherwise, will allow for any number 273 of includes via --include (this toggle is to change behavior 274 for dprepro) 275 276 Also adds a --simple-parser mode 277 """ 278 279 parser = argparse.ArgumentParser(\ 280 description=DESCRIPTION, 281 epilog=EPILOG, 282 formatter_class=argparse.RawDescriptionHelpFormatter) 283 284 parser.add_argument('--code',default='%',metavar='CHAR', 285 help='["%(default)s"] Specify the string to delineate a single code line') 286 parser.add_argument('--code-block',default='{% %}',metavar='"OPEN CLOSE"', 287 help=('["%(default)s"] Specify the open and close of a code block. NOTE: ' 288 'the inner-most character must *not* be any of "{}[]()"')) 289 parser.add_argument('--inline',default='{ }',metavar='"OPEN CLOSE"', 290 help=('["%(default)s"] Specify the open and close of inline ' 291 'code/variables to print')) # out of order but makes more sense 292 293 if dprepro: 294 parser.add_argument('--simple-parser',action='store_true', 295 help='Always use the simple parser in %(prog)s rather than dakota.interfacing') 296 parser.add_argument('include', help='Include (parameter) file.') 297 else: 298 parser.add_argument('-I','--include',metavar='FILE',action='append',default=[], 299 help=('Specify a file to read before parsing input. ' 300 "Should be formatted with the same '--inline','--code', and/or '--code-block' " 301 "as the 'infile' template. " 302 'Note: All variables read from the --include will be take precedence ' 303 '(i.e. be immutable). You later make them mutable if necessary. ' 304 'Can specify more than one and they will be read in order. ' 305 )) 306 parser.add_argument('--dakota-include',metavar='FILE',action='append', 307 help=('Specify Dakota formatted files to load variables ' 308 'directly. As with `--include`, all variables will ' 309 'be immutable and can specify this flag multiple times. ' 310 'See include ordering. ' 311 'All ":" in variables names are converted to "_".')) 312 313 parser.add_argument('--json-include',metavar='FILE',action='append', 314 help=('Specify JSON formatted files to load variables ' 315 'directly. As with `--include`, all variables will ' 316 'be immutable. Can specify multiple. ' 317 'See include ordering')) 318 parser.add_argument('--python-include',metavar='FILE',action='append', 319 help=('Specify a python formatted file to read and use ' 320 'the resulting environment. NOTE: the file ' 321 'is read a regular python where indentation ' 322 'matters and blocks should *not* have an `end` ' 323 'statement (unlike in coode blocks). ' 324 'As with `--include`, all variables will ' 325 'be immutable and can specify this flag multiple times. ' 326 'See include ordering')) 327 328 329 parser.add_argument('--no-warn',action='store_false',default=True,dest='warn', 330 help = ('Silence warning messages.')) 331 332 parser.add_argument('--output-format',default='%0.10g',dest='fmt', 333 help=("['%(default)s'] Specify the default float format. Note that this can " 334 "be easily overridden inline as follows: `{'%%3.8e' %% param}`. " 335 "Specify in either %%-notation or {}.format() notation.")) 336 parser.add_argument('--var',metavar='"var=value"',action='append',default=[], 337 help = ('Specify variables to predefine. They will be defined as ' 338 'immutable. Use quotes to properly delineate')) 339 340 # Positional arguments. In reality, this is set this way so 341 # the help text will format correctly. We will rearrange arguments 342 # post-parsing so that all but the last two are command line. 343 344 # include is set above based on positional_include 345 parser.add_argument('infile', help='Specify the input file. Or set as `-` to read stdin') 346 parser.add_argument('outfile', nargs='?', 347 help='Specify the output file. Otherwise, will print to stdout') 348 349 ## dprepro fallbacks: 350 parser.add_argument('--left-delimiter',help=argparse.SUPPRESS) 351 parser.add_argument('--right-delimiter',help=argparse.SUPPRESS) 352 353 # Version 354 parser.add_argument('-v', '--version', action='version', 355 version='%(prog)s-' + __version__,help="Print the version and exit") 356 357 358 # Hidden debug 359 parser.add_argument('--debug',action='store_true',help=argparse.SUPPRESS) 360 361 # This sorts the optional arguments or each parser. 362 # It is a bit of a hack. The biggest issue is that this happens on every 363 # call but it takes about 10 microseconds 364 # Inspired by https://stackoverflow.com/a/12269358/3633154 365 for action_group in parser._action_groups: 366 # Make sure it is the **OPTIONAL** ones 367 if not all(len(action.option_strings) > 0 for action in action_group._group_actions): 368 continue 369 action_group._group_actions.sort(key=lambda action: # lower of the longest key 370 sorted(action.option_strings, 371 key=lambda a:-len(a))[0].lower()) 372 373 374 args = parser.parse_args(argv) 375 376 if args.debug: 377 global DEBUGCLI 378 DEBUGCLI = True 379 380 ########## Handle Dakota fallbacks 381 382 left,right = args.inline.split() 383 left = args.left_delimiter if args.left_delimiter else left 384 right = args.right_delimiter if args.right_delimiter else right 385 args.inline = left + ' ' + right 386 387 del args.left_delimiter 388 del args.right_delimiter 389 390 # Evaluate additional vars from command line (as immutable) 391 env = ImmutableValDict() 392 for addvar in args.var: 393 # TODO: support strings that contain = 394 addvar = addvar.split('=',2) 395 if len(addvar) != 2: 396 sys.stderr.write('ERROR: --var must be of the form `--var "var=value"`\n') 397 sys.exit(1) 398 key,val = addvar 399 400 key = key.strip() 401 # Try to convert it to a float. 402 try: 403 val = float(val) 404 except ValueError: 405 val = val.strip() 406 407 env[key] = Immutable(val) 408 409 # Read stdin if needed 410 if args.infile == '-': 411 args.infile = _touni(sys.stdin.read()) 412 elif not os.path.isfile(args.infile): 413 # pyprepro function can take an input file or text but the CLI 414 # should always be a file 415 print('ERROR: `infile` must be a file or `-` to read from stdin',file=sys.stderr) 416 sys.exit(1) 417 418 return args,env 419 420def _pyprepro_cli(argv): 421 """ 422 Actual CLI parser 423 """ 424 try: 425 args,env = _parse_cli(argv) 426 427 output = pyprepro(args.infile, 428 include_files=args.include, 429 json_include=args.json_include, 430 python_include=args.python_include, 431 dakota_include=args.dakota_include, 432 env=env, 433 fmt=args.fmt, 434 code=args.code, 435 code_block=args.code_block, 436 inline=args.inline, 437 warn=args.warn, 438 ) 439 except (NameError,BlockCharacterError,IncludeSyntaxError) as E: 440 if DEBUGCLI: 441 raise 442 sys.stderr.write(_error_msg(E)) 443 sys.exit(1) 444 445 if args.outfile is None: 446 sys.stdout.write(output) 447 else: 448 with open(args.outfile,'wt',encoding='utf8') as FF: 449 FF.write(output) 450 451########################################################################### 452############################# Helper Functions ############################ 453########################################################################### 454 455class IncludeSyntaxError(Exception): 456 pass 457 458class BlockCharacterError(Exception): 459 pass 460 461def _check_block_syntax(): 462 """ 463 Confirm that the open and closing blocks inner-most characters 464 are not any of "{}[]()" 465 """ 466 if BLOCK_START[-1] in "{}[]()" or BLOCK_CLOSE[0] in "{}[]()": 467 raise BlockCharacterError('Cannot have inner-most code block be any of "{}[]()" ') 468 469def _mult_replace(text,*A,**replacements): 470 """ 471 Simple tool to replace text with replacements dictionary. 472 Input can be either `param=val` or (param,val) tuples. 473 474 Can also invert if _invert=True 475 """ 476 invert = replacements.pop('_invert',False) 477 for item in A: 478 if isinstance(item,dict): 479 replacements.update(item) 480 481 for key,val in replacements.items(): 482 if invert: 483 val,key = key,val 484 text = text.replace(key,val) 485 return text 486 487def _formatter(*obj): 488 """ 489 Perform the formatting for output 490 """ 491 # Unexpand tuples 492 if len(obj) == 1: 493 obj = obj[0] 494 else: 495 return '(' + ','.join(_formatter(o) for o in obj) + ')' 496 497 # This is to catch a user error if the include is called wrong. It should be 498 # (with default syntax) 499 # {% include('file') %} 500 # and NOT 501 # {include('file')} 502 try: 503 if obj['__includesentinel']: 504 msg = ['Incorrect include syntax. Use "code-block" syntax, not "inline"'] 505 msg.append(" e.g.: BLOCK_START include('include_file.inp') BLOCK_CLOSE") 506 msg = _mult_replace('\n'.join(msg),BLOCK_START=BLOCK_START,BLOCK_CLOSE=BLOCK_CLOSE) 507 raise IncludeSyntaxError(msg) 508 except (KeyError,AttributeError,TypeError): 509 pass 510 511 if obj is None: 512 return '' 513 if isinstance(obj,Immutable): 514 obj = obj.val 515 if isinstance(obj,(unicode,str)): 516 return obj 517 if isinstance(obj,bytes): 518 return _formatter(_touni(obj)) 519 if isinstance(obj,bool): 520 return '{0}'.format(obj) # True or False 521 522 try: 523 if '%' in DEFAULT_FMT: 524 return DEFAULT_FMT % obj # numerical 525 elif '{' in DEFAULT_FMT: 526 return DEFAULT_FMT.format(obj) 527 except: pass 528 529 # See if it is numpy (w/o importing numpy) 530 if hasattr(obj,'tolist'): 531 obj = obj.tolist() 532 533 # Special case for lists of certain types 534 if isinstance(obj,list): 535 if len(obj) == 1: # Single item 536 return _formatter(obj[0]) 537 newobj = [] 538 for subobj in obj: 539 if not isinstance(subobj,(Immutable,unicode,str,bytes,bool,int,float)): 540 break 541 newobj.append(_formatter(subobj)) 542 else: # for-else only gets called if the for loop never had a break 543 return '[' + ', '.join(newobj) + ']' 544 545 # Fallback to pprint 546 try: 547 return pprint.pformat(obj,indent=1) 548 except: 549 pass 550 551 # give up! 552 return repr(obj) 553 554def _preparser(text): 555 """ 556 This is a PREPARSER before sending anything to Bottle. 557 558 It parses out inline syntax of `{ param = val }` so that it will still 559 define `param`. It will also make sure the evaluation is NOT inside 560 of %< and %} blocks (by parsing them out first). 561 562 It also handles escaped inline assigments 563 564 Can also handle complex siutations such as: 565 566 {p = 10} 567 start,{p = p+1},{p = p+1},{p = p+1},end 568 {p} 569 570 which will turn into the following. 571 572 \\ 573 {% p = 10 %} 574 { p } 575 start,\\ 576 {% p = p+1 %} 577 { p },\\ 578 {% p = p+1 %} 579 { p },\\ 580 {% p = p+1 %} 581 { p },end 582 {p} 583 584 and will (eventually) render 585 586 10 587 start,11,12,13,end 588 13 589 590 This will also fix assignments made such as 591 592 { ASV_1:fun1 = 1 } 593 594 to 595 596 { ASV_1_fun1 = 1 } 597 598 and can handle lines such as { p += 1 } 599 """ 600 # Clean up 601 text = _touni(text) 602 text = text.replace(u'\ufeff', '') # Remove BOM from windows 603 text = text.replace('\r','') # Remove `^M` characters 604 605 # Remove any code blocks and replace with random text 606 code_rep = defaultdict(lambda:_rnd_str(20)) # will return random string but store it 607 _,text = _delim_capture(text,'{0} {1}'.format(BLOCK_START,BLOCK_CLOSE), # delim_capture does NOT want re.escape 608 lambda t:code_rep[t]) 609 610 #if text != 'BLA': import ipdb;ipdb.set_trace() 611 612 # Convert single line expression "% expr" and convert them to "{% expr %}" 613 search = "^([\t\f ]*)LINE_START(.*)".replace('LINE_START',re.escape(LINE_START)) 614 replace = r"\1{0} \2 {1}".format(BLOCK_START,BLOCK_CLOSE) 615 text = re.sub(search,replace,text) 616 617 # and then remove them too! 618 _,text = _delim_capture(text,'{0} {1}'.format(BLOCK_START,BLOCK_CLOSE), # delim_capture does NOT want re.escape 619 lambda t:code_rep[t]) 620 621 ###### Bracket Escaping 622 # Apply escaping to things like '\{' --> "{" and "\\{" --> "\{" 623 # by replacing them with a variable. First, remove all inline, then find 624 # the offending lines, replace them, then add back in the inline 625 inline_rep = defaultdict(lambda:_rnd_str(20)) 626 _,text = _delim_capture(text, 627 '{0} {1}'.format(INLINE_START,INLINE_END), # do not use re escaped 628 lambda t:inline_rep[t]) 629 630 # Replace '\{' with a variable version of '{ _INLINE_START }'. Make sure it is not escaped 631 text = re.sub(r'(?<!\\)\\{0}'.format(re.escape(INLINE_START)), 632 r'{0} _INLINE_START {1}'.format(INLINE_START,INLINE_END), 633 text) 634 635 # replace '\\{' with '\{ _INLINE_START }' since it is escaped 636 text = re.sub(r'\\\\{0}'.format(re.escape(INLINE_START)), 637 r'{0}_eINLINE_START{1}'.format(INLINE_START,INLINE_END), 638 text) 639 640 # Replace '\}' with a variable version of '{ _INLINE_END }'. Make sure it is not escaped 641 text = re.sub(r'(?<!\\)\\{0}'.format(re.escape(INLINE_END)), 642 r'{0} _INLINE_END {1}'.format(INLINE_START,INLINE_END), 643 text) # reminder r"\\" will *still* be "\" to regex 644 645 # replace '\\{' with '\{ _INLINE_END }' since it is escaped 646 text = re.sub(r'\\\\{0}'.format(re.escape(INLINE_END)), 647 r'{0}_eINLINE_END{1}'.format(INLINE_START,INLINE_END), 648 text) 649 650 # Sub back in the other removed inline expressions 651 text = _mult_replace(text,inline_rep,_invert=True) 652 ###### /Bracket Escaping 653 654 # Apply _inline_fix to all inline assignments 655 _,text = _delim_capture(text, 656 '{0} {1}'.format(INLINE_START,INLINE_END), # do not use re escaped 657 _inline_fix) 658 659 660 # Re-add the code blocks with an inverted dict 661 return _mult_replace(text,code_rep,_invert=True) 662 663def _inline_fix(capture): 664 """ 665 Replace the matched line in a ROBUST manner to allow multiple definitions 666 on each line 667 """ 668 # Take EVERYTHING and then remove the outer. 669 670 match = capture[len(INLINE_START):-len(INLINE_END)].strip() # Remove open and close brackets 671 672 # Need to decide if this is a {param} or {var=param} 673 # But need to be careful for: 674 # 675 # {var = "text}"} 676 # {function(p="}")} 677 # 678 # Do this by splitting at '=' but make sure there are no 679 # disallowed characters. Check for assignment (+=) and comparison (<=) 680 # 681 # Also fixes lines such as {ASV_1:fun1 = 1}, {ASV_1:fun1} but will *ignore* 682 # {"ASV_1:fun1"} 683 684 def _fix_varnames(name): 685 """ 686 Fix variable names 687 * remove colons 688 * Add `i` to leading integers 689 """ 690 name = name.strip().replace(':','_') 691 if name[0] in '0123456789': 692 name = 'i' + name 693 return name 694 695 parts = match.split('=',1) 696 if len(parts) != 2: # *must* be just {param} 697 return capture # Do NOT fix since we dissallow variables like "A:B". 698 # They will already have been converted to "A_B" 699 700 var,val = parts # Can't be more than two 701 var = var.strip() 702 if any(c in var for c in ['"',"'",'(',')']): # something like {function(p="}")} 703 return capture # Do not fix. See above 704 705 opperator = '=' 706 707 # is it a modified assignment operator (e.g. "+=","<<=") but NOT comparison (e.g. "<="). 708 # Check first for assignment and ONLY then can you check for comparison. 709 assignment_mods = ['+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '>>', '<<'] # += -= *=, etc 710 comparison_mods = ['=','!','>','<'] # ==,!=, etc 711 712 is_assignment = False 713 for v in assignment_mods: 714 if var.endswith(v): 715 is_assignment = True 716 var = var[:-len(v)] 717 opperator = v + opperator 718 break 719 720 if not is_assignment: 721 for v in comparison_mods: 722 if var.endswith(v): 723 var = var[:-len(v)] 724 var = _fix_varnames(var) # { A <= 10 } and/or {A:1 <= 10} becomes {A_1<=10} 725 opperator = v + opperator 726 return INLINE_START + var + opperator + val + INLINE_END 727 728 # Fix disallowed var names 729 var = _fix_varnames(var) 730 731 # Set the value 732 return ''.join([r'\\','\n', 733 BLOCK_START,' ',var,opperator,val,' ',BLOCK_CLOSE,'\n', 734 INLINE_START,' ',var.strip(),' ',INLINE_END]) 735 736 737def _delim_capture(txt,delim,sub=None): 738 ''' 739 Combination of regex and some hacking to LAZY capture text between 740 the delims *while* accounting for quotes. 741 742 Returns the captured group INCLUDING the delimiters 743 744 For example, consider delim = "{% %}", it will handle: 745 '{%testing%}' --> {%testing%} (1) 746 '{%test"%}"ing%}' --> {%test"%}"ing%} (1) 747 '{%te"""%}" """sting%}' --> {%te"""%}" """sting%} (1) 748 '{%TE"%}"%}{%STING%}' --> {%TE"%}"%}, {%STING%} (2) 749 '"{%test"%}"ing%}"' --> {%test"%}"ing%} (1) 750 751 (notice it handles quotes around the matches) 752 753 This is an alternative to more complex regexes such as those discussed 754 in https://stackoverflow.com/a/22184202/3633154 755 756 inputs: 757 txt : The input text 758 delim : Space-separated delimiters. DO NOT re.escape them! 759 760 options: 761 sub : [None] text to replace the capture or function. 762 NOTE: if it is a function, it will be passed the string only 763 and *not* the SRE_Match object 764 765 returns: 766 captured: List of captured items NOT subbed 767 txt : Resulting txt (potentially with the subs) 768 ''' 769 770 # Algorithm: 771 # 1. Find the first opening of a block 772 # a. If none was found, add the rest of the text to the out 773 # and break 774 # b. Add all preceding text to the output and trim it off txt 775 # 2. Remove all quoted strings from remaining text 776 # 3. Split at the closing block. 777 # a. If not found, replace quoted txt, add to output, and break. 778 # This is a poorly formed file!!! 779 # 4. Replace quoted txt in both the capture block and remaining text. 780 # Also re-add the closing text since it was removed in split 781 # 5. Store capture block (and sub if applicable) 782 # 6. Continue until break 783 784 # Set up the regexes and the output 785 OPEN,CLOSE = delim.split() 786 rOPEN,rCLOSE = [re.escape(d) for d in (OPEN,CLOSE)] 787 reOPEN = re.compile(r'(?<!\\)' + rOPEN) # Checks for escape 788 reCLOSE = re.compile(rCLOSE) 789 790 reQUOTE = re.compile(r""" 791 '{3}(?:[^\\]|\\.|\n)+?'{3} # 3 single ticks 792 |\"{3}(?:[^\\]|\\.|\n)+?\"{3} # 3 double ticks 793 |\".+?\" # 1 double tick 794 |'.+?' # 1 single tick 795 """,flags=re.MULTILINE|re.DOTALL|re.UNICODE|re.VERBOSE) # Regex to capture quotes 796 797 outtxt = [] 798 captured = [] 799 800 while True: 801 match = reOPEN.search(txt) 802 if not match: 803 outtxt.append(txt) 804 break 805 806 outtxt.append(txt[:match.start()]) 807 txt = txt[match.start():] 808 809 # Remove all correctly quoted material (i.e. has matching quotes) 810 quote_rep = defaultdict(lambda:_rnd_str(20)) # will return random string but store it 811 txt = reQUOTE.sub(lambda m:quote_rep[m.group(0)],txt) # Replace quotes with random string 812 813 # Find the end 814 try: 815 cap,txt = reCLOSE.split(txt,1) 816 except ValueError: # There was no close. Restore txt and break 817 outtxt.append(_mult_replace(txt,quote_rep,_invert=True)) 818 break 819 820 # Restore both captured and txt 821 cap = _mult_replace(cap,quote_rep,_invert=True) + CLOSE 822 txt = _mult_replace(txt,quote_rep,_invert=True) 823 824 captured.append(cap) 825 826 # Apply sub and then add to outtxt 827 if sub is not None: 828 if callable(sub): # callabe 829 cap = sub(cap) 830 else: 831 cap = sub 832 outtxt.append(cap) 833 834 return captured,''.join(outtxt) 835 836def _error_msg(E): 837 msg = [] 838 err = E.__class__.__name__ 839 msg.append('Exception: {0}'.format(err)) 840 if hasattr(E,'filename'): 841 msg.append('Filename: {0}'.format(E.filename)) 842 if hasattr(E,'lineno'): 843 msg.append('Approximate Line Number: {0}'.format(E.lineno)) 844# if hasattr(E,'offset'): # Not reliable 845# msg.append('Column: {0}'.format(E.offset)) 846 if hasattr(E,'args') and len(E.args)>0: 847 msg.append('Message: {0}'.format(E.args[0])) 848 849 msg = 'Error occurred\n' + '\n'.join(' ' + l for l in msg) + '\n' 850 return msg 851###### Functions for inside templates 852 853def _vartxt(env,return_values=True,comment=None): 854 """ 855 small helper to print the variables in the environment. 856 857 If comment is set, will prepend all lines with the comment character 858 """ 859 subenv = dict((k,v) for k,v in env.items() if k not in INIT_VARS) 860 861 if return_values: 862 txt = pprint.pformat(subenv,indent=1) 863 else: 864 txt = pprint.pformat(list(subenv.keys()),indent=1) 865 866 if comment is None: 867 return txt 868 869 if not any(comment.endswith(c) for c in " \t"): 870 comment += ' ' # make sure ends with space 871 872 return '\n'.join(comment + t for t in txt.split('\n')) 873 874def _setfmt(fmt=None): 875 """ 876 (re)set the global formatting. If passed None, will reset to initial 877 """ 878 global DEFAULT_FMT 879 DEFAULT_FMT = fmt if fmt is not None else DEFAULT_FMT0 880 881def _vset(key,val,env=None): 882 """ 883 Used inside the templates (with partial(_vset,env=env) ) to set a variable 884 and also print the name. 885 """ 886 if env is None: 887 raise ValueError('Must specify an env') 888 env[key] = val 889 return '{0} = {1}'.format(key,env[key]) # use env[key] for val since it may be immutable 890 891 892####### This is the main driver of immutability inside of eval statements 893class ImmutableValDict(dict): 894 """ 895 A regular dict with the ability to set Immutable key and values. 896 897 For example: 898 D = ImmutableValDict() 899 D['a'] = Immutable(10) 900 D['a'] = 20 901 D['a'] == 20 # False 902 D['a'] == 10 # True 903 904 In the above, the key 'a' is not overritten. But, the value itself 905 may be mutable: 906 907 D = ImmutableValDict() 908 D['b'] = Immutable([1,2]) # Lists are mutable but 'b' will be fixed 909 D['b'].append(3) 910 D['b'] == [1,2,3] # True 911 912 Note, you *could* do: 913 914 D = ImmutableValDict() 915 obj = [1,2,3] 916 c = Immutable(obj) 917 D['c'] = c 918 919 # But note: 920 D['c'] is obj # True -- same object 921 c is obj # False 922 923 """ 924 def __init__(self, *args, **kwargs): 925 # This has to be overridden to call __setitem__ 926 self.__locked = set() # define first since update will use it 927 self.update(*args, **kwargs) 928 929 def __setitem__(self,key,item): 930 """ 931 Set the key but only the previously defined item is not 932 already immutable 933 """ 934 if isinstance(item,Mutable): # Check first since Mutable inherits Immutable 935 item = item.val 936 if key in self.immutables: 937 self.immutables.remove(key) 938 939 if key in self.immutables: 940 return 941 942 if isinstance(item,Immutable): 943 self.__locked.add(key) 944 item = item.val 945 super(ImmutableValDict,self).__setitem__(key,item) 946 947 def __delitem__(self,key): 948 if key in self.__locked: 949 self.immutables.remove(key) 950 super(ImmutableValDict,self).__delitem__(key) 951 952 def update(self, *args, **kwargs): 953 """ 954 Update the keys in the dictionary 955 """ 956 # This has to be overridden to call __setitem__ and to 957 # keep immutability of vars if the input is an ImmutableValDict 958 for k, v in dict(*args, **kwargs).items(): 959 self[k] = v 960 # Update the locked keys if args[0] is an ImmutableValDict 961 if len(args)>0 and isinstance(args[0],ImmutableValDict): 962 self.immutables.update(args[0].immutables) 963 964 @property 965 def immutables(self): 966 return self.__locked 967 968class Immutable: 969 """ 970 Container object for ImmutableValDict 971 """ 972 __slots__ = ('val',) 973 def __init__(self,val): 974 self.val = val 975 def __repr__(self): 976 return '(Immutable(' + self.val.__repr__() + ')' 977 __str__ = __repr__ 978 979class Mutable(Immutable): 980 """ 981 Container object for ImmutableValDict 982 """ 983 def __repr__(self): 984 return '(Mutable(' + self.val.__repr__() + ')' 985 986def _rnd_str(N=10): 987 CH = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' 988 return ''.join(random.choice(CH) for _ in range(N)) 989 990########################################################################### 991############################ dprepro functions ############################ 992########################################################################### 993# dprepro is designed to be called directly by dakota [1] and follows a 994# similar syntax. The biggest difference is that dprepro takes an include 995# file as a positional argument and that include file will *always* be 996# of one of two Dakota formats: 997# 998# val param 999# 1000# or 1001# 1002# {param = val} 1003# 1004# (the latter will work in aprepro iff the inline syntax is not changed 1005# 1006# [1]: http://dakota.sandia.gov 1007 1008def _add_di_paths(): 1009 di_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)),"..","share","dakota","Python")) 1010 sys.path.append(di_path) 1011 1012diwarning = """ 1013WARNING: dprepro could not find dakota.interfacing module. Make sure either 1014 the PYTHONPATH environment is correctly set and/or dprepro has not 1015 been moved from its original installed location 1016 1017 dprepro will fallback to defining all dakota settings in the 1018 environment. 1019""" # This is not called automatically 1020def convert_dakota(input_file): 1021 """ 1022 Convert files to be the correct format and return the env 1023 """ 1024 # Note: Dakota files can be 1025 # val param 1026 # by default or 1027 # { val = param } 1028 # in aprepro mode (regardless of the delinators set here) 1029 env = {} 1030 ### TMP 1031 if isinstance(input_file,(list,tuple)): 1032 assert len(input_file) == 1,"WARNING: Does not handle multple yet" 1033 input_file = input_file[0] 1034 ### /TMP 1035 1036 N = None 1037 # Use pyprepro's _touni since it is more robust to windows encoding 1038 with open(input_file,'rb') as F: 1039 lines = _touni(F.read()).strip().split('\n') 1040 1041 for n,line in enumerate(lines): 1042 line = line.strip() 1043 if len(line) == 0: 1044 continue 1045 if line.startswith('{'): # aprepro "{key = value}" 1046 line = line[1:-1] 1047 key,val = line.split('=',1) 1048 val = val.strip() 1049 if val[0] in ['"',"'"]: 1050 # it's a string with quotes 1051 val = val[1:-1] 1052 1053 else: # dakota " value key " 1054 # Need to split but also have to worry about string (and spaces in strings) 1055 # so do an rsplit 1056 val,key = line.rsplit(None,1) 1057 1058 try: 1059 val = float(val) 1060 except ValueError: 1061 val = val.strip() 1062 1063 key = key.strip() 1064 env[key] = val 1065 1066 # The first line is the parameters. Assume it can be read 1067 # but add a fallback if not 1068 if n == 0: 1069 try: 1070 N = int(val) 1071 except ValueError: # Could not be read 1072 N = float('inf') 1073 1074 # Only do the parameters themselves. Not the other ASV... stuff 1075 if n >= N: # n starts at 0 1076 break 1077 1078 return env 1079 1080 1081def _fix_param_name(param,warn=False): 1082 """ 1083 Fix param/key names to be valid python. If warn == True, will add a 1084 warning to stderr 1085 1086 1. Convert characters that are not alphanumeric or _ to _. Alphanumeric 1087 means not just ascii, but includes many Unicode characters. 1088 2. Python 2 allows only ascii alphanumeric (+ _) identifiers, so "normalize" 1089 everything to ascii. E.g. ñ -> n. 1090 """ 1091 param = _touni(param) # Ensure the string is unicode in case passed bytes 1092 1093 param0 = param # string are immutable so it won't be affect by changes below 1094 param = re.compile("\W",flags=re.UNICODE).sub('_',param) # Allow unicode on python2 (and compile first for 2.6) 1095 if re.match("\d",param[0],flags=re.UNICODE): 1096 param = 'i' + param 1097 while keyword.iskeyword(param): 1098 param += "_" 1099 1100 # unicode check for python2 1101 add_unicode_warn = False 1102 if not py3: 1103 import unicodedata 1104 param0u = param 1105 param = unicodedata.normalize('NFKD', param).encode('ascii','ignore') # https://www.peterbe.com/plog/unicode-to-ascii convert to ascii 1106 param = unicode(param) 1107 add_unicode_warn = param0u != param 1108 1109 if param0 != param and warn: 1110 txt = (u'WARNING: Paramater "{0}" is not a valid name.\n' 1111 u' Converted to "{1}"\n'.format(param0,param)) 1112 1113 if pyv < (2,7): 1114 txt = txt.encode('utf8') 1115 1116 sys.stderr.write(txt) 1117 1118 if add_unicode_warn: 1119 sys.stderr.write(' Unicode characters in variable name.\n' 1120 ' Must use python3!\n') 1121 1122 return param 1123 1124def _dprepro_cli(argv): 1125 """ 1126 CLI parser 1127 """ 1128 # Import dakota.interfacing here to avoid circular import 1129 _add_di_paths() 1130 try: 1131 import dakota.interfacing as di 1132 except ImportError: 1133 di = None 1134 1135 args,env = _parse_cli(argv,dprepro=True) 1136 1137 params = None 1138 results = None 1139 # Convert Dakota 1140 if di is None or args.simple_parser: 1141 if args.warn and not args.simple_parser: 1142 sys.stderr.write(diwarning + '\n') # print the error message 1143 env2 = convert_dakota(args.include) 1144 env.update(env2) 1145 else: 1146 try: 1147 params, results = di.read_parameters_file(parameters_file=args.include,results_file=di.UNNAMED) 1148 except di.ParamsFormatError as E: 1149 sys.stderr.write(_error_msg(E)) 1150 sys.exit(1) 1151 1152 env["DakotaParams"] = params 1153 for d, v in params.items(): 1154 env[d] = v 1155 env["DakotaResults"] = results 1156 for d, v in results.items(): 1157 env[d] = v 1158 1159 try: 1160 output = dprepro(include=env, 1161 template = args.infile, 1162 fmt=args.fmt, 1163 code=args.code, 1164 code_block=args.code_block, 1165 inline=args.inline, 1166 json_include=args.json_include, 1167 python_include=args.python_include, 1168 warn=args.warn 1169 ) 1170 except Exception as E: 1171 # the _template has a catch but this will be the last resort. 1172 if DEBUGCLI: 1173 raise 1174 1175 sys.stderr.write(_error_msg(E)) 1176 sys.exit(1) 1177 1178 if args.outfile is None: 1179 sys.stdout.write(output) 1180 else: 1181 with open(args.outfile,'wt',encoding='utf8') as FF: 1182 FF.write(output) 1183 1184def dprepro(include=None, template=None, output=None, fmt='%0.10g', code='%', 1185 code_block='{% %}', inline='{ }',warn=True,**kwargs): 1186 """Validate Dakota parameters and insert them into a template 1187 1188 Keyword Args: 1189 1190 include(dict): Items to make available for substitution 1191 template(str or IO object): Template. If it has .read(), will be 1192 treated like a file. Otherwise, assumed to contain a template. 1193 output(str or IO object): If None (the default), the substituted 1194 template will be returned as a string. If it has .write(), will 1195 be treated like a file. Otherwise, assumed to be the name of a file. 1196 fmt(str): Default format for numerical fields. Default: '%0.10g' 1197 code(str): Delimiter for a code line. Default: '%' 1198 code_block(str): Delimiters for a code block. Default: '{% %}' 1199 inline(str): Delimiters for inline substitution. Default: '{ }' 1200 warn(bool): Whether or not to warn the user of invalid parameter names 1201 1202 All additional parameters are passed to pyprepro (e.g. json_include, 1203 python_include) 1204 Returns: 1205 If no output is specified, returns a string containing the substituted 1206 template. 1207 """ 1208 # Process the Dakota input file and then call pyprepro 1209 1210 # Construct the env from parameters, results, and include 1211 env = ImmutableValDict() 1212 1213 if include is None: 1214 include = {} 1215 for key, val in include.items(): 1216 param0 = key 1217 param = _fix_param_name(param0,warn=warn) 1218 if param0 != param and warn: 1219 txt = """ Or, may be accessed via "DakotaParams['{0}']"\n""".format(param0) 1220 if sys.version_info < (2,7): 1221 txt = txt.encode('utf8') 1222 sys.stderr.write(txt) 1223 env[param] = Immutable(val) 1224 1225 # read in the template if needed 1226 use_template = template 1227 if hasattr(template,"read"): 1228 use_template = template.read() 1229 1230 # Call pyprepro engine 1231 output_string = pyprepro(tpl=use_template, 1232 env=env, 1233 fmt=fmt, 1234 code=code, 1235 code_block=code_block, 1236 inline=inline,**kwargs) 1237 1238 # Output 1239 if output is None: 1240 return output_string 1241 elif hasattr(output, "write"): 1242 output.write(output_string) 1243 else: 1244 with open(output,"wt") as f: 1245 f.write(output_string) 1246 1247########################################################################### 1248####################### BottlePy Extracted Functions ###################### 1249########################################################################### 1250# This is all pulled from Bottle with lots of little changes to make it work 1251# 1252# A NON-EXHAUSTIVE list of changes are below: 1253# 1254# Major: 1255# 1256# * Changed the default environment to ImmutableValDict 1257# * Added Immutable and Mutable functions to be passed it 1258# * All text is routed through _preparser (3 places...I think) 1259# * Ability to return the environment 1260# * Adjusted scope so that if a variable is parsed in an include, it is present 1261# in the parent. (the env object is passed in and NEVER copied) 1262# 1263# Minor: 1264# 1265# * {{ }} syntax to { } (though settable) 1266# * No HTML escaping 1267# * No caching 1268# * math namespace is imported 1269# * Simply decide if input is filename or string based on whether the file 1270# exists 1271# * Fix for local files with absolute system paths 1272# * Commented out rebase 1273# 1274############################################################################## 1275# Copyright (c) 2017, Marcel Hellkamp. 1276# 1277# Permission is hereby granted, free of charge, to any person obtaining a copy 1278# of this software and associated documentation files (the "Software"), to deal 1279# in the Software without restriction, including without limitation the rights 1280# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1281# copies of the Software, and to permit persons to whom the Software is 1282# furnished to do so, subject to the following conditions: 1283# 1284# The above copyright notice and this permission notice shall be included in 1285# all copies or substantial portions of the Software. 1286# 1287# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1288# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1289# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 1290# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 1291# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 1292# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 1293# THE SOFTWARE. 1294############################################################################# 1295 1296#TEMPLATE_PATH = "./" 1297TEMPLATES = {} 1298DEBUG = True # Will also turn off caching 1299 1300 1301class TemplateError(Exception): 1302 pass 1303 1304def _touni(s, enc=None, err='strict'): 1305 if enc is None: 1306 # This ordering is intensional since, anecdotally, some Windows-1252 will 1307 # be decodable as UTF-16. The chardet module is the "correct" answer 1308 # but we don't want to add the dependency 1309 enc = ['utf8','Windows-1252','utf16','ISO-8859-1',] 1310 1311 if isinstance(enc,(str,unicode)): 1312 enc = [enc] 1313 1314 if isinstance(s, bytes): 1315 for e in enc: 1316 try: 1317 return s.decode(e, err) 1318 except UnicodeDecodeError: 1319 pass 1320 1321 return unicode("" if s is None else s) 1322 1323class _cached_property(object): 1324 """ A property that is only computed once per instance and then replaces 1325 itself with an ordinary attribute. Deleting the attribute resets the 1326 property. """ 1327 1328 def __init__(self, func): 1329 _update_wrapper(self, func) 1330 self.func = func 1331 1332 def __get__(self, obj, cls): 1333 if obj is None: return self 1334 value = obj.__dict__[self.func.__name__] = self.func(obj) 1335 return value 1336# A bug in functools causes it to break if the wrapper is an instance method 1337def _update_wrapper(wrapper, wrapped, *a, **ka): 1338 try: 1339 functools.update_wrapper(wrapper, wrapped, *a, **ka) 1340 except AttributeError: 1341 pass 1342class _BaseTemplate(object): 1343 """ Base class and minimal API for _template adapters """ 1344 extensions = ['tpl', 'html', 'thtml', 'stpl'] 1345 settings = {} #used in prepare() 1346 defaults = {} #used in render() 1347 1348 def __init__(self, 1349 source=None, 1350 name=None, 1351 lookup=None, 1352 encoding='utf8', **settings): 1353 """ Create a new _template. 1354 If the source parameter (str or buffer) is missing, the name argument 1355 is used to guess a _template filename. Subclasses can assume that 1356 self.source and/or self.filename are set. Both are strings. 1357 The lookup, encoding and settings parameters are stored as instance 1358 variables. 1359 The lookup parameter stores a list containing directory paths. 1360 The encoding parameter should be used to decode byte strings or files. 1361 The settings parameter contains a dict for engine-specific settings. 1362 """ 1363 self.name = name 1364 self.source = _preparser(source.read()) if hasattr(source, 'read') else source 1365 self.filename = source.filename if hasattr(source, 'filename') else None 1366 self.lookup = [os.path.abspath(x) for x in lookup] if lookup else [] 1367 self.encoding = encoding 1368 self.settings = self.settings.copy() # Copy from class variable 1369 self.settings.update(settings) # Apply 1370 if not self.source and self.name: 1371 self.filename = self.search(self.name, self.lookup) 1372 if not self.filename: 1373 raise TemplateError('Template %s not found.' % repr(name)) 1374 if not self.source and not self.filename: 1375 raise TemplateError('No _template specified.') 1376 self.prepare(**self.settings) 1377 1378 @classmethod 1379 def search(cls, name, lookup=None): 1380 """ Search name in all directories specified in lookup. 1381 First without, then with common extensions. Return first hit. """ 1382 #if not lookup: 1383 # raise depr(0, 12, "Empty _template lookup path.", "Configure a _template lookup path.") 1384 #if os.path.isabs(name): 1385 # raise depr(0, 12, "Use of absolute path for _template name.", 1386 # "Refer to _templates with names or paths relative to the lookup path.") 1387 1388 # JW: Search full system name first: 1389 if os.path.isfile(name): 1390 return os.path.abspath(name) 1391 1392 for spath in lookup: 1393 spath = os.path.abspath(spath) + os.sep 1394 fname = os.path.abspath(os.path.join(spath, name)) 1395 if not fname.startswith(spath): continue 1396 if os.path.isfile(fname): return fname 1397 for ext in cls.extensions: 1398 if os.path.isfile('%s.%s' % (fname, ext)): 1399 return '%s.%s' % (fname, ext) 1400 1401 @classmethod 1402 def global_config(cls, key, *args): 1403 """ This reads or sets the global settings stored in class.settings. """ 1404 if args: 1405 cls.settings = cls.settings.copy() # Make settings local to class 1406 cls.settings[key] = args[0] 1407 else: 1408 return cls.settings[key] 1409 1410 def prepare(self, **options): 1411 """ Run preparations (parsing, caching, ...). 1412 It should be possible to call this again to refresh a _template or to 1413 update settings. 1414 """ 1415 raise NotImplementedError 1416 1417 def render(self, *args, **kwargs): 1418 """ Render the _template with the specified local variables and return 1419 a single byte or unicode string. If it is a byte string, the encoding 1420 must match self.encoding. This method must be thread-safe! 1421 Local variables may be provided in dictionaries (args) 1422 or directly, as keywords (kwargs). 1423 """ 1424 raise NotImplementedError 1425 1426 1427class _SimpleTemplate(_BaseTemplate): 1428 def prepare(self, 1429 escape_func=lambda a:a, 1430 noescape=True, 1431 syntax=None, **ka): 1432 self.cache = {} 1433 enc = self.encoding 1434 self._str = _formatter 1435 self._escape = lambda x: escape_func(_touni(x, enc)) 1436 self.syntax = syntax 1437 if noescape: 1438 self._str, self._escape = self._escape, self._str 1439 1440 @_cached_property 1441 def co(self): 1442 return compile(self.code, self.filename or '<string>', 'exec') 1443 1444 @_cached_property 1445 def code(self): 1446 source = self.source 1447 if not source: 1448 with open(self.filename, 'rb') as f: 1449 source = f.read() 1450 try: 1451 source, encoding = _touni(source), 'utf8' 1452 except UnicodeError: 1453 raise depr(0, 11, 'Unsupported _template encodings.', 'Use utf-8 for _templates.') 1454 source = _preparser(source) 1455 parser = _StplParser(source, encoding=encoding, syntax=self.syntax) 1456 code = parser.translate() 1457 self.encoding = parser.encoding 1458 return code 1459 1460 def _rebase(self, _env, _name=None, **kwargs): 1461 _env['_rebase'] = (_name, kwargs) 1462 1463 def _include(self, _env, _name=None, **kwargs): 1464 env = _env # Use the same namespace/environment rather than a copy 1465 env.update(kwargs) 1466 if _name not in self.cache: 1467 self.cache[_name] = self.__class__(name=_name, lookup=self.lookup, syntax=self.syntax) 1468 1469 r = self.cache[_name].execute(env['_stdout'], env) 1470 r['__includesentinel'] = True # This is to make sure the return of include 1471 # is not trying to be displayed 1472 return r 1473 1474 def execute(self, _stdout, kwargs): 1475 env = kwargs # Use the same namespace/environment rather than a copy 1476 1477 # Math + constants 1478 env.update( dict((k,v) for k,v in vars(math).items() if not k.startswith('__')) ) 1479 env.update({'tau':2*math.pi, 1480 'deg':180/math.pi, 1481 'rad':math.pi/180, 1482 'E':math.e, 1483 'PI':math.pi, 1484 'phi':(math.sqrt(5)+1)/2, 1485 }) 1486 1487 # Other helpful functions (esp. to make sure py3 works the same) 1488 env.update({'unicode':unicode, # set at top for py2 1489 'xrange':xrange, # "..." 1490 }) 1491 1492 # pyprepro Functions 1493 env.update({ 1494 '_stdout': _stdout, 1495 '_printlist': _stdout.extend, 1496 'include': functools.partial(self._include, env), 1497 #'rebase': functools.partial(self._rebase, env), 1498 #'_rebase': None, 1499 '_str': self._str, 1500 '_escape': self._escape, 1501 'get': env.get, 1502 #'setdefault': env.setdefault, 1503 'defined': env.__contains__, 1504 '_copy':copy.copy, 1505 # Added: 1506 'vset':functools.partial(_vset,env=env), 1507 'Immutable':Immutable, 1508 'Mutable':Mutable, 1509 'setfmt':_setfmt, 1510 'all_vars':lambda **k: _vartxt(env,return_values=True,**k), 1511 'all_var_names':lambda **k: _vartxt(env,return_values=False,**k), 1512 }) 1513 1514 # String literals of escape characters 1515 env.update({ 1516 '_BLOCK_START':BLOCK_START, 1517 '_BLOCK_CLOSE':BLOCK_CLOSE, 1518 '_LINE_START':LINE_START, 1519 '_INLINE_START':INLINE_START, 1520 '_eINLINE_START': '\\' + INLINE_START, 1521 '_INLINE_END':INLINE_END, 1522 '_eINLINE_END':'\\' + INLINE_END, 1523 }) 1524 1525 1526 exec_(self.co,env) 1527 1528 if env.get('_rebase'): 1529 subtpl, rargs = env.pop('_rebase') 1530 rargs['base'] = ''.join(_stdout) #copy stdout 1531 del _stdout[:] # clear stdout 1532 return self._include(env, subtpl, **rargs) 1533 return env 1534 1535 def render(self,env=None): 1536 """ Render the _template using keyword arguments as local variables. """ 1537 if env is None: 1538 env = ImmutableValDict() 1539 stdout = [] 1540 env = self.execute(stdout, env) 1541 return ''.join(stdout), env # Return both now 1542 1543 1544class StplSyntaxError(TemplateError):pass 1545 1546 1547class _StplParser(object): 1548 """ Parser for stpl _templates. """ 1549 _re_cache = {} #: Cache for compiled re patterns 1550 1551 # This huge pile of voodoo magic splits python code into 8 different tokens. 1552 # We use the verbose (?x) regex mode to make this more manageable 1553 1554 _re_tok = _re_inl = r'''( # (?mx) will be added below for verbose and dotall mode 1555 [urbURB]* 1556 (?: ''(?!') 1557 |""(?!") 1558 |'{6} 1559 |"{6} 1560 |'(?:[^\\']|\\.)+?' 1561 |"(?:[^\\"]|\\.)+?" 1562 |'{3}(?:[^\\]|\\.|\n)+?'{3} 1563 |"{3}(?:[^\\]|\\.|\n)+?"{3} 1564 ) 1565 )''' 1566 1567 _re_inl = _re_tok.replace(r'|\n', '') # We re-use this string pattern later 1568 1569 _re_tok += r''' 1570 # 2: Comments (until end of line, but not the newline itself) 1571 |(\#.*) 1572 1573 # 3: Open and close (4) grouping tokens 1574 |([\[\{\(]) 1575 |([\]\}\)]) 1576 1577 # 5,6: Keywords that start or continue a python block (only start of line) 1578 |^([\ \t]*(?:if|for|while|with|try|def|class)\b) 1579 |^([\ \t]*(?:elif|else|except|finally)\b) 1580 1581 # 7: Our special 'end' keyword (but only if it stands alone) 1582 |((?:^|;)[\ \t]*end:{0,1}[\ \t]*(?=(?:%(block_close)s[\ \t]*)?\r?$|;|\#)) 1583 1584 # 8: A customizable end-of-code-block _template token (only end of line) 1585 |(%(block_close)s[\ \t]*(?=\r?$)) 1586 1587 # 9: And finally, a single newline. The 10th token is 'everything else' 1588 |(\r?\n) 1589 ''' 1590 1591 # Match the start tokens of code areas in a _template 1592 _re_split = r'''(?m)^[ \t]*(\\?)((%(line_start)s)|(%(block_start)s))''' 1593 # Match inline statements (may contain python strings) 1594 _re_inl = r'''%%(inline_start)s((?:%s|[^'"\n]+?)*?)%%(inline_end)s''' % _re_inl 1595 1596 # Add back in the flags to avoid the deprecation warning 1597 # verbose and dot-matches-newline mode 1598 _re_tok = '(?mx)' + _re_tok 1599 _re_inl = '(?mx)' + _re_inl 1600 1601 # default_syntax = '{% %} % { }' 1602 1603 def __init__(self, source, syntax=None, encoding='utf8'): 1604 self.source, self.encoding = _touni(source, encoding), encoding 1605 self.set_syntax(' '.join( [ BLOCK_START, 1606 BLOCK_CLOSE, 1607 LINE_START, 1608 INLINE_START, 1609 INLINE_END, 1610 ])) 1611 self.code_buffer, self.text_buffer = [], [] 1612 self.lineno, self.offset = 1, 0 1613 self.indent, self.indent_mod = 0, 0 1614 self.paren_depth = 0 1615 1616 def get_syntax(self): 1617 """ Tokens as a space separated string (default: {% %} % {{ }}) """ 1618 return self._syntax 1619 1620 def set_syntax(self, syntax): 1621 self._syntax = syntax 1622 self._tokens = syntax.split() 1623 if syntax not in self._re_cache: 1624 names = 'block_start block_close line_start inline_start inline_end' 1625 etokens = map(re.escape, self._tokens) 1626 pattern_vars = dict(zip(names.split(), etokens)) 1627 patterns = (self._re_split, self._re_tok, self._re_inl) 1628 patterns = [re.compile(p % pattern_vars) for p in patterns] 1629 self._re_cache[syntax] = patterns 1630 self.re_split, self.re_tok, self.re_inl = self._re_cache[syntax] 1631 1632 syntax = property(get_syntax, set_syntax) 1633 1634 def translate(self): 1635 if self.offset: raise RuntimeError('Parser is a one time instance.') 1636 while True: 1637 m = self.re_split.search(self.source, pos=self.offset) 1638 if m: 1639 text = self.source[self.offset:m.start()] 1640 self.text_buffer.append(text) 1641 self.offset = m.end() 1642 if m.group(1): # Escape syntax 1643 line, sep, _ = self.source[self.offset:].partition('\n') 1644 self.text_buffer.append(self.source[m.start():m.start(1)] + 1645 m.group(2) + line + sep) 1646 self.offset += len(line + sep) 1647 continue 1648 self.flush_text() 1649 self.offset += self.read_code(self.source[self.offset:], 1650 multiline=bool(m.group(4))) 1651 else: 1652 break 1653 self.text_buffer.append(self.source[self.offset:]) 1654 self.flush_text() 1655 return ''.join(self.code_buffer) 1656 1657 def read_code(self, pysource, multiline): 1658 code_line, comment = '', '' 1659 offset = 0 1660 while True: 1661 m = self.re_tok.search(pysource, pos=offset) 1662 if not m: 1663 code_line += pysource[offset:] 1664 offset = len(pysource) 1665 self.write_code(code_line.strip(), comment) 1666 break 1667 code_line += pysource[offset:m.start()] 1668 offset = m.end() 1669 _str, _com, _po, _pc, _blk1, _blk2, _end, _cend, _nl = m.groups() 1670 if self.paren_depth > 0 and (_blk1 or _blk2): # a if b else c 1671 code_line += _blk1 or _blk2 1672 continue 1673 if _str: # Python string 1674 code_line += _str 1675 elif _com: # Python comment (up to EOL) 1676 comment = _com 1677 if multiline and _com.strip().endswith(self._tokens[1]): 1678 multiline = False # Allow end-of-block in comments 1679 elif _po: # open parenthesis 1680 self.paren_depth += 1 1681 code_line += _po 1682 elif _pc: # close parenthesis 1683 if self.paren_depth > 0: 1684 # we could check for matching parentheses here, but it's 1685 # easier to leave that to python - just check counts 1686 self.paren_depth -= 1 1687 code_line += _pc 1688 elif _blk1: # Start-block keyword (if/for/while/def/try/...) 1689 code_line, self.indent_mod = _blk1, -1 1690 self.indent += 1 1691 elif _blk2: # Continue-block keyword (else/elif/except/...) 1692 code_line, self.indent_mod = _blk2, -1 1693 elif _end: # The non-standard 'end'-keyword (ends a block) 1694 self.indent -= 1 1695 elif _cend: # The end-code-block _template token (usually '%}') 1696 if multiline: multiline = False 1697 else: code_line += _cend 1698 else: # \n 1699 self.write_code(code_line.strip(), comment) 1700 self.lineno += 1 1701 code_line, comment, self.indent_mod = '', '', 0 1702 if not multiline: 1703 break 1704 1705 return offset 1706 1707 def flush_text(self): 1708 text = ''.join(self.text_buffer) 1709 del self.text_buffer[:] 1710 if not text: return 1711 parts, pos, nl = [], 0, '\\\n' + ' ' * self.indent 1712 for m in self.re_inl.finditer(text): 1713 prefix, pos = text[pos:m.start()], m.end() 1714 if prefix: 1715 parts.append(nl.join(map(repr, prefix.splitlines(True)))) 1716 if prefix.endswith('\n'): parts[-1] += nl 1717 parts.append(self.process_inline(m.group(1).strip())) 1718 if pos < len(text): 1719 prefix = text[pos:] 1720 lines = prefix.splitlines(True) 1721 if lines[-1].endswith('\\\\\n'): lines[-1] = lines[-1][:-3] 1722 elif lines[-1].endswith('\\\\\r\n'): lines[-1] = lines[-1][:-4] 1723 parts.append(nl.join(map(repr, lines))) 1724 code = '_printlist((%s,))' % ', '.join(parts) 1725 self.lineno += code.count('\n') + 1 1726 self.write_code(code) 1727 1728 @staticmethod 1729 def process_inline(chunk): 1730 if chunk[0] == '!': return '_str(%s)' % chunk[1:] 1731 return '_escape(%s)' % chunk 1732 1733 def write_code(self, line, comment=''): 1734 code = ' ' * (self.indent + self.indent_mod) 1735 code += line.lstrip() + comment + '\n' 1736 self.code_buffer.append(code) 1737 1738 1739def _template(tpl, env=None, return_env=False): 1740 """ 1741 Get a rendered _template as a string iterator. 1742 You can use a name, a filename or a _template string as first parameter. 1743 Template rendering arguments can be passed as dictionaries 1744 or directly (as keyword arguments). 1745 """ 1746 try: 1747 if env is None: 1748 env = ImmutableValDict() 1749 1750 # This was changed to first see if the file exists. If it does, 1751 # it is assumed to be a path. Otherwise, assumed it to be text 1752 1753 settings = {} 1754 tpl = _touni(tpl) 1755 1756 # Try to determine if it is a file or a template string 1757 1758 isfile = False 1759 try: 1760 if os.path.exists(tpl): 1761 isfile = True 1762 except:pass # Catch any kind of error 1763 1764 if not isfile: # template string 1765 lookup = ['./'] # Just have the lookup be in this path 1766 tpl = _preparser(tpl) 1767 tpl_obj = _SimpleTemplate(source=tpl, lookup=lookup, **settings) 1768 else: # template file 1769 # set the lookup. It goes in order so first check directory 1770 # of the original template and then the current. 1771 lookup = [os.path.dirname(tpl) + '/.','./'] 1772 tpl_obj = _SimpleTemplate(name=tpl, lookup=lookup, **settings) 1773 1774 # Added the option to return the environment, but this is really not needed 1775 # if env is set. 1776 1777 rendered,env = tpl_obj.render(env) 1778 1779 if not return_env: 1780 return rendered 1781 return rendered,env 1782 except Exception as E: 1783 if CLI_MODE and not DEBUGCLI: 1784 msg = _error_msg(E) 1785 sys.stderr.write(msg) 1786 sys.exit(1) 1787 else: 1788 raise 1789########################### six extracted codes ########################### 1790# This is pulled from the python six module (see links below) to work 1791# around some python 2.7.4 issues 1792# Links: 1793# https://github.com/benjaminp/six 1794# https://pypi.python.org/pypi/six 1795# http://pythonhosted.org/six/ 1796############################################################################## 1797# Copyright (c) 2010-2018 Benjamin Peterson 1798# 1799# Permission is hereby granted, free of charge, to any person obtaining a copy 1800# of this software and associated documentation files (the "Software"), to deal 1801# in the Software without restriction, including without limitation the rights 1802# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1803# copies of the Software, and to permit persons to whom the Software is 1804# furnished to do so, subject to the following conditions: 1805# 1806# The above copyright notice and this permission notice shall be included in 1807# all copies or substantial portions of the Software. 1808# 1809# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1810# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1811# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 1812# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 1813# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 1814# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 1815# THE SOFTWARE. 1816############################################################################# 1817 1818if py3: 1819 exec('exec_ = exec') 1820else: 1821 def exec_(_code_, _globs_=None, _locs_=None): 1822 """Execute code in a namespace.""" 1823 if _globs_ is None: 1824 frame = sys._getframe(1) 1825 _globs_ = frame.f_globals 1826 if _locs_ is None: 1827 _locs_ = frame.f_locals 1828 del frame 1829 elif _locs_ is None: 1830 _locs_ = _globs_ 1831 exec("""exec _code_ in _globs_, _locs_""") 1832 1833############################################################################## 1834 1835# Global set of keys from an empty execution: 1836INIT_VARS = set(_template('BLA',return_env=True)[-1].keys()) 1837 1838def main(): 1839 global CLI_MODE 1840 CLI_MODE = True 1841 cmdname = sys.argv[0].lower() 1842 path, execname = os.path.split(cmdname) 1843 if execname.startswith('dprepro'): 1844 sys.exit(_dprepro_cli(sys.argv[1:])) 1845 sys.exit(_pyprepro_cli(sys.argv[1:])) 1846 1847# When called via command line 1848if __name__ == '__main__': 1849 main() 1850 1851 1852 1853 1854 1855