1# mako/parsetree.py 2# Copyright 2006-2020 the Mako authors and contributors <see AUTHORS file> 3# 4# This module is part of Mako and is released under 5# the MIT License: http://www.opensource.org/licenses/mit-license.php 6 7"""defines the parse tree components for Mako templates.""" 8 9import re 10 11from mako import ast 12from mako import compat 13from mako import exceptions 14from mako import filters 15from mako import util 16 17 18class Node(object): 19 20 """base class for a Node in the parse tree.""" 21 22 def __init__(self, source, lineno, pos, filename): 23 self.source = source 24 self.lineno = lineno 25 self.pos = pos 26 self.filename = filename 27 28 @property 29 def exception_kwargs(self): 30 return { 31 "source": self.source, 32 "lineno": self.lineno, 33 "pos": self.pos, 34 "filename": self.filename, 35 } 36 37 def get_children(self): 38 return [] 39 40 def accept_visitor(self, visitor): 41 def traverse(node): 42 for n in node.get_children(): 43 n.accept_visitor(visitor) 44 45 method = getattr(visitor, "visit" + self.__class__.__name__, traverse) 46 method(self) 47 48 49class TemplateNode(Node): 50 51 """a 'container' node that stores the overall collection of nodes.""" 52 53 def __init__(self, filename): 54 super(TemplateNode, self).__init__("", 0, 0, filename) 55 self.nodes = [] 56 self.page_attributes = {} 57 58 def get_children(self): 59 return self.nodes 60 61 def __repr__(self): 62 return "TemplateNode(%s, %r)" % ( 63 util.sorted_dict_repr(self.page_attributes), 64 self.nodes, 65 ) 66 67 68class ControlLine(Node): 69 70 """defines a control line, a line-oriented python line or end tag. 71 72 e.g.:: 73 74 % if foo: 75 (markup) 76 % endif 77 78 """ 79 80 has_loop_context = False 81 82 def __init__(self, keyword, isend, text, **kwargs): 83 super(ControlLine, self).__init__(**kwargs) 84 self.text = text 85 self.keyword = keyword 86 self.isend = isend 87 self.is_primary = keyword in ["for", "if", "while", "try", "with"] 88 self.nodes = [] 89 if self.isend: 90 self._declared_identifiers = [] 91 self._undeclared_identifiers = [] 92 else: 93 code = ast.PythonFragment(text, **self.exception_kwargs) 94 self._declared_identifiers = code.declared_identifiers 95 self._undeclared_identifiers = code.undeclared_identifiers 96 97 def get_children(self): 98 return self.nodes 99 100 def declared_identifiers(self): 101 return self._declared_identifiers 102 103 def undeclared_identifiers(self): 104 return self._undeclared_identifiers 105 106 def is_ternary(self, keyword): 107 """return true if the given keyword is a ternary keyword 108 for this ControlLine""" 109 110 return keyword in { 111 "if": set(["else", "elif"]), 112 "try": set(["except", "finally"]), 113 "for": set(["else"]), 114 }.get(self.keyword, []) 115 116 def __repr__(self): 117 return "ControlLine(%r, %r, %r, %r)" % ( 118 self.keyword, 119 self.text, 120 self.isend, 121 (self.lineno, self.pos), 122 ) 123 124 125class Text(Node): 126 127 """defines plain text in the template.""" 128 129 def __init__(self, content, **kwargs): 130 super(Text, self).__init__(**kwargs) 131 self.content = content 132 133 def __repr__(self): 134 return "Text(%r, %r)" % (self.content, (self.lineno, self.pos)) 135 136 137class Code(Node): 138 139 """defines a Python code block, either inline or module level. 140 141 e.g.:: 142 143 inline: 144 <% 145 x = 12 146 %> 147 148 module level: 149 <%! 150 import logger 151 %> 152 153 """ 154 155 def __init__(self, text, ismodule, **kwargs): 156 super(Code, self).__init__(**kwargs) 157 self.text = text 158 self.ismodule = ismodule 159 self.code = ast.PythonCode(text, **self.exception_kwargs) 160 161 def declared_identifiers(self): 162 return self.code.declared_identifiers 163 164 def undeclared_identifiers(self): 165 return self.code.undeclared_identifiers 166 167 def __repr__(self): 168 return "Code(%r, %r, %r)" % ( 169 self.text, 170 self.ismodule, 171 (self.lineno, self.pos), 172 ) 173 174 175class Comment(Node): 176 177 """defines a comment line. 178 179 # this is a comment 180 181 """ 182 183 def __init__(self, text, **kwargs): 184 super(Comment, self).__init__(**kwargs) 185 self.text = text 186 187 def __repr__(self): 188 return "Comment(%r, %r)" % (self.text, (self.lineno, self.pos)) 189 190 191class Expression(Node): 192 193 """defines an inline expression. 194 195 ${x+y} 196 197 """ 198 199 def __init__(self, text, escapes, **kwargs): 200 super(Expression, self).__init__(**kwargs) 201 self.text = text 202 self.escapes = escapes 203 self.escapes_code = ast.ArgumentList(escapes, **self.exception_kwargs) 204 self.code = ast.PythonCode(text, **self.exception_kwargs) 205 206 def declared_identifiers(self): 207 return [] 208 209 def undeclared_identifiers(self): 210 # TODO: make the "filter" shortcut list configurable at parse/gen time 211 return self.code.undeclared_identifiers.union( 212 self.escapes_code.undeclared_identifiers.difference( 213 set(filters.DEFAULT_ESCAPES.keys()) 214 ) 215 ).difference(self.code.declared_identifiers) 216 217 def __repr__(self): 218 return "Expression(%r, %r, %r)" % ( 219 self.text, 220 self.escapes_code.args, 221 (self.lineno, self.pos), 222 ) 223 224 225class _TagMeta(type): 226 227 """metaclass to allow Tag to produce a subclass according to 228 its keyword""" 229 230 _classmap = {} 231 232 def __init__(cls, clsname, bases, dict_): 233 if getattr(cls, "__keyword__", None) is not None: 234 cls._classmap[cls.__keyword__] = cls 235 super(_TagMeta, cls).__init__(clsname, bases, dict_) 236 237 def __call__(cls, keyword, attributes, **kwargs): 238 if ":" in keyword: 239 ns, defname = keyword.split(":") 240 return type.__call__( 241 CallNamespaceTag, ns, defname, attributes, **kwargs 242 ) 243 244 try: 245 cls = _TagMeta._classmap[keyword] 246 except KeyError: 247 raise exceptions.CompileException( 248 "No such tag: '%s'" % keyword, 249 source=kwargs["source"], 250 lineno=kwargs["lineno"], 251 pos=kwargs["pos"], 252 filename=kwargs["filename"], 253 ) 254 return type.__call__(cls, keyword, attributes, **kwargs) 255 256 257class Tag(compat.with_metaclass(_TagMeta, Node)): 258 """abstract base class for tags. 259 260 e.g.:: 261 262 <%sometag/> 263 264 <%someothertag> 265 stuff 266 </%someothertag> 267 268 """ 269 270 __keyword__ = None 271 272 def __init__( 273 self, 274 keyword, 275 attributes, 276 expressions, 277 nonexpressions, 278 required, 279 **kwargs 280 ): 281 r"""construct a new Tag instance. 282 283 this constructor not called directly, and is only called 284 by subclasses. 285 286 :param keyword: the tag keyword 287 288 :param attributes: raw dictionary of attribute key/value pairs 289 290 :param expressions: a set of identifiers that are legal attributes, 291 which can also contain embedded expressions 292 293 :param nonexpressions: a set of identifiers that are legal 294 attributes, which cannot contain embedded expressions 295 296 :param \**kwargs: 297 other arguments passed to the Node superclass (lineno, pos) 298 299 """ 300 super(Tag, self).__init__(**kwargs) 301 self.keyword = keyword 302 self.attributes = attributes 303 self._parse_attributes(expressions, nonexpressions) 304 missing = [r for r in required if r not in self.parsed_attributes] 305 if len(missing): 306 raise exceptions.CompileException( 307 "Missing attribute(s): %s" 308 % ",".join([repr(m) for m in missing]), 309 **self.exception_kwargs 310 ) 311 self.parent = None 312 self.nodes = [] 313 314 def is_root(self): 315 return self.parent is None 316 317 def get_children(self): 318 return self.nodes 319 320 def _parse_attributes(self, expressions, nonexpressions): 321 undeclared_identifiers = set() 322 self.parsed_attributes = {} 323 for key in self.attributes: 324 if key in expressions: 325 expr = [] 326 for x in re.compile(r"(\${.+?})", re.S).split( 327 self.attributes[key] 328 ): 329 m = re.compile(r"^\${(.+?)}$", re.S).match(x) 330 if m: 331 code = ast.PythonCode( 332 m.group(1).rstrip(), **self.exception_kwargs 333 ) 334 # we aren't discarding "declared_identifiers" here, 335 # which we do so that list comprehension-declared 336 # variables aren't counted. As yet can't find a 337 # condition that requires it here. 338 undeclared_identifiers = undeclared_identifiers.union( 339 code.undeclared_identifiers 340 ) 341 expr.append("(%s)" % m.group(1)) 342 else: 343 if x: 344 expr.append(repr(x)) 345 self.parsed_attributes[key] = " + ".join(expr) or repr("") 346 elif key in nonexpressions: 347 if re.search(r"\${.+?}", self.attributes[key]): 348 raise exceptions.CompileException( 349 "Attibute '%s' in tag '%s' does not allow embedded " 350 "expressions" % (key, self.keyword), 351 **self.exception_kwargs 352 ) 353 self.parsed_attributes[key] = repr(self.attributes[key]) 354 else: 355 raise exceptions.CompileException( 356 "Invalid attribute for tag '%s': '%s'" 357 % (self.keyword, key), 358 **self.exception_kwargs 359 ) 360 self.expression_undeclared_identifiers = undeclared_identifiers 361 362 def declared_identifiers(self): 363 return [] 364 365 def undeclared_identifiers(self): 366 return self.expression_undeclared_identifiers 367 368 def __repr__(self): 369 return "%s(%r, %s, %r, %r)" % ( 370 self.__class__.__name__, 371 self.keyword, 372 util.sorted_dict_repr(self.attributes), 373 (self.lineno, self.pos), 374 self.nodes, 375 ) 376 377 378class IncludeTag(Tag): 379 __keyword__ = "include" 380 381 def __init__(self, keyword, attributes, **kwargs): 382 super(IncludeTag, self).__init__( 383 keyword, 384 attributes, 385 ("file", "import", "args"), 386 (), 387 ("file",), 388 **kwargs 389 ) 390 self.page_args = ast.PythonCode( 391 "__DUMMY(%s)" % attributes.get("args", ""), **self.exception_kwargs 392 ) 393 394 def declared_identifiers(self): 395 return [] 396 397 def undeclared_identifiers(self): 398 identifiers = self.page_args.undeclared_identifiers.difference( 399 set(["__DUMMY"]) 400 ).difference(self.page_args.declared_identifiers) 401 return identifiers.union( 402 super(IncludeTag, self).undeclared_identifiers() 403 ) 404 405 406class NamespaceTag(Tag): 407 __keyword__ = "namespace" 408 409 def __init__(self, keyword, attributes, **kwargs): 410 super(NamespaceTag, self).__init__( 411 keyword, 412 attributes, 413 ("file",), 414 ("name", "inheritable", "import", "module"), 415 (), 416 **kwargs 417 ) 418 419 self.name = attributes.get("name", "__anon_%s" % hex(abs(id(self)))) 420 if "name" not in attributes and "import" not in attributes: 421 raise exceptions.CompileException( 422 "'name' and/or 'import' attributes are required " 423 "for <%namespace>", 424 **self.exception_kwargs 425 ) 426 if "file" in attributes and "module" in attributes: 427 raise exceptions.CompileException( 428 "<%namespace> may only have one of 'file' or 'module'", 429 **self.exception_kwargs 430 ) 431 432 def declared_identifiers(self): 433 return [] 434 435 436class TextTag(Tag): 437 __keyword__ = "text" 438 439 def __init__(self, keyword, attributes, **kwargs): 440 super(TextTag, self).__init__( 441 keyword, attributes, (), ("filter"), (), **kwargs 442 ) 443 self.filter_args = ast.ArgumentList( 444 attributes.get("filter", ""), **self.exception_kwargs 445 ) 446 447 def undeclared_identifiers(self): 448 return self.filter_args.undeclared_identifiers.difference( 449 filters.DEFAULT_ESCAPES.keys() 450 ).union(self.expression_undeclared_identifiers) 451 452 453class DefTag(Tag): 454 __keyword__ = "def" 455 456 def __init__(self, keyword, attributes, **kwargs): 457 expressions = ["buffered", "cached"] + [ 458 c for c in attributes if c.startswith("cache_") 459 ] 460 461 super(DefTag, self).__init__( 462 keyword, 463 attributes, 464 expressions, 465 ("name", "filter", "decorator"), 466 ("name",), 467 **kwargs 468 ) 469 name = attributes["name"] 470 if re.match(r"^[\w_]+$", name): 471 raise exceptions.CompileException( 472 "Missing parenthesis in %def", **self.exception_kwargs 473 ) 474 self.function_decl = ast.FunctionDecl( 475 "def " + name + ":pass", **self.exception_kwargs 476 ) 477 self.name = self.function_decl.funcname 478 self.decorator = attributes.get("decorator", "") 479 self.filter_args = ast.ArgumentList( 480 attributes.get("filter", ""), **self.exception_kwargs 481 ) 482 483 is_anonymous = False 484 is_block = False 485 486 @property 487 def funcname(self): 488 return self.function_decl.funcname 489 490 def get_argument_expressions(self, **kw): 491 return self.function_decl.get_argument_expressions(**kw) 492 493 def declared_identifiers(self): 494 return self.function_decl.allargnames 495 496 def undeclared_identifiers(self): 497 res = [] 498 for c in self.function_decl.defaults: 499 res += list( 500 ast.PythonCode( 501 c, **self.exception_kwargs 502 ).undeclared_identifiers 503 ) 504 return ( 505 set(res) 506 .union( 507 self.filter_args.undeclared_identifiers.difference( 508 filters.DEFAULT_ESCAPES.keys() 509 ) 510 ) 511 .union(self.expression_undeclared_identifiers) 512 .difference(self.function_decl.allargnames) 513 ) 514 515 516class BlockTag(Tag): 517 __keyword__ = "block" 518 519 def __init__(self, keyword, attributes, **kwargs): 520 expressions = ["buffered", "cached", "args"] + [ 521 c for c in attributes if c.startswith("cache_") 522 ] 523 524 super(BlockTag, self).__init__( 525 keyword, 526 attributes, 527 expressions, 528 ("name", "filter", "decorator"), 529 (), 530 **kwargs 531 ) 532 name = attributes.get("name") 533 if name and not re.match(r"^[\w_]+$", name): 534 raise exceptions.CompileException( 535 "%block may not specify an argument signature", 536 **self.exception_kwargs 537 ) 538 if not name and attributes.get("args", None): 539 raise exceptions.CompileException( 540 "Only named %blocks may specify args", **self.exception_kwargs 541 ) 542 self.body_decl = ast.FunctionArgs( 543 attributes.get("args", ""), **self.exception_kwargs 544 ) 545 546 self.name = name 547 self.decorator = attributes.get("decorator", "") 548 self.filter_args = ast.ArgumentList( 549 attributes.get("filter", ""), **self.exception_kwargs 550 ) 551 552 is_block = True 553 554 @property 555 def is_anonymous(self): 556 return self.name is None 557 558 @property 559 def funcname(self): 560 return self.name or "__M_anon_%d" % (self.lineno,) 561 562 def get_argument_expressions(self, **kw): 563 return self.body_decl.get_argument_expressions(**kw) 564 565 def declared_identifiers(self): 566 return self.body_decl.allargnames 567 568 def undeclared_identifiers(self): 569 return ( 570 self.filter_args.undeclared_identifiers.difference( 571 filters.DEFAULT_ESCAPES.keys() 572 ) 573 ).union(self.expression_undeclared_identifiers) 574 575 576class CallTag(Tag): 577 __keyword__ = "call" 578 579 def __init__(self, keyword, attributes, **kwargs): 580 super(CallTag, self).__init__( 581 keyword, attributes, ("args"), ("expr",), ("expr",), **kwargs 582 ) 583 self.expression = attributes["expr"] 584 self.code = ast.PythonCode(self.expression, **self.exception_kwargs) 585 self.body_decl = ast.FunctionArgs( 586 attributes.get("args", ""), **self.exception_kwargs 587 ) 588 589 def declared_identifiers(self): 590 return self.code.declared_identifiers.union(self.body_decl.allargnames) 591 592 def undeclared_identifiers(self): 593 return self.code.undeclared_identifiers.difference( 594 self.code.declared_identifiers 595 ) 596 597 598class CallNamespaceTag(Tag): 599 def __init__(self, namespace, defname, attributes, **kwargs): 600 super(CallNamespaceTag, self).__init__( 601 namespace + ":" + defname, 602 attributes, 603 tuple(attributes.keys()) + ("args",), 604 (), 605 (), 606 **kwargs 607 ) 608 609 self.expression = "%s.%s(%s)" % ( 610 namespace, 611 defname, 612 ",".join( 613 [ 614 "%s=%s" % (k, v) 615 for k, v in self.parsed_attributes.items() 616 if k != "args" 617 ] 618 ), 619 ) 620 self.code = ast.PythonCode(self.expression, **self.exception_kwargs) 621 self.body_decl = ast.FunctionArgs( 622 attributes.get("args", ""), **self.exception_kwargs 623 ) 624 625 def declared_identifiers(self): 626 return self.code.declared_identifiers.union(self.body_decl.allargnames) 627 628 def undeclared_identifiers(self): 629 return self.code.undeclared_identifiers.difference( 630 self.code.declared_identifiers 631 ) 632 633 634class InheritTag(Tag): 635 __keyword__ = "inherit" 636 637 def __init__(self, keyword, attributes, **kwargs): 638 super(InheritTag, self).__init__( 639 keyword, attributes, ("file",), (), ("file",), **kwargs 640 ) 641 642 643class PageTag(Tag): 644 __keyword__ = "page" 645 646 def __init__(self, keyword, attributes, **kwargs): 647 expressions = [ 648 "cached", 649 "args", 650 "expression_filter", 651 "enable_loop", 652 ] + [c for c in attributes if c.startswith("cache_")] 653 654 super(PageTag, self).__init__( 655 keyword, attributes, expressions, (), (), **kwargs 656 ) 657 self.body_decl = ast.FunctionArgs( 658 attributes.get("args", ""), **self.exception_kwargs 659 ) 660 self.filter_args = ast.ArgumentList( 661 attributes.get("expression_filter", ""), **self.exception_kwargs 662 ) 663 664 def declared_identifiers(self): 665 return self.body_decl.allargnames 666