1"""Compiles nodes from the parser into Python code.""" 2import typing as t 3from contextlib import contextmanager 4from functools import update_wrapper 5from io import StringIO 6from itertools import chain 7from keyword import iskeyword as is_python_keyword 8 9from markupsafe import escape 10from markupsafe import Markup 11 12from . import nodes 13from .exceptions import TemplateAssertionError 14from .idtracking import Symbols 15from .idtracking import VAR_LOAD_ALIAS 16from .idtracking import VAR_LOAD_PARAMETER 17from .idtracking import VAR_LOAD_RESOLVE 18from .idtracking import VAR_LOAD_UNDEFINED 19from .nodes import EvalContext 20from .optimizer import Optimizer 21from .utils import _PassArg 22from .utils import concat 23from .visitor import NodeVisitor 24 25if t.TYPE_CHECKING: 26 import typing_extensions as te 27 from .environment import Environment 28 29F = t.TypeVar("F", bound=t.Callable[..., t.Any]) 30 31operators = { 32 "eq": "==", 33 "ne": "!=", 34 "gt": ">", 35 "gteq": ">=", 36 "lt": "<", 37 "lteq": "<=", 38 "in": "in", 39 "notin": "not in", 40} 41 42 43def optimizeconst(f: F) -> F: 44 def new_func( 45 self: "CodeGenerator", node: nodes.Expr, frame: "Frame", **kwargs: t.Any 46 ) -> t.Any: 47 # Only optimize if the frame is not volatile 48 if self.optimizer is not None and not frame.eval_ctx.volatile: 49 new_node = self.optimizer.visit(node, frame.eval_ctx) 50 51 if new_node != node: 52 return self.visit(new_node, frame) 53 54 return f(self, node, frame, **kwargs) 55 56 return update_wrapper(t.cast(F, new_func), f) 57 58 59def _make_binop(op: str) -> t.Callable[["CodeGenerator", nodes.BinExpr, "Frame"], None]: 60 @optimizeconst 61 def visitor(self: "CodeGenerator", node: nodes.BinExpr, frame: Frame) -> None: 62 if ( 63 self.environment.sandboxed 64 and op in self.environment.intercepted_binops # type: ignore 65 ): 66 self.write(f"environment.call_binop(context, {op!r}, ") 67 self.visit(node.left, frame) 68 self.write(", ") 69 self.visit(node.right, frame) 70 else: 71 self.write("(") 72 self.visit(node.left, frame) 73 self.write(f" {op} ") 74 self.visit(node.right, frame) 75 76 self.write(")") 77 78 return visitor 79 80 81def _make_unop( 82 op: str, 83) -> t.Callable[["CodeGenerator", nodes.UnaryExpr, "Frame"], None]: 84 @optimizeconst 85 def visitor(self: "CodeGenerator", node: nodes.UnaryExpr, frame: Frame) -> None: 86 if ( 87 self.environment.sandboxed 88 and op in self.environment.intercepted_unops # type: ignore 89 ): 90 self.write(f"environment.call_unop(context, {op!r}, ") 91 self.visit(node.node, frame) 92 else: 93 self.write("(" + op) 94 self.visit(node.node, frame) 95 96 self.write(")") 97 98 return visitor 99 100 101def generate( 102 node: nodes.Template, 103 environment: "Environment", 104 name: t.Optional[str], 105 filename: t.Optional[str], 106 stream: t.Optional[t.TextIO] = None, 107 defer_init: bool = False, 108 optimized: bool = True, 109) -> t.Optional[str]: 110 """Generate the python source for a node tree.""" 111 if not isinstance(node, nodes.Template): 112 raise TypeError("Can't compile non template nodes") 113 114 generator = environment.code_generator_class( 115 environment, name, filename, stream, defer_init, optimized 116 ) 117 generator.visit(node) 118 119 if stream is None: 120 return generator.stream.getvalue() # type: ignore 121 122 return None 123 124 125def has_safe_repr(value: t.Any) -> bool: 126 """Does the node have a safe representation?""" 127 if value is None or value is NotImplemented or value is Ellipsis: 128 return True 129 130 if type(value) in {bool, int, float, complex, range, str, Markup}: 131 return True 132 133 if type(value) in {tuple, list, set, frozenset}: 134 return all(has_safe_repr(v) for v in value) 135 136 if type(value) is dict: 137 return all(has_safe_repr(k) and has_safe_repr(v) for k, v in value.items()) 138 139 return False 140 141 142def find_undeclared( 143 nodes: t.Iterable[nodes.Node], names: t.Iterable[str] 144) -> t.Set[str]: 145 """Check if the names passed are accessed undeclared. The return value 146 is a set of all the undeclared names from the sequence of names found. 147 """ 148 visitor = UndeclaredNameVisitor(names) 149 try: 150 for node in nodes: 151 visitor.visit(node) 152 except VisitorExit: 153 pass 154 return visitor.undeclared 155 156 157class MacroRef: 158 def __init__(self, node: t.Union[nodes.Macro, nodes.CallBlock]) -> None: 159 self.node = node 160 self.accesses_caller = False 161 self.accesses_kwargs = False 162 self.accesses_varargs = False 163 164 165class Frame: 166 """Holds compile time information for us.""" 167 168 def __init__( 169 self, 170 eval_ctx: EvalContext, 171 parent: t.Optional["Frame"] = None, 172 level: t.Optional[int] = None, 173 ) -> None: 174 self.eval_ctx = eval_ctx 175 176 # the parent of this frame 177 self.parent = parent 178 179 if parent is None: 180 self.symbols = Symbols(level=level) 181 182 # in some dynamic inheritance situations the compiler needs to add 183 # write tests around output statements. 184 self.require_output_check = False 185 186 # inside some tags we are using a buffer rather than yield statements. 187 # this for example affects {% filter %} or {% macro %}. If a frame 188 # is buffered this variable points to the name of the list used as 189 # buffer. 190 self.buffer: t.Optional[str] = None 191 192 # the name of the block we're in, otherwise None. 193 self.block: t.Optional[str] = None 194 195 else: 196 self.symbols = Symbols(parent.symbols, level=level) 197 self.require_output_check = parent.require_output_check 198 self.buffer = parent.buffer 199 self.block = parent.block 200 201 # a toplevel frame is the root + soft frames such as if conditions. 202 self.toplevel = False 203 204 # the root frame is basically just the outermost frame, so no if 205 # conditions. This information is used to optimize inheritance 206 # situations. 207 self.rootlevel = False 208 209 # variables set inside of loops and blocks should not affect outer frames, 210 # but they still needs to be kept track of as part of the active context. 211 self.loop_frame = False 212 self.block_frame = False 213 214 # track whether the frame is being used in an if-statement or conditional 215 # expression as it determines which errors should be raised during runtime 216 # or compile time. 217 self.soft_frame = False 218 219 def copy(self) -> "Frame": 220 """Create a copy of the current one.""" 221 rv = t.cast(Frame, object.__new__(self.__class__)) 222 rv.__dict__.update(self.__dict__) 223 rv.symbols = self.symbols.copy() 224 return rv 225 226 def inner(self, isolated: bool = False) -> "Frame": 227 """Return an inner frame.""" 228 if isolated: 229 return Frame(self.eval_ctx, level=self.symbols.level + 1) 230 return Frame(self.eval_ctx, self) 231 232 def soft(self) -> "Frame": 233 """Return a soft frame. A soft frame may not be modified as 234 standalone thing as it shares the resources with the frame it 235 was created of, but it's not a rootlevel frame any longer. 236 237 This is only used to implement if-statements and conditional 238 expressions. 239 """ 240 rv = self.copy() 241 rv.rootlevel = False 242 rv.soft_frame = True 243 return rv 244 245 __copy__ = copy 246 247 248class VisitorExit(RuntimeError): 249 """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" 250 251 252class DependencyFinderVisitor(NodeVisitor): 253 """A visitor that collects filter and test calls.""" 254 255 def __init__(self) -> None: 256 self.filters: t.Set[str] = set() 257 self.tests: t.Set[str] = set() 258 259 def visit_Filter(self, node: nodes.Filter) -> None: 260 self.generic_visit(node) 261 self.filters.add(node.name) 262 263 def visit_Test(self, node: nodes.Test) -> None: 264 self.generic_visit(node) 265 self.tests.add(node.name) 266 267 def visit_Block(self, node: nodes.Block) -> None: 268 """Stop visiting at blocks.""" 269 270 271class UndeclaredNameVisitor(NodeVisitor): 272 """A visitor that checks if a name is accessed without being 273 declared. This is different from the frame visitor as it will 274 not stop at closure frames. 275 """ 276 277 def __init__(self, names: t.Iterable[str]) -> None: 278 self.names = set(names) 279 self.undeclared: t.Set[str] = set() 280 281 def visit_Name(self, node: nodes.Name) -> None: 282 if node.ctx == "load" and node.name in self.names: 283 self.undeclared.add(node.name) 284 if self.undeclared == self.names: 285 raise VisitorExit() 286 else: 287 self.names.discard(node.name) 288 289 def visit_Block(self, node: nodes.Block) -> None: 290 """Stop visiting a blocks.""" 291 292 293class CompilerExit(Exception): 294 """Raised if the compiler encountered a situation where it just 295 doesn't make sense to further process the code. Any block that 296 raises such an exception is not further processed. 297 """ 298 299 300class CodeGenerator(NodeVisitor): 301 def __init__( 302 self, 303 environment: "Environment", 304 name: t.Optional[str], 305 filename: t.Optional[str], 306 stream: t.Optional[t.TextIO] = None, 307 defer_init: bool = False, 308 optimized: bool = True, 309 ) -> None: 310 if stream is None: 311 stream = StringIO() 312 self.environment = environment 313 self.name = name 314 self.filename = filename 315 self.stream = stream 316 self.created_block_context = False 317 self.defer_init = defer_init 318 self.optimizer: t.Optional[Optimizer] = None 319 320 if optimized: 321 self.optimizer = Optimizer(environment) 322 323 # aliases for imports 324 self.import_aliases: t.Dict[str, str] = {} 325 326 # a registry for all blocks. Because blocks are moved out 327 # into the global python scope they are registered here 328 self.blocks: t.Dict[str, nodes.Block] = {} 329 330 # the number of extends statements so far 331 self.extends_so_far = 0 332 333 # some templates have a rootlevel extends. In this case we 334 # can safely assume that we're a child template and do some 335 # more optimizations. 336 self.has_known_extends = False 337 338 # the current line number 339 self.code_lineno = 1 340 341 # registry of all filters and tests (global, not block local) 342 self.tests: t.Dict[str, str] = {} 343 self.filters: t.Dict[str, str] = {} 344 345 # the debug information 346 self.debug_info: t.List[t.Tuple[int, int]] = [] 347 self._write_debug_info: t.Optional[int] = None 348 349 # the number of new lines before the next write() 350 self._new_lines = 0 351 352 # the line number of the last written statement 353 self._last_line = 0 354 355 # true if nothing was written so far. 356 self._first_write = True 357 358 # used by the `temporary_identifier` method to get new 359 # unique, temporary identifier 360 self._last_identifier = 0 361 362 # the current indentation 363 self._indentation = 0 364 365 # Tracks toplevel assignments 366 self._assign_stack: t.List[t.Set[str]] = [] 367 368 # Tracks parameter definition blocks 369 self._param_def_block: t.List[t.Set[str]] = [] 370 371 # Tracks the current context. 372 self._context_reference_stack = ["context"] 373 374 @property 375 def optimized(self) -> bool: 376 return self.optimizer is not None 377 378 # -- Various compilation helpers 379 380 def fail(self, msg: str, lineno: int) -> "te.NoReturn": 381 """Fail with a :exc:`TemplateAssertionError`.""" 382 raise TemplateAssertionError(msg, lineno, self.name, self.filename) 383 384 def temporary_identifier(self) -> str: 385 """Get a new unique identifier.""" 386 self._last_identifier += 1 387 return f"t_{self._last_identifier}" 388 389 def buffer(self, frame: Frame) -> None: 390 """Enable buffering for the frame from that point onwards.""" 391 frame.buffer = self.temporary_identifier() 392 self.writeline(f"{frame.buffer} = []") 393 394 def return_buffer_contents( 395 self, frame: Frame, force_unescaped: bool = False 396 ) -> None: 397 """Return the buffer contents of the frame.""" 398 if not force_unescaped: 399 if frame.eval_ctx.volatile: 400 self.writeline("if context.eval_ctx.autoescape:") 401 self.indent() 402 self.writeline(f"return Markup(concat({frame.buffer}))") 403 self.outdent() 404 self.writeline("else:") 405 self.indent() 406 self.writeline(f"return concat({frame.buffer})") 407 self.outdent() 408 return 409 elif frame.eval_ctx.autoescape: 410 self.writeline(f"return Markup(concat({frame.buffer}))") 411 return 412 self.writeline(f"return concat({frame.buffer})") 413 414 def indent(self) -> None: 415 """Indent by one.""" 416 self._indentation += 1 417 418 def outdent(self, step: int = 1) -> None: 419 """Outdent by step.""" 420 self._indentation -= step 421 422 def start_write(self, frame: Frame, node: t.Optional[nodes.Node] = None) -> None: 423 """Yield or write into the frame buffer.""" 424 if frame.buffer is None: 425 self.writeline("yield ", node) 426 else: 427 self.writeline(f"{frame.buffer}.append(", node) 428 429 def end_write(self, frame: Frame) -> None: 430 """End the writing process started by `start_write`.""" 431 if frame.buffer is not None: 432 self.write(")") 433 434 def simple_write( 435 self, s: str, frame: Frame, node: t.Optional[nodes.Node] = None 436 ) -> None: 437 """Simple shortcut for start_write + write + end_write.""" 438 self.start_write(frame, node) 439 self.write(s) 440 self.end_write(frame) 441 442 def blockvisit(self, nodes: t.Iterable[nodes.Node], frame: Frame) -> None: 443 """Visit a list of nodes as block in a frame. If the current frame 444 is no buffer a dummy ``if 0: yield None`` is written automatically. 445 """ 446 try: 447 self.writeline("pass") 448 for node in nodes: 449 self.visit(node, frame) 450 except CompilerExit: 451 pass 452 453 def write(self, x: str) -> None: 454 """Write a string into the output stream.""" 455 if self._new_lines: 456 if not self._first_write: 457 self.stream.write("\n" * self._new_lines) 458 self.code_lineno += self._new_lines 459 if self._write_debug_info is not None: 460 self.debug_info.append((self._write_debug_info, self.code_lineno)) 461 self._write_debug_info = None 462 self._first_write = False 463 self.stream.write(" " * self._indentation) 464 self._new_lines = 0 465 self.stream.write(x) 466 467 def writeline( 468 self, x: str, node: t.Optional[nodes.Node] = None, extra: int = 0 469 ) -> None: 470 """Combination of newline and write.""" 471 self.newline(node, extra) 472 self.write(x) 473 474 def newline(self, node: t.Optional[nodes.Node] = None, extra: int = 0) -> None: 475 """Add one or more newlines before the next write.""" 476 self._new_lines = max(self._new_lines, 1 + extra) 477 if node is not None and node.lineno != self._last_line: 478 self._write_debug_info = node.lineno 479 self._last_line = node.lineno 480 481 def signature( 482 self, 483 node: t.Union[nodes.Call, nodes.Filter, nodes.Test], 484 frame: Frame, 485 extra_kwargs: t.Optional[t.Mapping[str, t.Any]] = None, 486 ) -> None: 487 """Writes a function call to the stream for the current node. 488 A leading comma is added automatically. The extra keyword 489 arguments may not include python keywords otherwise a syntax 490 error could occur. The extra keyword arguments should be given 491 as python dict. 492 """ 493 # if any of the given keyword arguments is a python keyword 494 # we have to make sure that no invalid call is created. 495 kwarg_workaround = any( 496 is_python_keyword(t.cast(str, k)) 497 for k in chain((x.key for x in node.kwargs), extra_kwargs or ()) 498 ) 499 500 for arg in node.args: 501 self.write(", ") 502 self.visit(arg, frame) 503 504 if not kwarg_workaround: 505 for kwarg in node.kwargs: 506 self.write(", ") 507 self.visit(kwarg, frame) 508 if extra_kwargs is not None: 509 for key, value in extra_kwargs.items(): 510 self.write(f", {key}={value}") 511 if node.dyn_args: 512 self.write(", *") 513 self.visit(node.dyn_args, frame) 514 515 if kwarg_workaround: 516 if node.dyn_kwargs is not None: 517 self.write(", **dict({") 518 else: 519 self.write(", **{") 520 for kwarg in node.kwargs: 521 self.write(f"{kwarg.key!r}: ") 522 self.visit(kwarg.value, frame) 523 self.write(", ") 524 if extra_kwargs is not None: 525 for key, value in extra_kwargs.items(): 526 self.write(f"{key!r}: {value}, ") 527 if node.dyn_kwargs is not None: 528 self.write("}, **") 529 self.visit(node.dyn_kwargs, frame) 530 self.write(")") 531 else: 532 self.write("}") 533 534 elif node.dyn_kwargs is not None: 535 self.write(", **") 536 self.visit(node.dyn_kwargs, frame) 537 538 def pull_dependencies(self, nodes: t.Iterable[nodes.Node]) -> None: 539 """Find all filter and test names used in the template and 540 assign them to variables in the compiled namespace. Checking 541 that the names are registered with the environment is done when 542 compiling the Filter and Test nodes. If the node is in an If or 543 CondExpr node, the check is done at runtime instead. 544 545 .. versionchanged:: 3.0 546 Filters and tests in If and CondExpr nodes are checked at 547 runtime instead of compile time. 548 """ 549 visitor = DependencyFinderVisitor() 550 551 for node in nodes: 552 visitor.visit(node) 553 554 for id_map, names, dependency in (self.filters, visitor.filters, "filters"), ( 555 self.tests, 556 visitor.tests, 557 "tests", 558 ): 559 for name in names: 560 if name not in id_map: 561 id_map[name] = self.temporary_identifier() 562 563 # add check during runtime that dependencies used inside of executed 564 # blocks are defined, as this step may be skipped during compile time 565 self.writeline("try:") 566 self.indent() 567 self.writeline(f"{id_map[name]} = environment.{dependency}[{name!r}]") 568 self.outdent() 569 self.writeline("except KeyError:") 570 self.indent() 571 self.writeline("@internalcode") 572 self.writeline(f"def {id_map[name]}(*unused):") 573 self.indent() 574 self.writeline( 575 f'raise TemplateRuntimeError("No {dependency[:-1]}' 576 f' named {name!r} found.")' 577 ) 578 self.outdent() 579 self.outdent() 580 581 def enter_frame(self, frame: Frame) -> None: 582 undefs = [] 583 for target, (action, param) in frame.symbols.loads.items(): 584 if action == VAR_LOAD_PARAMETER: 585 pass 586 elif action == VAR_LOAD_RESOLVE: 587 self.writeline(f"{target} = {self.get_resolve_func()}({param!r})") 588 elif action == VAR_LOAD_ALIAS: 589 self.writeline(f"{target} = {param}") 590 elif action == VAR_LOAD_UNDEFINED: 591 undefs.append(target) 592 else: 593 raise NotImplementedError("unknown load instruction") 594 if undefs: 595 self.writeline(f"{' = '.join(undefs)} = missing") 596 597 def leave_frame(self, frame: Frame, with_python_scope: bool = False) -> None: 598 if not with_python_scope: 599 undefs = [] 600 for target in frame.symbols.loads: 601 undefs.append(target) 602 if undefs: 603 self.writeline(f"{' = '.join(undefs)} = missing") 604 605 def choose_async(self, async_value: str = "async ", sync_value: str = "") -> str: 606 return async_value if self.environment.is_async else sync_value 607 608 def func(self, name: str) -> str: 609 return f"{self.choose_async()}def {name}" 610 611 def macro_body( 612 self, node: t.Union[nodes.Macro, nodes.CallBlock], frame: Frame 613 ) -> t.Tuple[Frame, MacroRef]: 614 """Dump the function def of a macro or call block.""" 615 frame = frame.inner() 616 frame.symbols.analyze_node(node) 617 macro_ref = MacroRef(node) 618 619 explicit_caller = None 620 skip_special_params = set() 621 args = [] 622 623 for idx, arg in enumerate(node.args): 624 if arg.name == "caller": 625 explicit_caller = idx 626 if arg.name in ("kwargs", "varargs"): 627 skip_special_params.add(arg.name) 628 args.append(frame.symbols.ref(arg.name)) 629 630 undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) 631 632 if "caller" in undeclared: 633 # In older Jinja versions there was a bug that allowed caller 634 # to retain the special behavior even if it was mentioned in 635 # the argument list. However thankfully this was only really 636 # working if it was the last argument. So we are explicitly 637 # checking this now and error out if it is anywhere else in 638 # the argument list. 639 if explicit_caller is not None: 640 try: 641 node.defaults[explicit_caller - len(node.args)] 642 except IndexError: 643 self.fail( 644 "When defining macros or call blocks the " 645 'special "caller" argument must be omitted ' 646 "or be given a default.", 647 node.lineno, 648 ) 649 else: 650 args.append(frame.symbols.declare_parameter("caller")) 651 macro_ref.accesses_caller = True 652 if "kwargs" in undeclared and "kwargs" not in skip_special_params: 653 args.append(frame.symbols.declare_parameter("kwargs")) 654 macro_ref.accesses_kwargs = True 655 if "varargs" in undeclared and "varargs" not in skip_special_params: 656 args.append(frame.symbols.declare_parameter("varargs")) 657 macro_ref.accesses_varargs = True 658 659 # macros are delayed, they never require output checks 660 frame.require_output_check = False 661 frame.symbols.analyze_node(node) 662 self.writeline(f"{self.func('macro')}({', '.join(args)}):", node) 663 self.indent() 664 665 self.buffer(frame) 666 self.enter_frame(frame) 667 668 self.push_parameter_definitions(frame) 669 for idx, arg in enumerate(node.args): 670 ref = frame.symbols.ref(arg.name) 671 self.writeline(f"if {ref} is missing:") 672 self.indent() 673 try: 674 default = node.defaults[idx - len(node.args)] 675 except IndexError: 676 self.writeline( 677 f'{ref} = undefined("parameter {arg.name!r} was not provided",' 678 f" name={arg.name!r})" 679 ) 680 else: 681 self.writeline(f"{ref} = ") 682 self.visit(default, frame) 683 self.mark_parameter_stored(ref) 684 self.outdent() 685 self.pop_parameter_definitions() 686 687 self.blockvisit(node.body, frame) 688 self.return_buffer_contents(frame, force_unescaped=True) 689 self.leave_frame(frame, with_python_scope=True) 690 self.outdent() 691 692 return frame, macro_ref 693 694 def macro_def(self, macro_ref: MacroRef, frame: Frame) -> None: 695 """Dump the macro definition for the def created by macro_body.""" 696 arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) 697 name = getattr(macro_ref.node, "name", None) 698 if len(macro_ref.node.args) == 1: 699 arg_tuple += "," 700 self.write( 701 f"Macro(environment, macro, {name!r}, ({arg_tuple})," 702 f" {macro_ref.accesses_kwargs!r}, {macro_ref.accesses_varargs!r}," 703 f" {macro_ref.accesses_caller!r}, context.eval_ctx.autoescape)" 704 ) 705 706 def position(self, node: nodes.Node) -> str: 707 """Return a human readable position for the node.""" 708 rv = f"line {node.lineno}" 709 if self.name is not None: 710 rv = f"{rv} in {self.name!r}" 711 return rv 712 713 def dump_local_context(self, frame: Frame) -> str: 714 items_kv = ", ".join( 715 f"{name!r}: {target}" 716 for name, target in frame.symbols.dump_stores().items() 717 ) 718 return f"{{{items_kv}}}" 719 720 def write_commons(self) -> None: 721 """Writes a common preamble that is used by root and block functions. 722 Primarily this sets up common local helpers and enforces a generator 723 through a dead branch. 724 """ 725 self.writeline("resolve = context.resolve_or_missing") 726 self.writeline("undefined = environment.undefined") 727 # always use the standard Undefined class for the implicit else of 728 # conditional expressions 729 self.writeline("cond_expr_undefined = Undefined") 730 self.writeline("if 0: yield None") 731 732 def push_parameter_definitions(self, frame: Frame) -> None: 733 """Pushes all parameter targets from the given frame into a local 734 stack that permits tracking of yet to be assigned parameters. In 735 particular this enables the optimization from `visit_Name` to skip 736 undefined expressions for parameters in macros as macros can reference 737 otherwise unbound parameters. 738 """ 739 self._param_def_block.append(frame.symbols.dump_param_targets()) 740 741 def pop_parameter_definitions(self) -> None: 742 """Pops the current parameter definitions set.""" 743 self._param_def_block.pop() 744 745 def mark_parameter_stored(self, target: str) -> None: 746 """Marks a parameter in the current parameter definitions as stored. 747 This will skip the enforced undefined checks. 748 """ 749 if self._param_def_block: 750 self._param_def_block[-1].discard(target) 751 752 def push_context_reference(self, target: str) -> None: 753 self._context_reference_stack.append(target) 754 755 def pop_context_reference(self) -> None: 756 self._context_reference_stack.pop() 757 758 def get_context_ref(self) -> str: 759 return self._context_reference_stack[-1] 760 761 def get_resolve_func(self) -> str: 762 target = self._context_reference_stack[-1] 763 if target == "context": 764 return "resolve" 765 return f"{target}.resolve" 766 767 def derive_context(self, frame: Frame) -> str: 768 return f"{self.get_context_ref()}.derived({self.dump_local_context(frame)})" 769 770 def parameter_is_undeclared(self, target: str) -> bool: 771 """Checks if a given target is an undeclared parameter.""" 772 if not self._param_def_block: 773 return False 774 return target in self._param_def_block[-1] 775 776 def push_assign_tracking(self) -> None: 777 """Pushes a new layer for assignment tracking.""" 778 self._assign_stack.append(set()) 779 780 def pop_assign_tracking(self, frame: Frame) -> None: 781 """Pops the topmost level for assignment tracking and updates the 782 context variables if necessary. 783 """ 784 vars = self._assign_stack.pop() 785 if ( 786 not frame.block_frame 787 and not frame.loop_frame 788 and not frame.toplevel 789 or not vars 790 ): 791 return 792 public_names = [x for x in vars if x[:1] != "_"] 793 if len(vars) == 1: 794 name = next(iter(vars)) 795 ref = frame.symbols.ref(name) 796 if frame.loop_frame: 797 self.writeline(f"_loop_vars[{name!r}] = {ref}") 798 return 799 if frame.block_frame: 800 self.writeline(f"_block_vars[{name!r}] = {ref}") 801 return 802 self.writeline(f"context.vars[{name!r}] = {ref}") 803 else: 804 if frame.loop_frame: 805 self.writeline("_loop_vars.update({") 806 elif frame.block_frame: 807 self.writeline("_block_vars.update({") 808 else: 809 self.writeline("context.vars.update({") 810 for idx, name in enumerate(vars): 811 if idx: 812 self.write(", ") 813 ref = frame.symbols.ref(name) 814 self.write(f"{name!r}: {ref}") 815 self.write("})") 816 if not frame.block_frame and not frame.loop_frame and public_names: 817 if len(public_names) == 1: 818 self.writeline(f"context.exported_vars.add({public_names[0]!r})") 819 else: 820 names_str = ", ".join(map(repr, public_names)) 821 self.writeline(f"context.exported_vars.update(({names_str}))") 822 823 # -- Statement Visitors 824 825 def visit_Template( 826 self, node: nodes.Template, frame: t.Optional[Frame] = None 827 ) -> None: 828 assert frame is None, "no root frame allowed" 829 eval_ctx = EvalContext(self.environment, self.name) 830 831 from .runtime import exported, async_exported 832 833 if self.environment.is_async: 834 exported_names = sorted(exported + async_exported) 835 else: 836 exported_names = sorted(exported) 837 838 self.writeline("from __future__ import generator_stop") # Python < 3.7 839 self.writeline("from jinja2.runtime import " + ", ".join(exported_names)) 840 841 # if we want a deferred initialization we cannot move the 842 # environment into a local name 843 envenv = "" if self.defer_init else ", environment=environment" 844 845 # do we have an extends tag at all? If not, we can save some 846 # overhead by just not processing any inheritance code. 847 have_extends = node.find(nodes.Extends) is not None 848 849 # find all blocks 850 for block in node.find_all(nodes.Block): 851 if block.name in self.blocks: 852 self.fail(f"block {block.name!r} defined twice", block.lineno) 853 self.blocks[block.name] = block 854 855 # find all imports and import them 856 for import_ in node.find_all(nodes.ImportedName): 857 if import_.importname not in self.import_aliases: 858 imp = import_.importname 859 self.import_aliases[imp] = alias = self.temporary_identifier() 860 if "." in imp: 861 module, obj = imp.rsplit(".", 1) 862 self.writeline(f"from {module} import {obj} as {alias}") 863 else: 864 self.writeline(f"import {imp} as {alias}") 865 866 # add the load name 867 self.writeline(f"name = {self.name!r}") 868 869 # generate the root render function. 870 self.writeline( 871 f"{self.func('root')}(context, missing=missing{envenv}):", extra=1 872 ) 873 self.indent() 874 self.write_commons() 875 876 # process the root 877 frame = Frame(eval_ctx) 878 if "self" in find_undeclared(node.body, ("self",)): 879 ref = frame.symbols.declare_parameter("self") 880 self.writeline(f"{ref} = TemplateReference(context)") 881 frame.symbols.analyze_node(node) 882 frame.toplevel = frame.rootlevel = True 883 frame.require_output_check = have_extends and not self.has_known_extends 884 if have_extends: 885 self.writeline("parent_template = None") 886 self.enter_frame(frame) 887 self.pull_dependencies(node.body) 888 self.blockvisit(node.body, frame) 889 self.leave_frame(frame, with_python_scope=True) 890 self.outdent() 891 892 # make sure that the parent root is called. 893 if have_extends: 894 if not self.has_known_extends: 895 self.indent() 896 self.writeline("if parent_template is not None:") 897 self.indent() 898 if not self.environment.is_async: 899 self.writeline("yield from parent_template.root_render_func(context)") 900 else: 901 self.writeline( 902 "async for event in parent_template.root_render_func(context):" 903 ) 904 self.indent() 905 self.writeline("yield event") 906 self.outdent() 907 self.outdent(1 + (not self.has_known_extends)) 908 909 # at this point we now have the blocks collected and can visit them too. 910 for name, block in self.blocks.items(): 911 self.writeline( 912 f"{self.func('block_' + name)}(context, missing=missing{envenv}):", 913 block, 914 1, 915 ) 916 self.indent() 917 self.write_commons() 918 # It's important that we do not make this frame a child of the 919 # toplevel template. This would cause a variety of 920 # interesting issues with identifier tracking. 921 block_frame = Frame(eval_ctx) 922 block_frame.block_frame = True 923 undeclared = find_undeclared(block.body, ("self", "super")) 924 if "self" in undeclared: 925 ref = block_frame.symbols.declare_parameter("self") 926 self.writeline(f"{ref} = TemplateReference(context)") 927 if "super" in undeclared: 928 ref = block_frame.symbols.declare_parameter("super") 929 self.writeline(f"{ref} = context.super({name!r}, block_{name})") 930 block_frame.symbols.analyze_node(block) 931 block_frame.block = name 932 self.writeline("_block_vars = {}") 933 self.enter_frame(block_frame) 934 self.pull_dependencies(block.body) 935 self.blockvisit(block.body, block_frame) 936 self.leave_frame(block_frame, with_python_scope=True) 937 self.outdent() 938 939 blocks_kv_str = ", ".join(f"{x!r}: block_{x}" for x in self.blocks) 940 self.writeline(f"blocks = {{{blocks_kv_str}}}", extra=1) 941 debug_kv_str = "&".join(f"{k}={v}" for k, v in self.debug_info) 942 self.writeline(f"debug_info = {debug_kv_str!r}") 943 944 def visit_Block(self, node: nodes.Block, frame: Frame) -> None: 945 """Call a block and register it for the template.""" 946 level = 0 947 if frame.toplevel: 948 # if we know that we are a child template, there is no need to 949 # check if we are one 950 if self.has_known_extends: 951 return 952 if self.extends_so_far > 0: 953 self.writeline("if parent_template is None:") 954 self.indent() 955 level += 1 956 957 if node.scoped: 958 context = self.derive_context(frame) 959 else: 960 context = self.get_context_ref() 961 962 if node.required: 963 self.writeline(f"if len(context.blocks[{node.name!r}]) <= 1:", node) 964 self.indent() 965 self.writeline( 966 f'raise TemplateRuntimeError("Required block {node.name!r} not found")', 967 node, 968 ) 969 self.outdent() 970 971 if not self.environment.is_async and frame.buffer is None: 972 self.writeline( 973 f"yield from context.blocks[{node.name!r}][0]({context})", node 974 ) 975 else: 976 self.writeline( 977 f"{self.choose_async()}for event in" 978 f" context.blocks[{node.name!r}][0]({context}):", 979 node, 980 ) 981 self.indent() 982 self.simple_write("event", frame) 983 self.outdent() 984 985 self.outdent(level) 986 987 def visit_Extends(self, node: nodes.Extends, frame: Frame) -> None: 988 """Calls the extender.""" 989 if not frame.toplevel: 990 self.fail("cannot use extend from a non top-level scope", node.lineno) 991 992 # if the number of extends statements in general is zero so 993 # far, we don't have to add a check if something extended 994 # the template before this one. 995 if self.extends_so_far > 0: 996 997 # if we have a known extends we just add a template runtime 998 # error into the generated code. We could catch that at compile 999 # time too, but i welcome it not to confuse users by throwing the 1000 # same error at different times just "because we can". 1001 if not self.has_known_extends: 1002 self.writeline("if parent_template is not None:") 1003 self.indent() 1004 self.writeline('raise TemplateRuntimeError("extended multiple times")') 1005 1006 # if we have a known extends already we don't need that code here 1007 # as we know that the template execution will end here. 1008 if self.has_known_extends: 1009 raise CompilerExit() 1010 else: 1011 self.outdent() 1012 1013 self.writeline("parent_template = environment.get_template(", node) 1014 self.visit(node.template, frame) 1015 self.write(f", {self.name!r})") 1016 self.writeline("for name, parent_block in parent_template.blocks.items():") 1017 self.indent() 1018 self.writeline("context.blocks.setdefault(name, []).append(parent_block)") 1019 self.outdent() 1020 1021 # if this extends statement was in the root level we can take 1022 # advantage of that information and simplify the generated code 1023 # in the top level from this point onwards 1024 if frame.rootlevel: 1025 self.has_known_extends = True 1026 1027 # and now we have one more 1028 self.extends_so_far += 1 1029 1030 def visit_Include(self, node: nodes.Include, frame: Frame) -> None: 1031 """Handles includes.""" 1032 if node.ignore_missing: 1033 self.writeline("try:") 1034 self.indent() 1035 1036 func_name = "get_or_select_template" 1037 if isinstance(node.template, nodes.Const): 1038 if isinstance(node.template.value, str): 1039 func_name = "get_template" 1040 elif isinstance(node.template.value, (tuple, list)): 1041 func_name = "select_template" 1042 elif isinstance(node.template, (nodes.Tuple, nodes.List)): 1043 func_name = "select_template" 1044 1045 self.writeline(f"template = environment.{func_name}(", node) 1046 self.visit(node.template, frame) 1047 self.write(f", {self.name!r})") 1048 if node.ignore_missing: 1049 self.outdent() 1050 self.writeline("except TemplateNotFound:") 1051 self.indent() 1052 self.writeline("pass") 1053 self.outdent() 1054 self.writeline("else:") 1055 self.indent() 1056 1057 skip_event_yield = False 1058 if node.with_context: 1059 self.writeline( 1060 f"{self.choose_async()}for event in template.root_render_func(" 1061 "template.new_context(context.get_all(), True," 1062 f" {self.dump_local_context(frame)})):" 1063 ) 1064 elif self.environment.is_async: 1065 self.writeline( 1066 "for event in (await template._get_default_module_async())" 1067 "._body_stream:" 1068 ) 1069 else: 1070 self.writeline("yield from template._get_default_module()._body_stream") 1071 skip_event_yield = True 1072 1073 if not skip_event_yield: 1074 self.indent() 1075 self.simple_write("event", frame) 1076 self.outdent() 1077 1078 if node.ignore_missing: 1079 self.outdent() 1080 1081 def _import_common( 1082 self, node: t.Union[nodes.Import, nodes.FromImport], frame: Frame 1083 ) -> None: 1084 self.write(f"{self.choose_async('await ')}environment.get_template(") 1085 self.visit(node.template, frame) 1086 self.write(f", {self.name!r}).") 1087 1088 if node.with_context: 1089 f_name = f"make_module{self.choose_async('_async')}" 1090 self.write( 1091 f"{f_name}(context.get_all(), True, {self.dump_local_context(frame)})" 1092 ) 1093 elif self.environment.is_async: 1094 self.write("_get_default_module_async()") 1095 else: 1096 self.write("_get_default_module(context)") 1097 1098 def visit_Import(self, node: nodes.Import, frame: Frame) -> None: 1099 """Visit regular imports.""" 1100 self.writeline(f"{frame.symbols.ref(node.target)} = ", node) 1101 if frame.toplevel: 1102 self.write(f"context.vars[{node.target!r}] = ") 1103 1104 self._import_common(node, frame) 1105 1106 if frame.toplevel and not node.target.startswith("_"): 1107 self.writeline(f"context.exported_vars.discard({node.target!r})") 1108 1109 def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None: 1110 """Visit named imports.""" 1111 self.newline(node) 1112 self.write("included_template = ") 1113 self._import_common(node, frame) 1114 var_names = [] 1115 discarded_names = [] 1116 for name in node.names: 1117 if isinstance(name, tuple): 1118 name, alias = name 1119 else: 1120 alias = name 1121 self.writeline( 1122 f"{frame.symbols.ref(alias)} =" 1123 f" getattr(included_template, {name!r}, missing)" 1124 ) 1125 self.writeline(f"if {frame.symbols.ref(alias)} is missing:") 1126 self.indent() 1127 message = ( 1128 "the template {included_template.__name__!r}" 1129 f" (imported on {self.position(node)})" 1130 f" does not export the requested name {name!r}" 1131 ) 1132 self.writeline( 1133 f"{frame.symbols.ref(alias)} = undefined(f{message!r}, name={name!r})" 1134 ) 1135 self.outdent() 1136 if frame.toplevel: 1137 var_names.append(alias) 1138 if not alias.startswith("_"): 1139 discarded_names.append(alias) 1140 1141 if var_names: 1142 if len(var_names) == 1: 1143 name = var_names[0] 1144 self.writeline(f"context.vars[{name!r}] = {frame.symbols.ref(name)}") 1145 else: 1146 names_kv = ", ".join( 1147 f"{name!r}: {frame.symbols.ref(name)}" for name in var_names 1148 ) 1149 self.writeline(f"context.vars.update({{{names_kv}}})") 1150 if discarded_names: 1151 if len(discarded_names) == 1: 1152 self.writeline(f"context.exported_vars.discard({discarded_names[0]!r})") 1153 else: 1154 names_str = ", ".join(map(repr, discarded_names)) 1155 self.writeline( 1156 f"context.exported_vars.difference_update(({names_str}))" 1157 ) 1158 1159 def visit_For(self, node: nodes.For, frame: Frame) -> None: 1160 loop_frame = frame.inner() 1161 loop_frame.loop_frame = True 1162 test_frame = frame.inner() 1163 else_frame = frame.inner() 1164 1165 # try to figure out if we have an extended loop. An extended loop 1166 # is necessary if the loop is in recursive mode if the special loop 1167 # variable is accessed in the body if the body is a scoped block. 1168 extended_loop = ( 1169 node.recursive 1170 or "loop" 1171 in find_undeclared(node.iter_child_nodes(only=("body",)), ("loop",)) 1172 or any(block.scoped for block in node.find_all(nodes.Block)) 1173 ) 1174 1175 loop_ref = None 1176 if extended_loop: 1177 loop_ref = loop_frame.symbols.declare_parameter("loop") 1178 1179 loop_frame.symbols.analyze_node(node, for_branch="body") 1180 if node.else_: 1181 else_frame.symbols.analyze_node(node, for_branch="else") 1182 1183 if node.test: 1184 loop_filter_func = self.temporary_identifier() 1185 test_frame.symbols.analyze_node(node, for_branch="test") 1186 self.writeline(f"{self.func(loop_filter_func)}(fiter):", node.test) 1187 self.indent() 1188 self.enter_frame(test_frame) 1189 self.writeline(self.choose_async("async for ", "for ")) 1190 self.visit(node.target, loop_frame) 1191 self.write(" in ") 1192 self.write(self.choose_async("auto_aiter(fiter)", "fiter")) 1193 self.write(":") 1194 self.indent() 1195 self.writeline("if ", node.test) 1196 self.visit(node.test, test_frame) 1197 self.write(":") 1198 self.indent() 1199 self.writeline("yield ") 1200 self.visit(node.target, loop_frame) 1201 self.outdent(3) 1202 self.leave_frame(test_frame, with_python_scope=True) 1203 1204 # if we don't have an recursive loop we have to find the shadowed 1205 # variables at that point. Because loops can be nested but the loop 1206 # variable is a special one we have to enforce aliasing for it. 1207 if node.recursive: 1208 self.writeline( 1209 f"{self.func('loop')}(reciter, loop_render_func, depth=0):", node 1210 ) 1211 self.indent() 1212 self.buffer(loop_frame) 1213 1214 # Use the same buffer for the else frame 1215 else_frame.buffer = loop_frame.buffer 1216 1217 # make sure the loop variable is a special one and raise a template 1218 # assertion error if a loop tries to write to loop 1219 if extended_loop: 1220 self.writeline(f"{loop_ref} = missing") 1221 1222 for name in node.find_all(nodes.Name): 1223 if name.ctx == "store" and name.name == "loop": 1224 self.fail( 1225 "Can't assign to special loop variable in for-loop target", 1226 name.lineno, 1227 ) 1228 1229 if node.else_: 1230 iteration_indicator = self.temporary_identifier() 1231 self.writeline(f"{iteration_indicator} = 1") 1232 1233 self.writeline(self.choose_async("async for ", "for "), node) 1234 self.visit(node.target, loop_frame) 1235 if extended_loop: 1236 self.write(f", {loop_ref} in {self.choose_async('Async')}LoopContext(") 1237 else: 1238 self.write(" in ") 1239 1240 if node.test: 1241 self.write(f"{loop_filter_func}(") 1242 if node.recursive: 1243 self.write("reciter") 1244 else: 1245 if self.environment.is_async and not extended_loop: 1246 self.write("auto_aiter(") 1247 self.visit(node.iter, frame) 1248 if self.environment.is_async and not extended_loop: 1249 self.write(")") 1250 if node.test: 1251 self.write(")") 1252 1253 if node.recursive: 1254 self.write(", undefined, loop_render_func, depth):") 1255 else: 1256 self.write(", undefined):" if extended_loop else ":") 1257 1258 self.indent() 1259 self.enter_frame(loop_frame) 1260 1261 self.writeline("_loop_vars = {}") 1262 self.blockvisit(node.body, loop_frame) 1263 if node.else_: 1264 self.writeline(f"{iteration_indicator} = 0") 1265 self.outdent() 1266 self.leave_frame( 1267 loop_frame, with_python_scope=node.recursive and not node.else_ 1268 ) 1269 1270 if node.else_: 1271 self.writeline(f"if {iteration_indicator}:") 1272 self.indent() 1273 self.enter_frame(else_frame) 1274 self.blockvisit(node.else_, else_frame) 1275 self.leave_frame(else_frame) 1276 self.outdent() 1277 1278 # if the node was recursive we have to return the buffer contents 1279 # and start the iteration code 1280 if node.recursive: 1281 self.return_buffer_contents(loop_frame) 1282 self.outdent() 1283 self.start_write(frame, node) 1284 self.write(f"{self.choose_async('await ')}loop(") 1285 if self.environment.is_async: 1286 self.write("auto_aiter(") 1287 self.visit(node.iter, frame) 1288 if self.environment.is_async: 1289 self.write(")") 1290 self.write(", loop)") 1291 self.end_write(frame) 1292 1293 def visit_If(self, node: nodes.If, frame: Frame) -> None: 1294 if_frame = frame.soft() 1295 self.writeline("if ", node) 1296 self.visit(node.test, if_frame) 1297 self.write(":") 1298 self.indent() 1299 self.blockvisit(node.body, if_frame) 1300 self.outdent() 1301 for elif_ in node.elif_: 1302 self.writeline("elif ", elif_) 1303 self.visit(elif_.test, if_frame) 1304 self.write(":") 1305 self.indent() 1306 self.blockvisit(elif_.body, if_frame) 1307 self.outdent() 1308 if node.else_: 1309 self.writeline("else:") 1310 self.indent() 1311 self.blockvisit(node.else_, if_frame) 1312 self.outdent() 1313 1314 def visit_Macro(self, node: nodes.Macro, frame: Frame) -> None: 1315 macro_frame, macro_ref = self.macro_body(node, frame) 1316 self.newline() 1317 if frame.toplevel: 1318 if not node.name.startswith("_"): 1319 self.write(f"context.exported_vars.add({node.name!r})") 1320 self.writeline(f"context.vars[{node.name!r}] = ") 1321 self.write(f"{frame.symbols.ref(node.name)} = ") 1322 self.macro_def(macro_ref, macro_frame) 1323 1324 def visit_CallBlock(self, node: nodes.CallBlock, frame: Frame) -> None: 1325 call_frame, macro_ref = self.macro_body(node, frame) 1326 self.writeline("caller = ") 1327 self.macro_def(macro_ref, call_frame) 1328 self.start_write(frame, node) 1329 self.visit_Call(node.call, frame, forward_caller=True) 1330 self.end_write(frame) 1331 1332 def visit_FilterBlock(self, node: nodes.FilterBlock, frame: Frame) -> None: 1333 filter_frame = frame.inner() 1334 filter_frame.symbols.analyze_node(node) 1335 self.enter_frame(filter_frame) 1336 self.buffer(filter_frame) 1337 self.blockvisit(node.body, filter_frame) 1338 self.start_write(frame, node) 1339 self.visit_Filter(node.filter, filter_frame) 1340 self.end_write(frame) 1341 self.leave_frame(filter_frame) 1342 1343 def visit_With(self, node: nodes.With, frame: Frame) -> None: 1344 with_frame = frame.inner() 1345 with_frame.symbols.analyze_node(node) 1346 self.enter_frame(with_frame) 1347 for target, expr in zip(node.targets, node.values): 1348 self.newline() 1349 self.visit(target, with_frame) 1350 self.write(" = ") 1351 self.visit(expr, frame) 1352 self.blockvisit(node.body, with_frame) 1353 self.leave_frame(with_frame) 1354 1355 def visit_ExprStmt(self, node: nodes.ExprStmt, frame: Frame) -> None: 1356 self.newline(node) 1357 self.visit(node.node, frame) 1358 1359 class _FinalizeInfo(t.NamedTuple): 1360 const: t.Optional[t.Callable[..., str]] 1361 src: t.Optional[str] 1362 1363 @staticmethod 1364 def _default_finalize(value: t.Any) -> t.Any: 1365 """The default finalize function if the environment isn't 1366 configured with one. Or, if the environment has one, this is 1367 called on that function's output for constants. 1368 """ 1369 return str(value) 1370 1371 _finalize: t.Optional[_FinalizeInfo] = None 1372 1373 def _make_finalize(self) -> _FinalizeInfo: 1374 """Build the finalize function to be used on constants and at 1375 runtime. Cached so it's only created once for all output nodes. 1376 1377 Returns a ``namedtuple`` with the following attributes: 1378 1379 ``const`` 1380 A function to finalize constant data at compile time. 1381 1382 ``src`` 1383 Source code to output around nodes to be evaluated at 1384 runtime. 1385 """ 1386 if self._finalize is not None: 1387 return self._finalize 1388 1389 finalize: t.Optional[t.Callable[..., t.Any]] 1390 finalize = default = self._default_finalize 1391 src = None 1392 1393 if self.environment.finalize: 1394 src = "environment.finalize(" 1395 env_finalize = self.environment.finalize 1396 pass_arg = { 1397 _PassArg.context: "context", 1398 _PassArg.eval_context: "context.eval_ctx", 1399 _PassArg.environment: "environment", 1400 }.get( 1401 _PassArg.from_obj(env_finalize) # type: ignore 1402 ) 1403 finalize = None 1404 1405 if pass_arg is None: 1406 1407 def finalize(value: t.Any) -> t.Any: 1408 return default(env_finalize(value)) 1409 1410 else: 1411 src = f"{src}{pass_arg}, " 1412 1413 if pass_arg == "environment": 1414 1415 def finalize(value: t.Any) -> t.Any: 1416 return default(env_finalize(self.environment, value)) 1417 1418 self._finalize = self._FinalizeInfo(finalize, src) 1419 return self._finalize 1420 1421 def _output_const_repr(self, group: t.Iterable[t.Any]) -> str: 1422 """Given a group of constant values converted from ``Output`` 1423 child nodes, produce a string to write to the template module 1424 source. 1425 """ 1426 return repr(concat(group)) 1427 1428 def _output_child_to_const( 1429 self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo 1430 ) -> str: 1431 """Try to optimize a child of an ``Output`` node by trying to 1432 convert it to constant, finalized data at compile time. 1433 1434 If :exc:`Impossible` is raised, the node is not constant and 1435 will be evaluated at runtime. Any other exception will also be 1436 evaluated at runtime for easier debugging. 1437 """ 1438 const = node.as_const(frame.eval_ctx) 1439 1440 if frame.eval_ctx.autoescape: 1441 const = escape(const) 1442 1443 # Template data doesn't go through finalize. 1444 if isinstance(node, nodes.TemplateData): 1445 return str(const) 1446 1447 return finalize.const(const) # type: ignore 1448 1449 def _output_child_pre( 1450 self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo 1451 ) -> None: 1452 """Output extra source code before visiting a child of an 1453 ``Output`` node. 1454 """ 1455 if frame.eval_ctx.volatile: 1456 self.write("(escape if context.eval_ctx.autoescape else str)(") 1457 elif frame.eval_ctx.autoescape: 1458 self.write("escape(") 1459 else: 1460 self.write("str(") 1461 1462 if finalize.src is not None: 1463 self.write(finalize.src) 1464 1465 def _output_child_post( 1466 self, node: nodes.Expr, frame: Frame, finalize: _FinalizeInfo 1467 ) -> None: 1468 """Output extra source code after visiting a child of an 1469 ``Output`` node. 1470 """ 1471 self.write(")") 1472 1473 if finalize.src is not None: 1474 self.write(")") 1475 1476 def visit_Output(self, node: nodes.Output, frame: Frame) -> None: 1477 # If an extends is active, don't render outside a block. 1478 if frame.require_output_check: 1479 # A top-level extends is known to exist at compile time. 1480 if self.has_known_extends: 1481 return 1482 1483 self.writeline("if parent_template is None:") 1484 self.indent() 1485 1486 finalize = self._make_finalize() 1487 body: t.List[t.Union[t.List[t.Any], nodes.Expr]] = [] 1488 1489 # Evaluate constants at compile time if possible. Each item in 1490 # body will be either a list of static data or a node to be 1491 # evaluated at runtime. 1492 for child in node.nodes: 1493 try: 1494 if not ( 1495 # If the finalize function requires runtime context, 1496 # constants can't be evaluated at compile time. 1497 finalize.const 1498 # Unless it's basic template data that won't be 1499 # finalized anyway. 1500 or isinstance(child, nodes.TemplateData) 1501 ): 1502 raise nodes.Impossible() 1503 1504 const = self._output_child_to_const(child, frame, finalize) 1505 except (nodes.Impossible, Exception): 1506 # The node was not constant and needs to be evaluated at 1507 # runtime. Or another error was raised, which is easier 1508 # to debug at runtime. 1509 body.append(child) 1510 continue 1511 1512 if body and isinstance(body[-1], list): 1513 body[-1].append(const) 1514 else: 1515 body.append([const]) 1516 1517 if frame.buffer is not None: 1518 if len(body) == 1: 1519 self.writeline(f"{frame.buffer}.append(") 1520 else: 1521 self.writeline(f"{frame.buffer}.extend((") 1522 1523 self.indent() 1524 1525 for item in body: 1526 if isinstance(item, list): 1527 # A group of constant data to join and output. 1528 val = self._output_const_repr(item) 1529 1530 if frame.buffer is None: 1531 self.writeline("yield " + val) 1532 else: 1533 self.writeline(val + ",") 1534 else: 1535 if frame.buffer is None: 1536 self.writeline("yield ", item) 1537 else: 1538 self.newline(item) 1539 1540 # A node to be evaluated at runtime. 1541 self._output_child_pre(item, frame, finalize) 1542 self.visit(item, frame) 1543 self._output_child_post(item, frame, finalize) 1544 1545 if frame.buffer is not None: 1546 self.write(",") 1547 1548 if frame.buffer is not None: 1549 self.outdent() 1550 self.writeline(")" if len(body) == 1 else "))") 1551 1552 if frame.require_output_check: 1553 self.outdent() 1554 1555 def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: 1556 self.push_assign_tracking() 1557 self.newline(node) 1558 self.visit(node.target, frame) 1559 self.write(" = ") 1560 self.visit(node.node, frame) 1561 self.pop_assign_tracking(frame) 1562 1563 def visit_AssignBlock(self, node: nodes.AssignBlock, frame: Frame) -> None: 1564 self.push_assign_tracking() 1565 block_frame = frame.inner() 1566 # This is a special case. Since a set block always captures we 1567 # will disable output checks. This way one can use set blocks 1568 # toplevel even in extended templates. 1569 block_frame.require_output_check = False 1570 block_frame.symbols.analyze_node(node) 1571 self.enter_frame(block_frame) 1572 self.buffer(block_frame) 1573 self.blockvisit(node.body, block_frame) 1574 self.newline(node) 1575 self.visit(node.target, frame) 1576 self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") 1577 if node.filter is not None: 1578 self.visit_Filter(node.filter, block_frame) 1579 else: 1580 self.write(f"concat({block_frame.buffer})") 1581 self.write(")") 1582 self.pop_assign_tracking(frame) 1583 self.leave_frame(block_frame) 1584 1585 # -- Expression Visitors 1586 1587 def visit_Name(self, node: nodes.Name, frame: Frame) -> None: 1588 if node.ctx == "store" and ( 1589 frame.toplevel or frame.loop_frame or frame.block_frame 1590 ): 1591 if self._assign_stack: 1592 self._assign_stack[-1].add(node.name) 1593 ref = frame.symbols.ref(node.name) 1594 1595 # If we are looking up a variable we might have to deal with the 1596 # case where it's undefined. We can skip that case if the load 1597 # instruction indicates a parameter which are always defined. 1598 if node.ctx == "load": 1599 load = frame.symbols.find_load(ref) 1600 if not ( 1601 load is not None 1602 and load[0] == VAR_LOAD_PARAMETER 1603 and not self.parameter_is_undeclared(ref) 1604 ): 1605 self.write( 1606 f"(undefined(name={node.name!r}) if {ref} is missing else {ref})" 1607 ) 1608 return 1609 1610 self.write(ref) 1611 1612 def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: 1613 # NSRefs can only be used to store values; since they use the normal 1614 # `foo.bar` notation they will be parsed as a normal attribute access 1615 # when used anywhere but in a `set` context 1616 ref = frame.symbols.ref(node.name) 1617 self.writeline(f"if not isinstance({ref}, Namespace):") 1618 self.indent() 1619 self.writeline( 1620 "raise TemplateRuntimeError" 1621 '("cannot assign attribute on non-namespace object")' 1622 ) 1623 self.outdent() 1624 self.writeline(f"{ref}[{node.attr!r}]") 1625 1626 def visit_Const(self, node: nodes.Const, frame: Frame) -> None: 1627 val = node.as_const(frame.eval_ctx) 1628 if isinstance(val, float): 1629 self.write(str(val)) 1630 else: 1631 self.write(repr(val)) 1632 1633 def visit_TemplateData(self, node: nodes.TemplateData, frame: Frame) -> None: 1634 try: 1635 self.write(repr(node.as_const(frame.eval_ctx))) 1636 except nodes.Impossible: 1637 self.write( 1638 f"(Markup if context.eval_ctx.autoescape else identity)({node.data!r})" 1639 ) 1640 1641 def visit_Tuple(self, node: nodes.Tuple, frame: Frame) -> None: 1642 self.write("(") 1643 idx = -1 1644 for idx, item in enumerate(node.items): 1645 if idx: 1646 self.write(", ") 1647 self.visit(item, frame) 1648 self.write(",)" if idx == 0 else ")") 1649 1650 def visit_List(self, node: nodes.List, frame: Frame) -> None: 1651 self.write("[") 1652 for idx, item in enumerate(node.items): 1653 if idx: 1654 self.write(", ") 1655 self.visit(item, frame) 1656 self.write("]") 1657 1658 def visit_Dict(self, node: nodes.Dict, frame: Frame) -> None: 1659 self.write("{") 1660 for idx, item in enumerate(node.items): 1661 if idx: 1662 self.write(", ") 1663 self.visit(item.key, frame) 1664 self.write(": ") 1665 self.visit(item.value, frame) 1666 self.write("}") 1667 1668 visit_Add = _make_binop("+") 1669 visit_Sub = _make_binop("-") 1670 visit_Mul = _make_binop("*") 1671 visit_Div = _make_binop("/") 1672 visit_FloorDiv = _make_binop("//") 1673 visit_Pow = _make_binop("**") 1674 visit_Mod = _make_binop("%") 1675 visit_And = _make_binop("and") 1676 visit_Or = _make_binop("or") 1677 visit_Pos = _make_unop("+") 1678 visit_Neg = _make_unop("-") 1679 visit_Not = _make_unop("not ") 1680 1681 @optimizeconst 1682 def visit_Concat(self, node: nodes.Concat, frame: Frame) -> None: 1683 if frame.eval_ctx.volatile: 1684 func_name = "(markup_join if context.eval_ctx.volatile else str_join)" 1685 elif frame.eval_ctx.autoescape: 1686 func_name = "markup_join" 1687 else: 1688 func_name = "str_join" 1689 self.write(f"{func_name}((") 1690 for arg in node.nodes: 1691 self.visit(arg, frame) 1692 self.write(", ") 1693 self.write("))") 1694 1695 @optimizeconst 1696 def visit_Compare(self, node: nodes.Compare, frame: Frame) -> None: 1697 self.write("(") 1698 self.visit(node.expr, frame) 1699 for op in node.ops: 1700 self.visit(op, frame) 1701 self.write(")") 1702 1703 def visit_Operand(self, node: nodes.Operand, frame: Frame) -> None: 1704 self.write(f" {operators[node.op]} ") 1705 self.visit(node.expr, frame) 1706 1707 @optimizeconst 1708 def visit_Getattr(self, node: nodes.Getattr, frame: Frame) -> None: 1709 if self.environment.is_async: 1710 self.write("(await auto_await(") 1711 1712 self.write("environment.getattr(") 1713 self.visit(node.node, frame) 1714 self.write(f", {node.attr!r})") 1715 1716 if self.environment.is_async: 1717 self.write("))") 1718 1719 @optimizeconst 1720 def visit_Getitem(self, node: nodes.Getitem, frame: Frame) -> None: 1721 # slices bypass the environment getitem method. 1722 if isinstance(node.arg, nodes.Slice): 1723 self.visit(node.node, frame) 1724 self.write("[") 1725 self.visit(node.arg, frame) 1726 self.write("]") 1727 else: 1728 if self.environment.is_async: 1729 self.write("(await auto_await(") 1730 1731 self.write("environment.getitem(") 1732 self.visit(node.node, frame) 1733 self.write(", ") 1734 self.visit(node.arg, frame) 1735 self.write(")") 1736 1737 if self.environment.is_async: 1738 self.write("))") 1739 1740 def visit_Slice(self, node: nodes.Slice, frame: Frame) -> None: 1741 if node.start is not None: 1742 self.visit(node.start, frame) 1743 self.write(":") 1744 if node.stop is not None: 1745 self.visit(node.stop, frame) 1746 if node.step is not None: 1747 self.write(":") 1748 self.visit(node.step, frame) 1749 1750 @contextmanager 1751 def _filter_test_common( 1752 self, node: t.Union[nodes.Filter, nodes.Test], frame: Frame, is_filter: bool 1753 ) -> t.Iterator[None]: 1754 if self.environment.is_async: 1755 self.write("await auto_await(") 1756 1757 if is_filter: 1758 self.write(f"{self.filters[node.name]}(") 1759 func = self.environment.filters.get(node.name) 1760 else: 1761 self.write(f"{self.tests[node.name]}(") 1762 func = self.environment.tests.get(node.name) 1763 1764 # When inside an If or CondExpr frame, allow the filter to be 1765 # undefined at compile time and only raise an error if it's 1766 # actually called at runtime. See pull_dependencies. 1767 if func is None and not frame.soft_frame: 1768 type_name = "filter" if is_filter else "test" 1769 self.fail(f"No {type_name} named {node.name!r}.", node.lineno) 1770 1771 pass_arg = { 1772 _PassArg.context: "context", 1773 _PassArg.eval_context: "context.eval_ctx", 1774 _PassArg.environment: "environment", 1775 }.get( 1776 _PassArg.from_obj(func) # type: ignore 1777 ) 1778 1779 if pass_arg is not None: 1780 self.write(f"{pass_arg}, ") 1781 1782 # Back to the visitor function to handle visiting the target of 1783 # the filter or test. 1784 yield 1785 1786 self.signature(node, frame) 1787 self.write(")") 1788 1789 if self.environment.is_async: 1790 self.write(")") 1791 1792 @optimizeconst 1793 def visit_Filter(self, node: nodes.Filter, frame: Frame) -> None: 1794 with self._filter_test_common(node, frame, True): 1795 # if the filter node is None we are inside a filter block 1796 # and want to write to the current buffer 1797 if node.node is not None: 1798 self.visit(node.node, frame) 1799 elif frame.eval_ctx.volatile: 1800 self.write( 1801 f"(Markup(concat({frame.buffer}))" 1802 f" if context.eval_ctx.autoescape else concat({frame.buffer}))" 1803 ) 1804 elif frame.eval_ctx.autoescape: 1805 self.write(f"Markup(concat({frame.buffer}))") 1806 else: 1807 self.write(f"concat({frame.buffer})") 1808 1809 @optimizeconst 1810 def visit_Test(self, node: nodes.Test, frame: Frame) -> None: 1811 with self._filter_test_common(node, frame, False): 1812 self.visit(node.node, frame) 1813 1814 @optimizeconst 1815 def visit_CondExpr(self, node: nodes.CondExpr, frame: Frame) -> None: 1816 frame = frame.soft() 1817 1818 def write_expr2() -> None: 1819 if node.expr2 is not None: 1820 self.visit(node.expr2, frame) 1821 return 1822 1823 self.write( 1824 f'cond_expr_undefined("the inline if-expression on' 1825 f" {self.position(node)} evaluated to false and no else" 1826 f' section was defined.")' 1827 ) 1828 1829 self.write("(") 1830 self.visit(node.expr1, frame) 1831 self.write(" if ") 1832 self.visit(node.test, frame) 1833 self.write(" else ") 1834 write_expr2() 1835 self.write(")") 1836 1837 @optimizeconst 1838 def visit_Call( 1839 self, node: nodes.Call, frame: Frame, forward_caller: bool = False 1840 ) -> None: 1841 if self.environment.is_async: 1842 self.write("await auto_await(") 1843 if self.environment.sandboxed: 1844 self.write("environment.call(context, ") 1845 else: 1846 self.write("context.call(") 1847 self.visit(node.node, frame) 1848 extra_kwargs = {"caller": "caller"} if forward_caller else None 1849 loop_kwargs = {"_loop_vars": "_loop_vars"} if frame.loop_frame else {} 1850 block_kwargs = {"_block_vars": "_block_vars"} if frame.block_frame else {} 1851 if extra_kwargs: 1852 extra_kwargs.update(loop_kwargs, **block_kwargs) 1853 elif loop_kwargs or block_kwargs: 1854 extra_kwargs = dict(loop_kwargs, **block_kwargs) 1855 self.signature(node, frame, extra_kwargs) 1856 self.write(")") 1857 if self.environment.is_async: 1858 self.write(")") 1859 1860 def visit_Keyword(self, node: nodes.Keyword, frame: Frame) -> None: 1861 self.write(node.key + "=") 1862 self.visit(node.value, frame) 1863 1864 # -- Unused nodes for extensions 1865 1866 def visit_MarkSafe(self, node: nodes.MarkSafe, frame: Frame) -> None: 1867 self.write("Markup(") 1868 self.visit(node.expr, frame) 1869 self.write(")") 1870 1871 def visit_MarkSafeIfAutoescape( 1872 self, node: nodes.MarkSafeIfAutoescape, frame: Frame 1873 ) -> None: 1874 self.write("(Markup if context.eval_ctx.autoescape else identity)(") 1875 self.visit(node.expr, frame) 1876 self.write(")") 1877 1878 def visit_EnvironmentAttribute( 1879 self, node: nodes.EnvironmentAttribute, frame: Frame 1880 ) -> None: 1881 self.write("environment." + node.name) 1882 1883 def visit_ExtensionAttribute( 1884 self, node: nodes.ExtensionAttribute, frame: Frame 1885 ) -> None: 1886 self.write(f"environment.extensions[{node.identifier!r}].{node.name}") 1887 1888 def visit_ImportedName(self, node: nodes.ImportedName, frame: Frame) -> None: 1889 self.write(self.import_aliases[node.importname]) 1890 1891 def visit_InternalName(self, node: nodes.InternalName, frame: Frame) -> None: 1892 self.write(node.name) 1893 1894 def visit_ContextReference( 1895 self, node: nodes.ContextReference, frame: Frame 1896 ) -> None: 1897 self.write("context") 1898 1899 def visit_DerivedContextReference( 1900 self, node: nodes.DerivedContextReference, frame: Frame 1901 ) -> None: 1902 self.write(self.derive_context(frame)) 1903 1904 def visit_Continue(self, node: nodes.Continue, frame: Frame) -> None: 1905 self.writeline("continue", node) 1906 1907 def visit_Break(self, node: nodes.Break, frame: Frame) -> None: 1908 self.writeline("break", node) 1909 1910 def visit_Scope(self, node: nodes.Scope, frame: Frame) -> None: 1911 scope_frame = frame.inner() 1912 scope_frame.symbols.analyze_node(node) 1913 self.enter_frame(scope_frame) 1914 self.blockvisit(node.body, scope_frame) 1915 self.leave_frame(scope_frame) 1916 1917 def visit_OverlayScope(self, node: nodes.OverlayScope, frame: Frame) -> None: 1918 ctx = self.temporary_identifier() 1919 self.writeline(f"{ctx} = {self.derive_context(frame)}") 1920 self.writeline(f"{ctx}.vars = ") 1921 self.visit(node.context, frame) 1922 self.push_context_reference(ctx) 1923 1924 scope_frame = frame.inner(isolated=True) 1925 scope_frame.symbols.analyze_node(node) 1926 self.enter_frame(scope_frame) 1927 self.blockvisit(node.body, scope_frame) 1928 self.leave_frame(scope_frame) 1929 self.pop_context_reference() 1930 1931 def visit_EvalContextModifier( 1932 self, node: nodes.EvalContextModifier, frame: Frame 1933 ) -> None: 1934 for keyword in node.options: 1935 self.writeline(f"context.eval_ctx.{keyword.key} = ") 1936 self.visit(keyword.value, frame) 1937 try: 1938 val = keyword.value.as_const(frame.eval_ctx) 1939 except nodes.Impossible: 1940 frame.eval_ctx.volatile = True 1941 else: 1942 setattr(frame.eval_ctx, keyword.key, val) 1943 1944 def visit_ScopedEvalContextModifier( 1945 self, node: nodes.ScopedEvalContextModifier, frame: Frame 1946 ) -> None: 1947 old_ctx_name = self.temporary_identifier() 1948 saved_ctx = frame.eval_ctx.save() 1949 self.writeline(f"{old_ctx_name} = context.eval_ctx.save()") 1950 self.visit_EvalContextModifier(node, frame) 1951 for child in node.body: 1952 self.visit(child, frame) 1953 frame.eval_ctx.revert(saved_ctx) 1954 self.writeline(f"context.eval_ctx.revert({old_ctx_name})") 1955