1"""Parse tokens from the lexer into nodes for the compiler.""" 2import typing 3import typing as t 4 5from . import nodes 6from .exceptions import TemplateAssertionError 7from .exceptions import TemplateSyntaxError 8from .lexer import describe_token 9from .lexer import describe_token_expr 10 11if t.TYPE_CHECKING: 12 import typing_extensions as te 13 from .environment import Environment 14 15_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include) 16_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock) 17 18_statement_keywords = frozenset( 19 [ 20 "for", 21 "if", 22 "block", 23 "extends", 24 "print", 25 "macro", 26 "include", 27 "from", 28 "import", 29 "set", 30 "with", 31 "autoescape", 32 ] 33) 34_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"]) 35 36_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = { 37 "add": nodes.Add, 38 "sub": nodes.Sub, 39 "mul": nodes.Mul, 40 "div": nodes.Div, 41 "floordiv": nodes.FloorDiv, 42 "mod": nodes.Mod, 43} 44 45 46class Parser: 47 """This is the central parsing class Jinja uses. It's passed to 48 extensions and can be used to parse expressions or statements. 49 """ 50 51 def __init__( 52 self, 53 environment: "Environment", 54 source: str, 55 name: t.Optional[str] = None, 56 filename: t.Optional[str] = None, 57 state: t.Optional[str] = None, 58 ) -> None: 59 self.environment = environment 60 self.stream = environment._tokenize(source, name, filename, state) 61 self.name = name 62 self.filename = filename 63 self.closed = False 64 self.extensions: t.Dict[ 65 str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]] 66 ] = {} 67 for extension in environment.iter_extensions(): 68 for tag in extension.tags: 69 self.extensions[tag] = extension.parse 70 self._last_identifier = 0 71 self._tag_stack: t.List[str] = [] 72 self._end_token_stack: t.List[t.Tuple[str, ...]] = [] 73 74 def fail( 75 self, 76 msg: str, 77 lineno: t.Optional[int] = None, 78 exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError, 79 ) -> "te.NoReturn": 80 """Convenience method that raises `exc` with the message, passed 81 line number or last line number as well as the current name and 82 filename. 83 """ 84 if lineno is None: 85 lineno = self.stream.current.lineno 86 raise exc(msg, lineno, self.name, self.filename) 87 88 def _fail_ut_eof( 89 self, 90 name: t.Optional[str], 91 end_token_stack: t.List[t.Tuple[str, ...]], 92 lineno: t.Optional[int], 93 ) -> "te.NoReturn": 94 expected: t.Set[str] = set() 95 for exprs in end_token_stack: 96 expected.update(map(describe_token_expr, exprs)) 97 if end_token_stack: 98 currently_looking: t.Optional[str] = " or ".join( 99 map(repr, map(describe_token_expr, end_token_stack[-1])) 100 ) 101 else: 102 currently_looking = None 103 104 if name is None: 105 message = ["Unexpected end of template."] 106 else: 107 message = [f"Encountered unknown tag {name!r}."] 108 109 if currently_looking: 110 if name is not None and name in expected: 111 message.append( 112 "You probably made a nesting mistake. Jinja is expecting this tag," 113 f" but currently looking for {currently_looking}." 114 ) 115 else: 116 message.append( 117 f"Jinja was looking for the following tags: {currently_looking}." 118 ) 119 120 if self._tag_stack: 121 message.append( 122 "The innermost block that needs to be closed is" 123 f" {self._tag_stack[-1]!r}." 124 ) 125 126 self.fail(" ".join(message), lineno) 127 128 def fail_unknown_tag( 129 self, name: str, lineno: t.Optional[int] = None 130 ) -> "te.NoReturn": 131 """Called if the parser encounters an unknown tag. Tries to fail 132 with a human readable error message that could help to identify 133 the problem. 134 """ 135 self._fail_ut_eof(name, self._end_token_stack, lineno) 136 137 def fail_eof( 138 self, 139 end_tokens: t.Optional[t.Tuple[str, ...]] = None, 140 lineno: t.Optional[int] = None, 141 ) -> "te.NoReturn": 142 """Like fail_unknown_tag but for end of template situations.""" 143 stack = list(self._end_token_stack) 144 if end_tokens is not None: 145 stack.append(end_tokens) 146 self._fail_ut_eof(None, stack, lineno) 147 148 def is_tuple_end( 149 self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None 150 ) -> bool: 151 """Are we at the end of a tuple?""" 152 if self.stream.current.type in ("variable_end", "block_end", "rparen"): 153 return True 154 elif extra_end_rules is not None: 155 return self.stream.current.test_any(extra_end_rules) # type: ignore 156 return False 157 158 def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName: 159 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" 160 self._last_identifier += 1 161 rv = object.__new__(nodes.InternalName) 162 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno) 163 return rv # type: ignore 164 165 def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]: 166 """Parse a single statement.""" 167 token = self.stream.current 168 if token.type != "name": 169 self.fail("tag name expected", token.lineno) 170 self._tag_stack.append(token.value) 171 pop_tag = True 172 try: 173 if token.value in _statement_keywords: 174 f = getattr(self, f"parse_{self.stream.current.value}") 175 return f() # type: ignore 176 if token.value == "call": 177 return self.parse_call_block() 178 if token.value == "filter": 179 return self.parse_filter_block() 180 ext = self.extensions.get(token.value) 181 if ext is not None: 182 return ext(self) 183 184 # did not work out, remove the token we pushed by accident 185 # from the stack so that the unknown tag fail function can 186 # produce a proper error message. 187 self._tag_stack.pop() 188 pop_tag = False 189 self.fail_unknown_tag(token.value, token.lineno) 190 finally: 191 if pop_tag: 192 self._tag_stack.pop() 193 194 def parse_statements( 195 self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False 196 ) -> t.List[nodes.Node]: 197 """Parse multiple statements into a list until one of the end tokens 198 is reached. This is used to parse the body of statements as it also 199 parses template data if appropriate. The parser checks first if the 200 current token is a colon and skips it if there is one. Then it checks 201 for the block end and parses until if one of the `end_tokens` is 202 reached. Per default the active token in the stream at the end of 203 the call is the matched end token. If this is not wanted `drop_needle` 204 can be set to `True` and the end token is removed. 205 """ 206 # the first token may be a colon for python compatibility 207 self.stream.skip_if("colon") 208 209 # in the future it would be possible to add whole code sections 210 # by adding some sort of end of statement token and parsing those here. 211 self.stream.expect("block_end") 212 result = self.subparse(end_tokens) 213 214 # we reached the end of the template too early, the subparser 215 # does not check for this, so we do that now 216 if self.stream.current.type == "eof": 217 self.fail_eof(end_tokens) 218 219 if drop_needle: 220 next(self.stream) 221 return result 222 223 def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]: 224 """Parse an assign statement.""" 225 lineno = next(self.stream).lineno 226 target = self.parse_assign_target(with_namespace=True) 227 if self.stream.skip_if("assign"): 228 expr = self.parse_tuple() 229 return nodes.Assign(target, expr, lineno=lineno) 230 filter_node = self.parse_filter(None) 231 body = self.parse_statements(("name:endset",), drop_needle=True) 232 return nodes.AssignBlock(target, filter_node, body, lineno=lineno) 233 234 def parse_for(self) -> nodes.For: 235 """Parse a for loop.""" 236 lineno = self.stream.expect("name:for").lineno 237 target = self.parse_assign_target(extra_end_rules=("name:in",)) 238 self.stream.expect("name:in") 239 iter = self.parse_tuple( 240 with_condexpr=False, extra_end_rules=("name:recursive",) 241 ) 242 test = None 243 if self.stream.skip_if("name:if"): 244 test = self.parse_expression() 245 recursive = self.stream.skip_if("name:recursive") 246 body = self.parse_statements(("name:endfor", "name:else")) 247 if next(self.stream).value == "endfor": 248 else_ = [] 249 else: 250 else_ = self.parse_statements(("name:endfor",), drop_needle=True) 251 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) 252 253 def parse_if(self) -> nodes.If: 254 """Parse an if construct.""" 255 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) 256 while True: 257 node.test = self.parse_tuple(with_condexpr=False) 258 node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) 259 node.elif_ = [] 260 node.else_ = [] 261 token = next(self.stream) 262 if token.test("name:elif"): 263 node = nodes.If(lineno=self.stream.current.lineno) 264 result.elif_.append(node) 265 continue 266 elif token.test("name:else"): 267 result.else_ = self.parse_statements(("name:endif",), drop_needle=True) 268 break 269 return result 270 271 def parse_with(self) -> nodes.With: 272 node = nodes.With(lineno=next(self.stream).lineno) 273 targets: t.List[nodes.Expr] = [] 274 values: t.List[nodes.Expr] = [] 275 while self.stream.current.type != "block_end": 276 if targets: 277 self.stream.expect("comma") 278 target = self.parse_assign_target() 279 target.set_ctx("param") 280 targets.append(target) 281 self.stream.expect("assign") 282 values.append(self.parse_expression()) 283 node.targets = targets 284 node.values = values 285 node.body = self.parse_statements(("name:endwith",), drop_needle=True) 286 return node 287 288 def parse_autoescape(self) -> nodes.Scope: 289 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) 290 node.options = [nodes.Keyword("autoescape", self.parse_expression())] 291 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) 292 return nodes.Scope([node]) 293 294 def parse_block(self) -> nodes.Block: 295 node = nodes.Block(lineno=next(self.stream).lineno) 296 node.name = self.stream.expect("name").value 297 node.scoped = self.stream.skip_if("name:scoped") 298 node.required = self.stream.skip_if("name:required") 299 300 # common problem people encounter when switching from django 301 # to jinja. we do not support hyphens in block names, so let's 302 # raise a nicer error message in that case. 303 if self.stream.current.type == "sub": 304 self.fail( 305 "Block names in Jinja have to be valid Python identifiers and may not" 306 " contain hyphens, use an underscore instead." 307 ) 308 309 node.body = self.parse_statements(("name:endblock",), drop_needle=True) 310 311 # enforce that required blocks only contain whitespace or comments 312 # by asserting that the body, if not empty, is just TemplateData nodes 313 # with whitespace data 314 if node.required and not all( 315 isinstance(child, nodes.TemplateData) and child.data.isspace() 316 for body in node.body 317 for child in body.nodes # type: ignore 318 ): 319 self.fail("Required blocks can only contain comments or whitespace") 320 321 self.stream.skip_if("name:" + node.name) 322 return node 323 324 def parse_extends(self) -> nodes.Extends: 325 node = nodes.Extends(lineno=next(self.stream).lineno) 326 node.template = self.parse_expression() 327 return node 328 329 def parse_import_context( 330 self, node: _ImportInclude, default: bool 331 ) -> _ImportInclude: 332 if self.stream.current.test_any( 333 "name:with", "name:without" 334 ) and self.stream.look().test("name:context"): 335 node.with_context = next(self.stream).value == "with" 336 self.stream.skip() 337 else: 338 node.with_context = default 339 return node 340 341 def parse_include(self) -> nodes.Include: 342 node = nodes.Include(lineno=next(self.stream).lineno) 343 node.template = self.parse_expression() 344 if self.stream.current.test("name:ignore") and self.stream.look().test( 345 "name:missing" 346 ): 347 node.ignore_missing = True 348 self.stream.skip(2) 349 else: 350 node.ignore_missing = False 351 return self.parse_import_context(node, True) 352 353 def parse_import(self) -> nodes.Import: 354 node = nodes.Import(lineno=next(self.stream).lineno) 355 node.template = self.parse_expression() 356 self.stream.expect("name:as") 357 node.target = self.parse_assign_target(name_only=True).name 358 return self.parse_import_context(node, False) 359 360 def parse_from(self) -> nodes.FromImport: 361 node = nodes.FromImport(lineno=next(self.stream).lineno) 362 node.template = self.parse_expression() 363 self.stream.expect("name:import") 364 node.names = [] 365 366 def parse_context() -> bool: 367 if ( 368 self.stream.current.value 369 in { 370 "with", 371 "without", 372 } 373 and self.stream.look().test("name:context") 374 ): 375 node.with_context = next(self.stream).value == "with" 376 self.stream.skip() 377 return True 378 return False 379 380 while True: 381 if node.names: 382 self.stream.expect("comma") 383 if self.stream.current.type == "name": 384 if parse_context(): 385 break 386 target = self.parse_assign_target(name_only=True) 387 if target.name.startswith("_"): 388 self.fail( 389 "names starting with an underline can not be imported", 390 target.lineno, 391 exc=TemplateAssertionError, 392 ) 393 if self.stream.skip_if("name:as"): 394 alias = self.parse_assign_target(name_only=True) 395 node.names.append((target.name, alias.name)) 396 else: 397 node.names.append(target.name) 398 if parse_context() or self.stream.current.type != "comma": 399 break 400 else: 401 self.stream.expect("name") 402 if not hasattr(node, "with_context"): 403 node.with_context = False 404 return node 405 406 def parse_signature(self, node: _MacroCall) -> None: 407 args = node.args = [] 408 defaults = node.defaults = [] 409 self.stream.expect("lparen") 410 while self.stream.current.type != "rparen": 411 if args: 412 self.stream.expect("comma") 413 arg = self.parse_assign_target(name_only=True) 414 arg.set_ctx("param") 415 if self.stream.skip_if("assign"): 416 defaults.append(self.parse_expression()) 417 elif defaults: 418 self.fail("non-default argument follows default argument") 419 args.append(arg) 420 self.stream.expect("rparen") 421 422 def parse_call_block(self) -> nodes.CallBlock: 423 node = nodes.CallBlock(lineno=next(self.stream).lineno) 424 if self.stream.current.type == "lparen": 425 self.parse_signature(node) 426 else: 427 node.args = [] 428 node.defaults = [] 429 430 call_node = self.parse_expression() 431 if not isinstance(call_node, nodes.Call): 432 self.fail("expected call", node.lineno) 433 node.call = call_node 434 node.body = self.parse_statements(("name:endcall",), drop_needle=True) 435 return node 436 437 def parse_filter_block(self) -> nodes.FilterBlock: 438 node = nodes.FilterBlock(lineno=next(self.stream).lineno) 439 node.filter = self.parse_filter(None, start_inline=True) # type: ignore 440 node.body = self.parse_statements(("name:endfilter",), drop_needle=True) 441 return node 442 443 def parse_macro(self) -> nodes.Macro: 444 node = nodes.Macro(lineno=next(self.stream).lineno) 445 node.name = self.parse_assign_target(name_only=True).name 446 self.parse_signature(node) 447 node.body = self.parse_statements(("name:endmacro",), drop_needle=True) 448 return node 449 450 def parse_print(self) -> nodes.Output: 451 node = nodes.Output(lineno=next(self.stream).lineno) 452 node.nodes = [] 453 while self.stream.current.type != "block_end": 454 if node.nodes: 455 self.stream.expect("comma") 456 node.nodes.append(self.parse_expression()) 457 return node 458 459 @typing.overload 460 def parse_assign_target( 461 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ... 462 ) -> nodes.Name: 463 ... 464 465 @typing.overload 466 def parse_assign_target( 467 self, 468 with_tuple: bool = True, 469 name_only: bool = False, 470 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 471 with_namespace: bool = False, 472 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: 473 ... 474 475 def parse_assign_target( 476 self, 477 with_tuple: bool = True, 478 name_only: bool = False, 479 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 480 with_namespace: bool = False, 481 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: 482 """Parse an assignment target. As Jinja allows assignments to 483 tuples, this function can parse all allowed assignment targets. Per 484 default assignments to tuples are parsed, that can be disable however 485 by setting `with_tuple` to `False`. If only assignments to names are 486 wanted `name_only` can be set to `True`. The `extra_end_rules` 487 parameter is forwarded to the tuple parsing function. If 488 `with_namespace` is enabled, a namespace assignment may be parsed. 489 """ 490 target: nodes.Expr 491 492 if with_namespace and self.stream.look().type == "dot": 493 token = self.stream.expect("name") 494 next(self.stream) # dot 495 attr = self.stream.expect("name") 496 target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) 497 elif name_only: 498 token = self.stream.expect("name") 499 target = nodes.Name(token.value, "store", lineno=token.lineno) 500 else: 501 if with_tuple: 502 target = self.parse_tuple( 503 simplified=True, extra_end_rules=extra_end_rules 504 ) 505 else: 506 target = self.parse_primary() 507 508 target.set_ctx("store") 509 510 if not target.can_assign(): 511 self.fail( 512 f"can't assign to {type(target).__name__.lower()!r}", target.lineno 513 ) 514 515 return target # type: ignore 516 517 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr: 518 """Parse an expression. Per default all expressions are parsed, if 519 the optional `with_condexpr` parameter is set to `False` conditional 520 expressions are not parsed. 521 """ 522 if with_condexpr: 523 return self.parse_condexpr() 524 return self.parse_or() 525 526 def parse_condexpr(self) -> nodes.Expr: 527 lineno = self.stream.current.lineno 528 expr1 = self.parse_or() 529 expr3: t.Optional[nodes.Expr] 530 531 while self.stream.skip_if("name:if"): 532 expr2 = self.parse_or() 533 if self.stream.skip_if("name:else"): 534 expr3 = self.parse_condexpr() 535 else: 536 expr3 = None 537 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) 538 lineno = self.stream.current.lineno 539 return expr1 540 541 def parse_or(self) -> nodes.Expr: 542 lineno = self.stream.current.lineno 543 left = self.parse_and() 544 while self.stream.skip_if("name:or"): 545 right = self.parse_and() 546 left = nodes.Or(left, right, lineno=lineno) 547 lineno = self.stream.current.lineno 548 return left 549 550 def parse_and(self) -> nodes.Expr: 551 lineno = self.stream.current.lineno 552 left = self.parse_not() 553 while self.stream.skip_if("name:and"): 554 right = self.parse_not() 555 left = nodes.And(left, right, lineno=lineno) 556 lineno = self.stream.current.lineno 557 return left 558 559 def parse_not(self) -> nodes.Expr: 560 if self.stream.current.test("name:not"): 561 lineno = next(self.stream).lineno 562 return nodes.Not(self.parse_not(), lineno=lineno) 563 return self.parse_compare() 564 565 def parse_compare(self) -> nodes.Expr: 566 lineno = self.stream.current.lineno 567 expr = self.parse_math1() 568 ops = [] 569 while True: 570 token_type = self.stream.current.type 571 if token_type in _compare_operators: 572 next(self.stream) 573 ops.append(nodes.Operand(token_type, self.parse_math1())) 574 elif self.stream.skip_if("name:in"): 575 ops.append(nodes.Operand("in", self.parse_math1())) 576 elif self.stream.current.test("name:not") and self.stream.look().test( 577 "name:in" 578 ): 579 self.stream.skip(2) 580 ops.append(nodes.Operand("notin", self.parse_math1())) 581 else: 582 break 583 lineno = self.stream.current.lineno 584 if not ops: 585 return expr 586 return nodes.Compare(expr, ops, lineno=lineno) 587 588 def parse_math1(self) -> nodes.Expr: 589 lineno = self.stream.current.lineno 590 left = self.parse_concat() 591 while self.stream.current.type in ("add", "sub"): 592 cls = _math_nodes[self.stream.current.type] 593 next(self.stream) 594 right = self.parse_concat() 595 left = cls(left, right, lineno=lineno) 596 lineno = self.stream.current.lineno 597 return left 598 599 def parse_concat(self) -> nodes.Expr: 600 lineno = self.stream.current.lineno 601 args = [self.parse_math2()] 602 while self.stream.current.type == "tilde": 603 next(self.stream) 604 args.append(self.parse_math2()) 605 if len(args) == 1: 606 return args[0] 607 return nodes.Concat(args, lineno=lineno) 608 609 def parse_math2(self) -> nodes.Expr: 610 lineno = self.stream.current.lineno 611 left = self.parse_pow() 612 while self.stream.current.type in ("mul", "div", "floordiv", "mod"): 613 cls = _math_nodes[self.stream.current.type] 614 next(self.stream) 615 right = self.parse_pow() 616 left = cls(left, right, lineno=lineno) 617 lineno = self.stream.current.lineno 618 return left 619 620 def parse_pow(self) -> nodes.Expr: 621 lineno = self.stream.current.lineno 622 left = self.parse_unary() 623 while self.stream.current.type == "pow": 624 next(self.stream) 625 right = self.parse_unary() 626 left = nodes.Pow(left, right, lineno=lineno) 627 lineno = self.stream.current.lineno 628 return left 629 630 def parse_unary(self, with_filter: bool = True) -> nodes.Expr: 631 token_type = self.stream.current.type 632 lineno = self.stream.current.lineno 633 node: nodes.Expr 634 635 if token_type == "sub": 636 next(self.stream) 637 node = nodes.Neg(self.parse_unary(False), lineno=lineno) 638 elif token_type == "add": 639 next(self.stream) 640 node = nodes.Pos(self.parse_unary(False), lineno=lineno) 641 else: 642 node = self.parse_primary() 643 node = self.parse_postfix(node) 644 if with_filter: 645 node = self.parse_filter_expr(node) 646 return node 647 648 def parse_primary(self) -> nodes.Expr: 649 token = self.stream.current 650 node: nodes.Expr 651 if token.type == "name": 652 if token.value in ("true", "false", "True", "False"): 653 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) 654 elif token.value in ("none", "None"): 655 node = nodes.Const(None, lineno=token.lineno) 656 else: 657 node = nodes.Name(token.value, "load", lineno=token.lineno) 658 next(self.stream) 659 elif token.type == "string": 660 next(self.stream) 661 buf = [token.value] 662 lineno = token.lineno 663 while self.stream.current.type == "string": 664 buf.append(self.stream.current.value) 665 next(self.stream) 666 node = nodes.Const("".join(buf), lineno=lineno) 667 elif token.type in ("integer", "float"): 668 next(self.stream) 669 node = nodes.Const(token.value, lineno=token.lineno) 670 elif token.type == "lparen": 671 next(self.stream) 672 node = self.parse_tuple(explicit_parentheses=True) 673 self.stream.expect("rparen") 674 elif token.type == "lbracket": 675 node = self.parse_list() 676 elif token.type == "lbrace": 677 node = self.parse_dict() 678 else: 679 self.fail(f"unexpected {describe_token(token)!r}", token.lineno) 680 return node 681 682 def parse_tuple( 683 self, 684 simplified: bool = False, 685 with_condexpr: bool = True, 686 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 687 explicit_parentheses: bool = False, 688 ) -> t.Union[nodes.Tuple, nodes.Expr]: 689 """Works like `parse_expression` but if multiple expressions are 690 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. 691 This method could also return a regular expression instead of a tuple 692 if no commas where found. 693 694 The default parsing mode is a full tuple. If `simplified` is `True` 695 only names and literals are parsed. The `no_condexpr` parameter is 696 forwarded to :meth:`parse_expression`. 697 698 Because tuples do not require delimiters and may end in a bogus comma 699 an extra hint is needed that marks the end of a tuple. For example 700 for loops support tuples between `for` and `in`. In that case the 701 `extra_end_rules` is set to ``['name:in']``. 702 703 `explicit_parentheses` is true if the parsing was triggered by an 704 expression in parentheses. This is used to figure out if an empty 705 tuple is a valid expression or not. 706 """ 707 lineno = self.stream.current.lineno 708 if simplified: 709 parse = self.parse_primary 710 elif with_condexpr: 711 parse = self.parse_expression 712 else: 713 714 def parse() -> nodes.Expr: 715 return self.parse_expression(with_condexpr=False) 716 717 args: t.List[nodes.Expr] = [] 718 is_tuple = False 719 720 while True: 721 if args: 722 self.stream.expect("comma") 723 if self.is_tuple_end(extra_end_rules): 724 break 725 args.append(parse()) 726 if self.stream.current.type == "comma": 727 is_tuple = True 728 else: 729 break 730 lineno = self.stream.current.lineno 731 732 if not is_tuple: 733 if args: 734 return args[0] 735 736 # if we don't have explicit parentheses, an empty tuple is 737 # not a valid expression. This would mean nothing (literally 738 # nothing) in the spot of an expression would be an empty 739 # tuple. 740 if not explicit_parentheses: 741 self.fail( 742 "Expected an expression," 743 f" got {describe_token(self.stream.current)!r}" 744 ) 745 746 return nodes.Tuple(args, "load", lineno=lineno) 747 748 def parse_list(self) -> nodes.List: 749 token = self.stream.expect("lbracket") 750 items: t.List[nodes.Expr] = [] 751 while self.stream.current.type != "rbracket": 752 if items: 753 self.stream.expect("comma") 754 if self.stream.current.type == "rbracket": 755 break 756 items.append(self.parse_expression()) 757 self.stream.expect("rbracket") 758 return nodes.List(items, lineno=token.lineno) 759 760 def parse_dict(self) -> nodes.Dict: 761 token = self.stream.expect("lbrace") 762 items: t.List[nodes.Pair] = [] 763 while self.stream.current.type != "rbrace": 764 if items: 765 self.stream.expect("comma") 766 if self.stream.current.type == "rbrace": 767 break 768 key = self.parse_expression() 769 self.stream.expect("colon") 770 value = self.parse_expression() 771 items.append(nodes.Pair(key, value, lineno=key.lineno)) 772 self.stream.expect("rbrace") 773 return nodes.Dict(items, lineno=token.lineno) 774 775 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr: 776 while True: 777 token_type = self.stream.current.type 778 if token_type == "dot" or token_type == "lbracket": 779 node = self.parse_subscript(node) 780 # calls are valid both after postfix expressions (getattr 781 # and getitem) as well as filters and tests 782 elif token_type == "lparen": 783 node = self.parse_call(node) 784 else: 785 break 786 return node 787 788 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr: 789 while True: 790 token_type = self.stream.current.type 791 if token_type == "pipe": 792 node = self.parse_filter(node) # type: ignore 793 elif token_type == "name" and self.stream.current.value == "is": 794 node = self.parse_test(node) 795 # calls are valid both after postfix expressions (getattr 796 # and getitem) as well as filters and tests 797 elif token_type == "lparen": 798 node = self.parse_call(node) 799 else: 800 break 801 return node 802 803 def parse_subscript( 804 self, node: nodes.Expr 805 ) -> t.Union[nodes.Getattr, nodes.Getitem]: 806 token = next(self.stream) 807 arg: nodes.Expr 808 809 if token.type == "dot": 810 attr_token = self.stream.current 811 next(self.stream) 812 if attr_token.type == "name": 813 return nodes.Getattr( 814 node, attr_token.value, "load", lineno=token.lineno 815 ) 816 elif attr_token.type != "integer": 817 self.fail("expected name or number", attr_token.lineno) 818 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) 819 return nodes.Getitem(node, arg, "load", lineno=token.lineno) 820 if token.type == "lbracket": 821 args: t.List[nodes.Expr] = [] 822 while self.stream.current.type != "rbracket": 823 if args: 824 self.stream.expect("comma") 825 args.append(self.parse_subscribed()) 826 self.stream.expect("rbracket") 827 if len(args) == 1: 828 arg = args[0] 829 else: 830 arg = nodes.Tuple(args, "load", lineno=token.lineno) 831 return nodes.Getitem(node, arg, "load", lineno=token.lineno) 832 self.fail("expected subscript expression", token.lineno) 833 834 def parse_subscribed(self) -> nodes.Expr: 835 lineno = self.stream.current.lineno 836 args: t.List[t.Optional[nodes.Expr]] 837 838 if self.stream.current.type == "colon": 839 next(self.stream) 840 args = [None] 841 else: 842 node = self.parse_expression() 843 if self.stream.current.type != "colon": 844 return node 845 next(self.stream) 846 args = [node] 847 848 if self.stream.current.type == "colon": 849 args.append(None) 850 elif self.stream.current.type not in ("rbracket", "comma"): 851 args.append(self.parse_expression()) 852 else: 853 args.append(None) 854 855 if self.stream.current.type == "colon": 856 next(self.stream) 857 if self.stream.current.type not in ("rbracket", "comma"): 858 args.append(self.parse_expression()) 859 else: 860 args.append(None) 861 else: 862 args.append(None) 863 864 return nodes.Slice(lineno=lineno, *args) 865 866 def parse_call_args(self) -> t.Tuple: 867 token = self.stream.expect("lparen") 868 args = [] 869 kwargs = [] 870 dyn_args = None 871 dyn_kwargs = None 872 require_comma = False 873 874 def ensure(expr: bool) -> None: 875 if not expr: 876 self.fail("invalid syntax for function call expression", token.lineno) 877 878 while self.stream.current.type != "rparen": 879 if require_comma: 880 self.stream.expect("comma") 881 882 # support for trailing comma 883 if self.stream.current.type == "rparen": 884 break 885 886 if self.stream.current.type == "mul": 887 ensure(dyn_args is None and dyn_kwargs is None) 888 next(self.stream) 889 dyn_args = self.parse_expression() 890 elif self.stream.current.type == "pow": 891 ensure(dyn_kwargs is None) 892 next(self.stream) 893 dyn_kwargs = self.parse_expression() 894 else: 895 if ( 896 self.stream.current.type == "name" 897 and self.stream.look().type == "assign" 898 ): 899 # Parsing a kwarg 900 ensure(dyn_kwargs is None) 901 key = self.stream.current.value 902 self.stream.skip(2) 903 value = self.parse_expression() 904 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) 905 else: 906 # Parsing an arg 907 ensure(dyn_args is None and dyn_kwargs is None and not kwargs) 908 args.append(self.parse_expression()) 909 910 require_comma = True 911 912 self.stream.expect("rparen") 913 return args, kwargs, dyn_args, dyn_kwargs 914 915 def parse_call(self, node: nodes.Expr) -> nodes.Call: 916 # The lparen will be expected in parse_call_args, but the lineno 917 # needs to be recorded before the stream is advanced. 918 token = self.stream.current 919 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 920 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) 921 922 def parse_filter( 923 self, node: t.Optional[nodes.Expr], start_inline: bool = False 924 ) -> t.Optional[nodes.Expr]: 925 while self.stream.current.type == "pipe" or start_inline: 926 if not start_inline: 927 next(self.stream) 928 token = self.stream.expect("name") 929 name = token.value 930 while self.stream.current.type == "dot": 931 next(self.stream) 932 name += "." + self.stream.expect("name").value 933 if self.stream.current.type == "lparen": 934 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 935 else: 936 args = [] 937 kwargs = [] 938 dyn_args = dyn_kwargs = None 939 node = nodes.Filter( 940 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno 941 ) 942 start_inline = False 943 return node 944 945 def parse_test(self, node: nodes.Expr) -> nodes.Expr: 946 token = next(self.stream) 947 if self.stream.current.test("name:not"): 948 next(self.stream) 949 negated = True 950 else: 951 negated = False 952 name = self.stream.expect("name").value 953 while self.stream.current.type == "dot": 954 next(self.stream) 955 name += "." + self.stream.expect("name").value 956 dyn_args = dyn_kwargs = None 957 kwargs = [] 958 if self.stream.current.type == "lparen": 959 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 960 elif ( 961 self.stream.current.type 962 in { 963 "name", 964 "string", 965 "integer", 966 "float", 967 "lparen", 968 "lbracket", 969 "lbrace", 970 } 971 and not self.stream.current.test_any("name:else", "name:or", "name:and") 972 ): 973 if self.stream.current.test("name:is"): 974 self.fail("You cannot chain multiple tests with is") 975 arg_node = self.parse_primary() 976 arg_node = self.parse_postfix(arg_node) 977 args = [arg_node] 978 else: 979 args = [] 980 node = nodes.Test( 981 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno 982 ) 983 if negated: 984 node = nodes.Not(node, lineno=token.lineno) 985 return node 986 987 def subparse( 988 self, end_tokens: t.Optional[t.Tuple[str, ...]] = None 989 ) -> t.List[nodes.Node]: 990 body: t.List[nodes.Node] = [] 991 data_buffer: t.List[nodes.Node] = [] 992 add_data = data_buffer.append 993 994 if end_tokens is not None: 995 self._end_token_stack.append(end_tokens) 996 997 def flush_data() -> None: 998 if data_buffer: 999 lineno = data_buffer[0].lineno 1000 body.append(nodes.Output(data_buffer[:], lineno=lineno)) 1001 del data_buffer[:] 1002 1003 try: 1004 while self.stream: 1005 token = self.stream.current 1006 if token.type == "data": 1007 if token.value: 1008 add_data(nodes.TemplateData(token.value, lineno=token.lineno)) 1009 next(self.stream) 1010 elif token.type == "variable_begin": 1011 next(self.stream) 1012 add_data(self.parse_tuple(with_condexpr=True)) 1013 self.stream.expect("variable_end") 1014 elif token.type == "block_begin": 1015 flush_data() 1016 next(self.stream) 1017 if end_tokens is not None and self.stream.current.test_any( 1018 *end_tokens 1019 ): 1020 return body 1021 rv = self.parse_statement() 1022 if isinstance(rv, list): 1023 body.extend(rv) 1024 else: 1025 body.append(rv) 1026 self.stream.expect("block_end") 1027 else: 1028 raise AssertionError("internal parsing error") 1029 1030 flush_data() 1031 finally: 1032 if end_tokens is not None: 1033 self._end_token_stack.pop() 1034 return body 1035 1036 def parse(self) -> nodes.Template: 1037 """Parse the whole template into a `Template` node.""" 1038 result = nodes.Template(self.subparse(), lineno=1) 1039 result.set_environment(self.environment) 1040 return result 1041