Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Parse tokens from the lexer into nodes for the compiler."""
3import typing
4import typing as t
6from . import nodes
7from .exceptions import TemplateAssertionError
8from .exceptions import TemplateSyntaxError
9from .lexer import describe_token
10from .lexer import describe_token_expr
12if t.TYPE_CHECKING:
13 import typing_extensions as te
15 from .environment import Environment
17_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
18_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
20_statement_keywords = frozenset(
21 [
22 "for",
23 "if",
24 "block",
25 "extends",
26 "print",
27 "macro",
28 "include",
29 "from",
30 "import",
31 "set",
32 "with",
33 "autoescape",
34 ]
35)
36_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
38_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
39 "add": nodes.Add,
40 "sub": nodes.Sub,
41 "mul": nodes.Mul,
42 "div": nodes.Div,
43 "floordiv": nodes.FloorDiv,
44 "mod": nodes.Mod,
45}
48class Parser:
49 """This is the central parsing class Jinja uses. It's passed to
50 extensions and can be used to parse expressions or statements.
51 """
53 def __init__(
54 self,
55 environment: "Environment",
56 source: str,
57 name: t.Optional[str] = None,
58 filename: t.Optional[str] = None,
59 state: t.Optional[str] = None,
60 ) -> None:
61 self.environment = environment
62 self.stream = environment._tokenize(source, name, filename, state)
63 self.name = name
64 self.filename = filename
65 self.closed = False
66 self.extensions: t.Dict[
67 str, t.Callable[[Parser], t.Union[nodes.Node, t.List[nodes.Node]]]
68 ] = {}
69 for extension in environment.iter_extensions():
70 for tag in extension.tags:
71 self.extensions[tag] = extension.parse
72 self._last_identifier = 0
73 self._tag_stack: t.List[str] = []
74 self._end_token_stack: t.List[t.Tuple[str, ...]] = []
76 def fail(
77 self,
78 msg: str,
79 lineno: t.Optional[int] = None,
80 exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
81 ) -> "te.NoReturn":
82 """Convenience method that raises `exc` with the message, passed
83 line number or last line number as well as the current name and
84 filename.
85 """
86 if lineno is None:
87 lineno = self.stream.current.lineno
88 raise exc(msg, lineno, self.name, self.filename)
90 def _fail_ut_eof(
91 self,
92 name: t.Optional[str],
93 end_token_stack: t.List[t.Tuple[str, ...]],
94 lineno: t.Optional[int],
95 ) -> "te.NoReturn":
96 expected: t.Set[str] = set()
97 for exprs in end_token_stack:
98 expected.update(map(describe_token_expr, exprs))
99 if end_token_stack:
100 currently_looking: t.Optional[str] = " or ".join(
101 map(repr, map(describe_token_expr, end_token_stack[-1]))
102 )
103 else:
104 currently_looking = None
106 if name is None:
107 message = ["Unexpected end of template."]
108 else:
109 message = [f"Encountered unknown tag {name!r}."]
111 if currently_looking:
112 if name is not None and name in expected:
113 message.append(
114 "You probably made a nesting mistake. Jinja is expecting this tag,"
115 f" but currently looking for {currently_looking}."
116 )
117 else:
118 message.append(
119 f"Jinja was looking for the following tags: {currently_looking}."
120 )
122 if self._tag_stack:
123 message.append(
124 "The innermost block that needs to be closed is"
125 f" {self._tag_stack[-1]!r}."
126 )
128 self.fail(" ".join(message), lineno)
130 def fail_unknown_tag(
131 self, name: str, lineno: t.Optional[int] = None
132 ) -> "te.NoReturn":
133 """Called if the parser encounters an unknown tag. Tries to fail
134 with a human readable error message that could help to identify
135 the problem.
136 """
137 self._fail_ut_eof(name, self._end_token_stack, lineno)
139 def fail_eof(
140 self,
141 end_tokens: t.Optional[t.Tuple[str, ...]] = None,
142 lineno: t.Optional[int] = None,
143 ) -> "te.NoReturn":
144 """Like fail_unknown_tag but for end of template situations."""
145 stack = list(self._end_token_stack)
146 if end_tokens is not None:
147 stack.append(end_tokens)
148 self._fail_ut_eof(None, stack, lineno)
150 def is_tuple_end(
151 self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
152 ) -> bool:
153 """Are we at the end of a tuple?"""
154 if self.stream.current.type in ("variable_end", "block_end", "rparen"):
155 return True
156 elif extra_end_rules is not None:
157 return self.stream.current.test_any(extra_end_rules) # type: ignore
158 return False
160 def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
161 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
162 self._last_identifier += 1
163 rv = object.__new__(nodes.InternalName)
164 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
165 return rv
167 def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
168 """Parse a single statement."""
169 token = self.stream.current
170 if token.type != "name":
171 self.fail("tag name expected", token.lineno)
172 self._tag_stack.append(token.value)
173 pop_tag = True
174 try:
175 if token.value in _statement_keywords:
176 f = getattr(self, f"parse_{self.stream.current.value}")
177 return f() # type: ignore
178 if token.value == "call":
179 return self.parse_call_block()
180 if token.value == "filter":
181 return self.parse_filter_block()
182 ext = self.extensions.get(token.value)
183 if ext is not None:
184 return ext(self)
186 # did not work out, remove the token we pushed by accident
187 # from the stack so that the unknown tag fail function can
188 # produce a proper error message.
189 self._tag_stack.pop()
190 pop_tag = False
191 self.fail_unknown_tag(token.value, token.lineno)
192 finally:
193 if pop_tag:
194 self._tag_stack.pop()
196 def parse_statements(
197 self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
198 ) -> t.List[nodes.Node]:
199 """Parse multiple statements into a list until one of the end tokens
200 is reached. This is used to parse the body of statements as it also
201 parses template data if appropriate. The parser checks first if the
202 current token is a colon and skips it if there is one. Then it checks
203 for the block end and parses until if one of the `end_tokens` is
204 reached. Per default the active token in the stream at the end of
205 the call is the matched end token. If this is not wanted `drop_needle`
206 can be set to `True` and the end token is removed.
207 """
208 # the first token may be a colon for python compatibility
209 self.stream.skip_if("colon")
211 # in the future it would be possible to add whole code sections
212 # by adding some sort of end of statement token and parsing those here.
213 self.stream.expect("block_end")
214 result = self.subparse(end_tokens)
216 # we reached the end of the template too early, the subparser
217 # does not check for this, so we do that now
218 if self.stream.current.type == "eof":
219 self.fail_eof(end_tokens)
221 if drop_needle:
222 next(self.stream)
223 return result
225 def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
226 """Parse an assign statement."""
227 lineno = next(self.stream).lineno
228 target = self.parse_assign_target(with_namespace=True)
229 if self.stream.skip_if("assign"):
230 expr = self.parse_tuple()
231 return nodes.Assign(target, expr, lineno=lineno)
232 filter_node = self.parse_filter(None)
233 body = self.parse_statements(("name:endset",), drop_needle=True)
234 return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
236 def parse_for(self) -> nodes.For:
237 """Parse a for loop."""
238 lineno = self.stream.expect("name:for").lineno
239 target = self.parse_assign_target(extra_end_rules=("name:in",))
240 self.stream.expect("name:in")
241 iter = self.parse_tuple(
242 with_condexpr=False, extra_end_rules=("name:recursive",)
243 )
244 test = None
245 if self.stream.skip_if("name:if"):
246 test = self.parse_expression()
247 recursive = self.stream.skip_if("name:recursive")
248 body = self.parse_statements(("name:endfor", "name:else"))
249 if next(self.stream).value == "endfor":
250 else_ = []
251 else:
252 else_ = self.parse_statements(("name:endfor",), drop_needle=True)
253 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
255 def parse_if(self) -> nodes.If:
256 """Parse an if construct."""
257 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
258 while True:
259 node.test = self.parse_tuple(with_condexpr=False)
260 node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
261 node.elif_ = []
262 node.else_ = []
263 token = next(self.stream)
264 if token.test("name:elif"):
265 node = nodes.If(lineno=self.stream.current.lineno)
266 result.elif_.append(node)
267 continue
268 elif token.test("name:else"):
269 result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
270 break
271 return result
273 def parse_with(self) -> nodes.With:
274 node = nodes.With(lineno=next(self.stream).lineno)
275 targets: t.List[nodes.Expr] = []
276 values: t.List[nodes.Expr] = []
277 while self.stream.current.type != "block_end":
278 if targets:
279 self.stream.expect("comma")
280 target = self.parse_assign_target()
281 target.set_ctx("param")
282 targets.append(target)
283 self.stream.expect("assign")
284 values.append(self.parse_expression())
285 node.targets = targets
286 node.values = values
287 node.body = self.parse_statements(("name:endwith",), drop_needle=True)
288 return node
290 def parse_autoescape(self) -> nodes.Scope:
291 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
292 node.options = [nodes.Keyword("autoescape", self.parse_expression())]
293 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
294 return nodes.Scope([node])
296 def parse_block(self) -> nodes.Block:
297 node = nodes.Block(lineno=next(self.stream).lineno)
298 node.name = self.stream.expect("name").value
299 node.scoped = self.stream.skip_if("name:scoped")
300 node.required = self.stream.skip_if("name:required")
302 # common problem people encounter when switching from django
303 # to jinja. we do not support hyphens in block names, so let's
304 # raise a nicer error message in that case.
305 if self.stream.current.type == "sub":
306 self.fail(
307 "Block names in Jinja have to be valid Python identifiers and may not"
308 " contain hyphens, use an underscore instead."
309 )
311 node.body = self.parse_statements(("name:endblock",), drop_needle=True)
313 # enforce that required blocks only contain whitespace or comments
314 # by asserting that the body, if not empty, is just TemplateData nodes
315 # with whitespace data
316 if node.required:
317 for body_node in node.body:
318 if not isinstance(body_node, nodes.Output) or any(
319 not isinstance(output_node, nodes.TemplateData)
320 or not output_node.data.isspace()
321 for output_node in body_node.nodes
322 ):
323 self.fail("Required blocks can only contain comments or whitespace")
325 self.stream.skip_if("name:" + node.name)
326 return node
328 def parse_extends(self) -> nodes.Extends:
329 node = nodes.Extends(lineno=next(self.stream).lineno)
330 node.template = self.parse_expression()
331 return node
333 def parse_import_context(
334 self, node: _ImportInclude, default: bool
335 ) -> _ImportInclude:
336 if self.stream.current.test_any(
337 "name:with", "name:without"
338 ) and self.stream.look().test("name:context"):
339 node.with_context = next(self.stream).value == "with"
340 self.stream.skip()
341 else:
342 node.with_context = default
343 return node
345 def parse_include(self) -> nodes.Include:
346 node = nodes.Include(lineno=next(self.stream).lineno)
347 node.template = self.parse_expression()
348 if self.stream.current.test("name:ignore") and self.stream.look().test(
349 "name:missing"
350 ):
351 node.ignore_missing = True
352 self.stream.skip(2)
353 else:
354 node.ignore_missing = False
355 return self.parse_import_context(node, True)
357 def parse_import(self) -> nodes.Import:
358 node = nodes.Import(lineno=next(self.stream).lineno)
359 node.template = self.parse_expression()
360 self.stream.expect("name:as")
361 node.target = self.parse_assign_target(name_only=True).name
362 return self.parse_import_context(node, False)
364 def parse_from(self) -> nodes.FromImport:
365 node = nodes.FromImport(lineno=next(self.stream).lineno)
366 node.template = self.parse_expression()
367 self.stream.expect("name:import")
368 node.names = []
370 def parse_context() -> bool:
371 if self.stream.current.value in {
372 "with",
373 "without",
374 } and self.stream.look().test("name:context"):
375 node.with_context = next(self.stream).value == "with"
376 self.stream.skip()
377 return True
378 return False
380 while True:
381 if node.names:
382 self.stream.expect("comma")
383 if self.stream.current.type == "name":
384 if parse_context():
385 break
386 target = self.parse_assign_target(name_only=True)
387 if target.name.startswith("_"):
388 self.fail(
389 "names starting with an underline can not be imported",
390 target.lineno,
391 exc=TemplateAssertionError,
392 )
393 if self.stream.skip_if("name:as"):
394 alias = self.parse_assign_target(name_only=True)
395 node.names.append((target.name, alias.name))
396 else:
397 node.names.append(target.name)
398 if parse_context() or self.stream.current.type != "comma":
399 break
400 else:
401 self.stream.expect("name")
402 if not hasattr(node, "with_context"):
403 node.with_context = False
404 return node
406 def parse_signature(self, node: _MacroCall) -> None:
407 args = node.args = []
408 defaults = node.defaults = []
409 self.stream.expect("lparen")
410 while self.stream.current.type != "rparen":
411 if args:
412 self.stream.expect("comma")
413 arg = self.parse_assign_target(name_only=True)
414 arg.set_ctx("param")
415 if self.stream.skip_if("assign"):
416 defaults.append(self.parse_expression())
417 elif defaults:
418 self.fail("non-default argument follows default argument")
419 args.append(arg)
420 self.stream.expect("rparen")
422 def parse_call_block(self) -> nodes.CallBlock:
423 node = nodes.CallBlock(lineno=next(self.stream).lineno)
424 if self.stream.current.type == "lparen":
425 self.parse_signature(node)
426 else:
427 node.args = []
428 node.defaults = []
430 call_node = self.parse_expression()
431 if not isinstance(call_node, nodes.Call):
432 self.fail("expected call", node.lineno)
433 node.call = call_node
434 node.body = self.parse_statements(("name:endcall",), drop_needle=True)
435 return node
437 def parse_filter_block(self) -> nodes.FilterBlock:
438 node = nodes.FilterBlock(lineno=next(self.stream).lineno)
439 node.filter = self.parse_filter(None, start_inline=True) # type: ignore
440 node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
441 return node
443 def parse_macro(self) -> nodes.Macro:
444 node = nodes.Macro(lineno=next(self.stream).lineno)
445 node.name = self.parse_assign_target(name_only=True).name
446 self.parse_signature(node)
447 node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
448 return node
450 def parse_print(self) -> nodes.Output:
451 node = nodes.Output(lineno=next(self.stream).lineno)
452 node.nodes = []
453 while self.stream.current.type != "block_end":
454 if node.nodes:
455 self.stream.expect("comma")
456 node.nodes.append(self.parse_expression())
457 return node
459 @typing.overload
460 def parse_assign_target(
461 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
462 ) -> nodes.Name: ...
464 @typing.overload
465 def parse_assign_target(
466 self,
467 with_tuple: bool = True,
468 name_only: bool = False,
469 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
470 with_namespace: bool = False,
471 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: ...
473 def parse_assign_target(
474 self,
475 with_tuple: bool = True,
476 name_only: bool = False,
477 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
478 with_namespace: bool = False,
479 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
480 """Parse an assignment target. As Jinja allows assignments to
481 tuples, this function can parse all allowed assignment targets. Per
482 default assignments to tuples are parsed, that can be disable however
483 by setting `with_tuple` to `False`. If only assignments to names are
484 wanted `name_only` can be set to `True`. The `extra_end_rules`
485 parameter is forwarded to the tuple parsing function. If
486 `with_namespace` is enabled, a namespace assignment may be parsed.
487 """
488 target: nodes.Expr
490 if name_only:
491 token = self.stream.expect("name")
492 target = nodes.Name(token.value, "store", lineno=token.lineno)
493 else:
494 if with_tuple:
495 target = self.parse_tuple(
496 simplified=True,
497 extra_end_rules=extra_end_rules,
498 with_namespace=with_namespace,
499 )
500 else:
501 target = self.parse_primary(with_namespace=with_namespace)
503 target.set_ctx("store")
505 if not target.can_assign():
506 self.fail(
507 f"can't assign to {type(target).__name__.lower()!r}", target.lineno
508 )
510 return target # type: ignore
512 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
513 """Parse an expression. Per default all expressions are parsed, if
514 the optional `with_condexpr` parameter is set to `False` conditional
515 expressions are not parsed.
516 """
517 if with_condexpr:
518 return self.parse_condexpr()
519 return self.parse_or()
521 def parse_condexpr(self) -> nodes.Expr:
522 lineno = self.stream.current.lineno
523 expr1 = self.parse_or()
524 expr3: t.Optional[nodes.Expr]
526 while self.stream.skip_if("name:if"):
527 expr2 = self.parse_or()
528 if self.stream.skip_if("name:else"):
529 expr3 = self.parse_condexpr()
530 else:
531 expr3 = None
532 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
533 lineno = self.stream.current.lineno
534 return expr1
536 def parse_or(self) -> nodes.Expr:
537 lineno = self.stream.current.lineno
538 left = self.parse_and()
539 while self.stream.skip_if("name:or"):
540 right = self.parse_and()
541 left = nodes.Or(left, right, lineno=lineno)
542 lineno = self.stream.current.lineno
543 return left
545 def parse_and(self) -> nodes.Expr:
546 lineno = self.stream.current.lineno
547 left = self.parse_not()
548 while self.stream.skip_if("name:and"):
549 right = self.parse_not()
550 left = nodes.And(left, right, lineno=lineno)
551 lineno = self.stream.current.lineno
552 return left
554 def parse_not(self) -> nodes.Expr:
555 if self.stream.current.test("name:not"):
556 lineno = next(self.stream).lineno
557 return nodes.Not(self.parse_not(), lineno=lineno)
558 return self.parse_compare()
560 def parse_compare(self) -> nodes.Expr:
561 lineno = self.stream.current.lineno
562 expr = self.parse_math1()
563 ops = []
564 while True:
565 token_type = self.stream.current.type
566 if token_type in _compare_operators:
567 next(self.stream)
568 ops.append(nodes.Operand(token_type, self.parse_math1()))
569 elif self.stream.skip_if("name:in"):
570 ops.append(nodes.Operand("in", self.parse_math1()))
571 elif self.stream.current.test("name:not") and self.stream.look().test(
572 "name:in"
573 ):
574 self.stream.skip(2)
575 ops.append(nodes.Operand("notin", self.parse_math1()))
576 else:
577 break
578 lineno = self.stream.current.lineno
579 if not ops:
580 return expr
581 return nodes.Compare(expr, ops, lineno=lineno)
583 def parse_math1(self) -> nodes.Expr:
584 lineno = self.stream.current.lineno
585 left = self.parse_concat()
586 while self.stream.current.type in ("add", "sub"):
587 cls = _math_nodes[self.stream.current.type]
588 next(self.stream)
589 right = self.parse_concat()
590 left = cls(left, right, lineno=lineno)
591 lineno = self.stream.current.lineno
592 return left
594 def parse_concat(self) -> nodes.Expr:
595 lineno = self.stream.current.lineno
596 args = [self.parse_math2()]
597 while self.stream.current.type == "tilde":
598 next(self.stream)
599 args.append(self.parse_math2())
600 if len(args) == 1:
601 return args[0]
602 return nodes.Concat(args, lineno=lineno)
604 def parse_math2(self) -> nodes.Expr:
605 lineno = self.stream.current.lineno
606 left = self.parse_pow()
607 while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
608 cls = _math_nodes[self.stream.current.type]
609 next(self.stream)
610 right = self.parse_pow()
611 left = cls(left, right, lineno=lineno)
612 lineno = self.stream.current.lineno
613 return left
615 def parse_pow(self) -> nodes.Expr:
616 lineno = self.stream.current.lineno
617 left = self.parse_unary()
618 while self.stream.current.type == "pow":
619 next(self.stream)
620 right = self.parse_unary()
621 left = nodes.Pow(left, right, lineno=lineno)
622 lineno = self.stream.current.lineno
623 return left
625 def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
626 token_type = self.stream.current.type
627 lineno = self.stream.current.lineno
628 node: nodes.Expr
630 if token_type == "sub":
631 next(self.stream)
632 node = nodes.Neg(self.parse_unary(False), lineno=lineno)
633 elif token_type == "add":
634 next(self.stream)
635 node = nodes.Pos(self.parse_unary(False), lineno=lineno)
636 else:
637 node = self.parse_primary()
638 node = self.parse_postfix(node)
639 if with_filter:
640 node = self.parse_filter_expr(node)
641 return node
643 def parse_primary(self, with_namespace: bool = False) -> nodes.Expr:
644 """Parse a name or literal value. If ``with_namespace`` is enabled, also
645 parse namespace attr refs, for use in assignments."""
646 token = self.stream.current
647 node: nodes.Expr
648 if token.type == "name":
649 next(self.stream)
650 if token.value in ("true", "false", "True", "False"):
651 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
652 elif token.value in ("none", "None"):
653 node = nodes.Const(None, lineno=token.lineno)
654 elif with_namespace and self.stream.current.type == "dot":
655 # If namespace attributes are allowed at this point, and the next
656 # token is a dot, produce a namespace reference.
657 next(self.stream)
658 attr = self.stream.expect("name")
659 node = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
660 else:
661 node = nodes.Name(token.value, "load", lineno=token.lineno)
662 elif token.type == "string":
663 next(self.stream)
664 buf = [token.value]
665 lineno = token.lineno
666 while self.stream.current.type == "string":
667 buf.append(self.stream.current.value)
668 next(self.stream)
669 node = nodes.Const("".join(buf), lineno=lineno)
670 elif token.type in ("integer", "float"):
671 next(self.stream)
672 node = nodes.Const(token.value, lineno=token.lineno)
673 elif token.type == "lparen":
674 next(self.stream)
675 node = self.parse_tuple(explicit_parentheses=True)
676 self.stream.expect("rparen")
677 elif token.type == "lbracket":
678 node = self.parse_list()
679 elif token.type == "lbrace":
680 node = self.parse_dict()
681 else:
682 self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
683 return node
685 def parse_tuple(
686 self,
687 simplified: bool = False,
688 with_condexpr: bool = True,
689 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
690 explicit_parentheses: bool = False,
691 with_namespace: bool = False,
692 ) -> t.Union[nodes.Tuple, nodes.Expr]:
693 """Works like `parse_expression` but if multiple expressions are
694 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
695 This method could also return a regular expression instead of a tuple
696 if no commas where found.
698 The default parsing mode is a full tuple. If `simplified` is `True`
699 only names and literals are parsed; ``with_namespace`` allows namespace
700 attr refs as well. The `no_condexpr` parameter is forwarded to
701 :meth:`parse_expression`.
703 Because tuples do not require delimiters and may end in a bogus comma
704 an extra hint is needed that marks the end of a tuple. For example
705 for loops support tuples between `for` and `in`. In that case the
706 `extra_end_rules` is set to ``['name:in']``.
708 `explicit_parentheses` is true if the parsing was triggered by an
709 expression in parentheses. This is used to figure out if an empty
710 tuple is a valid expression or not.
711 """
712 lineno = self.stream.current.lineno
713 if simplified:
715 def parse() -> nodes.Expr:
716 return self.parse_primary(with_namespace=with_namespace)
718 else:
720 def parse() -> nodes.Expr:
721 return self.parse_expression(with_condexpr=with_condexpr)
723 args: t.List[nodes.Expr] = []
724 is_tuple = False
726 while True:
727 if args:
728 self.stream.expect("comma")
729 if self.is_tuple_end(extra_end_rules):
730 break
731 args.append(parse())
732 if self.stream.current.type == "comma":
733 is_tuple = True
734 else:
735 break
736 lineno = self.stream.current.lineno
738 if not is_tuple:
739 if args:
740 return args[0]
742 # if we don't have explicit parentheses, an empty tuple is
743 # not a valid expression. This would mean nothing (literally
744 # nothing) in the spot of an expression would be an empty
745 # tuple.
746 if not explicit_parentheses:
747 self.fail(
748 "Expected an expression,"
749 f" got {describe_token(self.stream.current)!r}"
750 )
752 return nodes.Tuple(args, "load", lineno=lineno)
754 def parse_list(self) -> nodes.List:
755 token = self.stream.expect("lbracket")
756 items: t.List[nodes.Expr] = []
757 while self.stream.current.type != "rbracket":
758 if items:
759 self.stream.expect("comma")
760 if self.stream.current.type == "rbracket":
761 break
762 items.append(self.parse_expression())
763 self.stream.expect("rbracket")
764 return nodes.List(items, lineno=token.lineno)
766 def parse_dict(self) -> nodes.Dict:
767 token = self.stream.expect("lbrace")
768 items: t.List[nodes.Pair] = []
769 while self.stream.current.type != "rbrace":
770 if items:
771 self.stream.expect("comma")
772 if self.stream.current.type == "rbrace":
773 break
774 key = self.parse_expression()
775 self.stream.expect("colon")
776 value = self.parse_expression()
777 items.append(nodes.Pair(key, value, lineno=key.lineno))
778 self.stream.expect("rbrace")
779 return nodes.Dict(items, lineno=token.lineno)
781 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
782 while True:
783 token_type = self.stream.current.type
784 if token_type == "dot" or token_type == "lbracket":
785 node = self.parse_subscript(node)
786 # calls are valid both after postfix expressions (getattr
787 # and getitem) as well as filters and tests
788 elif token_type == "lparen":
789 node = self.parse_call(node)
790 else:
791 break
792 return node
794 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
795 while True:
796 token_type = self.stream.current.type
797 if token_type == "pipe":
798 node = self.parse_filter(node) # type: ignore
799 elif token_type == "name" and self.stream.current.value == "is":
800 node = self.parse_test(node)
801 # calls are valid both after postfix expressions (getattr
802 # and getitem) as well as filters and tests
803 elif token_type == "lparen":
804 node = self.parse_call(node)
805 else:
806 break
807 return node
809 def parse_subscript(
810 self, node: nodes.Expr
811 ) -> t.Union[nodes.Getattr, nodes.Getitem]:
812 token = next(self.stream)
813 arg: nodes.Expr
815 if token.type == "dot":
816 attr_token = self.stream.current
817 next(self.stream)
818 if attr_token.type == "name":
819 return nodes.Getattr(
820 node, attr_token.value, "load", lineno=token.lineno
821 )
822 elif attr_token.type != "integer":
823 self.fail("expected name or number", attr_token.lineno)
824 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
825 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
826 if token.type == "lbracket":
827 args: t.List[nodes.Expr] = []
828 while self.stream.current.type != "rbracket":
829 if args:
830 self.stream.expect("comma")
831 args.append(self.parse_subscribed())
832 self.stream.expect("rbracket")
833 if len(args) == 1:
834 arg = args[0]
835 else:
836 arg = nodes.Tuple(args, "load", lineno=token.lineno)
837 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
838 self.fail("expected subscript expression", token.lineno)
840 def parse_subscribed(self) -> nodes.Expr:
841 lineno = self.stream.current.lineno
842 args: t.List[t.Optional[nodes.Expr]]
844 if self.stream.current.type == "colon":
845 next(self.stream)
846 args = [None]
847 else:
848 node = self.parse_expression()
849 if self.stream.current.type != "colon":
850 return node
851 next(self.stream)
852 args = [node]
854 if self.stream.current.type == "colon":
855 args.append(None)
856 elif self.stream.current.type not in ("rbracket", "comma"):
857 args.append(self.parse_expression())
858 else:
859 args.append(None)
861 if self.stream.current.type == "colon":
862 next(self.stream)
863 if self.stream.current.type not in ("rbracket", "comma"):
864 args.append(self.parse_expression())
865 else:
866 args.append(None)
867 else:
868 args.append(None)
870 return nodes.Slice(lineno=lineno, *args) # noqa: B026
872 def parse_call_args(
873 self,
874 ) -> t.Tuple[
875 t.List[nodes.Expr],
876 t.List[nodes.Keyword],
877 t.Optional[nodes.Expr],
878 t.Optional[nodes.Expr],
879 ]:
880 token = self.stream.expect("lparen")
881 args = []
882 kwargs = []
883 dyn_args = None
884 dyn_kwargs = None
885 require_comma = False
887 def ensure(expr: bool) -> None:
888 if not expr:
889 self.fail("invalid syntax for function call expression", token.lineno)
891 while self.stream.current.type != "rparen":
892 if require_comma:
893 self.stream.expect("comma")
895 # support for trailing comma
896 if self.stream.current.type == "rparen":
897 break
899 if self.stream.current.type == "mul":
900 ensure(dyn_args is None and dyn_kwargs is None)
901 next(self.stream)
902 dyn_args = self.parse_expression()
903 elif self.stream.current.type == "pow":
904 ensure(dyn_kwargs is None)
905 next(self.stream)
906 dyn_kwargs = self.parse_expression()
907 else:
908 if (
909 self.stream.current.type == "name"
910 and self.stream.look().type == "assign"
911 ):
912 # Parsing a kwarg
913 ensure(dyn_kwargs is None)
914 key = self.stream.current.value
915 self.stream.skip(2)
916 value = self.parse_expression()
917 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
918 else:
919 # Parsing an arg
920 ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
921 args.append(self.parse_expression())
923 require_comma = True
925 self.stream.expect("rparen")
926 return args, kwargs, dyn_args, dyn_kwargs
928 def parse_call(self, node: nodes.Expr) -> nodes.Call:
929 # The lparen will be expected in parse_call_args, but the lineno
930 # needs to be recorded before the stream is advanced.
931 token = self.stream.current
932 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
933 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
935 def parse_filter(
936 self, node: t.Optional[nodes.Expr], start_inline: bool = False
937 ) -> t.Optional[nodes.Expr]:
938 while self.stream.current.type == "pipe" or start_inline:
939 if not start_inline:
940 next(self.stream)
941 token = self.stream.expect("name")
942 name = token.value
943 while self.stream.current.type == "dot":
944 next(self.stream)
945 name += "." + self.stream.expect("name").value
946 if self.stream.current.type == "lparen":
947 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
948 else:
949 args = []
950 kwargs = []
951 dyn_args = dyn_kwargs = None
952 node = nodes.Filter(
953 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
954 )
955 start_inline = False
956 return node
958 def parse_test(self, node: nodes.Expr) -> nodes.Expr:
959 token = next(self.stream)
960 if self.stream.current.test("name:not"):
961 next(self.stream)
962 negated = True
963 else:
964 negated = False
965 name = self.stream.expect("name").value
966 while self.stream.current.type == "dot":
967 next(self.stream)
968 name += "." + self.stream.expect("name").value
969 dyn_args = dyn_kwargs = None
970 kwargs: t.List[nodes.Keyword] = []
971 if self.stream.current.type == "lparen":
972 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
973 elif self.stream.current.type in {
974 "name",
975 "string",
976 "integer",
977 "float",
978 "lparen",
979 "lbracket",
980 "lbrace",
981 } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
982 if self.stream.current.test("name:is"):
983 self.fail("You cannot chain multiple tests with is")
984 arg_node = self.parse_primary()
985 arg_node = self.parse_postfix(arg_node)
986 args = [arg_node]
987 else:
988 args = []
989 node = nodes.Test(
990 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
991 )
992 if negated:
993 node = nodes.Not(node, lineno=token.lineno)
994 return node
996 def subparse(
997 self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
998 ) -> t.List[nodes.Node]:
999 body: t.List[nodes.Node] = []
1000 data_buffer: t.List[nodes.Node] = []
1001 add_data = data_buffer.append
1003 if end_tokens is not None:
1004 self._end_token_stack.append(end_tokens)
1006 def flush_data() -> None:
1007 if data_buffer:
1008 lineno = data_buffer[0].lineno
1009 body.append(nodes.Output(data_buffer[:], lineno=lineno))
1010 del data_buffer[:]
1012 try:
1013 while self.stream:
1014 token = self.stream.current
1015 if token.type == "data":
1016 if token.value:
1017 add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1018 next(self.stream)
1019 elif token.type == "variable_begin":
1020 next(self.stream)
1021 add_data(self.parse_tuple(with_condexpr=True))
1022 self.stream.expect("variable_end")
1023 elif token.type == "block_begin":
1024 flush_data()
1025 next(self.stream)
1026 if end_tokens is not None and self.stream.current.test_any(
1027 *end_tokens
1028 ):
1029 return body
1030 rv = self.parse_statement()
1031 if isinstance(rv, list):
1032 body.extend(rv)
1033 else:
1034 body.append(rv)
1035 self.stream.expect("block_end")
1036 else:
1037 raise AssertionError("internal parsing error")
1039 flush_data()
1040 finally:
1041 if end_tokens is not None:
1042 self._end_token_stack.pop()
1043 return body
1045 def parse(self) -> nodes.Template:
1046 """Parse the whole template into a `Template` node."""
1047 result = nodes.Template(self.subparse(), lineno=1)
1048 result.set_environment(self.environment)
1049 return result