Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/jinja2/parser.py: 92%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Parse tokens from the lexer into nodes for the compiler."""
3import typing
4import typing as t
6from . import nodes
7from .exceptions import TemplateAssertionError
8from .exceptions import TemplateSyntaxError
9from .lexer import describe_token
10from .lexer import describe_token_expr
12if t.TYPE_CHECKING:
13 import typing_extensions as te
15 from .environment import Environment
17_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
18_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
20_statement_keywords = frozenset(
21 [
22 "for",
23 "if",
24 "block",
25 "extends",
26 "print",
27 "macro",
28 "include",
29 "from",
30 "import",
31 "set",
32 "with",
33 "autoescape",
34 ]
35)
36_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
38_math_nodes: dict[str, type[nodes.Expr]] = {
39 "add": nodes.Add,
40 "sub": nodes.Sub,
41 "mul": nodes.Mul,
42 "div": nodes.Div,
43 "floordiv": nodes.FloorDiv,
44 "mod": nodes.Mod,
45}
48class Parser:
49 """This is the central parsing class Jinja uses. It's passed to
50 extensions and can be used to parse expressions or statements.
51 """
53 def __init__(
54 self,
55 environment: "Environment",
56 source: str,
57 name: str | None = None,
58 filename: str | None = None,
59 state: str | None = None,
60 ) -> None:
61 self.environment = environment
62 self.stream = environment._tokenize(source, name, filename, state)
63 self.name = name
64 self.filename = filename
65 self.closed = False
66 self.extensions: dict[
67 str, t.Callable[[Parser], nodes.Node | list[nodes.Node]]
68 ] = {}
69 for extension in environment.iter_extensions():
70 for tag in extension.tags:
71 self.extensions[tag] = extension.parse
72 self._last_identifier = 0
73 self._tag_stack: list[str] = []
74 self._end_token_stack: list[tuple[str, ...]] = []
76 def fail(
77 self,
78 msg: str,
79 lineno: int | None = None,
80 exc: type[TemplateSyntaxError] = TemplateSyntaxError,
81 ) -> "te.NoReturn":
82 """Convenience method that raises `exc` with the message, passed
83 line number or last line number as well as the current name and
84 filename.
85 """
86 if lineno is None:
87 lineno = self.stream.current.lineno
88 raise exc(msg, lineno, self.name, self.filename)
90 def _fail_ut_eof(
91 self,
92 name: str | None,
93 end_token_stack: list[tuple[str, ...]],
94 lineno: int | None,
95 ) -> "te.NoReturn":
96 expected: set[str] = set()
97 for exprs in end_token_stack:
98 expected.update(map(describe_token_expr, exprs))
99 if end_token_stack:
100 currently_looking: str | None = " or ".join(
101 map(repr, map(describe_token_expr, end_token_stack[-1]))
102 )
103 else:
104 currently_looking = None
106 if name is None:
107 message = ["Unexpected end of template."]
108 else:
109 message = [f"Encountered unknown tag {name!r}."]
111 if currently_looking:
112 if name is not None and name in expected:
113 message.append(
114 "You probably made a nesting mistake. Jinja is expecting this tag,"
115 f" but currently looking for {currently_looking}."
116 )
117 else:
118 message.append(
119 f"Jinja was looking for the following tags: {currently_looking}."
120 )
122 if self._tag_stack:
123 message.append(
124 "The innermost block that needs to be closed is"
125 f" {self._tag_stack[-1]!r}."
126 )
128 self.fail(" ".join(message), lineno)
130 def fail_unknown_tag(self, name: str, lineno: int | None = None) -> "te.NoReturn":
131 """Called if the parser encounters an unknown tag. Tries to fail
132 with a human readable error message that could help to identify
133 the problem.
134 """
135 self._fail_ut_eof(name, self._end_token_stack, lineno)
137 def fail_eof(
138 self,
139 end_tokens: tuple[str, ...] | None = None,
140 lineno: int | None = None,
141 ) -> "te.NoReturn":
142 """Like fail_unknown_tag but for end of template situations."""
143 stack = list(self._end_token_stack)
144 if end_tokens is not None:
145 stack.append(end_tokens)
146 self._fail_ut_eof(None, stack, lineno)
148 def is_tuple_end(self, extra_end_rules: tuple[str, ...] | None = None) -> bool:
149 """Are we at the end of a tuple?"""
150 if self.stream.current.type in ("variable_end", "block_end", "rparen"):
151 return True
152 elif extra_end_rules is not None:
153 return self.stream.current.test_any(extra_end_rules) # type: ignore
154 return False
156 def free_identifier(self, lineno: int | None = None) -> nodes.InternalName:
157 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
158 self._last_identifier += 1
159 rv = object.__new__(nodes.InternalName)
160 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
161 return rv
163 def parse_statement(self) -> nodes.Node | list[nodes.Node]:
164 """Parse a single statement."""
165 token = self.stream.current
166 if token.type != "name":
167 self.fail("tag name expected", token.lineno)
168 self._tag_stack.append(token.value)
169 pop_tag = True
170 try:
171 if token.value in _statement_keywords:
172 f = getattr(self, f"parse_{self.stream.current.value}")
173 return f() # type: ignore
174 if token.value == "call":
175 return self.parse_call_block()
176 if token.value == "filter":
177 return self.parse_filter_block()
178 ext = self.extensions.get(token.value)
179 if ext is not None:
180 return ext(self)
182 # did not work out, remove the token we pushed by accident
183 # from the stack so that the unknown tag fail function can
184 # produce a proper error message.
185 self._tag_stack.pop()
186 pop_tag = False
187 self.fail_unknown_tag(token.value, token.lineno)
188 finally:
189 if pop_tag:
190 self._tag_stack.pop()
192 def parse_statements(
193 self, end_tokens: tuple[str, ...], drop_needle: bool = False
194 ) -> list[nodes.Node]:
195 """Parse multiple statements into a list until one of the end tokens
196 is reached. This is used to parse the body of statements as it also
197 parses template data if appropriate. The parser checks first if the
198 current token is a colon and skips it if there is one. Then it checks
199 for the block end and parses until if one of the `end_tokens` is
200 reached. Per default the active token in the stream at the end of
201 the call is the matched end token. If this is not wanted `drop_needle`
202 can be set to `True` and the end token is removed.
203 """
204 # the first token may be a colon for python compatibility
205 self.stream.skip_if("colon")
207 # in the future it would be possible to add whole code sections
208 # by adding some sort of end of statement token and parsing those here.
209 self.stream.expect("block_end")
210 result = self.subparse(end_tokens)
212 # we reached the end of the template too early, the subparser
213 # does not check for this, so we do that now
214 if self.stream.current.type == "eof":
215 self.fail_eof(end_tokens)
217 if drop_needle:
218 next(self.stream)
219 return result
221 def parse_set(self) -> nodes.Assign | nodes.AssignBlock:
222 """Parse an assign statement."""
223 lineno = next(self.stream).lineno
224 target = self.parse_assign_target(with_namespace=True)
225 if self.stream.skip_if("assign"):
226 expr = self.parse_tuple()
227 return nodes.Assign(target, expr, lineno=lineno)
228 filter_node = self.parse_filter(None)
229 body = self.parse_statements(("name:endset",), drop_needle=True)
230 return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
232 def parse_for(self) -> nodes.For:
233 """Parse a for loop."""
234 lineno = self.stream.expect("name:for").lineno
235 target = self.parse_assign_target(extra_end_rules=("name:in",))
236 self.stream.expect("name:in")
237 iter = self.parse_tuple(
238 with_condexpr=False, extra_end_rules=("name:recursive",)
239 )
240 test = None
241 if self.stream.skip_if("name:if"):
242 test = self.parse_expression()
243 recursive = self.stream.skip_if("name:recursive")
244 body = self.parse_statements(("name:endfor", "name:else"))
245 if next(self.stream).value == "endfor":
246 else_ = []
247 else:
248 else_ = self.parse_statements(("name:endfor",), drop_needle=True)
249 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
251 def parse_if(self) -> nodes.If:
252 """Parse an if construct."""
253 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
254 while True:
255 node.test = self.parse_tuple(with_condexpr=False)
256 node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
257 node.elif_ = []
258 node.else_ = []
259 token = next(self.stream)
260 if token.test("name:elif"):
261 node = nodes.If(lineno=self.stream.current.lineno)
262 result.elif_.append(node)
263 continue
264 elif token.test("name:else"):
265 result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
266 break
267 return result
269 def parse_with(self) -> nodes.With:
270 node = nodes.With(lineno=next(self.stream).lineno)
271 targets: list[nodes.Expr] = []
272 values: list[nodes.Expr] = []
273 while self.stream.current.type != "block_end":
274 if targets:
275 self.stream.expect("comma")
276 target = self.parse_assign_target()
277 target.set_ctx("param")
278 targets.append(target)
279 self.stream.expect("assign")
280 values.append(self.parse_expression())
281 node.targets = targets
282 node.values = values
283 node.body = self.parse_statements(("name:endwith",), drop_needle=True)
284 return node
286 def parse_autoescape(self) -> nodes.Scope:
287 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
288 node.options = [nodes.Keyword("autoescape", self.parse_expression())]
289 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
290 return nodes.Scope([node])
292 def parse_block(self) -> nodes.Block:
293 node = nodes.Block(lineno=next(self.stream).lineno)
294 node.name = self.stream.expect("name").value
295 node.scoped = self.stream.skip_if("name:scoped")
296 node.required = self.stream.skip_if("name:required")
298 # common problem people encounter when switching from django
299 # to jinja. we do not support hyphens in block names, so let's
300 # raise a nicer error message in that case.
301 if self.stream.current.type == "sub":
302 self.fail(
303 "Block names in Jinja have to be valid Python identifiers and may not"
304 " contain hyphens, use an underscore instead."
305 )
307 node.body = self.parse_statements(("name:endblock",), drop_needle=True)
309 # enforce that required blocks only contain whitespace or comments
310 # by asserting that the body, if not empty, is just TemplateData nodes
311 # with whitespace data
312 if node.required:
313 for body_node in node.body:
314 if not isinstance(body_node, nodes.Output) or any(
315 not isinstance(output_node, nodes.TemplateData)
316 or not output_node.data.isspace()
317 for output_node in body_node.nodes
318 ):
319 self.fail("Required blocks can only contain comments or whitespace")
321 self.stream.skip_if("name:" + node.name)
322 return node
324 def parse_extends(self) -> nodes.Extends:
325 node = nodes.Extends(lineno=next(self.stream).lineno)
326 node.template = self.parse_expression()
327 return node
329 def parse_import_context(
330 self, node: _ImportInclude, default: bool
331 ) -> _ImportInclude:
332 if self.stream.current.test_any(
333 "name:with", "name:without"
334 ) and self.stream.look().test("name:context"):
335 node.with_context = next(self.stream).value == "with"
336 self.stream.skip()
337 else:
338 node.with_context = default
339 return node
341 def parse_include(self) -> nodes.Include:
342 node = nodes.Include(lineno=next(self.stream).lineno)
343 node.template = self.parse_expression()
344 if self.stream.current.test("name:ignore") and self.stream.look().test(
345 "name:missing"
346 ):
347 node.ignore_missing = True
348 self.stream.skip(2)
349 else:
350 node.ignore_missing = False
351 return self.parse_import_context(node, True)
353 def parse_import(self) -> nodes.Import:
354 node = nodes.Import(lineno=next(self.stream).lineno)
355 node.template = self.parse_expression()
356 self.stream.expect("name:as")
357 node.target = self.parse_assign_target(name_only=True).name
358 return self.parse_import_context(node, False)
360 def parse_from(self) -> nodes.FromImport:
361 node = nodes.FromImport(lineno=next(self.stream).lineno)
362 node.template = self.parse_expression()
363 self.stream.expect("name:import")
364 node.names = []
366 def parse_context() -> bool:
367 if self.stream.current.value in {
368 "with",
369 "without",
370 } and self.stream.look().test("name:context"):
371 node.with_context = next(self.stream).value == "with"
372 self.stream.skip()
373 return True
374 return False
376 while True:
377 if node.names:
378 self.stream.expect("comma")
379 if self.stream.current.type == "name":
380 if parse_context():
381 break
382 target = self.parse_assign_target(name_only=True)
383 if target.name.startswith("_"):
384 self.fail(
385 "names starting with an underline can not be imported",
386 target.lineno,
387 exc=TemplateAssertionError,
388 )
389 if self.stream.skip_if("name:as"):
390 alias = self.parse_assign_target(name_only=True)
391 node.names.append((target.name, alias.name))
392 else:
393 node.names.append(target.name)
394 if parse_context() or self.stream.current.type != "comma":
395 break
396 else:
397 self.stream.expect("name")
398 if not hasattr(node, "with_context"):
399 node.with_context = False
400 return node
402 def parse_signature(self, node: _MacroCall) -> None:
403 args = node.args = []
404 defaults = node.defaults = []
405 self.stream.expect("lparen")
406 while self.stream.current.type != "rparen":
407 if args:
408 self.stream.expect("comma")
409 arg = self.parse_assign_target(name_only=True)
410 arg.set_ctx("param")
411 if self.stream.skip_if("assign"):
412 defaults.append(self.parse_expression())
413 elif defaults:
414 self.fail("non-default argument follows default argument")
415 args.append(arg)
416 self.stream.expect("rparen")
418 def parse_call_block(self) -> nodes.CallBlock:
419 node = nodes.CallBlock(lineno=next(self.stream).lineno)
420 if self.stream.current.type == "lparen":
421 self.parse_signature(node)
422 else:
423 node.args = []
424 node.defaults = []
426 call_node = self.parse_expression()
427 if not isinstance(call_node, nodes.Call):
428 self.fail("expected call", node.lineno)
429 node.call = call_node
430 node.body = self.parse_statements(("name:endcall",), drop_needle=True)
431 return node
433 def parse_filter_block(self) -> nodes.FilterBlock:
434 node = nodes.FilterBlock(lineno=next(self.stream).lineno)
435 node.filter = self.parse_filter(None, start_inline=True) # type: ignore
436 node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
437 return node
439 def parse_macro(self) -> nodes.Macro:
440 node = nodes.Macro(lineno=next(self.stream).lineno)
441 node.name = self.parse_assign_target(name_only=True).name
442 self.parse_signature(node)
443 node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
444 return node
446 def parse_print(self) -> nodes.Output:
447 node = nodes.Output(lineno=next(self.stream).lineno)
448 node.nodes = []
449 while self.stream.current.type != "block_end":
450 if node.nodes:
451 self.stream.expect("comma")
452 node.nodes.append(self.parse_expression())
453 return node
455 @typing.overload
456 def parse_assign_target(
457 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
458 ) -> nodes.Name: ...
460 @typing.overload
461 def parse_assign_target(
462 self,
463 with_tuple: bool = True,
464 name_only: bool = False,
465 extra_end_rules: tuple[str, ...] | None = None,
466 with_namespace: bool = False,
467 ) -> nodes.NSRef | nodes.Name | nodes.Tuple: ...
469 def parse_assign_target(
470 self,
471 with_tuple: bool = True,
472 name_only: bool = False,
473 extra_end_rules: tuple[str, ...] | None = None,
474 with_namespace: bool = False,
475 ) -> nodes.NSRef | nodes.Name | nodes.Tuple:
476 """Parse an assignment target. As Jinja allows assignments to
477 tuples, this function can parse all allowed assignment targets. Per
478 default assignments to tuples are parsed, that can be disable however
479 by setting `with_tuple` to `False`. If only assignments to names are
480 wanted `name_only` can be set to `True`. The `extra_end_rules`
481 parameter is forwarded to the tuple parsing function. If
482 `with_namespace` is enabled, a namespace assignment may be parsed.
483 """
484 target: nodes.Expr
486 if name_only:
487 token = self.stream.expect("name")
488 target = nodes.Name(token.value, "store", lineno=token.lineno)
489 else:
490 if with_tuple:
491 target = self.parse_tuple(
492 simplified=True,
493 extra_end_rules=extra_end_rules,
494 with_namespace=with_namespace,
495 )
496 else:
497 target = self.parse_primary(with_namespace=with_namespace)
499 target.set_ctx("store")
501 if not target.can_assign():
502 self.fail(
503 f"can't assign to {type(target).__name__.lower()!r}", target.lineno
504 )
506 return target # type: ignore
508 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
509 """Parse an expression. Per default all expressions are parsed, if
510 the optional `with_condexpr` parameter is set to `False` conditional
511 expressions are not parsed.
512 """
513 if with_condexpr:
514 return self.parse_condexpr()
515 return self.parse_or()
517 def parse_condexpr(self) -> nodes.Expr:
518 lineno = self.stream.current.lineno
519 expr1 = self.parse_or()
520 expr3: nodes.Expr | None
522 while self.stream.skip_if("name:if"):
523 expr2 = self.parse_or()
524 if self.stream.skip_if("name:else"):
525 expr3 = self.parse_condexpr()
526 else:
527 expr3 = None
528 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
529 lineno = self.stream.current.lineno
530 return expr1
532 def parse_or(self) -> nodes.Expr:
533 lineno = self.stream.current.lineno
534 left = self.parse_and()
535 while self.stream.skip_if("name:or"):
536 right = self.parse_and()
537 left = nodes.Or(left, right, lineno=lineno)
538 lineno = self.stream.current.lineno
539 return left
541 def parse_and(self) -> nodes.Expr:
542 lineno = self.stream.current.lineno
543 left = self.parse_not()
544 while self.stream.skip_if("name:and"):
545 right = self.parse_not()
546 left = nodes.And(left, right, lineno=lineno)
547 lineno = self.stream.current.lineno
548 return left
550 def parse_not(self) -> nodes.Expr:
551 if self.stream.current.test("name:not"):
552 lineno = next(self.stream).lineno
553 return nodes.Not(self.parse_not(), lineno=lineno)
554 return self.parse_compare()
556 def parse_compare(self) -> nodes.Expr:
557 lineno = self.stream.current.lineno
558 expr = self.parse_math1()
559 ops = []
560 while True:
561 token_type = self.stream.current.type
562 if token_type in _compare_operators:
563 next(self.stream)
564 ops.append(nodes.Operand(token_type, self.parse_math1()))
565 elif self.stream.skip_if("name:in"):
566 ops.append(nodes.Operand("in", self.parse_math1()))
567 elif self.stream.current.test("name:not") and self.stream.look().test(
568 "name:in"
569 ):
570 self.stream.skip(2)
571 ops.append(nodes.Operand("notin", self.parse_math1()))
572 else:
573 break
574 lineno = self.stream.current.lineno
575 if not ops:
576 return expr
577 return nodes.Compare(expr, ops, lineno=lineno)
579 def parse_math1(self) -> nodes.Expr:
580 lineno = self.stream.current.lineno
581 left = self.parse_concat()
582 while self.stream.current.type in ("add", "sub"):
583 cls = _math_nodes[self.stream.current.type]
584 next(self.stream)
585 right = self.parse_concat()
586 left = cls(left, right, lineno=lineno)
587 lineno = self.stream.current.lineno
588 return left
590 def parse_concat(self) -> nodes.Expr:
591 lineno = self.stream.current.lineno
592 args = [self.parse_math2()]
593 while self.stream.current.type == "tilde":
594 next(self.stream)
595 args.append(self.parse_math2())
596 if len(args) == 1:
597 return args[0]
598 return nodes.Concat(args, lineno=lineno)
600 def parse_math2(self) -> nodes.Expr:
601 lineno = self.stream.current.lineno
602 left = self.parse_pow()
603 while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
604 cls = _math_nodes[self.stream.current.type]
605 next(self.stream)
606 right = self.parse_pow()
607 left = cls(left, right, lineno=lineno)
608 lineno = self.stream.current.lineno
609 return left
611 def parse_pow(self) -> nodes.Expr:
612 lineno = self.stream.current.lineno
613 left = self.parse_unary()
614 while self.stream.current.type == "pow":
615 next(self.stream)
616 right = self.parse_unary()
617 left = nodes.Pow(left, right, lineno=lineno)
618 lineno = self.stream.current.lineno
619 return left
621 def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
622 token_type = self.stream.current.type
623 lineno = self.stream.current.lineno
624 node: nodes.Expr
626 if token_type == "sub":
627 next(self.stream)
628 node = nodes.Neg(self.parse_unary(False), lineno=lineno)
629 elif token_type == "add":
630 next(self.stream)
631 node = nodes.Pos(self.parse_unary(False), lineno=lineno)
632 else:
633 node = self.parse_primary()
634 node = self.parse_postfix(node)
635 if with_filter:
636 node = self.parse_filter_expr(node)
637 return node
639 def parse_primary(self, with_namespace: bool = False) -> nodes.Expr:
640 """Parse a name or literal value. If ``with_namespace`` is enabled, also
641 parse namespace attr refs, for use in assignments."""
642 token = self.stream.current
643 node: nodes.Expr
644 if token.type == "name":
645 next(self.stream)
646 if token.value in ("true", "false", "True", "False"):
647 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
648 elif token.value in ("none", "None"):
649 node = nodes.Const(None, lineno=token.lineno)
650 elif with_namespace and self.stream.current.type == "dot":
651 # If namespace attributes are allowed at this point, and the next
652 # token is a dot, produce a namespace reference.
653 next(self.stream)
654 attr = self.stream.expect("name")
655 node = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
656 else:
657 node = nodes.Name(token.value, "load", lineno=token.lineno)
658 elif token.type == "string":
659 next(self.stream)
660 buf = [token.value]
661 lineno = token.lineno
662 while self.stream.current.type == "string":
663 buf.append(self.stream.current.value)
664 next(self.stream)
665 node = nodes.Const("".join(buf), lineno=lineno)
666 elif token.type in ("integer", "float"):
667 next(self.stream)
668 node = nodes.Const(token.value, lineno=token.lineno)
669 elif token.type == "lparen":
670 next(self.stream)
671 node = self.parse_tuple(explicit_parentheses=True)
672 self.stream.expect("rparen")
673 elif token.type == "lbracket":
674 node = self.parse_list()
675 elif token.type == "lbrace":
676 node = self.parse_dict()
677 else:
678 self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
679 return node
681 def parse_tuple(
682 self,
683 simplified: bool = False,
684 with_condexpr: bool = True,
685 extra_end_rules: tuple[str, ...] | None = None,
686 explicit_parentheses: bool = False,
687 with_namespace: bool = False,
688 ) -> nodes.Tuple | nodes.Expr:
689 """Works like `parse_expression` but if multiple expressions are
690 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
691 This method could also return a regular expression instead of a tuple
692 if no commas where found.
694 The default parsing mode is a full tuple. If `simplified` is `True`
695 only names and literals are parsed; ``with_namespace`` allows namespace
696 attr refs as well. The `no_condexpr` parameter is forwarded to
697 :meth:`parse_expression`.
699 Because tuples do not require delimiters and may end in a bogus comma
700 an extra hint is needed that marks the end of a tuple. For example
701 for loops support tuples between `for` and `in`. In that case the
702 `extra_end_rules` is set to ``['name:in']``.
704 `explicit_parentheses` is true if the parsing was triggered by an
705 expression in parentheses. This is used to figure out if an empty
706 tuple is a valid expression or not.
707 """
708 lineno = self.stream.current.lineno
709 if simplified:
711 def parse() -> nodes.Expr:
712 return self.parse_primary(with_namespace=with_namespace)
714 else:
716 def parse() -> nodes.Expr:
717 return self.parse_expression(with_condexpr=with_condexpr)
719 args: list[nodes.Expr] = []
720 is_tuple = False
722 while True:
723 if args:
724 self.stream.expect("comma")
725 if self.is_tuple_end(extra_end_rules):
726 break
727 args.append(parse())
728 if self.stream.current.type == "comma":
729 is_tuple = True
730 else:
731 break
732 lineno = self.stream.current.lineno
734 if not is_tuple:
735 if args:
736 return args[0]
738 # if we don't have explicit parentheses, an empty tuple is
739 # not a valid expression. This would mean nothing (literally
740 # nothing) in the spot of an expression would be an empty
741 # tuple.
742 if not explicit_parentheses:
743 self.fail(
744 "Expected an expression,"
745 f" got {describe_token(self.stream.current)!r}"
746 )
748 return nodes.Tuple(args, "load", lineno=lineno)
750 def parse_list(self) -> nodes.List:
751 token = self.stream.expect("lbracket")
752 items: list[nodes.Expr] = []
753 while self.stream.current.type != "rbracket":
754 if items:
755 self.stream.expect("comma")
756 if self.stream.current.type == "rbracket":
757 break
758 items.append(self.parse_expression())
759 self.stream.expect("rbracket")
760 return nodes.List(items, lineno=token.lineno)
762 def parse_dict(self) -> nodes.Dict:
763 token = self.stream.expect("lbrace")
764 items: list[nodes.Pair] = []
765 while self.stream.current.type != "rbrace":
766 if items:
767 self.stream.expect("comma")
768 if self.stream.current.type == "rbrace":
769 break
770 key = self.parse_expression()
771 self.stream.expect("colon")
772 value = self.parse_expression()
773 items.append(nodes.Pair(key, value, lineno=key.lineno))
774 self.stream.expect("rbrace")
775 return nodes.Dict(items, lineno=token.lineno)
777 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
778 while True:
779 token_type = self.stream.current.type
780 if token_type == "dot" or token_type == "lbracket":
781 node = self.parse_subscript(node)
782 # calls are valid both after postfix expressions (getattr
783 # and getitem) as well as filters and tests
784 elif token_type == "lparen":
785 node = self.parse_call(node)
786 else:
787 break
788 return node
790 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
791 while True:
792 token_type = self.stream.current.type
793 if token_type == "pipe":
794 node = self.parse_filter(node) # type: ignore
795 elif token_type == "name" and self.stream.current.value == "is":
796 node = self.parse_test(node)
797 # calls are valid both after postfix expressions (getattr
798 # and getitem) as well as filters and tests
799 elif token_type == "lparen":
800 node = self.parse_call(node)
801 else:
802 break
803 return node
805 def parse_subscript(self, node: nodes.Expr) -> nodes.Getattr | nodes.Getitem:
806 token = next(self.stream)
807 arg: nodes.Expr
809 if token.type == "dot":
810 attr_token = self.stream.current
811 next(self.stream)
812 if attr_token.type == "name":
813 return nodes.Getattr(
814 node, attr_token.value, "load", lineno=token.lineno
815 )
816 elif attr_token.type != "integer":
817 self.fail("expected name or number", attr_token.lineno)
818 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
819 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
820 if token.type == "lbracket":
821 args: list[nodes.Expr] = []
822 while self.stream.current.type != "rbracket":
823 if args:
824 self.stream.expect("comma")
825 args.append(self.parse_subscribed())
826 self.stream.expect("rbracket")
827 if len(args) == 1:
828 arg = args[0]
829 else:
830 arg = nodes.Tuple(args, "load", lineno=token.lineno)
831 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
832 self.fail("expected subscript expression", token.lineno)
834 def parse_subscribed(self) -> nodes.Expr:
835 lineno = self.stream.current.lineno
836 args: list[nodes.Expr | None]
838 if self.stream.current.type == "colon":
839 next(self.stream)
840 args = [None]
841 else:
842 node = self.parse_expression()
843 if self.stream.current.type != "colon":
844 return node
845 next(self.stream)
846 args = [node]
848 if self.stream.current.type == "colon":
849 args.append(None)
850 elif self.stream.current.type not in ("rbracket", "comma"):
851 args.append(self.parse_expression())
852 else:
853 args.append(None)
855 if self.stream.current.type == "colon":
856 next(self.stream)
857 if self.stream.current.type not in ("rbracket", "comma"):
858 args.append(self.parse_expression())
859 else:
860 args.append(None)
861 else:
862 args.append(None)
864 return nodes.Slice(lineno=lineno, *args) # noqa: B026
866 def parse_call_args(
867 self,
868 ) -> tuple[
869 list[nodes.Expr],
870 list[nodes.Keyword],
871 nodes.Expr | None,
872 nodes.Expr | None,
873 ]:
874 token = self.stream.expect("lparen")
875 args = []
876 kwargs = []
877 dyn_args = None
878 dyn_kwargs = None
879 require_comma = False
881 def ensure(expr: bool) -> None:
882 if not expr:
883 self.fail("invalid syntax for function call expression", token.lineno)
885 while self.stream.current.type != "rparen":
886 if require_comma:
887 self.stream.expect("comma")
889 # support for trailing comma
890 if self.stream.current.type == "rparen":
891 break
893 if self.stream.current.type == "mul":
894 ensure(dyn_args is None and dyn_kwargs is None)
895 next(self.stream)
896 dyn_args = self.parse_expression()
897 elif self.stream.current.type == "pow":
898 ensure(dyn_kwargs is None)
899 next(self.stream)
900 dyn_kwargs = self.parse_expression()
901 else:
902 if (
903 self.stream.current.type == "name"
904 and self.stream.look().type == "assign"
905 ):
906 # Parsing a kwarg
907 ensure(dyn_kwargs is None)
908 key = self.stream.current.value
909 self.stream.skip(2)
910 value = self.parse_expression()
911 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
912 else:
913 # Parsing an arg
914 ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
915 args.append(self.parse_expression())
917 require_comma = True
919 self.stream.expect("rparen")
920 return args, kwargs, dyn_args, dyn_kwargs
922 def parse_call(self, node: nodes.Expr) -> nodes.Call:
923 # The lparen will be expected in parse_call_args, but the lineno
924 # needs to be recorded before the stream is advanced.
925 token = self.stream.current
926 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
927 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
929 def parse_filter(
930 self, node: nodes.Expr | None, start_inline: bool = False
931 ) -> nodes.Expr | None:
932 while self.stream.current.type == "pipe" or start_inline:
933 if not start_inline:
934 next(self.stream)
935 token = self.stream.expect("name")
936 name = token.value
937 while self.stream.current.type == "dot":
938 next(self.stream)
939 name += "." + self.stream.expect("name").value
940 if self.stream.current.type == "lparen":
941 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
942 else:
943 args = []
944 kwargs = []
945 dyn_args = dyn_kwargs = None
946 node = nodes.Filter(
947 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
948 )
949 start_inline = False
950 return node
952 def parse_test(self, node: nodes.Expr) -> nodes.Expr:
953 token = next(self.stream)
954 if self.stream.current.test("name:not"):
955 next(self.stream)
956 negated = True
957 else:
958 negated = False
959 name = self.stream.expect("name").value
960 while self.stream.current.type == "dot":
961 next(self.stream)
962 name += "." + self.stream.expect("name").value
963 dyn_args = dyn_kwargs = None
964 kwargs: list[nodes.Keyword] = []
965 if self.stream.current.type == "lparen":
966 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
967 elif self.stream.current.type in {
968 "name",
969 "string",
970 "integer",
971 "float",
972 "lparen",
973 "lbracket",
974 "lbrace",
975 } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
976 if self.stream.current.test("name:is"):
977 self.fail("You cannot chain multiple tests with is")
978 arg_node = self.parse_primary()
979 arg_node = self.parse_postfix(arg_node)
980 args = [arg_node]
981 else:
982 args = []
983 node = nodes.Test(
984 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
985 )
986 if negated:
987 node = nodes.Not(node, lineno=token.lineno)
988 return node
990 def subparse(self, end_tokens: tuple[str, ...] | None = None) -> list[nodes.Node]:
991 body: list[nodes.Node] = []
992 data_buffer: list[nodes.Node] = []
993 add_data = data_buffer.append
995 if end_tokens is not None:
996 self._end_token_stack.append(end_tokens)
998 def flush_data() -> None:
999 if data_buffer:
1000 lineno = data_buffer[0].lineno
1001 body.append(nodes.Output(data_buffer[:], lineno=lineno))
1002 del data_buffer[:]
1004 try:
1005 while self.stream:
1006 token = self.stream.current
1007 if token.type == "data":
1008 if token.value:
1009 add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1010 next(self.stream)
1011 elif token.type == "variable_begin":
1012 next(self.stream)
1013 add_data(self.parse_tuple(with_condexpr=True))
1014 self.stream.expect("variable_end")
1015 elif token.type == "block_begin":
1016 flush_data()
1017 next(self.stream)
1018 if end_tokens is not None and self.stream.current.test_any(
1019 *end_tokens
1020 ):
1021 return body
1022 rv = self.parse_statement()
1023 if isinstance(rv, list):
1024 body.extend(rv)
1025 else:
1026 body.append(rv)
1027 self.stream.expect("block_end")
1028 else:
1029 raise AssertionError("internal parsing error")
1031 flush_data()
1032 finally:
1033 if end_tokens is not None:
1034 self._end_token_stack.pop()
1035 return body
1037 def parse(self) -> nodes.Template:
1038 """Parse the whole template into a `Template` node."""
1039 result = nodes.Template(self.subparse(), lineno=1)
1040 result.set_environment(self.environment)
1041 return result