Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/jinja2/parser.py: 84%
663 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:15 +0000
1"""Parse tokens from the lexer into nodes for the compiler."""
2import typing
3import typing as t
5from . import nodes
6from .exceptions import TemplateAssertionError
7from .exceptions import TemplateSyntaxError
8from .lexer import describe_token
9from .lexer import describe_token_expr
11if t.TYPE_CHECKING:
12 import typing_extensions as te
13 from .environment import Environment
15_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
16_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
18_statement_keywords = frozenset(
19 [
20 "for",
21 "if",
22 "block",
23 "extends",
24 "print",
25 "macro",
26 "include",
27 "from",
28 "import",
29 "set",
30 "with",
31 "autoescape",
32 ]
33)
34_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
36_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
37 "add": nodes.Add,
38 "sub": nodes.Sub,
39 "mul": nodes.Mul,
40 "div": nodes.Div,
41 "floordiv": nodes.FloorDiv,
42 "mod": nodes.Mod,
43}
46class Parser:
47 """This is the central parsing class Jinja uses. It's passed to
48 extensions and can be used to parse expressions or statements.
49 """
51 def __init__(
52 self,
53 environment: "Environment",
54 source: str,
55 name: t.Optional[str] = None,
56 filename: t.Optional[str] = None,
57 state: t.Optional[str] = None,
58 ) -> None:
59 self.environment = environment
60 self.stream = environment._tokenize(source, name, filename, state)
61 self.name = name
62 self.filename = filename
63 self.closed = False
64 self.extensions: t.Dict[
65 str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]]
66 ] = {}
67 for extension in environment.iter_extensions():
68 for tag in extension.tags:
69 self.extensions[tag] = extension.parse
70 self._last_identifier = 0
71 self._tag_stack: t.List[str] = []
72 self._end_token_stack: t.List[t.Tuple[str, ...]] = []
74 def fail(
75 self,
76 msg: str,
77 lineno: t.Optional[int] = None,
78 exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
79 ) -> "te.NoReturn":
80 """Convenience method that raises `exc` with the message, passed
81 line number or last line number as well as the current name and
82 filename.
83 """
84 if lineno is None:
85 lineno = self.stream.current.lineno
86 raise exc(msg, lineno, self.name, self.filename)
88 def _fail_ut_eof(
89 self,
90 name: t.Optional[str],
91 end_token_stack: t.List[t.Tuple[str, ...]],
92 lineno: t.Optional[int],
93 ) -> "te.NoReturn":
94 expected: t.Set[str] = set()
95 for exprs in end_token_stack:
96 expected.update(map(describe_token_expr, exprs))
97 if end_token_stack:
98 currently_looking: t.Optional[str] = " or ".join(
99 map(repr, map(describe_token_expr, end_token_stack[-1]))
100 )
101 else:
102 currently_looking = None
104 if name is None:
105 message = ["Unexpected end of template."]
106 else:
107 message = [f"Encountered unknown tag {name!r}."]
109 if currently_looking:
110 if name is not None and name in expected:
111 message.append(
112 "You probably made a nesting mistake. Jinja is expecting this tag,"
113 f" but currently looking for {currently_looking}."
114 )
115 else:
116 message.append(
117 f"Jinja was looking for the following tags: {currently_looking}."
118 )
120 if self._tag_stack:
121 message.append(
122 "The innermost block that needs to be closed is"
123 f" {self._tag_stack[-1]!r}."
124 )
126 self.fail(" ".join(message), lineno)
128 def fail_unknown_tag(
129 self, name: str, lineno: t.Optional[int] = None
130 ) -> "te.NoReturn":
131 """Called if the parser encounters an unknown tag. Tries to fail
132 with a human readable error message that could help to identify
133 the problem.
134 """
135 self._fail_ut_eof(name, self._end_token_stack, lineno)
137 def fail_eof(
138 self,
139 end_tokens: t.Optional[t.Tuple[str, ...]] = None,
140 lineno: t.Optional[int] = None,
141 ) -> "te.NoReturn":
142 """Like fail_unknown_tag but for end of template situations."""
143 stack = list(self._end_token_stack)
144 if end_tokens is not None:
145 stack.append(end_tokens)
146 self._fail_ut_eof(None, stack, lineno)
148 def is_tuple_end(
149 self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
150 ) -> bool:
151 """Are we at the end of a tuple?"""
152 if self.stream.current.type in ("variable_end", "block_end", "rparen"):
153 return True
154 elif extra_end_rules is not None:
155 return self.stream.current.test_any(extra_end_rules) # type: ignore
156 return False
158 def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
159 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
160 self._last_identifier += 1
161 rv = object.__new__(nodes.InternalName)
162 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
163 return rv
165 def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
166 """Parse a single statement."""
167 token = self.stream.current
168 if token.type != "name":
169 self.fail("tag name expected", token.lineno)
170 self._tag_stack.append(token.value)
171 pop_tag = True
172 try:
173 if token.value in _statement_keywords:
174 f = getattr(self, f"parse_{self.stream.current.value}")
175 return f() # type: ignore
176 if token.value == "call":
177 return self.parse_call_block()
178 if token.value == "filter":
179 return self.parse_filter_block()
180 ext = self.extensions.get(token.value)
181 if ext is not None:
182 return ext(self)
184 # did not work out, remove the token we pushed by accident
185 # from the stack so that the unknown tag fail function can
186 # produce a proper error message.
187 self._tag_stack.pop()
188 pop_tag = False
189 self.fail_unknown_tag(token.value, token.lineno)
190 finally:
191 if pop_tag:
192 self._tag_stack.pop()
194 def parse_statements(
195 self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
196 ) -> t.List[nodes.Node]:
197 """Parse multiple statements into a list until one of the end tokens
198 is reached. This is used to parse the body of statements as it also
199 parses template data if appropriate. The parser checks first if the
200 current token is a colon and skips it if there is one. Then it checks
201 for the block end and parses until if one of the `end_tokens` is
202 reached. Per default the active token in the stream at the end of
203 the call is the matched end token. If this is not wanted `drop_needle`
204 can be set to `True` and the end token is removed.
205 """
206 # the first token may be a colon for python compatibility
207 self.stream.skip_if("colon")
209 # in the future it would be possible to add whole code sections
210 # by adding some sort of end of statement token and parsing those here.
211 self.stream.expect("block_end")
212 result = self.subparse(end_tokens)
214 # we reached the end of the template too early, the subparser
215 # does not check for this, so we do that now
216 if self.stream.current.type == "eof":
217 self.fail_eof(end_tokens)
219 if drop_needle:
220 next(self.stream)
221 return result
223 def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
224 """Parse an assign statement."""
225 lineno = next(self.stream).lineno
226 target = self.parse_assign_target(with_namespace=True)
227 if self.stream.skip_if("assign"):
228 expr = self.parse_tuple()
229 return nodes.Assign(target, expr, lineno=lineno)
230 filter_node = self.parse_filter(None)
231 body = self.parse_statements(("name:endset",), drop_needle=True)
232 return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
234 def parse_for(self) -> nodes.For:
235 """Parse a for loop."""
236 lineno = self.stream.expect("name:for").lineno
237 target = self.parse_assign_target(extra_end_rules=("name:in",))
238 self.stream.expect("name:in")
239 iter = self.parse_tuple(
240 with_condexpr=False, extra_end_rules=("name:recursive",)
241 )
242 test = None
243 if self.stream.skip_if("name:if"):
244 test = self.parse_expression()
245 recursive = self.stream.skip_if("name:recursive")
246 body = self.parse_statements(("name:endfor", "name:else"))
247 if next(self.stream).value == "endfor":
248 else_ = []
249 else:
250 else_ = self.parse_statements(("name:endfor",), drop_needle=True)
251 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
253 def parse_if(self) -> nodes.If:
254 """Parse an if construct."""
255 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
256 while True:
257 node.test = self.parse_tuple(with_condexpr=False)
258 node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
259 node.elif_ = []
260 node.else_ = []
261 token = next(self.stream)
262 if token.test("name:elif"):
263 node = nodes.If(lineno=self.stream.current.lineno)
264 result.elif_.append(node)
265 continue
266 elif token.test("name:else"):
267 result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
268 break
269 return result
271 def parse_with(self) -> nodes.With:
272 node = nodes.With(lineno=next(self.stream).lineno)
273 targets: t.List[nodes.Expr] = []
274 values: t.List[nodes.Expr] = []
275 while self.stream.current.type != "block_end":
276 if targets:
277 self.stream.expect("comma")
278 target = self.parse_assign_target()
279 target.set_ctx("param")
280 targets.append(target)
281 self.stream.expect("assign")
282 values.append(self.parse_expression())
283 node.targets = targets
284 node.values = values
285 node.body = self.parse_statements(("name:endwith",), drop_needle=True)
286 return node
288 def parse_autoescape(self) -> nodes.Scope:
289 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
290 node.options = [nodes.Keyword("autoescape", self.parse_expression())]
291 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
292 return nodes.Scope([node])
294 def parse_block(self) -> nodes.Block:
295 node = nodes.Block(lineno=next(self.stream).lineno)
296 node.name = self.stream.expect("name").value
297 node.scoped = self.stream.skip_if("name:scoped")
298 node.required = self.stream.skip_if("name:required")
300 # common problem people encounter when switching from django
301 # to jinja. we do not support hyphens in block names, so let's
302 # raise a nicer error message in that case.
303 if self.stream.current.type == "sub":
304 self.fail(
305 "Block names in Jinja have to be valid Python identifiers and may not"
306 " contain hyphens, use an underscore instead."
307 )
309 node.body = self.parse_statements(("name:endblock",), drop_needle=True)
311 # enforce that required blocks only contain whitespace or comments
312 # by asserting that the body, if not empty, is just TemplateData nodes
313 # with whitespace data
314 if node.required:
315 for body_node in node.body:
316 if not isinstance(body_node, nodes.Output) or any(
317 not isinstance(output_node, nodes.TemplateData)
318 or not output_node.data.isspace()
319 for output_node in body_node.nodes
320 ):
321 self.fail("Required blocks can only contain comments or whitespace")
323 self.stream.skip_if("name:" + node.name)
324 return node
326 def parse_extends(self) -> nodes.Extends:
327 node = nodes.Extends(lineno=next(self.stream).lineno)
328 node.template = self.parse_expression()
329 return node
331 def parse_import_context(
332 self, node: _ImportInclude, default: bool
333 ) -> _ImportInclude:
334 if self.stream.current.test_any(
335 "name:with", "name:without"
336 ) and self.stream.look().test("name:context"):
337 node.with_context = next(self.stream).value == "with"
338 self.stream.skip()
339 else:
340 node.with_context = default
341 return node
343 def parse_include(self) -> nodes.Include:
344 node = nodes.Include(lineno=next(self.stream).lineno)
345 node.template = self.parse_expression()
346 if self.stream.current.test("name:ignore") and self.stream.look().test(
347 "name:missing"
348 ):
349 node.ignore_missing = True
350 self.stream.skip(2)
351 else:
352 node.ignore_missing = False
353 return self.parse_import_context(node, True)
355 def parse_import(self) -> nodes.Import:
356 node = nodes.Import(lineno=next(self.stream).lineno)
357 node.template = self.parse_expression()
358 self.stream.expect("name:as")
359 node.target = self.parse_assign_target(name_only=True).name
360 return self.parse_import_context(node, False)
362 def parse_from(self) -> nodes.FromImport:
363 node = nodes.FromImport(lineno=next(self.stream).lineno)
364 node.template = self.parse_expression()
365 self.stream.expect("name:import")
366 node.names = []
368 def parse_context() -> bool:
369 if self.stream.current.value in {
370 "with",
371 "without",
372 } and self.stream.look().test("name:context"):
373 node.with_context = next(self.stream).value == "with"
374 self.stream.skip()
375 return True
376 return False
378 while True:
379 if node.names:
380 self.stream.expect("comma")
381 if self.stream.current.type == "name":
382 if parse_context():
383 break
384 target = self.parse_assign_target(name_only=True)
385 if target.name.startswith("_"):
386 self.fail(
387 "names starting with an underline can not be imported",
388 target.lineno,
389 exc=TemplateAssertionError,
390 )
391 if self.stream.skip_if("name:as"):
392 alias = self.parse_assign_target(name_only=True)
393 node.names.append((target.name, alias.name))
394 else:
395 node.names.append(target.name)
396 if parse_context() or self.stream.current.type != "comma":
397 break
398 else:
399 self.stream.expect("name")
400 if not hasattr(node, "with_context"):
401 node.with_context = False
402 return node
404 def parse_signature(self, node: _MacroCall) -> None:
405 args = node.args = []
406 defaults = node.defaults = []
407 self.stream.expect("lparen")
408 while self.stream.current.type != "rparen":
409 if args:
410 self.stream.expect("comma")
411 arg = self.parse_assign_target(name_only=True)
412 arg.set_ctx("param")
413 if self.stream.skip_if("assign"):
414 defaults.append(self.parse_expression())
415 elif defaults:
416 self.fail("non-default argument follows default argument")
417 args.append(arg)
418 self.stream.expect("rparen")
420 def parse_call_block(self) -> nodes.CallBlock:
421 node = nodes.CallBlock(lineno=next(self.stream).lineno)
422 if self.stream.current.type == "lparen":
423 self.parse_signature(node)
424 else:
425 node.args = []
426 node.defaults = []
428 call_node = self.parse_expression()
429 if not isinstance(call_node, nodes.Call):
430 self.fail("expected call", node.lineno)
431 node.call = call_node
432 node.body = self.parse_statements(("name:endcall",), drop_needle=True)
433 return node
435 def parse_filter_block(self) -> nodes.FilterBlock:
436 node = nodes.FilterBlock(lineno=next(self.stream).lineno)
437 node.filter = self.parse_filter(None, start_inline=True) # type: ignore
438 node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
439 return node
441 def parse_macro(self) -> nodes.Macro:
442 node = nodes.Macro(lineno=next(self.stream).lineno)
443 node.name = self.parse_assign_target(name_only=True).name
444 self.parse_signature(node)
445 node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
446 return node
448 def parse_print(self) -> nodes.Output:
449 node = nodes.Output(lineno=next(self.stream).lineno)
450 node.nodes = []
451 while self.stream.current.type != "block_end":
452 if node.nodes:
453 self.stream.expect("comma")
454 node.nodes.append(self.parse_expression())
455 return node
457 @typing.overload
458 def parse_assign_target(
459 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
460 ) -> nodes.Name:
461 ...
463 @typing.overload
464 def parse_assign_target(
465 self,
466 with_tuple: bool = True,
467 name_only: bool = False,
468 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
469 with_namespace: bool = False,
470 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
471 ...
473 def parse_assign_target(
474 self,
475 with_tuple: bool = True,
476 name_only: bool = False,
477 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
478 with_namespace: bool = False,
479 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
480 """Parse an assignment target. As Jinja allows assignments to
481 tuples, this function can parse all allowed assignment targets. Per
482 default assignments to tuples are parsed, that can be disable however
483 by setting `with_tuple` to `False`. If only assignments to names are
484 wanted `name_only` can be set to `True`. The `extra_end_rules`
485 parameter is forwarded to the tuple parsing function. If
486 `with_namespace` is enabled, a namespace assignment may be parsed.
487 """
488 target: nodes.Expr
490 if with_namespace and self.stream.look().type == "dot":
491 token = self.stream.expect("name")
492 next(self.stream) # dot
493 attr = self.stream.expect("name")
494 target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
495 elif name_only:
496 token = self.stream.expect("name")
497 target = nodes.Name(token.value, "store", lineno=token.lineno)
498 else:
499 if with_tuple:
500 target = self.parse_tuple(
501 simplified=True, extra_end_rules=extra_end_rules
502 )
503 else:
504 target = self.parse_primary()
506 target.set_ctx("store")
508 if not target.can_assign():
509 self.fail(
510 f"can't assign to {type(target).__name__.lower()!r}", target.lineno
511 )
513 return target # type: ignore
515 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
516 """Parse an expression. Per default all expressions are parsed, if
517 the optional `with_condexpr` parameter is set to `False` conditional
518 expressions are not parsed.
519 """
520 if with_condexpr:
521 return self.parse_condexpr()
522 return self.parse_or()
524 def parse_condexpr(self) -> nodes.Expr:
525 lineno = self.stream.current.lineno
526 expr1 = self.parse_or()
527 expr3: t.Optional[nodes.Expr]
529 while self.stream.skip_if("name:if"):
530 expr2 = self.parse_or()
531 if self.stream.skip_if("name:else"):
532 expr3 = self.parse_condexpr()
533 else:
534 expr3 = None
535 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
536 lineno = self.stream.current.lineno
537 return expr1
539 def parse_or(self) -> nodes.Expr:
540 lineno = self.stream.current.lineno
541 left = self.parse_and()
542 while self.stream.skip_if("name:or"):
543 right = self.parse_and()
544 left = nodes.Or(left, right, lineno=lineno)
545 lineno = self.stream.current.lineno
546 return left
548 def parse_and(self) -> nodes.Expr:
549 lineno = self.stream.current.lineno
550 left = self.parse_not()
551 while self.stream.skip_if("name:and"):
552 right = self.parse_not()
553 left = nodes.And(left, right, lineno=lineno)
554 lineno = self.stream.current.lineno
555 return left
557 def parse_not(self) -> nodes.Expr:
558 if self.stream.current.test("name:not"):
559 lineno = next(self.stream).lineno
560 return nodes.Not(self.parse_not(), lineno=lineno)
561 return self.parse_compare()
563 def parse_compare(self) -> nodes.Expr:
564 lineno = self.stream.current.lineno
565 expr = self.parse_math1()
566 ops = []
567 while True:
568 token_type = self.stream.current.type
569 if token_type in _compare_operators:
570 next(self.stream)
571 ops.append(nodes.Operand(token_type, self.parse_math1()))
572 elif self.stream.skip_if("name:in"):
573 ops.append(nodes.Operand("in", self.parse_math1()))
574 elif self.stream.current.test("name:not") and self.stream.look().test(
575 "name:in"
576 ):
577 self.stream.skip(2)
578 ops.append(nodes.Operand("notin", self.parse_math1()))
579 else:
580 break
581 lineno = self.stream.current.lineno
582 if not ops:
583 return expr
584 return nodes.Compare(expr, ops, lineno=lineno)
586 def parse_math1(self) -> nodes.Expr:
587 lineno = self.stream.current.lineno
588 left = self.parse_concat()
589 while self.stream.current.type in ("add", "sub"):
590 cls = _math_nodes[self.stream.current.type]
591 next(self.stream)
592 right = self.parse_concat()
593 left = cls(left, right, lineno=lineno)
594 lineno = self.stream.current.lineno
595 return left
597 def parse_concat(self) -> nodes.Expr:
598 lineno = self.stream.current.lineno
599 args = [self.parse_math2()]
600 while self.stream.current.type == "tilde":
601 next(self.stream)
602 args.append(self.parse_math2())
603 if len(args) == 1:
604 return args[0]
605 return nodes.Concat(args, lineno=lineno)
607 def parse_math2(self) -> nodes.Expr:
608 lineno = self.stream.current.lineno
609 left = self.parse_pow()
610 while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
611 cls = _math_nodes[self.stream.current.type]
612 next(self.stream)
613 right = self.parse_pow()
614 left = cls(left, right, lineno=lineno)
615 lineno = self.stream.current.lineno
616 return left
618 def parse_pow(self) -> nodes.Expr:
619 lineno = self.stream.current.lineno
620 left = self.parse_unary()
621 while self.stream.current.type == "pow":
622 next(self.stream)
623 right = self.parse_unary()
624 left = nodes.Pow(left, right, lineno=lineno)
625 lineno = self.stream.current.lineno
626 return left
628 def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
629 token_type = self.stream.current.type
630 lineno = self.stream.current.lineno
631 node: nodes.Expr
633 if token_type == "sub":
634 next(self.stream)
635 node = nodes.Neg(self.parse_unary(False), lineno=lineno)
636 elif token_type == "add":
637 next(self.stream)
638 node = nodes.Pos(self.parse_unary(False), lineno=lineno)
639 else:
640 node = self.parse_primary()
641 node = self.parse_postfix(node)
642 if with_filter:
643 node = self.parse_filter_expr(node)
644 return node
646 def parse_primary(self) -> nodes.Expr:
647 token = self.stream.current
648 node: nodes.Expr
649 if token.type == "name":
650 if token.value in ("true", "false", "True", "False"):
651 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
652 elif token.value in ("none", "None"):
653 node = nodes.Const(None, lineno=token.lineno)
654 else:
655 node = nodes.Name(token.value, "load", lineno=token.lineno)
656 next(self.stream)
657 elif token.type == "string":
658 next(self.stream)
659 buf = [token.value]
660 lineno = token.lineno
661 while self.stream.current.type == "string":
662 buf.append(self.stream.current.value)
663 next(self.stream)
664 node = nodes.Const("".join(buf), lineno=lineno)
665 elif token.type in ("integer", "float"):
666 next(self.stream)
667 node = nodes.Const(token.value, lineno=token.lineno)
668 elif token.type == "lparen":
669 next(self.stream)
670 node = self.parse_tuple(explicit_parentheses=True)
671 self.stream.expect("rparen")
672 elif token.type == "lbracket":
673 node = self.parse_list()
674 elif token.type == "lbrace":
675 node = self.parse_dict()
676 else:
677 self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
678 return node
680 def parse_tuple(
681 self,
682 simplified: bool = False,
683 with_condexpr: bool = True,
684 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
685 explicit_parentheses: bool = False,
686 ) -> t.Union[nodes.Tuple, nodes.Expr]:
687 """Works like `parse_expression` but if multiple expressions are
688 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
689 This method could also return a regular expression instead of a tuple
690 if no commas where found.
692 The default parsing mode is a full tuple. If `simplified` is `True`
693 only names and literals are parsed. The `no_condexpr` parameter is
694 forwarded to :meth:`parse_expression`.
696 Because tuples do not require delimiters and may end in a bogus comma
697 an extra hint is needed that marks the end of a tuple. For example
698 for loops support tuples between `for` and `in`. In that case the
699 `extra_end_rules` is set to ``['name:in']``.
701 `explicit_parentheses` is true if the parsing was triggered by an
702 expression in parentheses. This is used to figure out if an empty
703 tuple is a valid expression or not.
704 """
705 lineno = self.stream.current.lineno
706 if simplified:
707 parse = self.parse_primary
708 elif with_condexpr:
709 parse = self.parse_expression
710 else:
712 def parse() -> nodes.Expr:
713 return self.parse_expression(with_condexpr=False)
715 args: t.List[nodes.Expr] = []
716 is_tuple = False
718 while True:
719 if args:
720 self.stream.expect("comma")
721 if self.is_tuple_end(extra_end_rules):
722 break
723 args.append(parse())
724 if self.stream.current.type == "comma":
725 is_tuple = True
726 else:
727 break
728 lineno = self.stream.current.lineno
730 if not is_tuple:
731 if args:
732 return args[0]
734 # if we don't have explicit parentheses, an empty tuple is
735 # not a valid expression. This would mean nothing (literally
736 # nothing) in the spot of an expression would be an empty
737 # tuple.
738 if not explicit_parentheses:
739 self.fail(
740 "Expected an expression,"
741 f" got {describe_token(self.stream.current)!r}"
742 )
744 return nodes.Tuple(args, "load", lineno=lineno)
746 def parse_list(self) -> nodes.List:
747 token = self.stream.expect("lbracket")
748 items: t.List[nodes.Expr] = []
749 while self.stream.current.type != "rbracket":
750 if items:
751 self.stream.expect("comma")
752 if self.stream.current.type == "rbracket":
753 break
754 items.append(self.parse_expression())
755 self.stream.expect("rbracket")
756 return nodes.List(items, lineno=token.lineno)
758 def parse_dict(self) -> nodes.Dict:
759 token = self.stream.expect("lbrace")
760 items: t.List[nodes.Pair] = []
761 while self.stream.current.type != "rbrace":
762 if items:
763 self.stream.expect("comma")
764 if self.stream.current.type == "rbrace":
765 break
766 key = self.parse_expression()
767 self.stream.expect("colon")
768 value = self.parse_expression()
769 items.append(nodes.Pair(key, value, lineno=key.lineno))
770 self.stream.expect("rbrace")
771 return nodes.Dict(items, lineno=token.lineno)
773 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
774 while True:
775 token_type = self.stream.current.type
776 if token_type == "dot" or token_type == "lbracket":
777 node = self.parse_subscript(node)
778 # calls are valid both after postfix expressions (getattr
779 # and getitem) as well as filters and tests
780 elif token_type == "lparen":
781 node = self.parse_call(node)
782 else:
783 break
784 return node
786 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
787 while True:
788 token_type = self.stream.current.type
789 if token_type == "pipe":
790 node = self.parse_filter(node) # type: ignore
791 elif token_type == "name" and self.stream.current.value == "is":
792 node = self.parse_test(node)
793 # calls are valid both after postfix expressions (getattr
794 # and getitem) as well as filters and tests
795 elif token_type == "lparen":
796 node = self.parse_call(node)
797 else:
798 break
799 return node
801 def parse_subscript(
802 self, node: nodes.Expr
803 ) -> t.Union[nodes.Getattr, nodes.Getitem]:
804 token = next(self.stream)
805 arg: nodes.Expr
807 if token.type == "dot":
808 attr_token = self.stream.current
809 next(self.stream)
810 if attr_token.type == "name":
811 return nodes.Getattr(
812 node, attr_token.value, "load", lineno=token.lineno
813 )
814 elif attr_token.type != "integer":
815 self.fail("expected name or number", attr_token.lineno)
816 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
817 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
818 if token.type == "lbracket":
819 args: t.List[nodes.Expr] = []
820 while self.stream.current.type != "rbracket":
821 if args:
822 self.stream.expect("comma")
823 args.append(self.parse_subscribed())
824 self.stream.expect("rbracket")
825 if len(args) == 1:
826 arg = args[0]
827 else:
828 arg = nodes.Tuple(args, "load", lineno=token.lineno)
829 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
830 self.fail("expected subscript expression", token.lineno)
832 def parse_subscribed(self) -> nodes.Expr:
833 lineno = self.stream.current.lineno
834 args: t.List[t.Optional[nodes.Expr]]
836 if self.stream.current.type == "colon":
837 next(self.stream)
838 args = [None]
839 else:
840 node = self.parse_expression()
841 if self.stream.current.type != "colon":
842 return node
843 next(self.stream)
844 args = [node]
846 if self.stream.current.type == "colon":
847 args.append(None)
848 elif self.stream.current.type not in ("rbracket", "comma"):
849 args.append(self.parse_expression())
850 else:
851 args.append(None)
853 if self.stream.current.type == "colon":
854 next(self.stream)
855 if self.stream.current.type not in ("rbracket", "comma"):
856 args.append(self.parse_expression())
857 else:
858 args.append(None)
859 else:
860 args.append(None)
862 return nodes.Slice(lineno=lineno, *args)
864 def parse_call_args(
865 self,
866 ) -> t.Tuple[
867 t.List[nodes.Expr],
868 t.List[nodes.Keyword],
869 t.Union[nodes.Expr, None],
870 t.Union[nodes.Expr, None],
871 ]:
872 token = self.stream.expect("lparen")
873 args = []
874 kwargs = []
875 dyn_args = None
876 dyn_kwargs = None
877 require_comma = False
879 def ensure(expr: bool) -> None:
880 if not expr:
881 self.fail("invalid syntax for function call expression", token.lineno)
883 while self.stream.current.type != "rparen":
884 if require_comma:
885 self.stream.expect("comma")
887 # support for trailing comma
888 if self.stream.current.type == "rparen":
889 break
891 if self.stream.current.type == "mul":
892 ensure(dyn_args is None and dyn_kwargs is None)
893 next(self.stream)
894 dyn_args = self.parse_expression()
895 elif self.stream.current.type == "pow":
896 ensure(dyn_kwargs is None)
897 next(self.stream)
898 dyn_kwargs = self.parse_expression()
899 else:
900 if (
901 self.stream.current.type == "name"
902 and self.stream.look().type == "assign"
903 ):
904 # Parsing a kwarg
905 ensure(dyn_kwargs is None)
906 key = self.stream.current.value
907 self.stream.skip(2)
908 value = self.parse_expression()
909 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
910 else:
911 # Parsing an arg
912 ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
913 args.append(self.parse_expression())
915 require_comma = True
917 self.stream.expect("rparen")
918 return args, kwargs, dyn_args, dyn_kwargs
920 def parse_call(self, node: nodes.Expr) -> nodes.Call:
921 # The lparen will be expected in parse_call_args, but the lineno
922 # needs to be recorded before the stream is advanced.
923 token = self.stream.current
924 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
925 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
927 def parse_filter(
928 self, node: t.Optional[nodes.Expr], start_inline: bool = False
929 ) -> t.Optional[nodes.Expr]:
930 while self.stream.current.type == "pipe" or start_inline:
931 if not start_inline:
932 next(self.stream)
933 token = self.stream.expect("name")
934 name = token.value
935 while self.stream.current.type == "dot":
936 next(self.stream)
937 name += "." + self.stream.expect("name").value
938 if self.stream.current.type == "lparen":
939 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
940 else:
941 args = []
942 kwargs = []
943 dyn_args = dyn_kwargs = None
944 node = nodes.Filter(
945 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
946 )
947 start_inline = False
948 return node
950 def parse_test(self, node: nodes.Expr) -> nodes.Expr:
951 token = next(self.stream)
952 if self.stream.current.test("name:not"):
953 next(self.stream)
954 negated = True
955 else:
956 negated = False
957 name = self.stream.expect("name").value
958 while self.stream.current.type == "dot":
959 next(self.stream)
960 name += "." + self.stream.expect("name").value
961 dyn_args = dyn_kwargs = None
962 kwargs: t.List[nodes.Keyword] = []
963 if self.stream.current.type == "lparen":
964 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
965 elif self.stream.current.type in {
966 "name",
967 "string",
968 "integer",
969 "float",
970 "lparen",
971 "lbracket",
972 "lbrace",
973 } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
974 if self.stream.current.test("name:is"):
975 self.fail("You cannot chain multiple tests with is")
976 arg_node = self.parse_primary()
977 arg_node = self.parse_postfix(arg_node)
978 args = [arg_node]
979 else:
980 args = []
981 node = nodes.Test(
982 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
983 )
984 if negated:
985 node = nodes.Not(node, lineno=token.lineno)
986 return node
988 def subparse(
989 self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
990 ) -> t.List[nodes.Node]:
991 body: t.List[nodes.Node] = []
992 data_buffer: t.List[nodes.Node] = []
993 add_data = data_buffer.append
995 if end_tokens is not None:
996 self._end_token_stack.append(end_tokens)
998 def flush_data() -> None:
999 if data_buffer:
1000 lineno = data_buffer[0].lineno
1001 body.append(nodes.Output(data_buffer[:], lineno=lineno))
1002 del data_buffer[:]
1004 try:
1005 while self.stream:
1006 token = self.stream.current
1007 if token.type == "data":
1008 if token.value:
1009 add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1010 next(self.stream)
1011 elif token.type == "variable_begin":
1012 next(self.stream)
1013 add_data(self.parse_tuple(with_condexpr=True))
1014 self.stream.expect("variable_end")
1015 elif token.type == "block_begin":
1016 flush_data()
1017 next(self.stream)
1018 if end_tokens is not None and self.stream.current.test_any(
1019 *end_tokens
1020 ):
1021 return body
1022 rv = self.parse_statement()
1023 if isinstance(rv, list):
1024 body.extend(rv)
1025 else:
1026 body.append(rv)
1027 self.stream.expect("block_end")
1028 else:
1029 raise AssertionError("internal parsing error")
1031 flush_data()
1032 finally:
1033 if end_tokens is not None:
1034 self._end_token_stack.pop()
1035 return body
1037 def parse(self) -> nodes.Template:
1038 """Parse the whole template into a `Template` node."""
1039 result = nodes.Template(self.subparse(), lineno=1)
1040 result.set_environment(self.environment)
1041 return result