Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/jinja2/parser.py: 10%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Parse tokens from the lexer into nodes for the compiler."""
3import typing
4import typing as t
6from . import nodes
7from .exceptions import TemplateAssertionError
8from .exceptions import TemplateSyntaxError
9from .lexer import describe_token
10from .lexer import describe_token_expr
12if t.TYPE_CHECKING:
13 import typing_extensions as te
15 from .environment import Environment
17_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
18_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
20_statement_keywords = frozenset(
21 [
22 "for",
23 "if",
24 "block",
25 "extends",
26 "print",
27 "macro",
28 "include",
29 "from",
30 "import",
31 "set",
32 "with",
33 "autoescape",
34 ]
35)
36_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
38_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
39 "add": nodes.Add,
40 "sub": nodes.Sub,
41 "mul": nodes.Mul,
42 "div": nodes.Div,
43 "floordiv": nodes.FloorDiv,
44 "mod": nodes.Mod,
45}
48class Parser:
49 """This is the central parsing class Jinja uses. It's passed to
50 extensions and can be used to parse expressions or statements.
51 """
53 def __init__(
54 self,
55 environment: "Environment",
56 source: str,
57 name: t.Optional[str] = None,
58 filename: t.Optional[str] = None,
59 state: t.Optional[str] = None,
60 ) -> None:
61 self.environment = environment
62 self.stream = environment._tokenize(source, name, filename, state)
63 self.name = name
64 self.filename = filename
65 self.closed = False
66 self.extensions: t.Dict[
67 str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]]
68 ] = {}
69 for extension in environment.iter_extensions():
70 for tag in extension.tags:
71 self.extensions[tag] = extension.parse
72 self._last_identifier = 0
73 self._tag_stack: t.List[str] = []
74 self._end_token_stack: t.List[t.Tuple[str, ...]] = []
76 def fail(
77 self,
78 msg: str,
79 lineno: t.Optional[int] = None,
80 exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
81 ) -> "te.NoReturn":
82 """Convenience method that raises `exc` with the message, passed
83 line number or last line number as well as the current name and
84 filename.
85 """
86 if lineno is None:
87 lineno = self.stream.current.lineno
88 raise exc(msg, lineno, self.name, self.filename)
90 def _fail_ut_eof(
91 self,
92 name: t.Optional[str],
93 end_token_stack: t.List[t.Tuple[str, ...]],
94 lineno: t.Optional[int],
95 ) -> "te.NoReturn":
96 expected: t.Set[str] = set()
97 for exprs in end_token_stack:
98 expected.update(map(describe_token_expr, exprs))
99 if end_token_stack:
100 currently_looking: t.Optional[str] = " or ".join(
101 map(repr, map(describe_token_expr, end_token_stack[-1]))
102 )
103 else:
104 currently_looking = None
106 if name is None:
107 message = ["Unexpected end of template."]
108 else:
109 message = [f"Encountered unknown tag {name!r}."]
111 if currently_looking:
112 if name is not None and name in expected:
113 message.append(
114 "You probably made a nesting mistake. Jinja is expecting this tag,"
115 f" but currently looking for {currently_looking}."
116 )
117 else:
118 message.append(
119 f"Jinja was looking for the following tags: {currently_looking}."
120 )
122 if self._tag_stack:
123 message.append(
124 "The innermost block that needs to be closed is"
125 f" {self._tag_stack[-1]!r}."
126 )
128 self.fail(" ".join(message), lineno)
130 def fail_unknown_tag(
131 self, name: str, lineno: t.Optional[int] = None
132 ) -> "te.NoReturn":
133 """Called if the parser encounters an unknown tag. Tries to fail
134 with a human readable error message that could help to identify
135 the problem.
136 """
137 self._fail_ut_eof(name, self._end_token_stack, lineno)
139 def fail_eof(
140 self,
141 end_tokens: t.Optional[t.Tuple[str, ...]] = None,
142 lineno: t.Optional[int] = None,
143 ) -> "te.NoReturn":
144 """Like fail_unknown_tag but for end of template situations."""
145 stack = list(self._end_token_stack)
146 if end_tokens is not None:
147 stack.append(end_tokens)
148 self._fail_ut_eof(None, stack, lineno)
150 def is_tuple_end(
151 self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
152 ) -> bool:
153 """Are we at the end of a tuple?"""
154 if self.stream.current.type in ("variable_end", "block_end", "rparen"):
155 return True
156 elif extra_end_rules is not None:
157 return self.stream.current.test_any(extra_end_rules) # type: ignore
158 return False
160 def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
161 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
162 self._last_identifier += 1
163 rv = object.__new__(nodes.InternalName)
164 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
165 return rv
167 def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
168 """Parse a single statement."""
169 token = self.stream.current
170 if token.type != "name":
171 self.fail("tag name expected", token.lineno)
172 self._tag_stack.append(token.value)
173 pop_tag = True
174 try:
175 if token.value in _statement_keywords:
176 f = getattr(self, f"parse_{self.stream.current.value}")
177 return f() # type: ignore
178 if token.value == "call":
179 return self.parse_call_block()
180 if token.value == "filter":
181 return self.parse_filter_block()
182 ext = self.extensions.get(token.value)
183 if ext is not None:
184 return ext(self)
186 # did not work out, remove the token we pushed by accident
187 # from the stack so that the unknown tag fail function can
188 # produce a proper error message.
189 self._tag_stack.pop()
190 pop_tag = False
191 self.fail_unknown_tag(token.value, token.lineno)
192 finally:
193 if pop_tag:
194 self._tag_stack.pop()
196 def parse_statements(
197 self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
198 ) -> t.List[nodes.Node]:
199 """Parse multiple statements into a list until one of the end tokens
200 is reached. This is used to parse the body of statements as it also
201 parses template data if appropriate. The parser checks first if the
202 current token is a colon and skips it if there is one. Then it checks
203 for the block end and parses until if one of the `end_tokens` is
204 reached. Per default the active token in the stream at the end of
205 the call is the matched end token. If this is not wanted `drop_needle`
206 can be set to `True` and the end token is removed.
207 """
208 # the first token may be a colon for python compatibility
209 self.stream.skip_if("colon")
211 # in the future it would be possible to add whole code sections
212 # by adding some sort of end of statement token and parsing those here.
213 self.stream.expect("block_end")
214 result = self.subparse(end_tokens)
216 # we reached the end of the template too early, the subparser
217 # does not check for this, so we do that now
218 if self.stream.current.type == "eof":
219 self.fail_eof(end_tokens)
221 if drop_needle:
222 next(self.stream)
223 return result
225 def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
226 """Parse an assign statement."""
227 lineno = next(self.stream).lineno
228 target = self.parse_assign_target(with_namespace=True)
229 if self.stream.skip_if("assign"):
230 expr = self.parse_tuple()
231 return nodes.Assign(target, expr, lineno=lineno)
232 filter_node = self.parse_filter(None)
233 body = self.parse_statements(("name:endset",), drop_needle=True)
234 return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
236 def parse_for(self) -> nodes.For:
237 """Parse a for loop."""
238 lineno = self.stream.expect("name:for").lineno
239 target = self.parse_assign_target(extra_end_rules=("name:in",))
240 self.stream.expect("name:in")
241 iter = self.parse_tuple(
242 with_condexpr=False, extra_end_rules=("name:recursive",)
243 )
244 test = None
245 if self.stream.skip_if("name:if"):
246 test = self.parse_expression()
247 recursive = self.stream.skip_if("name:recursive")
248 body = self.parse_statements(("name:endfor", "name:else"))
249 if next(self.stream).value == "endfor":
250 else_ = []
251 else:
252 else_ = self.parse_statements(("name:endfor",), drop_needle=True)
253 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
255 def parse_if(self) -> nodes.If:
256 """Parse an if construct."""
257 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
258 while True:
259 node.test = self.parse_tuple(with_condexpr=False)
260 node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
261 node.elif_ = []
262 node.else_ = []
263 token = next(self.stream)
264 if token.test("name:elif"):
265 node = nodes.If(lineno=self.stream.current.lineno)
266 result.elif_.append(node)
267 continue
268 elif token.test("name:else"):
269 result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
270 break
271 return result
273 def parse_with(self) -> nodes.With:
274 node = nodes.With(lineno=next(self.stream).lineno)
275 targets: t.List[nodes.Expr] = []
276 values: t.List[nodes.Expr] = []
277 while self.stream.current.type != "block_end":
278 if targets:
279 self.stream.expect("comma")
280 target = self.parse_assign_target()
281 target.set_ctx("param")
282 targets.append(target)
283 self.stream.expect("assign")
284 values.append(self.parse_expression())
285 node.targets = targets
286 node.values = values
287 node.body = self.parse_statements(("name:endwith",), drop_needle=True)
288 return node
290 def parse_autoescape(self) -> nodes.Scope:
291 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
292 node.options = [nodes.Keyword("autoescape", self.parse_expression())]
293 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
294 return nodes.Scope([node])
296 def parse_block(self) -> nodes.Block:
297 node = nodes.Block(lineno=next(self.stream).lineno)
298 node.name = self.stream.expect("name").value
299 node.scoped = self.stream.skip_if("name:scoped")
300 node.required = self.stream.skip_if("name:required")
302 # common problem people encounter when switching from django
303 # to jinja. we do not support hyphens in block names, so let's
304 # raise a nicer error message in that case.
305 if self.stream.current.type == "sub":
306 self.fail(
307 "Block names in Jinja have to be valid Python identifiers and may not"
308 " contain hyphens, use an underscore instead."
309 )
311 node.body = self.parse_statements(("name:endblock",), drop_needle=True)
313 # enforce that required blocks only contain whitespace or comments
314 # by asserting that the body, if not empty, is just TemplateData nodes
315 # with whitespace data
316 if node.required:
317 for body_node in node.body:
318 if not isinstance(body_node, nodes.Output) or any(
319 not isinstance(output_node, nodes.TemplateData)
320 or not output_node.data.isspace()
321 for output_node in body_node.nodes
322 ):
323 self.fail("Required blocks can only contain comments or whitespace")
325 self.stream.skip_if("name:" + node.name)
326 return node
328 def parse_extends(self) -> nodes.Extends:
329 node = nodes.Extends(lineno=next(self.stream).lineno)
330 node.template = self.parse_expression()
331 return node
333 def parse_import_context(
334 self, node: _ImportInclude, default: bool
335 ) -> _ImportInclude:
336 if self.stream.current.test_any(
337 "name:with", "name:without"
338 ) and self.stream.look().test("name:context"):
339 node.with_context = next(self.stream).value == "with"
340 self.stream.skip()
341 else:
342 node.with_context = default
343 return node
345 def parse_include(self) -> nodes.Include:
346 node = nodes.Include(lineno=next(self.stream).lineno)
347 node.template = self.parse_expression()
348 if self.stream.current.test("name:ignore") and self.stream.look().test(
349 "name:missing"
350 ):
351 node.ignore_missing = True
352 self.stream.skip(2)
353 else:
354 node.ignore_missing = False
355 return self.parse_import_context(node, True)
357 def parse_import(self) -> nodes.Import:
358 node = nodes.Import(lineno=next(self.stream).lineno)
359 node.template = self.parse_expression()
360 self.stream.expect("name:as")
361 node.target = self.parse_assign_target(name_only=True).name
362 return self.parse_import_context(node, False)
364 def parse_from(self) -> nodes.FromImport:
365 node = nodes.FromImport(lineno=next(self.stream).lineno)
366 node.template = self.parse_expression()
367 self.stream.expect("name:import")
368 node.names = []
370 def parse_context() -> bool:
371 if self.stream.current.value in {
372 "with",
373 "without",
374 } and self.stream.look().test("name:context"):
375 node.with_context = next(self.stream).value == "with"
376 self.stream.skip()
377 return True
378 return False
380 while True:
381 if node.names:
382 self.stream.expect("comma")
383 if self.stream.current.type == "name":
384 if parse_context():
385 break
386 target = self.parse_assign_target(name_only=True)
387 if target.name.startswith("_"):
388 self.fail(
389 "names starting with an underline can not be imported",
390 target.lineno,
391 exc=TemplateAssertionError,
392 )
393 if self.stream.skip_if("name:as"):
394 alias = self.parse_assign_target(name_only=True)
395 node.names.append((target.name, alias.name))
396 else:
397 node.names.append(target.name)
398 if parse_context() or self.stream.current.type != "comma":
399 break
400 else:
401 self.stream.expect("name")
402 if not hasattr(node, "with_context"):
403 node.with_context = False
404 return node
406 def parse_signature(self, node: _MacroCall) -> None:
407 args = node.args = []
408 defaults = node.defaults = []
409 self.stream.expect("lparen")
410 while self.stream.current.type != "rparen":
411 if args:
412 self.stream.expect("comma")
413 arg = self.parse_assign_target(name_only=True)
414 arg.set_ctx("param")
415 if self.stream.skip_if("assign"):
416 defaults.append(self.parse_expression())
417 elif defaults:
418 self.fail("non-default argument follows default argument")
419 args.append(arg)
420 self.stream.expect("rparen")
422 def parse_call_block(self) -> nodes.CallBlock:
423 node = nodes.CallBlock(lineno=next(self.stream).lineno)
424 if self.stream.current.type == "lparen":
425 self.parse_signature(node)
426 else:
427 node.args = []
428 node.defaults = []
430 call_node = self.parse_expression()
431 if not isinstance(call_node, nodes.Call):
432 self.fail("expected call", node.lineno)
433 node.call = call_node
434 node.body = self.parse_statements(("name:endcall",), drop_needle=True)
435 return node
437 def parse_filter_block(self) -> nodes.FilterBlock:
438 node = nodes.FilterBlock(lineno=next(self.stream).lineno)
439 node.filter = self.parse_filter(None, start_inline=True) # type: ignore
440 node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
441 return node
443 def parse_macro(self) -> nodes.Macro:
444 node = nodes.Macro(lineno=next(self.stream).lineno)
445 node.name = self.parse_assign_target(name_only=True).name
446 self.parse_signature(node)
447 node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
448 return node
450 def parse_print(self) -> nodes.Output:
451 node = nodes.Output(lineno=next(self.stream).lineno)
452 node.nodes = []
453 while self.stream.current.type != "block_end":
454 if node.nodes:
455 self.stream.expect("comma")
456 node.nodes.append(self.parse_expression())
457 return node
459 @typing.overload
460 def parse_assign_target(
461 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
462 ) -> nodes.Name: ...
464 @typing.overload
465 def parse_assign_target(
466 self,
467 with_tuple: bool = True,
468 name_only: bool = False,
469 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
470 with_namespace: bool = False,
471 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: ...
473 def parse_assign_target(
474 self,
475 with_tuple: bool = True,
476 name_only: bool = False,
477 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
478 with_namespace: bool = False,
479 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
480 """Parse an assignment target. As Jinja allows assignments to
481 tuples, this function can parse all allowed assignment targets. Per
482 default assignments to tuples are parsed, that can be disable however
483 by setting `with_tuple` to `False`. If only assignments to names are
484 wanted `name_only` can be set to `True`. The `extra_end_rules`
485 parameter is forwarded to the tuple parsing function. If
486 `with_namespace` is enabled, a namespace assignment may be parsed.
487 """
488 target: nodes.Expr
490 if with_namespace and self.stream.look().type == "dot":
491 token = self.stream.expect("name")
492 next(self.stream) # dot
493 attr = self.stream.expect("name")
494 target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
495 elif name_only:
496 token = self.stream.expect("name")
497 target = nodes.Name(token.value, "store", lineno=token.lineno)
498 else:
499 if with_tuple:
500 target = self.parse_tuple(
501 simplified=True, extra_end_rules=extra_end_rules
502 )
503 else:
504 target = self.parse_primary()
506 target.set_ctx("store")
508 if not target.can_assign():
509 self.fail(
510 f"can't assign to {type(target).__name__.lower()!r}", target.lineno
511 )
513 return target # type: ignore
515 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
516 """Parse an expression. Per default all expressions are parsed, if
517 the optional `with_condexpr` parameter is set to `False` conditional
518 expressions are not parsed.
519 """
520 if with_condexpr:
521 return self.parse_condexpr()
522 return self.parse_or()
524 def parse_condexpr(self) -> nodes.Expr:
525 lineno = self.stream.current.lineno
526 expr1 = self.parse_or()
527 expr3: t.Optional[nodes.Expr]
529 while self.stream.skip_if("name:if"):
530 expr2 = self.parse_or()
531 if self.stream.skip_if("name:else"):
532 expr3 = self.parse_condexpr()
533 else:
534 expr3 = None
535 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
536 lineno = self.stream.current.lineno
537 return expr1
539 def parse_or(self) -> nodes.Expr:
540 lineno = self.stream.current.lineno
541 left = self.parse_and()
542 while self.stream.skip_if("name:or"):
543 right = self.parse_and()
544 left = nodes.Or(left, right, lineno=lineno)
545 lineno = self.stream.current.lineno
546 return left
548 def parse_and(self) -> nodes.Expr:
549 lineno = self.stream.current.lineno
550 left = self.parse_not()
551 while self.stream.skip_if("name:and"):
552 right = self.parse_not()
553 left = nodes.And(left, right, lineno=lineno)
554 lineno = self.stream.current.lineno
555 return left
557 def parse_not(self) -> nodes.Expr:
558 if self.stream.current.test("name:not"):
559 lineno = next(self.stream).lineno
560 return nodes.Not(self.parse_not(), lineno=lineno)
561 return self.parse_compare()
563 def parse_compare(self) -> nodes.Expr:
564 lineno = self.stream.current.lineno
565 expr = self.parse_math1()
566 ops = []
567 while True:
568 token_type = self.stream.current.type
569 if token_type in _compare_operators:
570 next(self.stream)
571 ops.append(nodes.Operand(token_type, self.parse_math1()))
572 elif self.stream.skip_if("name:in"):
573 ops.append(nodes.Operand("in", self.parse_math1()))
574 elif self.stream.current.test("name:not") and self.stream.look().test(
575 "name:in"
576 ):
577 self.stream.skip(2)
578 ops.append(nodes.Operand("notin", self.parse_math1()))
579 else:
580 break
581 lineno = self.stream.current.lineno
582 if not ops:
583 return expr
584 return nodes.Compare(expr, ops, lineno=lineno)
586 def parse_math1(self) -> nodes.Expr:
587 lineno = self.stream.current.lineno
588 left = self.parse_concat()
589 while self.stream.current.type in ("add", "sub"):
590 cls = _math_nodes[self.stream.current.type]
591 next(self.stream)
592 right = self.parse_concat()
593 left = cls(left, right, lineno=lineno)
594 lineno = self.stream.current.lineno
595 return left
597 def parse_concat(self) -> nodes.Expr:
598 lineno = self.stream.current.lineno
599 args = [self.parse_math2()]
600 while self.stream.current.type == "tilde":
601 next(self.stream)
602 args.append(self.parse_math2())
603 if len(args) == 1:
604 return args[0]
605 return nodes.Concat(args, lineno=lineno)
607 def parse_math2(self) -> nodes.Expr:
608 lineno = self.stream.current.lineno
609 left = self.parse_pow()
610 while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
611 cls = _math_nodes[self.stream.current.type]
612 next(self.stream)
613 right = self.parse_pow()
614 left = cls(left, right, lineno=lineno)
615 lineno = self.stream.current.lineno
616 return left
618 def parse_pow(self) -> nodes.Expr:
619 lineno = self.stream.current.lineno
620 left = self.parse_unary()
621 while self.stream.current.type == "pow":
622 next(self.stream)
623 right = self.parse_unary()
624 left = nodes.Pow(left, right, lineno=lineno)
625 lineno = self.stream.current.lineno
626 return left
628 def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
629 token_type = self.stream.current.type
630 lineno = self.stream.current.lineno
631 node: nodes.Expr
633 if token_type == "sub":
634 next(self.stream)
635 node = nodes.Neg(self.parse_unary(False), lineno=lineno)
636 elif token_type == "add":
637 next(self.stream)
638 node = nodes.Pos(self.parse_unary(False), lineno=lineno)
639 else:
640 node = self.parse_primary()
641 node = self.parse_postfix(node)
642 if with_filter:
643 node = self.parse_filter_expr(node)
644 return node
646 def parse_primary(self) -> nodes.Expr:
647 token = self.stream.current
648 node: nodes.Expr
649 if token.type == "name":
650 if token.value in ("true", "false", "True", "False"):
651 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
652 elif token.value in ("none", "None"):
653 node = nodes.Const(None, lineno=token.lineno)
654 else:
655 node = nodes.Name(token.value, "load", lineno=token.lineno)
656 next(self.stream)
657 elif token.type == "string":
658 next(self.stream)
659 buf = [token.value]
660 lineno = token.lineno
661 while self.stream.current.type == "string":
662 buf.append(self.stream.current.value)
663 next(self.stream)
664 node = nodes.Const("".join(buf), lineno=lineno)
665 elif token.type in ("integer", "float"):
666 next(self.stream)
667 node = nodes.Const(token.value, lineno=token.lineno)
668 elif token.type == "lparen":
669 next(self.stream)
670 node = self.parse_tuple(explicit_parentheses=True)
671 self.stream.expect("rparen")
672 elif token.type == "lbracket":
673 node = self.parse_list()
674 elif token.type == "lbrace":
675 node = self.parse_dict()
676 else:
677 self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
678 return node
680 def parse_tuple(
681 self,
682 simplified: bool = False,
683 with_condexpr: bool = True,
684 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
685 explicit_parentheses: bool = False,
686 ) -> t.Union[nodes.Tuple, nodes.Expr]:
687 """Works like `parse_expression` but if multiple expressions are
688 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
689 This method could also return a regular expression instead of a tuple
690 if no commas where found.
692 The default parsing mode is a full tuple. If `simplified` is `True`
693 only names and literals are parsed. The `no_condexpr` parameter is
694 forwarded to :meth:`parse_expression`.
696 Because tuples do not require delimiters and may end in a bogus comma
697 an extra hint is needed that marks the end of a tuple. For example
698 for loops support tuples between `for` and `in`. In that case the
699 `extra_end_rules` is set to ``['name:in']``.
701 `explicit_parentheses` is true if the parsing was triggered by an
702 expression in parentheses. This is used to figure out if an empty
703 tuple is a valid expression or not.
704 """
705 lineno = self.stream.current.lineno
706 if simplified:
707 parse = self.parse_primary
708 elif with_condexpr:
709 parse = self.parse_expression
710 else:
712 def parse() -> nodes.Expr:
713 return self.parse_expression(with_condexpr=False)
715 args: t.List[nodes.Expr] = []
716 is_tuple = False
718 while True:
719 if args:
720 self.stream.expect("comma")
721 if self.is_tuple_end(extra_end_rules):
722 break
723 args.append(parse())
724 if self.stream.current.type == "comma":
725 is_tuple = True
726 else:
727 break
728 lineno = self.stream.current.lineno
730 if not is_tuple:
731 if args:
732 return args[0]
734 # if we don't have explicit parentheses, an empty tuple is
735 # not a valid expression. This would mean nothing (literally
736 # nothing) in the spot of an expression would be an empty
737 # tuple.
738 if not explicit_parentheses:
739 self.fail(
740 "Expected an expression,"
741 f" got {describe_token(self.stream.current)!r}"
742 )
744 return nodes.Tuple(args, "load", lineno=lineno)
746 def parse_list(self) -> nodes.List:
747 token = self.stream.expect("lbracket")
748 items: t.List[nodes.Expr] = []
749 while self.stream.current.type != "rbracket":
750 if items:
751 self.stream.expect("comma")
752 if self.stream.current.type == "rbracket":
753 break
754 items.append(self.parse_expression())
755 self.stream.expect("rbracket")
756 return nodes.List(items, lineno=token.lineno)
758 def parse_dict(self) -> nodes.Dict:
759 token = self.stream.expect("lbrace")
760 items: t.List[nodes.Pair] = []
761 while self.stream.current.type != "rbrace":
762 if items:
763 self.stream.expect("comma")
764 if self.stream.current.type == "rbrace":
765 break
766 key = self.parse_expression()
767 self.stream.expect("colon")
768 value = self.parse_expression()
769 items.append(nodes.Pair(key, value, lineno=key.lineno))
770 self.stream.expect("rbrace")
771 return nodes.Dict(items, lineno=token.lineno)
773 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
774 while True:
775 token_type = self.stream.current.type
776 if token_type == "dot" or token_type == "lbracket":
777 node = self.parse_subscript(node)
778 # calls are valid both after postfix expressions (getattr
779 # and getitem) as well as filters and tests
780 elif token_type == "lparen":
781 node = self.parse_call(node)
782 else:
783 break
784 return node
786 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
787 while True:
788 token_type = self.stream.current.type
789 if token_type == "pipe":
790 node = self.parse_filter(node) # type: ignore
791 elif token_type == "name" and self.stream.current.value == "is":
792 node = self.parse_test(node)
793 # calls are valid both after postfix expressions (getattr
794 # and getitem) as well as filters and tests
795 elif token_type == "lparen":
796 node = self.parse_call(node)
797 else:
798 break
799 return node
801 def parse_subscript(
802 self, node: nodes.Expr
803 ) -> t.Union[nodes.Getattr, nodes.Getitem]:
804 token = next(self.stream)
805 arg: nodes.Expr
807 if token.type == "dot":
808 attr_token = self.stream.current
809 next(self.stream)
810 if attr_token.type == "name":
811 return nodes.Getattr(
812 node, attr_token.value, "load", lineno=token.lineno
813 )
814 elif attr_token.type != "integer":
815 self.fail("expected name or number", attr_token.lineno)
816 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
817 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
818 if token.type == "lbracket":
819 args: t.List[nodes.Expr] = []
820 while self.stream.current.type != "rbracket":
821 if args:
822 self.stream.expect("comma")
823 args.append(self.parse_subscribed())
824 self.stream.expect("rbracket")
825 if len(args) == 1:
826 arg = args[0]
827 else:
828 arg = nodes.Tuple(args, "load", lineno=token.lineno)
829 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
830 self.fail("expected subscript expression", token.lineno)
832 def parse_subscribed(self) -> nodes.Expr:
833 lineno = self.stream.current.lineno
834 args: t.List[t.Optional[nodes.Expr]]
836 if self.stream.current.type == "colon":
837 next(self.stream)
838 args = [None]
839 else:
840 node = self.parse_expression()
841 if self.stream.current.type != "colon":
842 return node
843 next(self.stream)
844 args = [node]
846 if self.stream.current.type == "colon":
847 args.append(None)
848 elif self.stream.current.type not in ("rbracket", "comma"):
849 args.append(self.parse_expression())
850 else:
851 args.append(None)
853 if self.stream.current.type == "colon":
854 next(self.stream)
855 if self.stream.current.type not in ("rbracket", "comma"):
856 args.append(self.parse_expression())
857 else:
858 args.append(None)
859 else:
860 args.append(None)
862 return nodes.Slice(lineno=lineno, *args) # noqa: B026
864 def parse_call_args(
865 self,
866 ) -> t.Tuple[
867 t.List[nodes.Expr],
868 t.List[nodes.Keyword],
869 t.Optional[nodes.Expr],
870 t.Optional[nodes.Expr],
871 ]:
872 token = self.stream.expect("lparen")
873 args = []
874 kwargs = []
875 dyn_args = None
876 dyn_kwargs = None
877 require_comma = False
879 def ensure(expr: bool) -> None:
880 if not expr:
881 self.fail("invalid syntax for function call expression", token.lineno)
883 while self.stream.current.type != "rparen":
884 if require_comma:
885 self.stream.expect("comma")
887 # support for trailing comma
888 if self.stream.current.type == "rparen":
889 break
891 if self.stream.current.type == "mul":
892 ensure(dyn_args is None and dyn_kwargs is None)
893 next(self.stream)
894 dyn_args = self.parse_expression()
895 elif self.stream.current.type == "pow":
896 ensure(dyn_kwargs is None)
897 next(self.stream)
898 dyn_kwargs = self.parse_expression()
899 else:
900 if (
901 self.stream.current.type == "name"
902 and self.stream.look().type == "assign"
903 ):
904 # Parsing a kwarg
905 ensure(dyn_kwargs is None)
906 key = self.stream.current.value
907 self.stream.skip(2)
908 value = self.parse_expression()
909 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
910 else:
911 # Parsing an arg
912 ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
913 args.append(self.parse_expression())
915 require_comma = True
917 self.stream.expect("rparen")
918 return args, kwargs, dyn_args, dyn_kwargs
920 def parse_call(self, node: nodes.Expr) -> nodes.Call:
921 # The lparen will be expected in parse_call_args, but the lineno
922 # needs to be recorded before the stream is advanced.
923 token = self.stream.current
924 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
925 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
927 def parse_filter(
928 self, node: t.Optional[nodes.Expr], start_inline: bool = False
929 ) -> t.Optional[nodes.Expr]:
930 while self.stream.current.type == "pipe" or start_inline:
931 if not start_inline:
932 next(self.stream)
933 token = self.stream.expect("name")
934 name = token.value
935 while self.stream.current.type == "dot":
936 next(self.stream)
937 name += "." + self.stream.expect("name").value
938 if self.stream.current.type == "lparen":
939 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
940 else:
941 args = []
942 kwargs = []
943 dyn_args = dyn_kwargs = None
944 node = nodes.Filter(
945 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
946 )
947 start_inline = False
948 return node
950 def parse_test(self, node: nodes.Expr) -> nodes.Expr:
951 token = next(self.stream)
952 if self.stream.current.test("name:not"):
953 next(self.stream)
954 negated = True
955 else:
956 negated = False
957 name = self.stream.expect("name").value
958 while self.stream.current.type == "dot":
959 next(self.stream)
960 name += "." + self.stream.expect("name").value
961 dyn_args = dyn_kwargs = None
962 kwargs: t.List[nodes.Keyword] = []
963 if self.stream.current.type == "lparen":
964 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
965 elif self.stream.current.type in {
966 "name",
967 "string",
968 "integer",
969 "float",
970 "lparen",
971 "lbracket",
972 "lbrace",
973 } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
974 if self.stream.current.test("name:is"):
975 self.fail("You cannot chain multiple tests with is")
976 arg_node = self.parse_primary()
977 arg_node = self.parse_postfix(arg_node)
978 args = [arg_node]
979 else:
980 args = []
981 node = nodes.Test(
982 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
983 )
984 if negated:
985 node = nodes.Not(node, lineno=token.lineno)
986 return node
988 def subparse(
989 self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
990 ) -> t.List[nodes.Node]:
991 body: t.List[nodes.Node] = []
992 data_buffer: t.List[nodes.Node] = []
993 add_data = data_buffer.append
995 if end_tokens is not None:
996 self._end_token_stack.append(end_tokens)
998 def flush_data() -> None:
999 if data_buffer:
1000 lineno = data_buffer[0].lineno
1001 body.append(nodes.Output(data_buffer[:], lineno=lineno))
1002 del data_buffer[:]
1004 try:
1005 while self.stream:
1006 token = self.stream.current
1007 if token.type == "data":
1008 if token.value:
1009 add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1010 next(self.stream)
1011 elif token.type == "variable_begin":
1012 next(self.stream)
1013 add_data(self.parse_tuple(with_condexpr=True))
1014 self.stream.expect("variable_end")
1015 elif token.type == "block_begin":
1016 flush_data()
1017 next(self.stream)
1018 if end_tokens is not None and self.stream.current.test_any(
1019 *end_tokens
1020 ):
1021 return body
1022 rv = self.parse_statement()
1023 if isinstance(rv, list):
1024 body.extend(rv)
1025 else:
1026 body.append(rv)
1027 self.stream.expect("block_end")
1028 else:
1029 raise AssertionError("internal parsing error")
1031 flush_data()
1032 finally:
1033 if end_tokens is not None:
1034 self._end_token_stack.pop()
1035 return body
1037 def parse(self) -> nodes.Template:
1038 """Parse the whole template into a `Template` node."""
1039 result = nodes.Template(self.subparse(), lineno=1)
1040 result.set_environment(self.environment)
1041 return result