Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/_nodes/expression.py: 64%
1414 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 07:09 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
7import re
8from abc import ABC, abstractmethod
9from ast import literal_eval
10from contextlib import contextmanager
11from dataclasses import dataclass, field
12from enum import auto, Enum
13from tokenize import (
14 Floatnumber as FLOATNUMBER_RE,
15 Imagnumber as IMAGNUMBER_RE,
16 Intnumber as INTNUMBER_RE,
17)
18from typing import Callable, Generator, Optional, Sequence, Union
20from typing_extensions import Literal
22from libcst._add_slots import add_slots
23from libcst._maybe_sentinel import MaybeSentinel
24from libcst._nodes.base import CSTCodegenError, CSTNode, CSTValidationError
25from libcst._nodes.internal import (
26 CodegenState,
27 visit_optional,
28 visit_required,
29 visit_sentinel,
30 visit_sequence,
31)
32from libcst._nodes.op import (
33 AssignEqual,
34 BaseBinaryOp,
35 BaseBooleanOp,
36 BaseCompOp,
37 BaseUnaryOp,
38 Colon,
39 Comma,
40 Dot,
41 In,
42 Is,
43 IsNot,
44 Not,
45 NotIn,
46)
47from libcst._nodes.whitespace import BaseParenthesizableWhitespace, SimpleWhitespace
48from libcst._visitors import CSTVisitorT
51@add_slots
52@dataclass(frozen=True)
53class LeftSquareBracket(CSTNode):
54 """
55 Used by various nodes to denote a subscript or list section. This doesn't own
56 the whitespace to the left of it since this is owned by the parent node.
57 """
59 #: Any space that appears directly after this left square bracket.
60 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
62 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "LeftSquareBracket":
63 return LeftSquareBracket(
64 whitespace_after=visit_required(
65 self, "whitespace_after", self.whitespace_after, visitor
66 )
67 )
69 def _codegen_impl(self, state: CodegenState) -> None:
70 state.add_token("[")
71 self.whitespace_after._codegen(state)
74@add_slots
75@dataclass(frozen=True)
76class RightSquareBracket(CSTNode):
77 """
78 Used by various nodes to denote a subscript or list section. This doesn't own
79 the whitespace to the right of it since this is owned by the parent node.
80 """
82 #: Any space that appears directly before this right square bracket.
83 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
85 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "RightSquareBracket":
86 return RightSquareBracket(
87 whitespace_before=visit_required(
88 self, "whitespace_before", self.whitespace_before, visitor
89 )
90 )
92 def _codegen_impl(self, state: CodegenState) -> None:
93 self.whitespace_before._codegen(state)
94 state.add_token("]")
97@add_slots
98@dataclass(frozen=True)
99class LeftCurlyBrace(CSTNode):
100 """
101 Used by various nodes to denote a dict or set. This doesn't own the whitespace to
102 the left of it since this is owned by the parent node.
103 """
105 #: Any space that appears directly after this left curly brace.
106 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
108 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "LeftCurlyBrace":
109 return LeftCurlyBrace(
110 whitespace_after=visit_required(
111 self, "whitespace_after", self.whitespace_after, visitor
112 )
113 )
115 def _codegen_impl(self, state: CodegenState) -> None:
116 state.add_token("{")
117 self.whitespace_after._codegen(state)
120@add_slots
121@dataclass(frozen=True)
122class RightCurlyBrace(CSTNode):
123 """
124 Used by various nodes to denote a dict or set. This doesn't own the whitespace to
125 the right of it since this is owned by the parent node.
126 """
128 #: Any space that appears directly before this right curly brace.
129 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
131 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "RightCurlyBrace":
132 return RightCurlyBrace(
133 whitespace_before=visit_required(
134 self, "whitespace_before", self.whitespace_before, visitor
135 )
136 )
138 def _codegen_impl(self, state: CodegenState) -> None:
139 self.whitespace_before._codegen(state)
140 state.add_token("}")
143@add_slots
144@dataclass(frozen=True)
145class LeftParen(CSTNode):
146 """
147 Used by various nodes to denote a parenthesized section. This doesn't own
148 the whitespace to the left of it since this is owned by the parent node.
149 """
151 #: Any space that appears directly after this left parenthesis.
152 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
154 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "LeftParen":
155 return LeftParen(
156 whitespace_after=visit_required(
157 self, "whitespace_after", self.whitespace_after, visitor
158 )
159 )
161 def _codegen_impl(self, state: CodegenState) -> None:
162 state.add_token("(")
163 self.whitespace_after._codegen(state)
166@add_slots
167@dataclass(frozen=True)
168class RightParen(CSTNode):
169 """
170 Used by various nodes to denote a parenthesized section. This doesn't own
171 the whitespace to the right of it since this is owned by the parent node.
172 """
174 #: Any space that appears directly after this left parenthesis.
175 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
177 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "RightParen":
178 return RightParen(
179 whitespace_before=visit_required(
180 self, "whitespace_before", self.whitespace_before, visitor
181 )
182 )
184 def _codegen_impl(self, state: CodegenState) -> None:
185 self.whitespace_before._codegen(state)
186 state.add_token(")")
189@add_slots
190@dataclass(frozen=True)
191class Asynchronous(CSTNode):
192 """
193 Used by asynchronous function definitions, as well as ``async for`` and
194 ``async with``.
195 """
197 #: Any space that appears directly after this async keyword.
198 whitespace_after: SimpleWhitespace = SimpleWhitespace.field(" ")
200 def _validate(self) -> None:
201 if len(self.whitespace_after.value) < 1:
202 raise CSTValidationError("Must have at least one space after Asynchronous.")
204 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Asynchronous":
205 return Asynchronous(
206 whitespace_after=visit_required(
207 self, "whitespace_after", self.whitespace_after, visitor
208 )
209 )
211 def _codegen_impl(self, state: CodegenState) -> None:
212 with state.record_syntactic_position(self):
213 state.add_token("async")
214 self.whitespace_after._codegen(state)
217class _BaseParenthesizedNode(CSTNode, ABC):
218 """
219 We don't want to have another level of indirection for parenthesis in
220 our tree, since that makes us more of a CST than an AST. So, all the
221 expressions or atoms that can be wrapped in parenthesis will subclass
222 this to get that functionality.
223 """
225 __slots__ = ()
227 lpar: Sequence[LeftParen] = ()
228 # Sequence of parenthesis for precedence dictation.
229 rpar: Sequence[RightParen] = ()
231 def _validate(self) -> None:
232 if self.lpar and not self.rpar:
233 raise CSTValidationError("Cannot have left paren without right paren.")
234 if not self.lpar and self.rpar:
235 raise CSTValidationError("Cannot have right paren without left paren.")
236 if len(self.lpar) != len(self.rpar):
237 raise CSTValidationError("Cannot have unbalanced parens.")
239 @contextmanager
240 def _parenthesize(self, state: CodegenState) -> Generator[None, None, None]:
241 for lpar in self.lpar:
242 lpar._codegen(state)
243 with state.record_syntactic_position(self):
244 yield
245 for rpar in self.rpar:
246 rpar._codegen(state)
249class ExpressionPosition(Enum):
250 LEFT = auto()
251 RIGHT = auto()
254class BaseExpression(_BaseParenthesizedNode, ABC):
255 """
256 An base class for all expressions. :class:`BaseExpression` contains no fields.
257 """
259 __slots__ = ()
261 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
262 """
263 Returns true if this expression is safe to be use with a word operator
264 such as "not" without space between the operator an ourselves. Examples
265 where this is true are "not(True)", "(1)in[1,2,3]", etc. This base
266 function handles parenthesized nodes, but certain nodes such as tuples,
267 dictionaries and lists will override this to signifiy that they're always
268 safe.
269 """
271 return len(self.lpar) > 0 and len(self.rpar) > 0
273 def _check_left_right_word_concatenation_safety(
274 self,
275 position: ExpressionPosition,
276 left: "BaseExpression",
277 right: "BaseExpression",
278 ) -> bool:
279 if position == ExpressionPosition.RIGHT:
280 return left._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
281 if position == ExpressionPosition.LEFT:
282 return right._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
283 return False
286class BaseAssignTargetExpression(BaseExpression, ABC):
287 """
288 An expression that's valid on the left side of an assignment. That assignment may
289 be part an :class:`Assign` node, or it may be part of a number of other control
290 structures that perform an assignment, such as a :class:`For` loop.
292 Python's grammar defines all expression as valid in this position, but the AST
293 compiler further restricts the allowed types, which is what this type attempts to
294 express.
296 This is similar to a :class:`BaseDelTargetExpression`, but it also includes
297 :class:`StarredElement` as a valid node.
299 The set of valid nodes are defined as part of `CPython's AST context computation
300 <https://github.com/python/cpython/blob/v3.8.0a4/Python/ast.c#L1120>`_.
301 """
303 __slots__ = ()
306class BaseDelTargetExpression(BaseExpression, ABC):
307 """
308 An expression that's valid on the right side of a :class:`Del` statement.
310 Python's grammar defines all expression as valid in this position, but the AST
311 compiler further restricts the allowed types, which is what this type attempts to
312 express.
314 This is similar to a :class:`BaseAssignTargetExpression`, but it excludes
315 :class:`StarredElement`.
317 The set of valid nodes are defined as part of `CPython's AST context computation
318 <https://github.com/python/cpython/blob/v3.8.0a4/Python/ast.c#L1120>`_ and as part
319 of `CPython's bytecode compiler
320 <https://github.com/python/cpython/blob/v3.8.0a4/Python/compile.c#L4854>`_.
321 """
323 __slots__ = ()
326@add_slots
327@dataclass(frozen=True)
328class Name(BaseAssignTargetExpression, BaseDelTargetExpression):
329 """
330 A simple variable name. Names are typically used in the context of a variable
331 access, an assignment, or a deletion.
333 Dotted variable names (``a.b.c``) are represented with :class:`Attribute` nodes,
334 and subscripted variable names (``a[b]``) are represented with :class:`Subscript`
335 nodes.
336 """
338 #: The variable's name (or "identifier") as a string.
339 value: str
341 lpar: Sequence[LeftParen] = ()
342 #: Sequence of parenthesis for precedence dictation.
343 rpar: Sequence[RightParen] = ()
345 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Name":
346 return Name(
347 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
348 value=self.value,
349 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
350 )
352 def _validate(self) -> None:
353 super(Name, self)._validate()
354 if len(self.value) == 0:
355 raise CSTValidationError("Cannot have empty name identifier.")
356 if not self.value.isidentifier():
357 raise CSTValidationError(f"Name {self.value!r} is not a valid identifier.")
359 def _codegen_impl(self, state: CodegenState) -> None:
360 with self._parenthesize(state):
361 state.add_token(self.value)
364@add_slots
365@dataclass(frozen=True)
366class Ellipsis(BaseExpression):
367 """
368 An ellipsis ``...``. When used as an expression, it evaluates to the
369 `Ellipsis constant`_. Ellipsis are often used as placeholders in code or in
370 conjunction with :class:`SubscriptElement`.
372 .. _Ellipsis constant: https://docs.python.org/3/library/constants.html#Ellipsis
373 """
375 lpar: Sequence[LeftParen] = ()
377 #: Sequence of parenthesis for precedence dictation.
378 rpar: Sequence[RightParen] = ()
380 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Ellipsis":
381 return Ellipsis(
382 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
383 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
384 )
386 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
387 return True
389 def _codegen_impl(self, state: CodegenState) -> None:
390 with self._parenthesize(state):
391 state.add_token("...")
394class BaseNumber(BaseExpression, ABC):
395 """
396 A type such as :class:`Integer`, :class:`Float`, or :class:`Imaginary` that can be
397 used anywhere that you need to explicitly take any number type.
398 """
400 __slots__ = ()
402 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
403 """
404 Numbers are funny. The expression "5in [1,2,3,4,5]" is a valid expression
405 which evaluates to "True". So, encapsulate that here by allowing zero spacing
406 with the left hand side of an expression with a comparison operator.
407 """
408 if position == ExpressionPosition.LEFT:
409 return True
410 return super(BaseNumber, self)._safe_to_use_with_word_operator(position)
413@add_slots
414@dataclass(frozen=True)
415class Integer(BaseNumber):
416 #: A string representation of the integer, such as ``"100000"`` or ``100_000``.
417 #:
418 #: To convert this string representation to an ``int``, use the calculated
419 #: property :attr:`~Integer.evaluated_value`.
420 value: str
422 lpar: Sequence[LeftParen] = ()
423 #: Sequence of parenthesis for precedence dictation.
424 rpar: Sequence[RightParen] = ()
426 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Integer":
427 return Integer(
428 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
429 value=self.value,
430 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
431 )
433 def _validate(self) -> None:
434 super(Integer, self)._validate()
435 if not re.fullmatch(INTNUMBER_RE, self.value):
436 raise CSTValidationError("Number is not a valid integer.")
438 def _codegen_impl(self, state: CodegenState) -> None:
439 with self._parenthesize(state):
440 state.add_token(self.value)
442 @property
443 def evaluated_value(self) -> int:
444 """
445 Return an :func:`ast.literal_eval` evaluated int of :py:attr:`value`.
446 """
447 return literal_eval(self.value)
450@add_slots
451@dataclass(frozen=True)
452class Float(BaseNumber):
453 #: A string representation of the floating point number, such as ``"0.05"``,
454 #: ``".050"``, or ``"5e-2"``.
455 #:
456 #: To convert this string representation to an ``float``, use the calculated
457 #: property :attr:`~Float.evaluated_value`.
458 value: str
460 lpar: Sequence[LeftParen] = ()
461 #: Sequence of parenthesis for precedence dictation.
462 rpar: Sequence[RightParen] = ()
464 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Float":
465 return Float(
466 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
467 value=self.value,
468 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
469 )
471 def _validate(self) -> None:
472 super(Float, self)._validate()
473 if not re.fullmatch(FLOATNUMBER_RE, self.value):
474 raise CSTValidationError("Number is not a valid float.")
476 def _codegen_impl(self, state: CodegenState) -> None:
477 with self._parenthesize(state):
478 state.add_token(self.value)
480 @property
481 def evaluated_value(self) -> float:
482 """
483 Return an :func:`ast.literal_eval` evaluated float of :py:attr:`value`.
484 """
485 return literal_eval(self.value)
488@add_slots
489@dataclass(frozen=True)
490class Imaginary(BaseNumber):
491 #: A string representation of the imaginary (complex) number, such as ``"2j"``.
492 #:
493 #: To convert this string representation to an ``complex``, use the calculated
494 #: property :attr:`~Imaginary.evaluated_value`.
495 value: str
497 lpar: Sequence[LeftParen] = ()
498 #: Sequence of parenthesis for precedence dictation.
499 rpar: Sequence[RightParen] = ()
501 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Imaginary":
502 return Imaginary(
503 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
504 value=self.value,
505 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
506 )
508 def _validate(self) -> None:
509 super(Imaginary, self)._validate()
510 if not re.fullmatch(IMAGNUMBER_RE, self.value):
511 raise CSTValidationError("Number is not a valid imaginary.")
513 def _codegen_impl(self, state: CodegenState) -> None:
514 with self._parenthesize(state):
515 state.add_token(self.value)
517 @property
518 def evaluated_value(self) -> complex:
519 """
520 Return an :func:`ast.literal_eval` evaluated complex of :py:attr:`value`.
521 """
522 return literal_eval(self.value)
525class BaseString(BaseExpression, ABC):
526 """
527 A type that can be used anywhere that you need to take any string. This includes
528 :class:`SimpleString`, :class:`ConcatenatedString`, and :class:`FormattedString`.
529 """
531 __slots__ = ()
534StringQuoteLiteral = Literal['"', "'", '"""', "'''"]
537class _BasePrefixedString(BaseString, ABC):
538 __slots__ = ()
540 @property
541 def prefix(self) -> str:
542 """
543 Returns the string's prefix, if any exists.
545 See `String and Bytes literals
546 <https://docs.python.org/3.7/reference/lexical_analysis.html#string-and-bytes-literals>`_
547 for more information.
548 """
549 ...
551 @property
552 def quote(self) -> StringQuoteLiteral:
553 """
554 Returns the quotation used to denote the string. Can be either ``'``,
555 ``"``, ``'''`` or ``\"\"\"``.
556 """
557 ...
559 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
560 """
561 ``"a"in"abc`` is okay, but if you add a prefix, (e.g. ``b"a"inb"abc"``), the string
562 is no longer valid on the RHS of the word operator, because it's not clear where
563 the keyword ends and the prefix begins, unless it's parenthesized.
564 """
565 if position == ExpressionPosition.LEFT:
566 return True
567 elif self.prefix == "": # and position == ExpressionPosition.RIGHT
568 return True
569 else:
570 return super(_BasePrefixedString, self)._safe_to_use_with_word_operator(
571 position
572 )
575@add_slots
576@dataclass(frozen=True)
577class SimpleString(_BasePrefixedString):
578 """
579 Any sort of literal string expression that is not a :class:`FormattedString`
580 (f-string), including triple-quoted multi-line strings.
581 """
583 #: The texual representation of the string, including quotes, prefix characters, and
584 #: any escape characters present in the original source code , such as
585 #: ``r"my string\n"``. To remove the quotes and interpret any escape characters,
586 #: use the calculated property :attr:`~SimpleString.evaluated_value`.
587 value: str
589 lpar: Sequence[LeftParen] = ()
590 #: Sequence of parenthesis for precidence dictation.
591 rpar: Sequence[RightParen] = ()
593 def _validate(self) -> None:
594 super(SimpleString, self)._validate()
596 # Validate any prefix
597 prefix = self.prefix
598 if prefix not in ("", "r", "u", "b", "br", "rb"):
599 raise CSTValidationError("Invalid string prefix.")
600 prefixlen = len(prefix)
601 # Validate wrapping quotes
602 if len(self.value) < (prefixlen + 2):
603 raise CSTValidationError("String must have enclosing quotes.")
604 if (
605 self.value[prefixlen] not in ['"', "'"]
606 or self.value[prefixlen] != self.value[-1]
607 ):
608 raise CSTValidationError("String must have matching enclosing quotes.")
609 # Check validity of triple-quoted strings
610 if len(self.value) >= (prefixlen + 6):
611 if self.value[prefixlen] == self.value[prefixlen + 1]:
612 # We know this isn't an empty string, so there needs to be a third
613 # identical enclosing token.
614 if (
615 self.value[prefixlen] != self.value[prefixlen + 2]
616 or self.value[prefixlen] != self.value[-2]
617 or self.value[prefixlen] != self.value[-3]
618 ):
619 raise CSTValidationError(
620 "String must have matching enclosing quotes."
621 )
622 # We should check the contents as well, but this is pretty complicated,
623 # partially due to triple-quoted strings.
625 @property
626 def prefix(self) -> str:
627 """
628 Returns the string's prefix, if any exists. The prefix can be ``r``,
629 ``u``, ``b``, ``br`` or ``rb``.
630 """
632 prefix: str = ""
633 for c in self.value:
634 if c in ['"', "'"]:
635 break
636 prefix += c
637 return prefix.lower()
639 @property
640 def quote(self) -> StringQuoteLiteral:
641 """
642 Returns the quotation used to denote the string. Can be either ``'``,
643 ``"``, ``'''`` or ``\"\"\"``.
644 """
646 quote: str = ""
647 for char in self.value[len(self.prefix) :]:
648 if char not in {"'", '"'}:
649 break
650 if quote and char != quote[0]:
651 # This is no longer the same string quote
652 break
653 quote += char
655 if len(quote) == 2:
656 # Let's assume this is an empty string.
657 quote = quote[:1]
658 elif 3 < len(quote) <= 6:
659 # Let's assume this can be one of the following:
660 # >>> """"foo"""
661 # '"foo'
662 # >>> """""bar"""
663 # '""bar'
664 # >>> """"""
665 # ''
666 quote = quote[:3]
668 if len(quote) not in {1, 3}:
669 # We shouldn't get here due to construction validation logic,
670 # but handle the case anyway.
671 raise Exception(f"Invalid string {self.value}")
673 # pyre-ignore We know via the above validation that we will only
674 # ever return one of the four string literals.
675 return quote
677 @property
678 def raw_value(self) -> str:
679 """
680 Returns the raw value of the string as it appears in source, without
681 the beginning or end quotes and without the prefix. This is often
682 useful when constructing transforms which need to manipulate strings
683 in source code.
684 """
686 prefix_len = len(self.prefix)
687 quote_len = len(self.quote)
688 return self.value[(prefix_len + quote_len) : (-quote_len)]
690 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "SimpleString":
691 return SimpleString(
692 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
693 value=self.value,
694 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
695 )
697 def _codegen_impl(self, state: CodegenState) -> None:
698 with self._parenthesize(state):
699 state.add_token(self.value)
701 @property
702 def evaluated_value(self) -> Union[str, bytes]:
703 """
704 Return an :func:`ast.literal_eval` evaluated str of :py:attr:`value`.
705 """
706 return literal_eval(self.value)
709class BaseFormattedStringContent(CSTNode, ABC):
710 """
711 The base type for :class:`FormattedStringText` and
712 :class:`FormattedStringExpression`. A :class:`FormattedString` is composed of a
713 sequence of :class:`BaseFormattedStringContent` parts.
714 """
716 __slots__ = ()
719@add_slots
720@dataclass(frozen=True)
721class FormattedStringText(BaseFormattedStringContent):
722 """
723 Part of a :class:`FormattedString` that is not inside curly braces (``{`` or ``}``).
724 For example, in::
726 f"ab{cd}ef"
728 ``ab`` and ``ef`` are :class:`FormattedStringText` nodes, but ``{cd}`` is a
729 :class:`FormattedStringExpression`.
730 """
732 #: The raw string value, including any escape characters present in the source
733 #: code, not including any enclosing quotes.
734 value: str
736 def _visit_and_replace_children(
737 self, visitor: CSTVisitorT
738 ) -> "FormattedStringText":
739 return FormattedStringText(value=self.value)
741 def _codegen_impl(self, state: CodegenState) -> None:
742 state.add_token(self.value)
745@add_slots
746@dataclass(frozen=True)
747class FormattedStringExpression(BaseFormattedStringContent):
748 """
749 Part of a :class:`FormattedString` that is inside curly braces (``{`` or ``}``),
750 including the surrounding curly braces. For example, in::
752 f"ab{cd}ef"
754 ``{cd}`` is a :class:`FormattedStringExpression`, but ``ab`` and ``ef`` are
755 :class:`FormattedStringText` nodes.
757 An f-string expression may contain ``conversion`` and ``format_spec`` suffixes that
758 control how the expression is converted to a string. See `Python's language
759 reference
760 <https://docs.python.org/3/reference/lexical_analysis.html#formatted-string-literals>`__
761 for details.
762 """
764 #: The expression we will evaluate and render when generating the string.
765 expression: BaseExpression
767 #: An optional conversion specifier, such as ``!s``, ``!r`` or ``!a``.
768 conversion: Optional[str] = None
770 #: An optional format specifier following the `format specification mini-language
771 #: <https://docs.python.org/3/library/string.html#formatspec>`_.
772 format_spec: Optional[Sequence[BaseFormattedStringContent]] = None
774 #: Whitespace after the opening curly brace (``{``), but before the ``expression``.
775 whitespace_before_expression: BaseParenthesizableWhitespace = (
776 SimpleWhitespace.field("")
777 )
779 #: Whitespace after the ``expression``, but before the ``conversion``,
780 #: ``format_spec`` and the closing curly brace (``}``). Python does not
781 #: allow whitespace inside or after a ``conversion`` or ``format_spec``.
782 whitespace_after_expression: BaseParenthesizableWhitespace = SimpleWhitespace.field(
783 ""
784 )
786 #: Equal sign for formatted string expression uses self-documenting expressions,
787 #: such as ``f"{x=}"``. See the `Python 3.8 release notes
788 #: <https://docs.python.org/3/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging>`_.
789 equal: Optional[AssignEqual] = None
791 def _validate(self) -> None:
792 if self.conversion is not None and self.conversion not in ("s", "r", "a"):
793 raise CSTValidationError("Invalid f-string conversion.")
795 def _visit_and_replace_children(
796 self, visitor: CSTVisitorT
797 ) -> "FormattedStringExpression":
798 format_spec = self.format_spec
799 return FormattedStringExpression(
800 whitespace_before_expression=visit_required(
801 self,
802 "whitespace_before_expression",
803 self.whitespace_before_expression,
804 visitor,
805 ),
806 expression=visit_required(self, "expression", self.expression, visitor),
807 equal=visit_optional(self, "equal", self.equal, visitor),
808 whitespace_after_expression=visit_required(
809 self,
810 "whitespace_after_expression",
811 self.whitespace_after_expression,
812 visitor,
813 ),
814 conversion=self.conversion,
815 format_spec=(
816 visit_sequence(self, "format_spec", format_spec, visitor)
817 if format_spec is not None
818 else None
819 ),
820 )
822 def _codegen_impl(self, state: CodegenState) -> None:
823 state.add_token("{")
824 self.whitespace_before_expression._codegen(state)
825 self.expression._codegen(state)
826 equal = self.equal
827 if equal is not None:
828 equal._codegen(state)
829 self.whitespace_after_expression._codegen(state)
830 conversion = self.conversion
831 if conversion is not None:
832 state.add_token("!")
833 state.add_token(conversion)
834 format_spec = self.format_spec
835 if format_spec is not None:
836 state.add_token(":")
837 for spec in format_spec:
838 spec._codegen(state)
839 state.add_token("}")
842@add_slots
843@dataclass(frozen=True)
844class FormattedString(_BasePrefixedString):
845 """
846 An "f-string". These formatted strings are string literals prefixed by the letter
847 "f". An f-string may contain interpolated expressions inside curly braces (``{`` and
848 ``}``).
850 F-strings are defined in `PEP 498`_ and documented in `Python's language
851 reference
852 <https://docs.python.org/3/reference/lexical_analysis.html#formatted-string-literals>`__.
854 >>> import libcst as cst
855 >>> cst.parse_expression('f"ab{cd}ef"')
856 FormattedString(
857 parts=[
858 FormattedStringText(
859 value='ab',
860 ),
861 FormattedStringExpression(
862 expression=Name(
863 value='cd',
864 lpar=[],
865 rpar=[],
866 ),
867 conversion=None,
868 format_spec=None,
869 whitespace_before_expression=SimpleWhitespace(
870 value='',
871 ),
872 whitespace_after_expression=SimpleWhitespace(
873 value='',
874 ),
875 ),
876 FormattedStringText(
877 value='ef',
878 ),
879 ],
880 start='f"',
881 end='"',
882 lpar=[],
883 rpar=[],
884 )
886 .. _PEP 498: https://www.python.org/dev/peps/pep-0498/#specification
887 """
889 #: A formatted string is composed as a series of :class:`FormattedStringText` and
890 #: :class:`FormattedStringExpression` parts.
891 parts: Sequence[BaseFormattedStringContent]
893 #: The string prefix and the leading quote, such as ``f"``, ``F'``, ``fr"``, or
894 #: ``f"""``.
895 start: str = 'f"'
897 #: The trailing quote. This must match the type of quote used in ``start``.
898 end: Literal['"', "'", '"""', "'''"] = '"'
900 lpar: Sequence[LeftParen] = ()
901 #: Sequence of parenthesis for precidence dictation.
902 rpar: Sequence[RightParen] = ()
904 def _validate(self) -> None:
905 super(FormattedString, self)._validate()
907 # Validate any prefix
908 prefix = self.prefix
909 if prefix not in ("f", "fr", "rf"):
910 raise CSTValidationError("Invalid f-string prefix.")
912 # Validate wrapping quotes
913 starttoken = self.start[len(prefix) :]
914 if starttoken != self.end:
915 raise CSTValidationError("f-string must have matching enclosing quotes.")
917 # Validate valid wrapping quote usage
918 if starttoken not in ('"', "'", '"""', "'''"):
919 raise CSTValidationError("Invalid f-string enclosing quotes.")
921 @property
922 def prefix(self) -> str:
923 """
924 Returns the string's prefix, if any exists. The prefix can be ``f``,
925 ``fr``, or ``rf``.
926 """
928 prefix = ""
929 for c in self.start:
930 if c in ['"', "'"]:
931 break
932 prefix += c
933 return prefix.lower()
935 @property
936 def quote(self) -> StringQuoteLiteral:
937 """
938 Returns the quotation used to denote the string. Can be either ``'``,
939 ``"``, ``'''`` or ``\"\"\"``.
940 """
942 return self.end
944 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "FormattedString":
945 return FormattedString(
946 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
947 start=self.start,
948 parts=visit_sequence(self, "parts", self.parts, visitor),
949 end=self.end,
950 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
951 )
953 def _codegen_impl(self, state: CodegenState) -> None:
954 with self._parenthesize(state):
955 state.add_token(self.start)
956 for part in self.parts:
957 part._codegen(state)
958 state.add_token(self.end)
961@add_slots
962@dataclass(frozen=True)
963class ConcatenatedString(BaseString):
964 """
965 Represents an implicitly concatenated string, such as::
967 "abc" "def" == "abcdef"
969 .. warning::
970 This is different from two strings joined in a :class:`BinaryOperation` with an
971 :class:`Add` operator, and is `sometimes viewed as an antifeature of Python
972 <https://lwn.net/Articles/551426/>`_.
973 """
975 #: String on the left of the concatenation.
976 left: Union[SimpleString, FormattedString]
978 #: String on the right of the concatenation.
979 right: Union[SimpleString, FormattedString, "ConcatenatedString"]
981 lpar: Sequence[LeftParen] = ()
982 #: Sequence of parenthesis for precidence dictation.
983 rpar: Sequence[RightParen] = ()
985 #: Whitespace between the ``left`` and ``right`` substrings.
986 whitespace_between: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
988 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
989 if super(ConcatenatedString, self)._safe_to_use_with_word_operator(position):
990 # if we have parenthesis, we're safe.
991 return True
992 return self._check_left_right_word_concatenation_safety(
993 position, self.left, self.right
994 )
996 def _validate(self) -> None:
997 super(ConcatenatedString, self)._validate()
999 # Strings that are concatenated cannot have parens.
1000 if bool(self.left.lpar) or bool(self.left.rpar):
1001 raise CSTValidationError("Cannot concatenate parenthesized strings.")
1002 if bool(self.right.lpar) or bool(self.right.rpar):
1003 raise CSTValidationError("Cannot concatenate parenthesized strings.")
1005 # Cannot concatenate str and bytes
1006 leftbytes = "b" in self.left.prefix
1007 right = self.right
1008 if isinstance(right, ConcatenatedString):
1009 rightbytes = "b" in right.left.prefix
1010 elif isinstance(right, SimpleString):
1011 rightbytes = "b" in right.prefix
1012 elif isinstance(right, FormattedString):
1013 rightbytes = "b" in right.prefix
1014 else:
1015 raise Exception("Logic error!")
1016 if leftbytes != rightbytes:
1017 raise CSTValidationError("Cannot concatenate string and bytes.")
1019 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ConcatenatedString":
1020 return ConcatenatedString(
1021 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1022 left=visit_required(self, "left", self.left, visitor),
1023 whitespace_between=visit_required(
1024 self, "whitespace_between", self.whitespace_between, visitor
1025 ),
1026 right=visit_required(self, "right", self.right, visitor),
1027 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1028 )
1030 def _codegen_impl(self, state: CodegenState) -> None:
1031 with self._parenthesize(state):
1032 self.left._codegen(state)
1033 self.whitespace_between._codegen(state)
1034 self.right._codegen(state)
1036 @property
1037 def evaluated_value(self) -> Union[str, bytes, None]:
1038 """
1039 Return an :func:`ast.literal_eval` evaluated str of recursively concatenated :py:attr:`left` and :py:attr:`right`
1040 if and only if both :py:attr:`left` and :py:attr:`right` are composed by :class:`SimpleString` or :class:`ConcatenatedString`
1041 (:class:`FormattedString` cannot be evaluated).
1042 """
1043 left = self.left
1044 right = self.right
1045 if isinstance(left, FormattedString) or isinstance(right, FormattedString):
1046 return None
1047 left_val = left.evaluated_value
1048 right_val = right.evaluated_value
1049 if right_val is None:
1050 return None
1051 if isinstance(left_val, bytes) and isinstance(right_val, bytes):
1052 return left_val + right_val
1053 if isinstance(left_val, str) and isinstance(right_val, str):
1054 return left_val + right_val
1055 return None
1058@add_slots
1059@dataclass(frozen=True)
1060class ComparisonTarget(CSTNode):
1061 """
1062 A target for a :class:`Comparison`. Owns the comparison operator and the value to
1063 the right of the operator.
1064 """
1066 #: A comparison operator such as ``<``, ``>=``, ``==``, ``is``, or ``in``.
1067 operator: BaseCompOp
1069 #: The right hand side of the comparison operation.
1070 comparator: BaseExpression
1072 def _validate(self) -> None:
1073 # Validate operator spacing rules
1074 operator = self.operator
1075 if (
1076 isinstance(operator, (In, NotIn, Is, IsNot))
1077 and operator.whitespace_after.empty
1078 and not self.comparator._safe_to_use_with_word_operator(
1079 ExpressionPosition.RIGHT
1080 )
1081 ):
1082 raise CSTValidationError(
1083 "Must have at least one space around comparison operator."
1084 )
1086 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ComparisonTarget":
1087 return ComparisonTarget(
1088 operator=visit_required(self, "operator", self.operator, visitor),
1089 comparator=visit_required(self, "comparator", self.comparator, visitor),
1090 )
1092 def _codegen_impl(self, state: CodegenState) -> None:
1093 self.operator._codegen(state)
1094 self.comparator._codegen(state)
1097@add_slots
1098@dataclass(frozen=True)
1099class Comparison(BaseExpression):
1100 """
1101 A comparison between multiple values such as ``x < y``, ``x < y < z``, or
1102 ``x in [y, z]``. These comparisions typically result in boolean values.
1104 Unlike :class:`BinaryOperation` and :class:`BooleanOperation`, comparisons are not
1105 restricted to a left and right child. Instead they can contain an arbitrary number
1106 of :class:`ComparisonTarget` children.
1108 ``x < y < z`` is not equivalent to ``(x < y) < z`` or ``x < (y < z)``. Instead,
1109 it's roughly equivalent to ``x < y and y < z``.
1111 For more details, see `Python's documentation on comparisons
1112 <https://docs.python.org/3/reference/expressions.html#comparisons>`_.
1114 ::
1116 # x < y < z
1118 Comparison(
1119 Name("x"),
1120 [
1121 ComparisonTarget(LessThan(), Name("y")),
1122 ComparisonTarget(LessThan(), Name("z")),
1123 ],
1124 )
1125 """
1127 #: The first value in the full sequence of values to compare. This value will be
1128 #: compared against the first value in ``comparisions``.
1129 left: BaseExpression
1131 #: Pairs of :class:`BaseCompOp` operators and expression values to compare. These
1132 #: come after ``left``. Each value is compared against the value before and after
1133 #: itself in the sequence.
1134 comparisons: Sequence[ComparisonTarget]
1136 lpar: Sequence[LeftParen] = ()
1137 #: Sequence of parenthesis for precedence dictation.
1138 rpar: Sequence[RightParen] = ()
1140 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1141 if super(Comparison, self)._safe_to_use_with_word_operator(position):
1142 # we have parenthesis
1143 return True
1144 return self._check_left_right_word_concatenation_safety(
1145 position, self.left, self.comparisons[-1].comparator
1146 )
1148 def _validate(self) -> None:
1149 # Perform any validation on base type
1150 super(Comparison, self)._validate()
1152 if len(self.comparisons) == 0:
1153 raise CSTValidationError("Must have at least one ComparisonTarget.")
1155 # Validate operator spacing rules
1156 previous_comparator = self.left
1157 for target in self.comparisons:
1158 operator = target.operator
1159 if (
1160 isinstance(operator, (In, NotIn, Is, IsNot))
1161 and operator.whitespace_before.empty
1162 and not previous_comparator._safe_to_use_with_word_operator(
1163 ExpressionPosition.LEFT
1164 )
1165 ):
1166 raise CSTValidationError(
1167 "Must have at least one space around comparison operator."
1168 )
1169 previous_comparator = target.comparator
1171 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Comparison":
1172 return Comparison(
1173 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1174 left=visit_required(self, "left", self.left, visitor),
1175 comparisons=visit_sequence(self, "comparisons", self.comparisons, visitor),
1176 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1177 )
1179 def _codegen_impl(self, state: CodegenState) -> None:
1180 with self._parenthesize(state):
1181 self.left._codegen(state)
1182 for comp in self.comparisons:
1183 comp._codegen(state)
1186@add_slots
1187@dataclass(frozen=True)
1188class UnaryOperation(BaseExpression):
1189 """
1190 Any generic unary expression, such as ``not x`` or ``-x``. :class:`UnaryOperation`
1191 nodes apply a :class:`BaseUnaryOp` to an expression.
1192 """
1194 #: The unary operator that applies some operation (e.g. negation) to the
1195 #: ``expression``.
1196 operator: BaseUnaryOp
1198 #: The expression that should be transformed (e.g. negated) by the operator to
1199 #: create a new value.
1200 expression: BaseExpression
1202 lpar: Sequence[LeftParen] = ()
1203 #: Sequence of parenthesis for precedence dictation.
1204 rpar: Sequence[RightParen] = ()
1206 def _validate(self) -> None:
1207 # Perform any validation on base type
1208 super(UnaryOperation, self)._validate()
1210 if (
1211 isinstance(self.operator, Not)
1212 and self.operator.whitespace_after.empty
1213 and not self.expression._safe_to_use_with_word_operator(
1214 ExpressionPosition.RIGHT
1215 )
1216 ):
1217 raise CSTValidationError("Must have at least one space after not operator.")
1219 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "UnaryOperation":
1220 return UnaryOperation(
1221 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1222 operator=visit_required(self, "operator", self.operator, visitor),
1223 expression=visit_required(self, "expression", self.expression, visitor),
1224 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1225 )
1227 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1228 """
1229 As long as we aren't comprised of the Not unary operator, we are safe to use
1230 without space.
1231 """
1232 if super(UnaryOperation, self)._safe_to_use_with_word_operator(position):
1233 return True
1234 if position == ExpressionPosition.RIGHT:
1235 return not isinstance(self.operator, Not)
1236 if position == ExpressionPosition.LEFT:
1237 return self.expression._safe_to_use_with_word_operator(
1238 ExpressionPosition.LEFT
1239 )
1240 return False
1242 def _codegen_impl(self, state: CodegenState) -> None:
1243 with self._parenthesize(state):
1244 self.operator._codegen(state)
1245 self.expression._codegen(state)
1248@add_slots
1249@dataclass(frozen=True)
1250class BinaryOperation(BaseExpression):
1251 """
1252 An operation that combines two expression such as ``x << y`` or ``y + z``.
1253 :class:`BinaryOperation` nodes apply a :class:`BaseBinaryOp` to an expression.
1255 Binary operations do not include operations performed with :class:`BaseBooleanOp`
1256 nodes, such as ``and`` or ``or``. Instead, those operations are provided by
1257 :class:`BooleanOperation`.
1259 It also does not include support for comparision operators performed with
1260 :class:`BaseCompOp`, such as ``<``, ``>=``, ``==``, ``is``, or ``in``. Instead,
1261 those operations are provided by :class:`Comparison`.
1262 """
1264 #: The left hand side of the operation.
1265 left: BaseExpression
1267 #: The actual operator such as ``<<`` or ``+`` that combines the ``left`` and
1268 #: ``right`` expressions.
1269 operator: BaseBinaryOp
1271 #: The right hand side of the operation.
1272 right: BaseExpression
1274 lpar: Sequence[LeftParen] = ()
1275 #: Sequence of parenthesis for precedence dictation.
1276 rpar: Sequence[RightParen] = ()
1278 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "BinaryOperation":
1279 return BinaryOperation(
1280 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1281 left=visit_required(self, "left", self.left, visitor),
1282 operator=visit_required(self, "operator", self.operator, visitor),
1283 right=visit_required(self, "right", self.right, visitor),
1284 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1285 )
1287 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1288 if super(BinaryOperation, self)._safe_to_use_with_word_operator(position):
1289 return True
1290 return self._check_left_right_word_concatenation_safety(
1291 position, self.left, self.right
1292 )
1294 def _codegen_impl(self, state: CodegenState) -> None:
1295 with self._parenthesize(state):
1296 self.left._codegen(state)
1297 self.operator._codegen(state)
1298 self.right._codegen(state)
1301@add_slots
1302@dataclass(frozen=True)
1303class BooleanOperation(BaseExpression):
1304 """
1305 An operation that combines two booleans such as ``x or y`` or ``z and w``
1306 :class:`BooleanOperation` nodes apply a :class:`BaseBooleanOp` to an expression.
1308 Boolean operations do not include operations performed with :class:`BaseBinaryOp`
1309 nodes, such as ``+`` or ``<<``. Instead, those operations are provided by
1310 :class:`BinaryOperation`.
1312 It also does not include support for comparision operators performed with
1313 :class:`BaseCompOp`, such as ``<``, ``>=``, ``==``, ``is``, or ``in``. Instead,
1314 those operations are provided by :class:`Comparison`.
1315 """
1317 #: The left hand side of the operation.
1318 left: BaseExpression
1320 #: The actual operator such as ``and`` or ``or`` that combines the ``left`` and
1321 #: ``right`` expressions.
1322 operator: BaseBooleanOp
1324 #: The right hand side of the operation.
1325 right: BaseExpression
1327 lpar: Sequence[LeftParen] = ()
1328 #: Sequence of parenthesis for precedence dictation.
1329 rpar: Sequence[RightParen] = ()
1331 def _validate(self) -> None:
1332 # Paren validation and such
1333 super(BooleanOperation, self)._validate()
1334 # Validate spacing rules
1335 if (
1336 self.operator.whitespace_before.empty
1337 and not self.left._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
1338 ):
1339 raise CSTValidationError(
1340 "Must have at least one space around boolean operator."
1341 )
1342 if (
1343 self.operator.whitespace_after.empty
1344 and not self.right._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
1345 ):
1346 raise CSTValidationError(
1347 "Must have at least one space around boolean operator."
1348 )
1350 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "BooleanOperation":
1351 return BooleanOperation(
1352 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1353 left=visit_required(self, "left", self.left, visitor),
1354 operator=visit_required(self, "operator", self.operator, visitor),
1355 right=visit_required(self, "right", self.right, visitor),
1356 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1357 )
1359 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1360 if super(BooleanOperation, self)._safe_to_use_with_word_operator(position):
1361 return True
1362 return self._check_left_right_word_concatenation_safety(
1363 position, self.left, self.right
1364 )
1366 def _codegen_impl(self, state: CodegenState) -> None:
1367 with self._parenthesize(state):
1368 self.left._codegen(state)
1369 self.operator._codegen(state)
1370 self.right._codegen(state)
1373@add_slots
1374@dataclass(frozen=True)
1375class Attribute(BaseAssignTargetExpression, BaseDelTargetExpression):
1376 """
1377 An attribute reference, such as ``x.y``.
1379 Note that in the case of ``x.y.z``, the outer attribute will have an attr of ``z``
1380 and the value will be another :class:`Attribute` referencing the ``y`` attribute on
1381 ``x``::
1383 Attribute(
1384 value=Attribute(
1385 value=Name("x")
1386 attr=Name("y")
1387 ),
1388 attr=Name("z"),
1389 )
1390 """
1392 #: An expression which, when evaluated, will produce an object with ``attr`` as an
1393 #: attribute.
1394 value: BaseExpression
1396 #: The name of the attribute being accessed on the ``value`` object.
1397 attr: Name
1399 #: A separating dot. If there's whitespace between the ``value`` and ``attr``, this
1400 #: dot owns it.
1401 dot: Dot = Dot()
1403 lpar: Sequence[LeftParen] = ()
1404 #: Sequence of parenthesis for precedence dictation.
1405 rpar: Sequence[RightParen] = ()
1407 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Attribute":
1408 return Attribute(
1409 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1410 value=visit_required(self, "value", self.value, visitor),
1411 dot=visit_required(self, "dot", self.dot, visitor),
1412 attr=visit_required(self, "attr", self.attr, visitor),
1413 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1414 )
1416 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1417 if super(Attribute, self)._safe_to_use_with_word_operator(position):
1418 return True
1419 return self._check_left_right_word_concatenation_safety(
1420 position, self.value, self.attr
1421 )
1423 def _codegen_impl(self, state: CodegenState) -> None:
1424 with self._parenthesize(state):
1425 self.value._codegen(state)
1426 self.dot._codegen(state)
1427 self.attr._codegen(state)
1430class BaseSlice(CSTNode, ABC):
1431 """
1432 Any slice type that can slot into a :class:`SubscriptElement`.
1433 This node is purely for typing.
1434 """
1436 __slots__ = ()
1439@add_slots
1440@dataclass(frozen=True)
1441class Index(BaseSlice):
1442 """
1443 Any index as passed to a :class:`Subscript`. In ``x[2]``, this would be the ``2``
1444 value.
1445 """
1447 #: The index value itself.
1448 value: BaseExpression
1450 #: An optional string with an asterisk appearing before the name. This is
1451 #: expanded into variable number of positional arguments. See PEP-646
1452 star: Optional[Literal["*"]] = None
1454 #: Whitespace after the ``star`` (if it exists), but before the ``value``.
1455 whitespace_after_star: Optional[BaseParenthesizableWhitespace] = None
1457 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Index":
1458 return Index(
1459 star=self.star,
1460 whitespace_after_star=visit_optional(
1461 self, "whitespace_after_star", self.whitespace_after_star, visitor
1462 ),
1463 value=visit_required(self, "value", self.value, visitor),
1464 )
1466 def _codegen_impl(self, state: CodegenState) -> None:
1467 star = self.star
1468 if star is not None:
1469 state.add_token(star)
1470 ws = self.whitespace_after_star
1471 if ws is not None:
1472 ws._codegen(state)
1473 self.value._codegen(state)
1476@add_slots
1477@dataclass(frozen=True)
1478class Slice(BaseSlice):
1479 """
1480 Any slice operation in a :class:`Subscript`, such as ``1:``, ``2:3:4``, etc.
1482 Note that the grammar does NOT allow parenthesis around a slice so they are not
1483 supported here.
1484 """
1486 #: The lower bound in the slice, if present
1487 lower: Optional[BaseExpression]
1489 #: The upper bound in the slice, if present
1490 upper: Optional[BaseExpression]
1492 #: The step in the slice, if present
1493 step: Optional[BaseExpression] = None
1495 #: The first slice operator
1496 first_colon: Colon = Colon.field()
1498 #: The second slice operator, usually omitted
1499 second_colon: Union[Colon, MaybeSentinel] = MaybeSentinel.DEFAULT
1501 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Slice":
1502 return Slice(
1503 lower=visit_optional(self, "lower", self.lower, visitor),
1504 first_colon=visit_required(self, "first_colon", self.first_colon, visitor),
1505 upper=visit_optional(self, "upper", self.upper, visitor),
1506 second_colon=visit_sentinel(
1507 self, "second_colon", self.second_colon, visitor
1508 ),
1509 step=visit_optional(self, "step", self.step, visitor),
1510 )
1512 def _codegen_impl(self, state: CodegenState) -> None:
1513 lower = self.lower
1514 if lower is not None:
1515 lower._codegen(state)
1516 self.first_colon._codegen(state)
1517 upper = self.upper
1518 if upper is not None:
1519 upper._codegen(state)
1520 second_colon = self.second_colon
1521 if second_colon is MaybeSentinel.DEFAULT and self.step is not None:
1522 state.add_token(":")
1523 elif isinstance(second_colon, Colon):
1524 second_colon._codegen(state)
1525 step = self.step
1526 if step is not None:
1527 step._codegen(state)
1530@add_slots
1531@dataclass(frozen=True)
1532class SubscriptElement(CSTNode):
1533 """
1534 Part of a sequence of slices in a :class:`Subscript`, such as ``1:2, 3``. This is
1535 not used in Python's standard library, but it is used in some third-party
1536 libraries. For example, `NumPy uses it to select values and ranges from
1537 multi-dimensional arrays
1538 <https://docs.scipy.org/doc/numpy-1.10.1/user/basics.indexing.html>`_.
1539 """
1541 #: A slice or index that is part of a subscript.
1542 slice: BaseSlice
1544 #: A separating comma, with any whitespace it owns.
1545 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
1547 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "SubscriptElement":
1548 return SubscriptElement(
1549 slice=visit_required(self, "slice", self.slice, visitor),
1550 comma=visit_sentinel(self, "comma", self.comma, visitor),
1551 )
1553 def _codegen_impl(self, state: CodegenState, default_comma: bool = False) -> None:
1554 with state.record_syntactic_position(self):
1555 self.slice._codegen(state)
1557 comma = self.comma
1558 if comma is MaybeSentinel.DEFAULT and default_comma:
1559 state.add_token(", ")
1560 elif isinstance(comma, Comma):
1561 comma._codegen(state)
1564@add_slots
1565@dataclass(frozen=True)
1566class Subscript(BaseAssignTargetExpression, BaseDelTargetExpression):
1567 """
1568 A indexed subscript reference (:class:`Index`) such as ``x[2]``, a :class:`Slice`
1569 such as ``x[1:-1]``, or an extended slice (:class:`SubscriptElement`) such as ``x[1:2, 3]``.
1570 """
1572 #: The left-hand expression which, when evaluated, will be subscripted, such as
1573 #: ``x`` in ``x[2]``.
1574 value: BaseExpression
1576 #: The :class:`SubscriptElement` to extract from the ``value``.
1577 slice: Sequence[SubscriptElement]
1579 lbracket: LeftSquareBracket = LeftSquareBracket.field()
1580 #: Brackets after the ``value`` surrounding the ``slice``.
1581 rbracket: RightSquareBracket = RightSquareBracket.field()
1583 lpar: Sequence[LeftParen] = ()
1584 #: Sequence of parenthesis for precedence dictation.
1585 rpar: Sequence[RightParen] = ()
1587 #: Whitespace after the ``value``, but before the ``lbracket``.
1588 whitespace_after_value: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1590 def _validate(self) -> None:
1591 super(Subscript, self)._validate()
1592 # Validate valid commas
1593 if len(self.slice) < 1:
1594 raise CSTValidationError("Cannot have empty SubscriptElement.")
1596 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Subscript":
1597 return Subscript(
1598 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1599 value=visit_required(self, "value", self.value, visitor),
1600 whitespace_after_value=visit_required(
1601 self, "whitespace_after_value", self.whitespace_after_value, visitor
1602 ),
1603 lbracket=visit_required(self, "lbracket", self.lbracket, visitor),
1604 slice=visit_sequence(self, "slice", self.slice, visitor),
1605 rbracket=visit_required(self, "rbracket", self.rbracket, visitor),
1606 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1607 )
1609 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1610 if position == ExpressionPosition.LEFT:
1611 return True
1612 if super(Subscript, self)._safe_to_use_with_word_operator(position):
1613 return True
1614 if position == ExpressionPosition.RIGHT:
1615 return self.value._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
1616 return False
1618 def _codegen_impl(self, state: CodegenState) -> None:
1619 with self._parenthesize(state):
1620 self.value._codegen(state)
1621 self.whitespace_after_value._codegen(state)
1622 self.lbracket._codegen(state)
1623 lastslice = len(self.slice) - 1
1624 for i, slice in enumerate(self.slice):
1625 slice._codegen(state, default_comma=(i != lastslice))
1626 self.rbracket._codegen(state)
1629@add_slots
1630@dataclass(frozen=True)
1631class Annotation(CSTNode):
1632 """
1633 An annotation for a function (`PEP 3107`_) or on a variable (`PEP 526`_). Typically
1634 these are used in the context of type hints (`PEP 484`_), such as::
1636 # a variable with a type
1637 good_ideas: List[str] = []
1639 # a function with type annotations
1640 def concat(substrings: Sequence[str]) -> str:
1641 ...
1643 .. _PEP 3107: https://www.python.org/dev/peps/pep-3107/
1644 .. _PEP 526: https://www.python.org/dev/peps/pep-0526/
1645 .. _PEP 484: https://www.python.org/dev/peps/pep-0484/
1646 """
1648 #: The annotation's value itself. This is the part of the annotation after the
1649 #: colon or arrow.
1650 annotation: BaseExpression
1652 whitespace_before_indicator: Union[
1653 BaseParenthesizableWhitespace, MaybeSentinel
1654 ] = MaybeSentinel.DEFAULT
1655 whitespace_after_indicator: BaseParenthesizableWhitespace = SimpleWhitespace.field(
1656 " "
1657 )
1659 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Annotation":
1660 return Annotation(
1661 whitespace_before_indicator=visit_sentinel(
1662 self,
1663 "whitespace_before_indicator",
1664 self.whitespace_before_indicator,
1665 visitor,
1666 ),
1667 whitespace_after_indicator=visit_required(
1668 self,
1669 "whitespace_after_indicator",
1670 self.whitespace_after_indicator,
1671 visitor,
1672 ),
1673 annotation=visit_required(self, "annotation", self.annotation, visitor),
1674 )
1676 def _codegen_impl(
1677 self, state: CodegenState, default_indicator: Optional[str] = None
1678 ) -> None:
1679 # First, figure out the indicator which tells us default whitespace.
1680 if default_indicator is None:
1681 raise CSTCodegenError(
1682 "Must specify a concrete default_indicator if default used on indicator."
1683 )
1685 # Now, output the whitespace
1686 whitespace_before_indicator = self.whitespace_before_indicator
1687 if isinstance(whitespace_before_indicator, BaseParenthesizableWhitespace):
1688 whitespace_before_indicator._codegen(state)
1689 elif isinstance(whitespace_before_indicator, MaybeSentinel):
1690 if default_indicator == "->":
1691 state.add_token(" ")
1692 else:
1693 raise Exception("Logic error!")
1695 # Now, output the indicator and the rest of the annotation
1696 state.add_token(default_indicator)
1697 self.whitespace_after_indicator._codegen(state)
1699 with state.record_syntactic_position(self):
1700 self.annotation._codegen(state)
1703@add_slots
1704@dataclass(frozen=True)
1705class ParamStar(CSTNode):
1706 """
1707 A sentinel indicator on a :class:`Parameters` list to denote that the subsequent
1708 params are keyword-only args.
1710 This syntax is described in `PEP 3102`_.
1712 .. _PEP 3102: https://www.python.org/dev/peps/pep-3102/#specification
1713 """
1715 # Comma that comes after the star.
1716 comma: Comma = Comma.field(whitespace_after=SimpleWhitespace(" "))
1718 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ParamStar":
1719 return ParamStar(comma=visit_required(self, "comma", self.comma, visitor))
1721 def _codegen_impl(self, state: CodegenState) -> None:
1722 state.add_token("*")
1723 self.comma._codegen(state)
1726@add_slots
1727@dataclass(frozen=True)
1728class ParamSlash(CSTNode):
1729 """
1730 A sentinel indicator on a :class:`Parameters` list to denote that the previous
1731 params are positional-only args.
1733 This syntax is described in `PEP 570`_.
1735 .. _PEP 570: https://www.python.org/dev/peps/pep-0570/#specification
1736 """
1738 #: Optional comma that comes after the slash. This comma doesn't own the whitespace
1739 #: between ``/`` and ``,``.
1740 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
1742 #: Whitespace after the ``/`` character. This is captured here in case there is a
1743 #: comma.
1744 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1746 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ParamSlash":
1747 return ParamSlash(
1748 comma=visit_sentinel(self, "comma", self.comma, visitor),
1749 whitespace_after=visit_required(
1750 self, "whitespace_after", self.whitespace_after, visitor
1751 ),
1752 )
1754 def _codegen_impl(self, state: CodegenState, default_comma: bool = False) -> None:
1755 state.add_token("/")
1757 self.whitespace_after._codegen(state)
1758 comma = self.comma
1759 if comma is MaybeSentinel.DEFAULT and default_comma:
1760 state.add_token(", ")
1761 elif isinstance(comma, Comma):
1762 comma._codegen(state)
1765@add_slots
1766@dataclass(frozen=True)
1767class Param(CSTNode):
1768 """
1769 A positional or keyword argument in a :class:`Parameters` list. May contain an
1770 :class:`Annotation` and, in some cases, a ``default``.
1771 """
1773 #: The parameter name itself.
1774 name: Name
1776 #: Any optional :class:`Annotation`. These annotations are usually used as type
1777 #: hints.
1778 annotation: Optional[Annotation] = None
1780 #: The equal sign used to denote assignment if there is a default.
1781 equal: Union[AssignEqual, MaybeSentinel] = MaybeSentinel.DEFAULT
1783 #: Any optional default value, used when the argument is not supplied.
1784 default: Optional[BaseExpression] = None
1786 #: A trailing comma. If one is not provided, :class:`MaybeSentinel` will be
1787 #: replaced with a comma only if a comma is required.
1788 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
1790 #: Zero, one, or two asterisks appearing before name for :class:`Param`'s
1791 #: ``star_arg`` and ``star_kwarg``.
1792 star: Union[str, MaybeSentinel] = MaybeSentinel.DEFAULT
1794 #: The whitespace before ``name``. It will appear after ``star`` when a star
1795 #: exists.
1796 whitespace_after_star: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1798 #: The whitespace after this entire node.
1799 whitespace_after_param: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1801 def _validate(self) -> None:
1802 if self.default is None and isinstance(self.equal, AssignEqual):
1803 raise CSTValidationError(
1804 "Must have a default when specifying an AssignEqual."
1805 )
1806 if isinstance(self.star, str) and self.star not in ("", "*", "**"):
1807 raise CSTValidationError("Must specify either '', '*' or '**' for star.")
1809 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Param":
1810 return Param(
1811 star=self.star,
1812 whitespace_after_star=visit_required(
1813 self, "whitespace_after_star", self.whitespace_after_star, visitor
1814 ),
1815 name=visit_required(self, "name", self.name, visitor),
1816 annotation=visit_optional(self, "annotation", self.annotation, visitor),
1817 equal=visit_sentinel(self, "equal", self.equal, visitor),
1818 default=visit_optional(self, "default", self.default, visitor),
1819 comma=visit_sentinel(self, "comma", self.comma, visitor),
1820 whitespace_after_param=visit_required(
1821 self, "whitespace_after_param", self.whitespace_after_param, visitor
1822 ),
1823 )
1825 def _codegen_impl(
1826 self,
1827 state: CodegenState,
1828 default_star: Optional[str] = None,
1829 default_comma: bool = False,
1830 ) -> None:
1831 with state.record_syntactic_position(self):
1832 star = self.star
1833 if isinstance(star, MaybeSentinel):
1834 if default_star is None:
1835 raise CSTCodegenError(
1836 "Must specify a concrete default_star if default used on star."
1837 )
1838 star = default_star
1839 if isinstance(star, str):
1840 state.add_token(star)
1841 self.whitespace_after_star._codegen(state)
1842 self.name._codegen(state)
1844 annotation = self.annotation
1845 if annotation is not None:
1846 annotation._codegen(state, default_indicator=":")
1847 equal = self.equal
1848 if equal is MaybeSentinel.DEFAULT and self.default is not None:
1849 state.add_token(" = ")
1850 elif isinstance(equal, AssignEqual):
1851 equal._codegen(state)
1852 default = self.default
1853 if default is not None:
1854 default._codegen(state)
1855 comma = self.comma
1856 if comma is MaybeSentinel.DEFAULT and default_comma:
1857 state.add_token(", ")
1858 elif isinstance(comma, Comma):
1859 comma._codegen(state)
1861 self.whitespace_after_param._codegen(state)
1864@add_slots
1865@dataclass(frozen=True)
1866class Parameters(CSTNode):
1867 """
1868 A function or lambda parameter list.
1869 """
1871 #: Positional parameters, with or without defaults. Positional parameters
1872 #: with defaults must all be after those without defaults.
1873 params: Sequence[Param] = ()
1875 # Optional parameter that captures unspecified positional arguments or a sentinel
1876 # star that dictates parameters following are kwonly args.
1877 star_arg: Union[Param, ParamStar, MaybeSentinel] = MaybeSentinel.DEFAULT
1879 #: Keyword-only params that may or may not have defaults.
1880 kwonly_params: Sequence[Param] = ()
1882 #: Optional parameter that captures unspecified kwargs.
1883 star_kwarg: Optional[Param] = None
1885 #: Positional-only parameters, with or without defaults. Positional-only
1886 #: parameters with defaults must all be after those without defaults.
1887 posonly_params: Sequence[Param] = ()
1889 #: Optional sentinel that dictates parameters preceeding are positional-only
1890 #: args.
1891 posonly_ind: Union[ParamSlash, MaybeSentinel] = MaybeSentinel.DEFAULT
1893 def _validate_stars_sequence(self, vals: Sequence[Param], *, section: str) -> None:
1894 if len(vals) == 0:
1895 return
1896 for val in vals:
1897 if isinstance(val.star, str) and val.star != "":
1898 raise CSTValidationError(
1899 f"Expecting a star prefix of '' for {section} Param."
1900 )
1902 def _validate_posonly_ind(self) -> None:
1903 if isinstance(self.posonly_ind, ParamSlash) and len(self.posonly_params) == 0:
1904 raise CSTValidationError(
1905 "Must have at least one posonly param if ParamSlash is used."
1906 )
1908 def _validate_kwonly_star(self) -> None:
1909 if isinstance(self.star_arg, ParamStar) and len(self.kwonly_params) == 0:
1910 raise CSTValidationError(
1911 "Must have at least one kwonly param if ParamStar is used."
1912 )
1914 def _validate_defaults(self) -> None:
1915 seen_default = False
1916 # pyre-fixme[60]: Concatenation not yet support for multiple variadic
1917 # tuples: `*self.posonly_params, *self.params`.
1918 for param in (*self.posonly_params, *self.params):
1919 if param.default:
1920 # Mark that we've moved onto defaults
1921 if not seen_default:
1922 seen_default = True
1923 else:
1924 if seen_default:
1925 # We accidentally included a non-default after a default arg!
1926 raise CSTValidationError(
1927 "Cannot have param without defaults following a param with defaults."
1928 )
1929 star_arg = self.star_arg
1930 if isinstance(star_arg, Param) and star_arg.default is not None:
1931 raise CSTValidationError("Cannot have default for star_arg.")
1932 star_kwarg = self.star_kwarg
1933 if star_kwarg is not None and star_kwarg.default is not None:
1934 raise CSTValidationError("Cannot have default for star_kwarg.")
1936 def _validate_stars(self) -> None:
1937 if len(self.params) > 0:
1938 self._validate_stars_sequence(self.params, section="params")
1939 if len(self.posonly_params) > 0:
1940 self._validate_stars_sequence(self.posonly_params, section="posonly_params")
1941 star_arg = self.star_arg
1942 if (
1943 isinstance(star_arg, Param)
1944 and isinstance(star_arg.star, str)
1945 and star_arg.star != "*"
1946 ):
1947 raise CSTValidationError(
1948 "Expecting a star prefix of '*' for star_arg Param."
1949 )
1950 if len(self.kwonly_params) > 0:
1951 self._validate_stars_sequence(self.kwonly_params, section="kwonly_params")
1952 star_kwarg = self.star_kwarg
1953 if (
1954 star_kwarg is not None
1955 and isinstance(star_kwarg.star, str)
1956 and star_kwarg.star != "**"
1957 ):
1958 raise CSTValidationError(
1959 "Expecting a star prefix of '**' for star_kwarg Param."
1960 )
1962 def _validate(self) -> None:
1963 # Validate posonly_params slash placement semantics.
1964 self._validate_posonly_ind()
1965 # Validate kwonly_param star placement semantics.
1966 self._validate_kwonly_star()
1967 # Validate defaults semantics for params and star_arg/star_kwarg.
1968 self._validate_defaults()
1969 # Validate that we don't have random stars on non star_kwarg.
1970 self._validate_stars()
1972 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Parameters":
1973 return Parameters(
1974 posonly_params=visit_sequence(
1975 self, "posonly_params", self.posonly_params, visitor
1976 ),
1977 posonly_ind=visit_sentinel(self, "posonly_ind", self.posonly_ind, visitor),
1978 params=visit_sequence(self, "params", self.params, visitor),
1979 star_arg=visit_sentinel(self, "star_arg", self.star_arg, visitor),
1980 kwonly_params=visit_sequence(
1981 self, "kwonly_params", self.kwonly_params, visitor
1982 ),
1983 star_kwarg=visit_optional(self, "star_kwarg", self.star_kwarg, visitor),
1984 )
1986 def _codegen_impl(self, state: CodegenState) -> None: # noqa: C901
1987 # Compute the star existence first so we can ask about whether
1988 # each element is the last in the list or not.
1989 star_arg = self.star_arg
1990 if isinstance(star_arg, MaybeSentinel):
1991 starincluded = len(self.kwonly_params) > 0
1992 elif isinstance(star_arg, (Param, ParamStar)):
1993 starincluded = True
1994 else:
1995 starincluded = False
1996 # Render out the positional-only params first. They will always have trailing
1997 # commas because in order to have positional-only params, there must be a
1998 # slash afterwards.
1999 for i, param in enumerate(self.posonly_params):
2000 param._codegen(state, default_star="", default_comma=True)
2001 # Render out the positional-only indicator if necessary.
2002 more_values = (
2003 starincluded
2004 or len(self.params) > 0
2005 or len(self.kwonly_params) > 0
2006 or self.star_kwarg is not None
2007 )
2008 posonly_ind = self.posonly_ind
2009 if isinstance(posonly_ind, ParamSlash):
2010 # Its explicitly included, so render the version we have here which
2011 # might have spacing applied to its comma.
2012 posonly_ind._codegen(state, default_comma=more_values)
2013 elif len(self.posonly_params) > 0:
2014 if more_values:
2015 state.add_token("/, ")
2016 else:
2017 state.add_token("/")
2018 # Render out the params next, computing necessary trailing commas.
2019 lastparam = len(self.params) - 1
2020 more_values = (
2021 starincluded or len(self.kwonly_params) > 0 or self.star_kwarg is not None
2022 )
2023 for i, param in enumerate(self.params):
2024 param._codegen(
2025 state, default_star="", default_comma=(i < lastparam or more_values)
2026 )
2027 # Render out optional star sentinel if its explicitly included or
2028 # if we are inferring it from kwonly_params. Otherwise, render out the
2029 # optional star_arg.
2030 if isinstance(star_arg, MaybeSentinel):
2031 if starincluded:
2032 state.add_token("*, ")
2033 elif isinstance(star_arg, Param):
2034 more_values = len(self.kwonly_params) > 0 or self.star_kwarg is not None
2035 star_arg._codegen(state, default_star="*", default_comma=more_values)
2036 elif isinstance(star_arg, ParamStar):
2037 star_arg._codegen(state)
2038 # Render out the kwonly_args next, computing necessary trailing commas.
2039 lastparam = len(self.kwonly_params) - 1
2040 more_values = self.star_kwarg is not None
2041 for i, param in enumerate(self.kwonly_params):
2042 param._codegen(
2043 state, default_star="", default_comma=(i < lastparam or more_values)
2044 )
2045 # Finally, render out any optional star_kwarg
2046 star_kwarg = self.star_kwarg
2047 if star_kwarg is not None:
2048 star_kwarg._codegen(state, default_star="**", default_comma=False)
2051@add_slots
2052@dataclass(frozen=True)
2053class Lambda(BaseExpression):
2054 """
2055 A lambda expression that creates an anonymous function.
2057 ::
2059 Lambda(
2060 params=Parameters([Param(Name("arg"))]),
2061 body=Ellipsis(),
2062 )
2064 Represents the following code::
2066 lambda arg: ...
2068 Named functions statements are provided by :class:`FunctionDef`.
2069 """
2071 #: The arguments to the lambda. This is similar to the arguments on a
2072 #: :class:`FunctionDef`, however lambda arguments are not allowed to have an
2073 #: :class:`Annotation`.
2074 params: Parameters
2076 #: The value that the lambda computes and returns when called.
2077 body: BaseExpression
2079 #: The colon separating the parameters from the body.
2080 colon: Colon = Colon.field(whitespace_after=SimpleWhitespace(" "))
2082 lpar: Sequence[LeftParen] = ()
2083 #: Sequence of parenthesis for precedence dictation.
2084 rpar: Sequence[RightParen] = ()
2086 #: Whitespace after the lambda keyword, but before any argument or the colon.
2087 whitespace_after_lambda: Union[
2088 BaseParenthesizableWhitespace, MaybeSentinel
2089 ] = MaybeSentinel.DEFAULT
2091 def _validate(self) -> None:
2092 # Validate parents
2093 super(Lambda, self)._validate()
2094 # Sum up all parameters
2095 all_params = [
2096 *self.params.posonly_params,
2097 *self.params.params,
2098 *self.params.kwonly_params,
2099 ]
2100 star_arg = self.params.star_arg
2101 if isinstance(star_arg, Param):
2102 all_params.append(star_arg)
2103 star_kwarg = self.params.star_kwarg
2104 if star_kwarg is not None:
2105 all_params.append(star_kwarg)
2106 # Check for nonzero parameters because several checks care
2107 # about this.
2108 if len(all_params) > 0:
2109 for param in all_params:
2110 if param.annotation is not None:
2111 raise CSTValidationError(
2112 "Lambda params cannot have type annotations."
2113 )
2114 whitespace_after_lambda = self.whitespace_after_lambda
2115 if (
2116 isinstance(whitespace_after_lambda, BaseParenthesizableWhitespace)
2117 and whitespace_after_lambda.empty
2118 ):
2119 raise CSTValidationError(
2120 "Must have at least one space after lambda when specifying params"
2121 )
2123 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Lambda":
2124 return Lambda(
2125 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2126 whitespace_after_lambda=visit_sentinel(
2127 self, "whitespace_after_lambda", self.whitespace_after_lambda, visitor
2128 ),
2129 params=visit_required(self, "params", self.params, visitor),
2130 colon=visit_required(self, "colon", self.colon, visitor),
2131 body=visit_required(self, "body", self.body, visitor),
2132 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2133 )
2135 def _codegen_impl(self, state: CodegenState) -> None:
2136 with self._parenthesize(state):
2137 state.add_token("lambda")
2138 whitespace_after_lambda = self.whitespace_after_lambda
2139 if isinstance(whitespace_after_lambda, MaybeSentinel):
2140 if not (
2141 len(self.params.posonly_params) == 0
2142 and len(self.params.params) == 0
2143 and not isinstance(self.params.star_arg, Param)
2144 and len(self.params.kwonly_params) == 0
2145 and self.params.star_kwarg is None
2146 ):
2147 # We have one or more params, provide a space
2148 state.add_token(" ")
2149 elif isinstance(whitespace_after_lambda, BaseParenthesizableWhitespace):
2150 whitespace_after_lambda._codegen(state)
2151 self.params._codegen(state)
2152 self.colon._codegen(state)
2153 self.body._codegen(state)
2156@add_slots
2157@dataclass(frozen=True)
2158class Arg(CSTNode):
2159 """
2160 A single argument to a :class:`Call`.
2162 This supports named keyword arguments in the form of ``keyword=value`` and variable
2163 argument expansion using ``*args`` or ``**kwargs`` syntax.
2164 """
2166 #: The argument expression itself, not including a preceding keyword, or any of
2167 #: the surrounding the value, like a comma or asterisks.
2168 value: BaseExpression
2170 #: Optional keyword for the argument.
2171 keyword: Optional[Name] = None
2173 #: The equal sign used to denote assignment if there is a keyword.
2174 equal: Union[AssignEqual, MaybeSentinel] = MaybeSentinel.DEFAULT
2176 #: Any trailing comma.
2177 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2179 #: A string with zero, one, or two asterisks appearing before the name. These are
2180 #: expanded into variable number of positional or keyword arguments.
2181 star: Literal["", "*", "**"] = ""
2183 #: Whitespace after the ``star`` (if it exists), but before the ``keyword`` or
2184 #: ``value`` (if no keyword is provided).
2185 whitespace_after_star: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2186 #: Whitespace after this entire node. The :class:`Comma` node (if it exists) may
2187 #: also store some trailing whitespace.
2188 whitespace_after_arg: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2190 def _validate(self) -> None:
2191 if self.keyword is None and isinstance(self.equal, AssignEqual):
2192 raise CSTValidationError(
2193 "Must have a keyword when specifying an AssignEqual."
2194 )
2195 if self.star not in ("", "*", "**"):
2196 raise CSTValidationError("Must specify either '', '*' or '**' for star.")
2197 if self.star in ("*", "**") and self.keyword is not None:
2198 raise CSTValidationError("Cannot specify a star and a keyword together.")
2200 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Arg":
2201 return Arg(
2202 star=self.star,
2203 whitespace_after_star=visit_required(
2204 self, "whitespace_after_star", self.whitespace_after_star, visitor
2205 ),
2206 keyword=visit_optional(self, "keyword", self.keyword, visitor),
2207 equal=visit_sentinel(self, "equal", self.equal, visitor),
2208 value=visit_required(self, "value", self.value, visitor),
2209 comma=visit_sentinel(self, "comma", self.comma, visitor),
2210 whitespace_after_arg=visit_required(
2211 self, "whitespace_after_arg", self.whitespace_after_arg, visitor
2212 ),
2213 )
2215 def _codegen_impl(self, state: CodegenState, default_comma: bool = False) -> None:
2216 with state.record_syntactic_position(self):
2217 state.add_token(self.star)
2218 self.whitespace_after_star._codegen(state)
2219 keyword = self.keyword
2220 if keyword is not None:
2221 keyword._codegen(state)
2222 equal = self.equal
2223 if equal is MaybeSentinel.DEFAULT and self.keyword is not None:
2224 state.add_token(" = ")
2225 elif isinstance(equal, AssignEqual):
2226 equal._codegen(state)
2227 self.value._codegen(state)
2229 comma = self.comma
2230 if comma is MaybeSentinel.DEFAULT and default_comma:
2231 state.add_token(", ")
2232 elif isinstance(comma, Comma):
2233 comma._codegen(state)
2234 self.whitespace_after_arg._codegen(state)
2237class _BaseExpressionWithArgs(BaseExpression, ABC):
2238 """
2239 Arguments are complicated enough that we can't represent them easily
2240 in typing. So, we have common validation functions here.
2241 """
2243 __slots__ = ()
2245 #: Sequence of arguments that will be passed to the function call.
2246 args: Sequence[Arg] = ()
2248 def _check_kwargs_or_keywords(self, arg: Arg) -> None:
2249 """
2250 Validates that we only have a mix of "keyword=arg" and "**arg" expansion.
2251 """
2253 if arg.keyword is not None:
2254 # Valid, keyword argument
2255 return None
2256 elif arg.star == "**":
2257 # Valid, kwargs
2258 return None
2259 elif arg.star == "*":
2260 # Invalid, cannot have "*" follow "**"
2261 raise CSTValidationError(
2262 "Cannot have iterable argument unpacking after keyword argument unpacking."
2263 )
2264 else:
2265 # Invalid, cannot have positional argument follow **/keyword
2266 raise CSTValidationError(
2267 "Cannot have positional argument after keyword argument unpacking."
2268 )
2270 def _check_starred_or_keywords(
2271 self, arg: Arg
2272 ) -> Optional[Callable[[Arg], Callable[[Arg], None]]]:
2273 """
2274 Validates that we only have a mix of "*arg" expansion and "keyword=arg".
2275 """
2277 if arg.keyword is not None:
2278 # Valid, keyword argument
2279 return None
2280 elif arg.star == "**":
2281 # Valid, but we now no longer allow "*" args
2282 # pyre-fixme[7]: Expected `Optional[Callable[[Arg], Callable[...,
2283 # Any]]]` but got `Callable[[Arg], Optional[Callable[[Arg], Callable[...,
2284 # Any]]]]`.
2285 return self._check_kwargs_or_keywords
2286 elif arg.star == "*":
2287 # Valid, iterable unpacking
2288 return None
2289 else:
2290 # Invalid, cannot have positional argument follow **/keyword
2291 raise CSTValidationError(
2292 "Cannot have positional argument after keyword argument."
2293 )
2295 def _check_positional(
2296 self, arg: Arg
2297 ) -> Optional[Callable[[Arg], Callable[[Arg], Callable[[Arg], None]]]]:
2298 """
2299 Validates that we only have a mix of positional args and "*arg" expansion.
2300 """
2302 if arg.keyword is not None:
2303 # Valid, but this puts us into starred/keyword state
2304 # pyre-fixme[7]: Expected `Optional[Callable[[Arg], Callable[...,
2305 # Any]]]` but got `Callable[[Arg], Optional[Callable[[Arg], Callable[...,
2306 # Any]]]]`.
2307 return self._check_starred_or_keywords
2308 elif arg.star == "**":
2309 # Valid, but we skip states to kwargs/keywords
2310 # pyre-fixme[7]: Expected `Optional[Callable[[Arg], Callable[...,
2311 # Any]]]` but got `Callable[[Arg], Optional[Callable[[Arg], Callable[...,
2312 # Any]]]]`.
2313 return self._check_kwargs_or_keywords
2314 elif arg.star == "*":
2315 # Valid, iterator expansion
2316 return None
2317 else:
2318 # Valid, allowed to have positional arguments here
2319 return None
2321 # pyre-fixme[30]: Pyre gave up inferring some types - function `_validate` was
2322 # too complex.
2323 def _validate(self) -> None:
2324 # Validate any super-class stuff, whatever it may be.
2325 super()._validate()
2326 # Now, validate the weird intermingling rules for arguments by running
2327 # a small validator state machine. This works by passing each argument
2328 # to a validator function which can either raise an exception if it
2329 # detects an invalid sequence, return a new validator to be used for the
2330 # next arg, or return None to use the same validator. We could enforce
2331 # always returning ourselves instead of None but it ends up making the
2332 # functions themselves less readable. In this way, the current validator
2333 # function encodes the state we're in (positional state, iterable
2334 # expansion state, or dictionary expansion state).
2335 validator = self._check_positional
2336 for arg in self.args:
2337 validator = validator(arg) or validator
2340@add_slots
2341@dataclass(frozen=True)
2342class Call(_BaseExpressionWithArgs):
2343 """
2344 An expression representing a function call, such as ``do_math(1, 2)`` or
2345 ``picture.post_on_instagram()``.
2347 Function calls consist of a function name and a sequence of arguments wrapped in
2348 :class:`Arg` nodes.
2349 """
2351 #: The expression resulting in a callable that we are to call. Often a :class:`Name`
2352 #: or :class:`Attribute`.
2353 func: BaseExpression
2355 #: The arguments to pass to the resulting callable. These may be a mix of
2356 #: positional arguments, keyword arguments, or "starred" arguments.
2357 args: Sequence[Arg] = ()
2359 lpar: Sequence[LeftParen] = ()
2360 #: Sequence of parenthesis for precedence dictation. These are not the parenthesis
2361 #: before and after the list of ``args``, but rather arguments around the entire
2362 #: call expression, such as ``(( do_math(1, 2) ))``.
2363 rpar: Sequence[RightParen] = ()
2365 #: Whitespace after the ``func`` name, but before the opening parenthesis.
2366 whitespace_after_func: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2367 #: Whitespace after the opening parenthesis but before the first argument (if there
2368 #: are any). Whitespace after the last argument but before the closing parenthesis
2369 #: is owned by the last :class:`Arg` if it exists.
2370 whitespace_before_args: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2372 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2373 """
2374 Calls have a close paren on the right side regardless of whether they're
2375 parenthesized as a whole. As a result, they are safe to use directly against
2376 an adjacent node to the right.
2377 """
2378 if position == ExpressionPosition.LEFT:
2379 return True
2380 if super(Call, self)._safe_to_use_with_word_operator(position):
2381 return True
2382 if position == ExpressionPosition.RIGHT:
2383 return self.func._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
2384 return False
2386 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Call":
2387 return Call(
2388 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2389 func=visit_required(self, "func", self.func, visitor),
2390 whitespace_after_func=visit_required(
2391 self, "whitespace_after_func", self.whitespace_after_func, visitor
2392 ),
2393 whitespace_before_args=visit_required(
2394 self, "whitespace_before_args", self.whitespace_before_args, visitor
2395 ),
2396 args=visit_sequence(self, "args", self.args, visitor),
2397 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2398 )
2400 def _codegen_impl(self, state: CodegenState) -> None:
2401 with self._parenthesize(state):
2402 self.func._codegen(state)
2403 self.whitespace_after_func._codegen(state)
2404 state.add_token("(")
2405 self.whitespace_before_args._codegen(state)
2406 lastarg = len(self.args) - 1
2407 for i, arg in enumerate(self.args):
2408 arg._codegen(state, default_comma=(i != lastarg))
2409 state.add_token(")")
2412@add_slots
2413@dataclass(frozen=True)
2414class Await(BaseExpression):
2415 """
2416 An await expression. Await expressions are only valid inside the body of an
2417 asynchronous :class:`FunctionDef` or (as of Python 3.7) inside of an asynchronous
2418 :class:`GeneratorExp` nodes.
2419 """
2421 #: The actual expression we need to wait for.
2422 expression: BaseExpression
2424 lpar: Sequence[LeftParen] = ()
2425 #: Sequence of parenthesis for precedence dictation.
2426 rpar: Sequence[RightParen] = ()
2428 #: Whitespace that appears after the ``async`` keyword, but before the inner
2429 #: ``expression``.
2430 whitespace_after_await: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2432 def _validate(self) -> None:
2433 # Validate any super-class stuff, whatever it may be.
2434 super(Await, self)._validate()
2435 # Make sure we don't run identifiers together.
2436 if (
2437 self.whitespace_after_await.empty
2438 and not self.expression._safe_to_use_with_word_operator(
2439 ExpressionPosition.RIGHT
2440 )
2441 ):
2442 raise CSTValidationError("Must have at least one space after await")
2444 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Await":
2445 return Await(
2446 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2447 whitespace_after_await=visit_required(
2448 self, "whitespace_after_await", self.whitespace_after_await, visitor
2449 ),
2450 expression=visit_required(self, "expression", self.expression, visitor),
2451 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2452 )
2454 def _codegen_impl(self, state: CodegenState) -> None:
2455 with self._parenthesize(state):
2456 state.add_token("await")
2457 self.whitespace_after_await._codegen(state)
2458 self.expression._codegen(state)
2461@add_slots
2462@dataclass(frozen=True)
2463class IfExp(BaseExpression):
2464 """
2465 An if expression of the form ``body if test else orelse``.
2467 If statements are provided by :class:`If` and :class:`Else` nodes.
2468 """
2470 #: The test to perform.
2471 test: BaseExpression
2473 #: The expression to evaluate when the test is true.
2474 body: BaseExpression
2476 #: The expression to evaluate when the test is false.
2477 orelse: BaseExpression
2479 lpar: Sequence[LeftParen] = ()
2480 #: Sequence of parenthesis for precedence dictation.
2481 rpar: Sequence[RightParen] = ()
2483 #: Whitespace after the ``body`` expression, but before the ``if`` keyword.
2484 whitespace_before_if: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2486 #: Whitespace after the ``if`` keyword, but before the ``test`` clause.
2487 whitespace_after_if: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2489 #: Whitespace after the ``test`` expression, but before the ``else`` keyword.
2490 whitespace_before_else: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2492 #: Whitespace after the ``else`` keyword, but before the ``orelse`` expression.
2493 whitespace_after_else: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2495 def _validate(self) -> None:
2496 # Paren validation and such
2497 super(IfExp, self)._validate()
2498 # Validate spacing rules
2499 if (
2500 self.whitespace_before_if.empty
2501 and not self.body._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
2502 ):
2503 raise CSTValidationError(
2504 "Must have at least one space before 'if' keyword."
2505 )
2506 if (
2507 self.whitespace_after_if.empty
2508 and not self.test._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
2509 ):
2510 raise CSTValidationError("Must have at least one space after 'if' keyword.")
2511 if (
2512 self.whitespace_before_else.empty
2513 and not self.test._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
2514 ):
2515 raise CSTValidationError(
2516 "Must have at least one space before 'else' keyword."
2517 )
2518 if (
2519 self.whitespace_after_else.empty
2520 and not self.orelse._safe_to_use_with_word_operator(
2521 ExpressionPosition.RIGHT
2522 )
2523 ):
2524 raise CSTValidationError(
2525 "Must have at least one space after 'else' keyword."
2526 )
2528 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "IfExp":
2529 return IfExp(
2530 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2531 body=visit_required(self, "body", self.body, visitor),
2532 whitespace_before_if=visit_required(
2533 self, "whitespace_before_if", self.whitespace_before_if, visitor
2534 ),
2535 whitespace_after_if=visit_required(
2536 self, "whitespace_after_if", self.whitespace_after_if, visitor
2537 ),
2538 test=visit_required(self, "test", self.test, visitor),
2539 whitespace_before_else=visit_required(
2540 self, "whitespace_before_else", self.whitespace_before_else, visitor
2541 ),
2542 whitespace_after_else=visit_required(
2543 self, "whitespace_after_else", self.whitespace_after_else, visitor
2544 ),
2545 orelse=visit_required(self, "orelse", self.orelse, visitor),
2546 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2547 )
2549 def _codegen_impl(self, state: CodegenState) -> None:
2550 with self._parenthesize(state):
2551 self.body._codegen(state)
2552 self.whitespace_before_if._codegen(state)
2553 state.add_token("if")
2554 self.whitespace_after_if._codegen(state)
2555 self.test._codegen(state)
2556 self.whitespace_before_else._codegen(state)
2557 state.add_token("else")
2558 self.whitespace_after_else._codegen(state)
2559 self.orelse._codegen(state)
2562@add_slots
2563@dataclass(frozen=True)
2564class From(CSTNode):
2565 """
2566 A ``from x`` stanza in a :class:`Yield` or :class:`Raise`.
2567 """
2569 #: The expression that we are yielding/raising from.
2570 item: BaseExpression
2572 #: The whitespace at the very start of this node.
2573 whitespace_before_from: Union[
2574 BaseParenthesizableWhitespace, MaybeSentinel
2575 ] = MaybeSentinel.DEFAULT
2577 #: The whitespace after the ``from`` keyword, but before the ``item``.
2578 whitespace_after_from: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2580 def _validate(self) -> None:
2581 if (
2582 isinstance(self.whitespace_after_from, BaseParenthesizableWhitespace)
2583 and self.whitespace_after_from.empty
2584 and not self.item._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
2585 ):
2586 raise CSTValidationError(
2587 "Must have at least one space after 'from' keyword."
2588 )
2590 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "From":
2591 return From(
2592 whitespace_before_from=visit_sentinel(
2593 self, "whitespace_before_from", self.whitespace_before_from, visitor
2594 ),
2595 whitespace_after_from=visit_required(
2596 self, "whitespace_after_from", self.whitespace_after_from, visitor
2597 ),
2598 item=visit_required(self, "item", self.item, visitor),
2599 )
2601 def _codegen_impl(self, state: CodegenState, default_space: str = "") -> None:
2602 whitespace_before_from = self.whitespace_before_from
2603 if isinstance(whitespace_before_from, BaseParenthesizableWhitespace):
2604 whitespace_before_from._codegen(state)
2605 else:
2606 state.add_token(default_space)
2608 with state.record_syntactic_position(self):
2609 state.add_token("from")
2610 self.whitespace_after_from._codegen(state)
2611 self.item._codegen(state)
2614@add_slots
2615@dataclass(frozen=True)
2616class Yield(BaseExpression):
2617 """
2618 A yield expression similar to ``yield x`` or ``yield from fun()``.
2620 To learn more about the ways that yield can be used in generators, refer to
2621 `Python's language reference
2622 <https://docs.python.org/3/reference/expressions.html#yieldexpr>`__.
2623 """
2625 #: The value yielded from the generator, in the case of a :class:`From` clause, a
2626 #: sub-generator to iterate over.
2627 value: Optional[Union[BaseExpression, From]] = None
2629 lpar: Sequence[LeftParen] = ()
2630 #: Sequence of parenthesis for precedence dictation.
2631 rpar: Sequence[RightParen] = ()
2633 #: Whitespace after the ``yield`` keyword, but before the ``value``.
2634 whitespace_after_yield: Union[
2635 BaseParenthesizableWhitespace, MaybeSentinel
2636 ] = MaybeSentinel.DEFAULT
2638 def _validate(self) -> None:
2639 # Paren rules and such
2640 super(Yield, self)._validate()
2641 # Our own rules
2642 whitespace_after_yield = self.whitespace_after_yield
2643 if (
2644 isinstance(whitespace_after_yield, BaseParenthesizableWhitespace)
2645 and whitespace_after_yield.empty
2646 ):
2647 value = self.value
2648 if isinstance(value, From):
2649 raise CSTValidationError(
2650 "Must have at least one space after 'yield' keyword."
2651 )
2652 if isinstance(
2653 value, BaseExpression
2654 ) and not value._safe_to_use_with_word_operator(ExpressionPosition.RIGHT):
2655 raise CSTValidationError(
2656 "Must have at least one space after 'yield' keyword."
2657 )
2659 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Yield":
2660 return Yield(
2661 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2662 whitespace_after_yield=visit_sentinel(
2663 self, "whitespace_after_yield", self.whitespace_after_yield, visitor
2664 ),
2665 value=visit_optional(self, "value", self.value, visitor),
2666 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2667 )
2669 def _codegen_impl(self, state: CodegenState) -> None:
2670 with self._parenthesize(state):
2671 state.add_token("yield")
2672 whitespace_after_yield = self.whitespace_after_yield
2673 if isinstance(whitespace_after_yield, BaseParenthesizableWhitespace):
2674 whitespace_after_yield._codegen(state)
2675 else:
2676 # Only need a space after yield if there is a value to yield.
2677 if self.value is not None:
2678 state.add_token(" ")
2679 value = self.value
2680 if isinstance(value, From):
2681 value._codegen(state, default_space="")
2682 elif value is not None:
2683 value._codegen(state)
2686class _BaseElementImpl(CSTNode, ABC):
2687 """
2688 An internal base class for :class:`Element` and :class:`DictElement`.
2689 """
2691 __slots__ = ()
2693 value: BaseExpression
2694 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2696 def _codegen_comma(
2697 self,
2698 state: CodegenState,
2699 default_comma: bool = False,
2700 default_comma_whitespace: bool = False, # False for a single-item collection
2701 ) -> None:
2702 """
2703 Called by `_codegen_impl` in subclasses to generate the comma.
2704 """
2705 comma = self.comma
2706 if comma is MaybeSentinel.DEFAULT and default_comma:
2707 if default_comma_whitespace:
2708 state.add_token(", ")
2709 else:
2710 state.add_token(",")
2711 elif isinstance(comma, Comma):
2712 comma._codegen(state)
2714 @abstractmethod
2715 def _codegen_impl(
2716 self,
2717 state: CodegenState,
2718 default_comma: bool = False,
2719 default_comma_whitespace: bool = False, # False for a single-item collection
2720 ) -> None:
2721 ...
2724class BaseElement(_BaseElementImpl, ABC):
2725 """
2726 An element of a literal list, tuple, or set. For elements of a literal dict, see
2727 BaseDictElement.
2728 """
2730 __slots__ = ()
2733class BaseDictElement(_BaseElementImpl, ABC):
2734 """
2735 An element of a literal dict. For elements of a list, tuple, or set, see
2736 BaseElement.
2737 """
2739 __slots__ = ()
2742@add_slots
2743@dataclass(frozen=True)
2744class Element(BaseElement):
2745 """
2746 A simple value in a literal :class:`List`, :class:`Tuple`, or :class:`Set`.
2747 These a literal collection may also contain a :class:`StarredElement`.
2749 If you're using a literal :class:`Dict`, see :class:`DictElement` instead.
2750 """
2752 value: BaseExpression
2754 #: A trailing comma. By default, we'll only insert a comma if one is required.
2755 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2757 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Element":
2758 return Element(
2759 value=visit_required(self, "value", self.value, visitor),
2760 comma=visit_sentinel(self, "comma", self.comma, visitor),
2761 )
2763 def _codegen_impl(
2764 self,
2765 state: CodegenState,
2766 default_comma: bool = False,
2767 default_comma_whitespace: bool = False,
2768 ) -> None:
2769 with state.record_syntactic_position(self):
2770 self.value._codegen(state)
2771 self._codegen_comma(state, default_comma, default_comma_whitespace)
2774@add_slots
2775@dataclass(frozen=True)
2776class DictElement(BaseDictElement):
2777 """
2778 A simple ``key: value`` pair that represents a single entry in a literal
2779 :class:`Dict`. :class:`Dict` nodes may also contain a
2780 :class:`StarredDictElement`.
2782 If you're using a literal :class:`List`, :class:`Tuple`, or :class:`Set`,
2783 see :class:`Element` instead.
2784 """
2786 key: BaseExpression
2787 value: BaseExpression
2789 #: A trailing comma. By default, we'll only insert a comma if one is required.
2790 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2792 #: Whitespace after the key, but before the colon in ``key : value``.
2793 whitespace_before_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2794 #: Whitespace after the colon, but before the value in ``key : value``.
2795 whitespace_after_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2797 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "DictElement":
2798 return DictElement(
2799 key=visit_required(self, "key", self.key, visitor),
2800 whitespace_before_colon=visit_required(
2801 self, "whitespace_before_colon", self.whitespace_before_colon, visitor
2802 ),
2803 whitespace_after_colon=visit_required(
2804 self, "whitespace_after_colon", self.whitespace_after_colon, visitor
2805 ),
2806 value=visit_required(self, "value", self.value, visitor),
2807 comma=visit_sentinel(self, "comma", self.comma, visitor),
2808 )
2810 def _codegen_impl(
2811 self,
2812 state: CodegenState,
2813 default_comma: bool = False,
2814 default_comma_whitespace: bool = False,
2815 ) -> None:
2816 with state.record_syntactic_position(self):
2817 self.key._codegen(state)
2818 self.whitespace_before_colon._codegen(state)
2819 state.add_token(":")
2820 self.whitespace_after_colon._codegen(state)
2821 self.value._codegen(state)
2822 self._codegen_comma(state, default_comma, default_comma_whitespace)
2825@add_slots
2826@dataclass(frozen=True)
2827class StarredElement(BaseElement, BaseExpression, _BaseParenthesizedNode):
2828 """
2829 A starred ``*value`` element that expands to represent multiple values in a literal
2830 :class:`List`, :class:`Tuple`, or :class:`Set`.
2832 If you're using a literal :class:`Dict`, see :class:`StarredDictElement` instead.
2834 If this node owns parenthesis, those parenthesis wrap the leading asterisk, but not
2835 the trailing comma. For example::
2837 StarredElement(
2838 cst.Name("el"),
2839 comma=cst.Comma(),
2840 lpar=[cst.LeftParen()],
2841 rpar=[cst.RightParen()],
2842 )
2844 will generate::
2846 (*el),
2847 """
2849 value: BaseExpression
2851 #: A trailing comma. By default, we'll only insert a comma if one is required.
2852 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2854 #: Parenthesis at the beginning of the node, before the leading asterisk.
2855 lpar: Sequence[LeftParen] = ()
2856 #: Parentheses after the value, but before a comma (if there is one).
2857 rpar: Sequence[RightParen] = ()
2859 #: Whitespace between the leading asterisk and the value expression.
2860 whitespace_before_value: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2862 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "StarredElement":
2863 return StarredElement(
2864 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2865 whitespace_before_value=visit_required(
2866 self, "whitespace_before_value", self.whitespace_before_value, visitor
2867 ),
2868 value=visit_required(self, "value", self.value, visitor),
2869 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2870 comma=visit_sentinel(self, "comma", self.comma, visitor),
2871 )
2873 def _codegen_impl(
2874 self,
2875 state: CodegenState,
2876 default_comma: bool = False,
2877 default_comma_whitespace: bool = False,
2878 ) -> None:
2879 with self._parenthesize(state):
2880 state.add_token("*")
2881 self.whitespace_before_value._codegen(state)
2882 self.value._codegen(state)
2883 self._codegen_comma(state, default_comma, default_comma_whitespace)
2886@add_slots
2887@dataclass(frozen=True)
2888class StarredDictElement(BaseDictElement):
2889 """
2890 A starred ``**value`` element that expands to represent multiple values in a literal
2891 :class:`Dict`.
2893 If you're using a literal :class:`List`, :class:`Tuple`, or :class:`Set`,
2894 see :class:`StarredElement` instead.
2896 Unlike :class:`StarredElement`, this node does not own left or right parenthesis,
2897 but the ``value`` field may still contain parenthesis. This is due to some
2898 asymmetry in Python's grammar.
2899 """
2901 value: BaseExpression
2903 #: A trailing comma. By default, we'll only insert a comma if one is required.
2904 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2906 #: Whitespace between the leading asterisks and the value expression.
2907 whitespace_before_value: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2909 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "StarredDictElement":
2910 return StarredDictElement(
2911 whitespace_before_value=visit_required(
2912 self, "whitespace_before_value", self.whitespace_before_value, visitor
2913 ),
2914 value=visit_required(self, "value", self.value, visitor),
2915 comma=visit_sentinel(self, "comma", self.comma, visitor),
2916 )
2918 def _codegen_impl(
2919 self,
2920 state: CodegenState,
2921 default_comma: bool = False,
2922 default_comma_whitespace: bool = False,
2923 ) -> None:
2924 with state.record_syntactic_position(self):
2925 state.add_token("**")
2926 self.whitespace_before_value._codegen(state)
2927 self.value._codegen(state)
2928 self._codegen_comma(state, default_comma, default_comma_whitespace)
2931@add_slots
2932@dataclass(frozen=True)
2933class Tuple(BaseAssignTargetExpression, BaseDelTargetExpression):
2934 """
2935 An immutable literal tuple. Tuples are often (but not always) parenthesized.
2937 ::
2939 Tuple([
2940 Element(Integer("1")),
2941 Element(Integer("2")),
2942 StarredElement(Name("others")),
2943 ])
2945 generates the following code::
2947 (1, 2, *others)
2948 """
2950 #: A sequence containing all the :class:`Element` and :class:`StarredElement` nodes
2951 #: in the tuple.
2952 elements: Sequence[BaseElement]
2954 lpar: Sequence[LeftParen] = field(default_factory=lambda: (LeftParen(),))
2955 #: Sequence of parenthesis for precedence dictation.
2956 rpar: Sequence[RightParen] = field(default_factory=lambda: (RightParen(),))
2958 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2959 if super(Tuple, self)._safe_to_use_with_word_operator(position):
2960 # if we have parenthesis, we're safe.
2961 return True
2962 # elements[-1] and elements[0] must exist past this point, because
2963 # we're not parenthesized, meaning we must have at least one element.
2964 elements = self.elements
2965 if position == ExpressionPosition.LEFT:
2966 last_element = elements[-1]
2967 return (
2968 isinstance(last_element.comma, Comma)
2969 or (
2970 isinstance(last_element, StarredElement)
2971 and len(last_element.rpar) > 0
2972 )
2973 or last_element.value._safe_to_use_with_word_operator(position)
2974 )
2975 else: # ExpressionPosition.RIGHT
2976 first_element = elements[0]
2977 # starred elements are always safe because they begin with ( or *
2978 return isinstance(
2979 first_element, StarredElement
2980 ) or first_element.value._safe_to_use_with_word_operator(position)
2982 def _validate(self) -> None:
2983 # Paren validation and such
2984 super(Tuple, self)._validate()
2986 if len(self.elements) == 0:
2987 if len(self.lpar) == 0: # assumes len(lpar) == len(rpar), via superclass
2988 raise CSTValidationError(
2989 "A zero-length tuple must be wrapped in parentheses."
2990 )
2991 # Invalid commas aren't possible, because MaybeSentinel will ensure that there
2992 # is a comma where required.
2994 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Tuple":
2995 return Tuple(
2996 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2997 elements=visit_sequence(self, "elements", self.elements, visitor),
2998 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2999 )
3001 def _codegen_impl(self, state: CodegenState) -> None:
3002 with self._parenthesize(state):
3003 elements = self.elements
3004 if len(elements) == 1:
3005 elements[0]._codegen(
3006 state, default_comma=True, default_comma_whitespace=False
3007 )
3008 else:
3009 for idx, el in enumerate(elements):
3010 el._codegen(
3011 state,
3012 default_comma=(idx < len(elements) - 1),
3013 default_comma_whitespace=True,
3014 )
3017class BaseList(BaseExpression, ABC):
3018 """
3019 A base class for :class:`List` and :class:`ListComp`, which both result in a list
3020 object when evaluated.
3021 """
3023 __slots__ = ()
3025 lbracket: LeftSquareBracket = LeftSquareBracket.field()
3026 #: Brackets surrounding the list.
3027 rbracket: RightSquareBracket = RightSquareBracket.field()
3029 lpar: Sequence[LeftParen] = ()
3030 #: Sequence of parenthesis for precedence dictation.
3031 rpar: Sequence[RightParen] = ()
3033 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3034 return True
3036 @contextmanager
3037 def _bracketize(self, state: CodegenState) -> Generator[None, None, None]:
3038 self.lbracket._codegen(state)
3039 yield
3040 self.rbracket._codegen(state)
3043@add_slots
3044@dataclass(frozen=True)
3045class List(BaseList, BaseAssignTargetExpression, BaseDelTargetExpression):
3046 """
3047 A mutable literal list.
3049 ::
3051 List([
3052 Element(Integer("1")),
3053 Element(Integer("2")),
3054 StarredElement(Name("others")),
3055 ])
3057 generates the following code::
3059 [1, 2, *others]
3061 List comprehensions are represented with a :class:`ListComp` node.
3062 """
3064 #: A sequence containing all the :class:`Element` and :class:`StarredElement` nodes
3065 #: in the list.
3066 elements: Sequence[BaseElement]
3068 lbracket: LeftSquareBracket = LeftSquareBracket.field()
3069 #: Brackets surrounding the list.
3070 rbracket: RightSquareBracket = RightSquareBracket.field()
3072 lpar: Sequence[LeftParen] = ()
3073 #: Sequence of parenthesis for precedence dictation.
3074 rpar: Sequence[RightParen] = ()
3076 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "List":
3077 return List(
3078 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3079 lbracket=visit_required(self, "lbracket", self.lbracket, visitor),
3080 elements=visit_sequence(self, "elements", self.elements, visitor),
3081 rbracket=visit_required(self, "rbracket", self.rbracket, visitor),
3082 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3083 )
3085 def _codegen_impl(self, state: CodegenState) -> None:
3086 with self._parenthesize(state), self._bracketize(state):
3087 elements = self.elements
3088 for idx, el in enumerate(elements):
3089 el._codegen(
3090 state,
3091 default_comma=(idx < len(elements) - 1),
3092 default_comma_whitespace=True,
3093 )
3096class _BaseSetOrDict(BaseExpression, ABC):
3097 """
3098 An abstract base class for :class:`BaseSet` and :class:`BaseDict`.
3100 Literal sets and dicts are syntactically similar (hence this shared base class), but
3101 are semantically different. This base class is an implementation detail and
3102 shouldn't be exported.
3103 """
3105 __slots__ = ()
3107 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3108 #: Braces surrounding the set or dict.
3109 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3111 lpar: Sequence[LeftParen] = ()
3112 #: Sequence of parenthesis for precedence dictation.
3113 rpar: Sequence[RightParen] = ()
3115 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3116 return True
3118 # brace-ize seems like a very made-up word. And it is!
3119 @contextmanager
3120 def _braceize(self, state: CodegenState) -> Generator[None, None, None]:
3121 self.lbrace._codegen(state)
3122 yield
3123 self.rbrace._codegen(state)
3126class BaseSet(_BaseSetOrDict, ABC):
3127 """
3128 An abstract base class for :class:`Set` and :class:`SetComp`, which both result in
3129 a set object when evaluated.
3130 """
3132 __slots__ = ()
3135@add_slots
3136@dataclass(frozen=True)
3137class Set(BaseSet):
3138 """
3139 A mutable literal set.
3141 ::
3143 Set([
3144 Element(Integer("1")),
3145 Element(Integer("2")),
3146 StarredElement(Name("others")),
3147 ])
3149 generates the following code::
3151 {1, 2, *others}
3153 Set comprehensions are represented with a :class:`SetComp` node.
3154 """
3156 #: A sequence containing all the :class:`Element` and :class:`StarredElement` nodes
3157 #: in the set.
3158 elements: Sequence[BaseElement]
3160 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3161 #: Braces surrounding the set.
3162 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3164 lpar: Sequence[LeftParen] = ()
3165 #: Sequence of parenthesis for precedence dictation.
3166 rpar: Sequence[RightParen] = ()
3168 def _validate(self) -> None:
3169 super(Set, self)._validate()
3171 if len(self.elements) == 0:
3172 raise CSTValidationError(
3173 "A literal set must have at least one element. A zero-element set "
3174 + "would be syntatically ambiguous with an empty dict, `{}`."
3175 )
3177 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Set":
3178 return Set(
3179 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3180 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3181 elements=visit_sequence(self, "elements", self.elements, visitor),
3182 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3183 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3184 )
3186 def _codegen_impl(self, state: CodegenState) -> None:
3187 with self._parenthesize(state), self._braceize(state):
3188 elements = self.elements
3189 for idx, el in enumerate(elements):
3190 el._codegen(
3191 state,
3192 default_comma=(idx < len(elements) - 1),
3193 default_comma_whitespace=True,
3194 )
3197class BaseDict(_BaseSetOrDict, ABC):
3198 """
3199 An abstract base class for :class:`Dict` and :class:`DictComp`, which both result in
3200 a dict object when evaluated.
3201 """
3203 __slots__ = ()
3206@add_slots
3207@dataclass(frozen=True)
3208class Dict(BaseDict):
3209 """
3210 A literal dictionary. Key-value pairs are stored in ``elements`` using
3211 :class:`DictElement` nodes.
3213 It's possible to expand one dictionary into another, as in ``{k: v, **expanded}``.
3214 Expanded elements are stored as :class:`StarredDictElement` nodes.
3216 ::
3218 Dict([
3219 DictElement(Name("k1"), Name("v1")),
3220 DictElement(Name("k2"), Name("v2")),
3221 StarredDictElement(Name("expanded")),
3222 ])
3224 generates the following code::
3226 {k1: v1, k2: v2, **expanded}
3227 """
3229 elements: Sequence[BaseDictElement]
3230 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3231 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3232 lpar: Sequence[LeftParen] = ()
3233 rpar: Sequence[RightParen] = ()
3235 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Dict":
3236 return Dict(
3237 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3238 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3239 elements=visit_sequence(self, "elements", self.elements, visitor),
3240 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3241 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3242 )
3244 def _codegen_impl(self, state: CodegenState) -> None:
3245 with self._parenthesize(state), self._braceize(state):
3246 elements = self.elements
3247 for idx, el in enumerate(elements):
3248 el._codegen(
3249 state,
3250 default_comma=(idx < len(elements) - 1),
3251 default_comma_whitespace=True,
3252 )
3255@add_slots
3256@dataclass(frozen=True)
3257class CompFor(CSTNode):
3258 """
3259 One ``for`` clause in a :class:`BaseComp`, or a nested hierarchy of
3260 ``for`` clauses.
3262 Nested loops in comprehensions are difficult to get right, but they can be thought
3263 of as a flat representation of nested clauses.
3265 ``elt for a in b for c in d if e`` can be thought of as::
3267 for a in b:
3268 for c in d:
3269 if e:
3270 yield elt
3272 And that would form the following CST::
3274 ListComp(
3275 elt=Name("elt"),
3276 for_in=CompFor(
3277 target=Name("a"),
3278 iter=Name("b"),
3279 ifs=[],
3280 inner_comp_for=CompFor(
3281 target=Name("c"),
3282 iter=Name("d"),
3283 ifs=[
3284 CompIf(
3285 test=Name("e"),
3286 ),
3287 ],
3288 ),
3289 ),
3290 )
3292 Normal ``for`` statements are provided by :class:`For`.
3293 """
3295 #: The target to assign a value to in each iteration of the loop. This is different
3296 #: from :attr:`GeneratorExp.elt`, :attr:`ListComp.elt`, :attr:`SetComp.elt`, and
3297 #: ``key`` and ``value`` in :class:`DictComp`, because it doesn't directly effect
3298 #: the value of resulting generator, list, set, or dict.
3299 target: BaseAssignTargetExpression
3301 #: The value to iterate over. Every value in ``iter`` is stored in ``target``.
3302 iter: BaseExpression
3304 #: Zero or more conditional clauses that control this loop. If any of these tests
3305 #: fail, the ``target`` item is skipped.
3306 #:
3307 #: ::
3308 #:
3309 #: if a if b if c
3310 #:
3311 #: has similar semantics to::
3312 #:
3313 #: if a and b and c
3314 ifs: Sequence["CompIf"] = ()
3316 #: Another :class:`CompFor` node used to form nested loops. Nested comprehensions
3317 #: can be useful, but they tend to be difficult to read and write. As a result they
3318 #: are uncommon.
3319 inner_for_in: Optional["CompFor"] = None
3321 #: An optional async modifier that appears before the ``for`` keyword.
3322 asynchronous: Optional[Asynchronous] = None
3324 #: Whitespace that appears at the beginning of this node, before the ``for`` and
3325 #: ``async`` keywords.
3326 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3328 #: Whitespace appearing after the ``for`` keyword, but before the ``target``.
3329 whitespace_after_for: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3331 #: Whitespace appearing after the ``target``, but before the ``in`` keyword.
3332 whitespace_before_in: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3334 #: Whitespace appearing after the ``in`` keyword, but before the ``iter``.
3335 whitespace_after_in: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3337 def _validate(self) -> None:
3338 if (
3339 self.whitespace_after_for.empty
3340 and not self.target._safe_to_use_with_word_operator(
3341 ExpressionPosition.RIGHT
3342 )
3343 ):
3344 raise CSTValidationError(
3345 "Must have at least one space after 'for' keyword."
3346 )
3348 if (
3349 self.whitespace_before_in.empty
3350 and not self.target._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3351 ):
3352 raise CSTValidationError(
3353 "Must have at least one space before 'in' keyword."
3354 )
3356 if (
3357 self.whitespace_after_in.empty
3358 and not self.iter._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
3359 ):
3360 raise CSTValidationError("Must have at least one space after 'in' keyword.")
3362 prev_expr = self.iter
3363 for if_clause in self.ifs:
3364 if (
3365 if_clause.whitespace_before.empty
3366 and not prev_expr._safe_to_use_with_word_operator(
3367 ExpressionPosition.LEFT
3368 )
3369 ):
3370 raise CSTValidationError(
3371 "Must have at least one space before 'if' keyword."
3372 )
3373 prev_expr = if_clause.test
3375 inner_for_in = self.inner_for_in
3376 if (
3377 inner_for_in is not None
3378 and inner_for_in.whitespace_before.empty
3379 and not prev_expr._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3380 ):
3381 keyword = "async" if inner_for_in.asynchronous else "for"
3382 raise CSTValidationError(
3383 f"Must have at least one space before '{keyword}' keyword."
3384 )
3386 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "CompFor":
3387 return CompFor(
3388 whitespace_before=visit_required(
3389 self, "whitespace_before", self.whitespace_before, visitor
3390 ),
3391 asynchronous=visit_optional(
3392 self, "asynchronous", self.asynchronous, visitor
3393 ),
3394 whitespace_after_for=visit_required(
3395 self, "whitespace_after_for", self.whitespace_after_for, visitor
3396 ),
3397 target=visit_required(self, "target", self.target, visitor),
3398 whitespace_before_in=visit_required(
3399 self, "whitespace_before_in", self.whitespace_before_in, visitor
3400 ),
3401 whitespace_after_in=visit_required(
3402 self, "whitespace_after_in", self.whitespace_after_in, visitor
3403 ),
3404 iter=visit_required(self, "iter", self.iter, visitor),
3405 ifs=visit_sequence(self, "ifs", self.ifs, visitor),
3406 inner_for_in=visit_optional(
3407 self, "inner_for_in", self.inner_for_in, visitor
3408 ),
3409 )
3411 def _codegen_impl(self, state: CodegenState) -> None:
3412 self.whitespace_before._codegen(state)
3413 asynchronous = self.asynchronous
3414 if asynchronous is not None:
3415 asynchronous._codegen(state)
3416 state.add_token("for")
3417 self.whitespace_after_for._codegen(state)
3418 self.target._codegen(state)
3419 self.whitespace_before_in._codegen(state)
3420 state.add_token("in")
3421 self.whitespace_after_in._codegen(state)
3422 self.iter._codegen(state)
3423 ifs = self.ifs
3424 for if_clause in ifs:
3425 if_clause._codegen(state)
3426 inner_for_in = self.inner_for_in
3427 if inner_for_in is not None:
3428 inner_for_in._codegen(state)
3431@add_slots
3432@dataclass(frozen=True)
3433class CompIf(CSTNode):
3434 """
3435 A conditional clause in a :class:`CompFor`, used as part of a generator or
3436 comprehension expression.
3438 If the ``test`` fails, the current element in the :class:`CompFor` will be skipped.
3439 """
3441 #: An expression to evaluate. When interpreted, Python will coerce it to a boolean.
3442 test: BaseExpression
3444 #: Whitespace before the ``if`` keyword.
3445 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3447 #: Whitespace after the ``if`` keyword, but before the ``test`` expression.
3448 whitespace_before_test: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3450 def _validate(self) -> None:
3451 if (
3452 self.whitespace_before_test.empty
3453 and not self.test._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
3454 ):
3455 raise CSTValidationError("Must have at least one space after 'if' keyword.")
3457 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "CompIf":
3458 return CompIf(
3459 whitespace_before=visit_required(
3460 self, "whitespace_before", self.whitespace_before, visitor
3461 ),
3462 whitespace_before_test=visit_required(
3463 self, "whitespace_before_test", self.whitespace_before_test, visitor
3464 ),
3465 test=visit_required(self, "test", self.test, visitor),
3466 )
3468 def _codegen_impl(self, state: CodegenState) -> None:
3469 self.whitespace_before._codegen(state)
3470 state.add_token("if")
3471 self.whitespace_before_test._codegen(state)
3472 self.test._codegen(state)
3475class BaseComp(BaseExpression, ABC):
3476 """
3477 A base class for all comprehension and generator expressions, including
3478 :class:`GeneratorExp`, :class:`ListComp`, :class:`SetComp`, and :class:`DictComp`.
3479 """
3481 __slots__ = ()
3483 for_in: CompFor
3486class BaseSimpleComp(BaseComp, ABC):
3487 """
3488 The base class for :class:`ListComp`, :class:`SetComp`, and :class:`GeneratorExp`.
3489 :class:`DictComp` is not a :class:`BaseSimpleComp`, because it uses ``key`` and
3490 ``value``.
3491 """
3493 __slots__ = ()
3495 #: The expression evaluated during each iteration of the comprehension. This
3496 #: lexically comes before the ``for_in`` clause, but it is semantically the
3497 #: inner-most element, evaluated inside the ``for_in`` clause.
3498 elt: BaseAssignTargetExpression
3500 #: The ``for ... in ... if ...`` clause that lexically comes after ``elt``. This may
3501 #: be a nested structure for nested comprehensions. See :class:`CompFor` for
3502 #: details.
3503 for_in: CompFor
3505 def _validate(self) -> None:
3506 super(BaseSimpleComp, self)._validate()
3508 for_in = self.for_in
3509 if (
3510 for_in.whitespace_before.empty
3511 and not self.elt._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3512 ):
3513 keyword = "async" if for_in.asynchronous else "for"
3514 raise CSTValidationError(
3515 f"Must have at least one space before '{keyword}' keyword."
3516 )
3519@add_slots
3520@dataclass(frozen=True)
3521class GeneratorExp(BaseSimpleComp):
3522 """
3523 A generator expression. ``elt`` represents the value yielded for each item in
3524 :attr:`CompFor.iter`.
3526 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3527 :class:`CompFor` data structure inside ``for_in``.
3528 """
3530 #: The expression evaluated and yielded during each iteration of the generator.
3531 elt: BaseAssignTargetExpression
3533 #: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
3534 #: nested structure for nested comprehensions. See :class:`CompFor` for details.
3535 for_in: CompFor
3537 lpar: Sequence[LeftParen] = field(default_factory=lambda: (LeftParen(),))
3538 #: Sequence of parentheses for precedence dictation. Generator expressions must
3539 #: always be parenthesized. However, if a generator expression is the only argument
3540 #: inside a function call, the enclosing :class:`Call` node may own the parentheses
3541 #: instead.
3542 rpar: Sequence[RightParen] = field(default_factory=lambda: (RightParen(),))
3544 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3545 # Generators are always parenthesized
3546 return True
3548 # A note about validation: Generators must always be parenthesized, but it's
3549 # possible that this Generator node doesn't own those parenthesis (in the case of a
3550 # function call with a single generator argument).
3551 #
3552 # Therefore, there's no useful validation we can do here. In theory, our parent
3553 # could do the validation, but there's a ton of potential parents to a Generator, so
3554 # it's not worth the effort.
3556 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "GeneratorExp":
3557 return GeneratorExp(
3558 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3559 elt=visit_required(self, "elt", self.elt, visitor),
3560 for_in=visit_required(self, "for_in", self.for_in, visitor),
3561 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3562 )
3564 def _codegen_impl(self, state: CodegenState) -> None:
3565 with self._parenthesize(state):
3566 self.elt._codegen(state)
3567 self.for_in._codegen(state)
3570@add_slots
3571@dataclass(frozen=True)
3572class ListComp(BaseList, BaseSimpleComp):
3573 """
3574 A list comprehension. ``elt`` represents the value stored for each item in
3575 :attr:`CompFor.iter`.
3577 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3578 :class:`CompFor` data structure inside ``for_in``.
3579 """
3581 #: The expression evaluated and stored during each iteration of the comprehension.
3582 elt: BaseAssignTargetExpression
3584 #: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
3585 #: nested structure for nested comprehensions. See :class:`CompFor` for details.
3586 for_in: CompFor
3588 lbracket: LeftSquareBracket = LeftSquareBracket.field()
3589 #: Brackets surrounding the list comprehension.
3590 rbracket: RightSquareBracket = RightSquareBracket.field()
3592 lpar: Sequence[LeftParen] = ()
3593 #: Sequence of parenthesis for precedence dictation.
3594 rpar: Sequence[RightParen] = ()
3596 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ListComp":
3597 return ListComp(
3598 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3599 lbracket=visit_required(self, "lbracket", self.lbracket, visitor),
3600 elt=visit_required(self, "elt", self.elt, visitor),
3601 for_in=visit_required(self, "for_in", self.for_in, visitor),
3602 rbracket=visit_required(self, "rbracket", self.rbracket, visitor),
3603 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3604 )
3606 def _codegen_impl(self, state: CodegenState) -> None:
3607 with self._parenthesize(state), self._bracketize(state):
3608 self.elt._codegen(state)
3609 self.for_in._codegen(state)
3612@add_slots
3613@dataclass(frozen=True)
3614class SetComp(BaseSet, BaseSimpleComp):
3615 """
3616 A set comprehension. ``elt`` represents the value stored for each item in
3617 :attr:`CompFor.iter`.
3619 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3620 :class:`CompFor` data structure inside ``for_in``.
3621 """
3623 #: The expression evaluated and stored during each iteration of the comprehension.
3624 elt: BaseAssignTargetExpression
3626 #: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
3627 #: nested structure for nested comprehensions. See :class:`CompFor` for details.
3628 for_in: CompFor
3630 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3631 #: Braces surrounding the set comprehension.
3632 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3634 lpar: Sequence[LeftParen] = ()
3635 #: Sequence of parenthesis for precedence dictation.
3636 rpar: Sequence[RightParen] = ()
3638 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "SetComp":
3639 return SetComp(
3640 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3641 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3642 elt=visit_required(self, "elt", self.elt, visitor),
3643 for_in=visit_required(self, "for_in", self.for_in, visitor),
3644 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3645 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3646 )
3648 def _codegen_impl(self, state: CodegenState) -> None:
3649 with self._parenthesize(state), self._braceize(state):
3650 self.elt._codegen(state)
3651 self.for_in._codegen(state)
3654@add_slots
3655@dataclass(frozen=True)
3656class DictComp(BaseDict, BaseComp):
3657 """
3658 A dictionary comprehension. ``key`` and ``value`` represent the dictionary entry
3659 evaluated for each item.
3661 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3662 :class:`CompFor` data structure inside ``for_in``.
3663 """
3665 #: The key inserted into the dictionary during each iteration of the comprehension.
3666 key: BaseAssignTargetExpression
3667 #: The value associated with the ``key`` inserted into the dictionary during each
3668 #: iteration of the comprehension.
3669 value: BaseAssignTargetExpression
3671 #: The ``for ... in ... if ...`` clause that lexically comes after ``key`` and
3672 #: ``value``. This may be a nested structure for nested comprehensions. See
3673 #: :class:`CompFor` for details.
3674 for_in: CompFor
3676 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3677 #: Braces surrounding the dict comprehension.
3678 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3680 lpar: Sequence[LeftParen] = ()
3681 #: Sequence of parenthesis for precedence dictation.
3682 rpar: Sequence[RightParen] = ()
3684 #: Whitespace after the key, but before the colon in ``key : value``.
3685 whitespace_before_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
3686 #: Whitespace after the colon, but before the value in ``key : value``.
3687 whitespace_after_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3689 def _validate(self) -> None:
3690 super(DictComp, self)._validate()
3692 for_in = self.for_in
3693 if (
3694 for_in.whitespace_before.empty
3695 and not self.value._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3696 ):
3697 keyword = "async" if for_in.asynchronous else "for"
3698 raise CSTValidationError(
3699 f"Must have at least one space before '{keyword}' keyword."
3700 )
3702 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "DictComp":
3703 return DictComp(
3704 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3705 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3706 key=visit_required(self, "key", self.key, visitor),
3707 whitespace_before_colon=visit_required(
3708 self, "whitespace_before_colon", self.whitespace_before_colon, visitor
3709 ),
3710 whitespace_after_colon=visit_required(
3711 self, "whitespace_after_colon", self.whitespace_after_colon, visitor
3712 ),
3713 value=visit_required(self, "value", self.value, visitor),
3714 for_in=visit_required(self, "for_in", self.for_in, visitor),
3715 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3716 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3717 )
3719 def _codegen_impl(self, state: CodegenState) -> None:
3720 with self._parenthesize(state), self._braceize(state):
3721 self.key._codegen(state)
3722 self.whitespace_before_colon._codegen(state)
3723 state.add_token(":")
3724 self.whitespace_after_colon._codegen(state)
3725 self.value._codegen(state)
3726 self.for_in._codegen(state)
3729@add_slots
3730@dataclass(frozen=True)
3731class NamedExpr(BaseExpression):
3732 """
3733 An expression that is also an assignment, such as ``x := y + z``. Affectionately
3734 known as the walrus operator, this expression allows you to make an assignment
3735 inside an expression. This greatly simplifies loops::
3737 while line := read_some_line_or_none():
3738 do_thing_with_line(line)
3739 """
3741 #: The target that is being assigned to.
3742 target: BaseExpression
3744 #: The expression being assigned to the target.
3745 value: BaseExpression
3747 #: Sequence of parenthesis for precedence dictation.
3748 lpar: Sequence[LeftParen] = ()
3749 #: Sequence of parenthesis for precedence dictation.
3750 rpar: Sequence[RightParen] = ()
3752 #: Whitespace after the target, but before the walrus operator.
3753 whitespace_before_walrus: BaseParenthesizableWhitespace = SimpleWhitespace.field(
3754 " "
3755 )
3756 #: Whitespace after the walrus operator, but before the value.
3757 whitespace_after_walrus: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3759 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "NamedExpr":
3760 return NamedExpr(
3761 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3762 target=visit_required(self, "target", self.target, visitor),
3763 whitespace_before_walrus=visit_required(
3764 self, "whitespace_before_walrus", self.whitespace_before_walrus, visitor
3765 ),
3766 whitespace_after_walrus=visit_required(
3767 self, "whitespace_after_walrus", self.whitespace_after_walrus, visitor
3768 ),
3769 value=visit_required(self, "value", self.value, visitor),
3770 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3771 )
3773 def _codegen_impl(self, state: CodegenState) -> None:
3774 with self._parenthesize(state):
3775 self.target._codegen(state)
3776 self.whitespace_before_walrus._codegen(state)
3777 state.add_token(":=")
3778 self.whitespace_after_walrus._codegen(state)
3779 self.value._codegen(state)