Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/_nodes/expression.py: 64%
1432 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:43 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
7import re
8from abc import ABC, abstractmethod
9from ast import literal_eval
10from contextlib import contextmanager
11from dataclasses import dataclass, field
12from enum import auto, Enum
13from tokenize import (
14 Floatnumber as FLOATNUMBER_RE,
15 Imagnumber as IMAGNUMBER_RE,
16 Intnumber as INTNUMBER_RE,
17)
18from typing import Callable, Generator, Optional, Sequence, Union
20from typing_extensions import Literal
22from libcst._add_slots import add_slots
23from libcst._maybe_sentinel import MaybeSentinel
24from libcst._nodes.base import CSTCodegenError, CSTNode, CSTValidationError
25from libcst._nodes.internal import (
26 CodegenState,
27 visit_optional,
28 visit_required,
29 visit_sentinel,
30 visit_sequence,
31)
32from libcst._nodes.op import (
33 AssignEqual,
34 BaseBinaryOp,
35 BaseBooleanOp,
36 BaseCompOp,
37 BaseUnaryOp,
38 Colon,
39 Comma,
40 Dot,
41 In,
42 Is,
43 IsNot,
44 Not,
45 NotIn,
46)
47from libcst._nodes.whitespace import BaseParenthesizableWhitespace, SimpleWhitespace
48from libcst._visitors import CSTVisitorT
51@add_slots
52@dataclass(frozen=True)
53class LeftSquareBracket(CSTNode):
54 """
55 Used by various nodes to denote a subscript or list section. This doesn't own
56 the whitespace to the left of it since this is owned by the parent node.
57 """
59 #: Any space that appears directly after this left square bracket.
60 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
62 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "LeftSquareBracket":
63 return LeftSquareBracket(
64 whitespace_after=visit_required(
65 self, "whitespace_after", self.whitespace_after, visitor
66 )
67 )
69 def _codegen_impl(self, state: CodegenState) -> None:
70 state.add_token("[")
71 self.whitespace_after._codegen(state)
74@add_slots
75@dataclass(frozen=True)
76class RightSquareBracket(CSTNode):
77 """
78 Used by various nodes to denote a subscript or list section. This doesn't own
79 the whitespace to the right of it since this is owned by the parent node.
80 """
82 #: Any space that appears directly before this right square bracket.
83 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
85 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "RightSquareBracket":
86 return RightSquareBracket(
87 whitespace_before=visit_required(
88 self, "whitespace_before", self.whitespace_before, visitor
89 )
90 )
92 def _codegen_impl(self, state: CodegenState) -> None:
93 self.whitespace_before._codegen(state)
94 state.add_token("]")
97@add_slots
98@dataclass(frozen=True)
99class LeftCurlyBrace(CSTNode):
100 """
101 Used by various nodes to denote a dict or set. This doesn't own the whitespace to
102 the left of it since this is owned by the parent node.
103 """
105 #: Any space that appears directly after this left curly brace.
106 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
108 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "LeftCurlyBrace":
109 return LeftCurlyBrace(
110 whitespace_after=visit_required(
111 self, "whitespace_after", self.whitespace_after, visitor
112 )
113 )
115 def _codegen_impl(self, state: CodegenState) -> None:
116 state.add_token("{")
117 self.whitespace_after._codegen(state)
120@add_slots
121@dataclass(frozen=True)
122class RightCurlyBrace(CSTNode):
123 """
124 Used by various nodes to denote a dict or set. This doesn't own the whitespace to
125 the right of it since this is owned by the parent node.
126 """
128 #: Any space that appears directly before this right curly brace.
129 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
131 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "RightCurlyBrace":
132 return RightCurlyBrace(
133 whitespace_before=visit_required(
134 self, "whitespace_before", self.whitespace_before, visitor
135 )
136 )
138 def _codegen_impl(self, state: CodegenState) -> None:
139 self.whitespace_before._codegen(state)
140 state.add_token("}")
143@add_slots
144@dataclass(frozen=True)
145class LeftParen(CSTNode):
146 """
147 Used by various nodes to denote a parenthesized section. This doesn't own
148 the whitespace to the left of it since this is owned by the parent node.
149 """
151 #: Any space that appears directly after this left parenthesis.
152 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
154 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "LeftParen":
155 return LeftParen(
156 whitespace_after=visit_required(
157 self, "whitespace_after", self.whitespace_after, visitor
158 )
159 )
161 def _codegen_impl(self, state: CodegenState) -> None:
162 state.add_token("(")
163 self.whitespace_after._codegen(state)
166@add_slots
167@dataclass(frozen=True)
168class RightParen(CSTNode):
169 """
170 Used by various nodes to denote a parenthesized section. This doesn't own
171 the whitespace to the right of it since this is owned by the parent node.
172 """
174 #: Any space that appears directly after this left parenthesis.
175 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
177 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "RightParen":
178 return RightParen(
179 whitespace_before=visit_required(
180 self, "whitespace_before", self.whitespace_before, visitor
181 )
182 )
184 def _codegen_impl(self, state: CodegenState) -> None:
185 self.whitespace_before._codegen(state)
186 state.add_token(")")
189@add_slots
190@dataclass(frozen=True)
191class Asynchronous(CSTNode):
192 """
193 Used by asynchronous function definitions, as well as ``async for`` and
194 ``async with``.
195 """
197 #: Any space that appears directly after this async keyword.
198 whitespace_after: SimpleWhitespace = SimpleWhitespace.field(" ")
200 def _validate(self) -> None:
201 if len(self.whitespace_after.value) < 1:
202 raise CSTValidationError("Must have at least one space after Asynchronous.")
204 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Asynchronous":
205 return Asynchronous(
206 whitespace_after=visit_required(
207 self, "whitespace_after", self.whitespace_after, visitor
208 )
209 )
211 def _codegen_impl(self, state: CodegenState) -> None:
212 with state.record_syntactic_position(self):
213 state.add_token("async")
214 self.whitespace_after._codegen(state)
217class _BaseParenthesizedNode(CSTNode, ABC):
218 """
219 We don't want to have another level of indirection for parenthesis in
220 our tree, since that makes us more of a CST than an AST. So, all the
221 expressions or atoms that can be wrapped in parenthesis will subclass
222 this to get that functionality.
223 """
225 __slots__ = ()
227 lpar: Sequence[LeftParen] = ()
228 # Sequence of parenthesis for precedence dictation.
229 rpar: Sequence[RightParen] = ()
231 def _validate(self) -> None:
232 if self.lpar and not self.rpar:
233 raise CSTValidationError("Cannot have left paren without right paren.")
234 if not self.lpar and self.rpar:
235 raise CSTValidationError("Cannot have right paren without left paren.")
236 if len(self.lpar) != len(self.rpar):
237 raise CSTValidationError("Cannot have unbalanced parens.")
239 @contextmanager
240 def _parenthesize(self, state: CodegenState) -> Generator[None, None, None]:
241 for lpar in self.lpar:
242 lpar._codegen(state)
243 with state.record_syntactic_position(self):
244 yield
245 for rpar in self.rpar:
246 rpar._codegen(state)
249class ExpressionPosition(Enum):
250 LEFT = auto()
251 RIGHT = auto()
254class BaseExpression(_BaseParenthesizedNode, ABC):
255 """
256 An base class for all expressions. :class:`BaseExpression` contains no fields.
257 """
259 __slots__ = ()
261 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
262 """
263 Returns true if this expression is safe to be use with a word operator
264 such as "not" without space between the operator an ourselves. Examples
265 where this is true are "not(True)", "(1)in[1,2,3]", etc. This base
266 function handles parenthesized nodes, but certain nodes such as tuples,
267 dictionaries and lists will override this to signifiy that they're always
268 safe.
269 """
271 return len(self.lpar) > 0 and len(self.rpar) > 0
273 def _check_left_right_word_concatenation_safety(
274 self,
275 position: ExpressionPosition,
276 left: "BaseExpression",
277 right: "BaseExpression",
278 ) -> bool:
279 if position == ExpressionPosition.RIGHT:
280 return left._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
281 if position == ExpressionPosition.LEFT:
282 return right._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
283 return False
286class BaseAssignTargetExpression(BaseExpression, ABC):
287 """
288 An expression that's valid on the left side of an assignment. That assignment may
289 be part an :class:`Assign` node, or it may be part of a number of other control
290 structures that perform an assignment, such as a :class:`For` loop.
292 Python's grammar defines all expression as valid in this position, but the AST
293 compiler further restricts the allowed types, which is what this type attempts to
294 express.
296 This is similar to a :class:`BaseDelTargetExpression`, but it also includes
297 :class:`StarredElement` as a valid node.
299 The set of valid nodes are defined as part of `CPython's AST context computation
300 <https://github.com/python/cpython/blob/v3.8.0a4/Python/ast.c#L1120>`_.
301 """
303 __slots__ = ()
306class BaseDelTargetExpression(BaseExpression, ABC):
307 """
308 An expression that's valid on the right side of a :class:`Del` statement.
310 Python's grammar defines all expression as valid in this position, but the AST
311 compiler further restricts the allowed types, which is what this type attempts to
312 express.
314 This is similar to a :class:`BaseAssignTargetExpression`, but it excludes
315 :class:`StarredElement`.
317 The set of valid nodes are defined as part of `CPython's AST context computation
318 <https://github.com/python/cpython/blob/v3.8.0a4/Python/ast.c#L1120>`_ and as part
319 of `CPython's bytecode compiler
320 <https://github.com/python/cpython/blob/v3.8.0a4/Python/compile.c#L4854>`_.
321 """
323 __slots__ = ()
326@add_slots
327@dataclass(frozen=True)
328class Name(BaseAssignTargetExpression, BaseDelTargetExpression):
329 """
330 A simple variable name. Names are typically used in the context of a variable
331 access, an assignment, or a deletion.
333 Dotted variable names (``a.b.c``) are represented with :class:`Attribute` nodes,
334 and subscripted variable names (``a[b]``) are represented with :class:`Subscript`
335 nodes.
336 """
338 #: The variable's name (or "identifier") as a string.
339 value: str
341 lpar: Sequence[LeftParen] = ()
342 #: Sequence of parenthesis for precedence dictation.
343 rpar: Sequence[RightParen] = ()
345 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Name":
346 return Name(
347 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
348 value=self.value,
349 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
350 )
352 def _validate(self) -> None:
353 super(Name, self)._validate()
354 if len(self.value) == 0:
355 raise CSTValidationError("Cannot have empty name identifier.")
356 if not self.value.isidentifier():
357 raise CSTValidationError(f"Name {self.value!r} is not a valid identifier.")
359 def _codegen_impl(self, state: CodegenState) -> None:
360 with self._parenthesize(state):
361 state.add_token(self.value)
364@add_slots
365@dataclass(frozen=True)
366class Ellipsis(BaseExpression):
367 """
368 An ellipsis ``...``. When used as an expression, it evaluates to the
369 `Ellipsis constant`_. Ellipsis are often used as placeholders in code or in
370 conjunction with :class:`SubscriptElement`.
372 .. _Ellipsis constant: https://docs.python.org/3/library/constants.html#Ellipsis
373 """
375 lpar: Sequence[LeftParen] = ()
377 #: Sequence of parenthesis for precedence dictation.
378 rpar: Sequence[RightParen] = ()
380 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Ellipsis":
381 return Ellipsis(
382 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
383 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
384 )
386 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
387 return True
389 def _codegen_impl(self, state: CodegenState) -> None:
390 with self._parenthesize(state):
391 state.add_token("...")
394class BaseNumber(BaseExpression, ABC):
395 """
396 A type such as :class:`Integer`, :class:`Float`, or :class:`Imaginary` that can be
397 used anywhere that you need to explicitly take any number type.
398 """
400 __slots__ = ()
402 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
403 """
404 Numbers are funny. The expression "5in [1,2,3,4,5]" is a valid expression
405 which evaluates to "True". So, encapsulate that here by allowing zero spacing
406 with the left hand side of an expression with a comparison operator.
407 """
408 if position == ExpressionPosition.LEFT:
409 return True
410 return super(BaseNumber, self)._safe_to_use_with_word_operator(position)
413@add_slots
414@dataclass(frozen=True)
415class Integer(BaseNumber):
416 #: A string representation of the integer, such as ``"100000"`` or ``100_000``.
417 #:
418 #: To convert this string representation to an ``int``, use the calculated
419 #: property :attr:`~Integer.evaluated_value`.
420 value: str
422 lpar: Sequence[LeftParen] = ()
423 #: Sequence of parenthesis for precedence dictation.
424 rpar: Sequence[RightParen] = ()
426 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Integer":
427 return Integer(
428 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
429 value=self.value,
430 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
431 )
433 def _validate(self) -> None:
434 super(Integer, self)._validate()
435 if not re.fullmatch(INTNUMBER_RE, self.value):
436 raise CSTValidationError("Number is not a valid integer.")
438 def _codegen_impl(self, state: CodegenState) -> None:
439 with self._parenthesize(state):
440 state.add_token(self.value)
442 @property
443 def evaluated_value(self) -> int:
444 """
445 Return an :func:`ast.literal_eval` evaluated int of :py:attr:`value`.
446 """
447 return literal_eval(self.value)
450@add_slots
451@dataclass(frozen=True)
452class Float(BaseNumber):
453 #: A string representation of the floating point number, such as ``"0.05"``,
454 #: ``".050"``, or ``"5e-2"``.
455 #:
456 #: To convert this string representation to an ``float``, use the calculated
457 #: property :attr:`~Float.evaluated_value`.
458 value: str
460 lpar: Sequence[LeftParen] = ()
461 #: Sequence of parenthesis for precedence dictation.
462 rpar: Sequence[RightParen] = ()
464 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Float":
465 return Float(
466 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
467 value=self.value,
468 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
469 )
471 def _validate(self) -> None:
472 super(Float, self)._validate()
473 if not re.fullmatch(FLOATNUMBER_RE, self.value):
474 raise CSTValidationError("Number is not a valid float.")
476 def _codegen_impl(self, state: CodegenState) -> None:
477 with self._parenthesize(state):
478 state.add_token(self.value)
480 @property
481 def evaluated_value(self) -> float:
482 """
483 Return an :func:`ast.literal_eval` evaluated float of :py:attr:`value`.
484 """
485 return literal_eval(self.value)
488@add_slots
489@dataclass(frozen=True)
490class Imaginary(BaseNumber):
491 #: A string representation of the imaginary (complex) number, such as ``"2j"``.
492 #:
493 #: To convert this string representation to an ``complex``, use the calculated
494 #: property :attr:`~Imaginary.evaluated_value`.
495 value: str
497 lpar: Sequence[LeftParen] = ()
498 #: Sequence of parenthesis for precedence dictation.
499 rpar: Sequence[RightParen] = ()
501 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Imaginary":
502 return Imaginary(
503 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
504 value=self.value,
505 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
506 )
508 def _validate(self) -> None:
509 super(Imaginary, self)._validate()
510 if not re.fullmatch(IMAGNUMBER_RE, self.value):
511 raise CSTValidationError("Number is not a valid imaginary.")
513 def _codegen_impl(self, state: CodegenState) -> None:
514 with self._parenthesize(state):
515 state.add_token(self.value)
517 @property
518 def evaluated_value(self) -> complex:
519 """
520 Return an :func:`ast.literal_eval` evaluated complex of :py:attr:`value`.
521 """
522 return literal_eval(self.value)
525class BaseString(BaseExpression, ABC):
526 """
527 A type that can be used anywhere that you need to take any string. This includes
528 :class:`SimpleString`, :class:`ConcatenatedString`, and :class:`FormattedString`.
529 """
531 __slots__ = ()
534StringQuoteLiteral = Literal['"', "'", '"""', "'''"]
537class _BasePrefixedString(BaseString, ABC):
538 __slots__ = ()
540 @property
541 def prefix(self) -> str:
542 """
543 Returns the string's prefix, if any exists.
545 See `String and Bytes literals
546 <https://docs.python.org/3.7/reference/lexical_analysis.html#string-and-bytes-literals>`_
547 for more information.
548 """
549 ...
551 @property
552 def quote(self) -> StringQuoteLiteral:
553 """
554 Returns the quotation used to denote the string. Can be either ``'``,
555 ``"``, ``'''`` or ``\"\"\"``.
556 """
557 ...
559 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
560 """
561 ``"a"in"abc`` is okay, but if you add a prefix, (e.g. ``b"a"inb"abc"``), the string
562 is no longer valid on the RHS of the word operator, because it's not clear where
563 the keyword ends and the prefix begins, unless it's parenthesized.
564 """
565 if position == ExpressionPosition.LEFT:
566 return True
567 elif self.prefix == "": # and position == ExpressionPosition.RIGHT
568 return True
569 else:
570 return super(_BasePrefixedString, self)._safe_to_use_with_word_operator(
571 position
572 )
575@add_slots
576@dataclass(frozen=True)
577class SimpleString(_BasePrefixedString):
578 """
579 Any sort of literal string expression that is not a :class:`FormattedString`
580 (f-string), including triple-quoted multi-line strings.
581 """
583 #: The texual representation of the string, including quotes, prefix characters, and
584 #: any escape characters present in the original source code , such as
585 #: ``r"my string\n"``. To remove the quotes and interpret any escape characters,
586 #: use the calculated property :attr:`~SimpleString.evaluated_value`.
587 value: str
589 lpar: Sequence[LeftParen] = ()
590 #: Sequence of parenthesis for precidence dictation.
591 rpar: Sequence[RightParen] = ()
593 def _validate(self) -> None:
594 super(SimpleString, self)._validate()
596 # Validate any prefix
597 prefix = self.prefix
598 if prefix not in ("", "r", "u", "b", "br", "rb"):
599 raise CSTValidationError("Invalid string prefix.")
600 prefixlen = len(prefix)
601 # Validate wrapping quotes
602 if len(self.value) < (prefixlen + 2):
603 raise CSTValidationError("String must have enclosing quotes.")
604 if (
605 self.value[prefixlen] not in ['"', "'"]
606 or self.value[prefixlen] != self.value[-1]
607 ):
608 raise CSTValidationError("String must have matching enclosing quotes.")
609 # Check validity of triple-quoted strings
610 if len(self.value) >= (prefixlen + 6):
611 if self.value[prefixlen] == self.value[prefixlen + 1]:
612 # We know this isn't an empty string, so there needs to be a third
613 # identical enclosing token.
614 if (
615 self.value[prefixlen] != self.value[prefixlen + 2]
616 or self.value[prefixlen] != self.value[-2]
617 or self.value[prefixlen] != self.value[-3]
618 ):
619 raise CSTValidationError(
620 "String must have matching enclosing quotes."
621 )
622 # We should check the contents as well, but this is pretty complicated,
623 # partially due to triple-quoted strings.
625 @property
626 def prefix(self) -> str:
627 """
628 Returns the string's prefix, if any exists. The prefix can be ``r``,
629 ``u``, ``b``, ``br`` or ``rb``.
630 """
632 prefix: str = ""
633 for c in self.value:
634 if c in ['"', "'"]:
635 break
636 prefix += c
637 return prefix.lower()
639 @property
640 def quote(self) -> StringQuoteLiteral:
641 """
642 Returns the quotation used to denote the string. Can be either ``'``,
643 ``"``, ``'''`` or ``\"\"\"``.
644 """
646 quote: str = ""
647 for char in self.value[len(self.prefix) :]:
648 if char not in {"'", '"'}:
649 break
650 if quote and char != quote[0]:
651 # This is no longer the same string quote
652 break
653 quote += char
655 if len(quote) == 2:
656 # Let's assume this is an empty string.
657 quote = quote[:1]
658 elif 3 < len(quote) <= 6:
659 # Let's assume this can be one of the following:
660 # >>> """"foo"""
661 # '"foo'
662 # >>> """""bar"""
663 # '""bar'
664 # >>> """"""
665 # ''
666 quote = quote[:3]
668 if len(quote) not in {1, 3}:
669 # We shouldn't get here due to construction validation logic,
670 # but handle the case anyway.
671 raise Exception(f"Invalid string {self.value}")
673 # pyre-ignore We know via the above validation that we will only
674 # ever return one of the four string literals.
675 return quote
677 @property
678 def raw_value(self) -> str:
679 """
680 Returns the raw value of the string as it appears in source, without
681 the beginning or end quotes and without the prefix. This is often
682 useful when constructing transforms which need to manipulate strings
683 in source code.
684 """
686 prefix_len = len(self.prefix)
687 quote_len = len(self.quote)
688 return self.value[(prefix_len + quote_len) : (-quote_len)]
690 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "SimpleString":
691 return SimpleString(
692 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
693 value=self.value,
694 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
695 )
697 def _codegen_impl(self, state: CodegenState) -> None:
698 with self._parenthesize(state):
699 state.add_token(self.value)
701 @property
702 def evaluated_value(self) -> Union[str, bytes]:
703 """
704 Return an :func:`ast.literal_eval` evaluated str of :py:attr:`value`.
705 """
706 return literal_eval(self.value)
709class BaseFormattedStringContent(CSTNode, ABC):
710 """
711 The base type for :class:`FormattedStringText` and
712 :class:`FormattedStringExpression`. A :class:`FormattedString` is composed of a
713 sequence of :class:`BaseFormattedStringContent` parts.
714 """
716 __slots__ = ()
719@add_slots
720@dataclass(frozen=True)
721class FormattedStringText(BaseFormattedStringContent):
722 """
723 Part of a :class:`FormattedString` that is not inside curly braces (``{`` or ``}``).
724 For example, in::
726 f"ab{cd}ef"
728 ``ab`` and ``ef`` are :class:`FormattedStringText` nodes, but ``{cd}`` is a
729 :class:`FormattedStringExpression`.
730 """
732 #: The raw string value, including any escape characters present in the source
733 #: code, not including any enclosing quotes.
734 value: str
736 def _visit_and_replace_children(
737 self, visitor: CSTVisitorT
738 ) -> "FormattedStringText":
739 return FormattedStringText(value=self.value)
741 def _codegen_impl(self, state: CodegenState) -> None:
742 state.add_token(self.value)
745@add_slots
746@dataclass(frozen=True)
747class FormattedStringExpression(BaseFormattedStringContent):
748 """
749 Part of a :class:`FormattedString` that is inside curly braces (``{`` or ``}``),
750 including the surrounding curly braces. For example, in::
752 f"ab{cd}ef"
754 ``{cd}`` is a :class:`FormattedStringExpression`, but ``ab`` and ``ef`` are
755 :class:`FormattedStringText` nodes.
757 An f-string expression may contain ``conversion`` and ``format_spec`` suffixes that
758 control how the expression is converted to a string. See `Python's language
759 reference
760 <https://docs.python.org/3/reference/lexical_analysis.html#formatted-string-literals>`__
761 for details.
762 """
764 #: The expression we will evaluate and render when generating the string.
765 expression: BaseExpression
767 #: An optional conversion specifier, such as ``!s``, ``!r`` or ``!a``.
768 conversion: Optional[str] = None
770 #: An optional format specifier following the `format specification mini-language
771 #: <https://docs.python.org/3/library/string.html#formatspec>`_.
772 format_spec: Optional[Sequence[BaseFormattedStringContent]] = None
774 #: Whitespace after the opening curly brace (``{``), but before the ``expression``.
775 whitespace_before_expression: BaseParenthesizableWhitespace = (
776 SimpleWhitespace.field("")
777 )
779 #: Whitespace after the ``expression``, but before the ``conversion``,
780 #: ``format_spec`` and the closing curly brace (``}``). Python does not
781 #: allow whitespace inside or after a ``conversion`` or ``format_spec``.
782 whitespace_after_expression: BaseParenthesizableWhitespace = SimpleWhitespace.field(
783 ""
784 )
786 #: Equal sign for formatted string expression uses self-documenting expressions,
787 #: such as ``f"{x=}"``. See the `Python 3.8 release notes
788 #: <https://docs.python.org/3/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging>`_.
789 equal: Optional[AssignEqual] = None
791 def _validate(self) -> None:
792 if self.conversion is not None and self.conversion not in ("s", "r", "a"):
793 raise CSTValidationError("Invalid f-string conversion.")
795 def _visit_and_replace_children(
796 self, visitor: CSTVisitorT
797 ) -> "FormattedStringExpression":
798 format_spec = self.format_spec
799 return FormattedStringExpression(
800 whitespace_before_expression=visit_required(
801 self,
802 "whitespace_before_expression",
803 self.whitespace_before_expression,
804 visitor,
805 ),
806 expression=visit_required(self, "expression", self.expression, visitor),
807 equal=visit_optional(self, "equal", self.equal, visitor),
808 whitespace_after_expression=visit_required(
809 self,
810 "whitespace_after_expression",
811 self.whitespace_after_expression,
812 visitor,
813 ),
814 conversion=self.conversion,
815 format_spec=(
816 visit_sequence(self, "format_spec", format_spec, visitor)
817 if format_spec is not None
818 else None
819 ),
820 )
822 def _codegen_impl(self, state: CodegenState) -> None:
823 state.add_token("{")
824 self.whitespace_before_expression._codegen(state)
825 self.expression._codegen(state)
826 equal = self.equal
827 if equal is not None:
828 equal._codegen(state)
829 self.whitespace_after_expression._codegen(state)
830 conversion = self.conversion
831 if conversion is not None:
832 state.add_token("!")
833 state.add_token(conversion)
834 format_spec = self.format_spec
835 if format_spec is not None:
836 state.add_token(":")
837 for spec in format_spec:
838 spec._codegen(state)
839 state.add_token("}")
842@add_slots
843@dataclass(frozen=True)
844class FormattedString(_BasePrefixedString):
845 """
846 An "f-string". These formatted strings are string literals prefixed by the letter
847 "f". An f-string may contain interpolated expressions inside curly braces (``{`` and
848 ``}``).
850 F-strings are defined in `PEP 498`_ and documented in `Python's language
851 reference
852 <https://docs.python.org/3/reference/lexical_analysis.html#formatted-string-literals>`__.
854 >>> import libcst as cst
855 >>> cst.parse_expression('f"ab{cd}ef"')
856 FormattedString(
857 parts=[
858 FormattedStringText(
859 value='ab',
860 ),
861 FormattedStringExpression(
862 expression=Name(
863 value='cd',
864 lpar=[],
865 rpar=[],
866 ),
867 conversion=None,
868 format_spec=None,
869 whitespace_before_expression=SimpleWhitespace(
870 value='',
871 ),
872 whitespace_after_expression=SimpleWhitespace(
873 value='',
874 ),
875 ),
876 FormattedStringText(
877 value='ef',
878 ),
879 ],
880 start='f"',
881 end='"',
882 lpar=[],
883 rpar=[],
884 )
886 .. _PEP 498: https://www.python.org/dev/peps/pep-0498/#specification
887 """
889 #: A formatted string is composed as a series of :class:`FormattedStringText` and
890 #: :class:`FormattedStringExpression` parts.
891 parts: Sequence[BaseFormattedStringContent]
893 #: The string prefix and the leading quote, such as ``f"``, ``F'``, ``fr"``, or
894 #: ``f"""``.
895 start: str = 'f"'
897 #: The trailing quote. This must match the type of quote used in ``start``.
898 end: Literal['"', "'", '"""', "'''"] = '"'
900 lpar: Sequence[LeftParen] = ()
901 #: Sequence of parenthesis for precidence dictation.
902 rpar: Sequence[RightParen] = ()
904 def _validate(self) -> None:
905 super(FormattedString, self)._validate()
907 # Validate any prefix
908 prefix = self.prefix
909 if prefix not in ("f", "fr", "rf"):
910 raise CSTValidationError("Invalid f-string prefix.")
912 # Validate wrapping quotes
913 starttoken = self.start[len(prefix) :]
914 if starttoken != self.end:
915 raise CSTValidationError("f-string must have matching enclosing quotes.")
917 # Validate valid wrapping quote usage
918 if starttoken not in ('"', "'", '"""', "'''"):
919 raise CSTValidationError("Invalid f-string enclosing quotes.")
921 @property
922 def prefix(self) -> str:
923 """
924 Returns the string's prefix, if any exists. The prefix can be ``f``,
925 ``fr``, or ``rf``.
926 """
928 prefix = ""
929 for c in self.start:
930 if c in ['"', "'"]:
931 break
932 prefix += c
933 return prefix.lower()
935 @property
936 def quote(self) -> StringQuoteLiteral:
937 """
938 Returns the quotation used to denote the string. Can be either ``'``,
939 ``"``, ``'''`` or ``\"\"\"``.
940 """
942 return self.end
944 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "FormattedString":
945 return FormattedString(
946 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
947 start=self.start,
948 parts=visit_sequence(self, "parts", self.parts, visitor),
949 end=self.end,
950 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
951 )
953 def _codegen_impl(self, state: CodegenState) -> None:
954 with self._parenthesize(state):
955 state.add_token(self.start)
956 for part in self.parts:
957 part._codegen(state)
958 state.add_token(self.end)
961@add_slots
962@dataclass(frozen=True)
963class ConcatenatedString(BaseString):
964 """
965 Represents an implicitly concatenated string, such as::
967 "abc" "def" == "abcdef"
969 .. warning::
970 This is different from two strings joined in a :class:`BinaryOperation` with an
971 :class:`Add` operator, and is `sometimes viewed as an antifeature of Python
972 <https://lwn.net/Articles/551426/>`_.
973 """
975 #: String on the left of the concatenation.
976 left: Union[SimpleString, FormattedString]
978 #: String on the right of the concatenation.
979 right: Union[SimpleString, FormattedString, "ConcatenatedString"]
981 lpar: Sequence[LeftParen] = ()
982 #: Sequence of parenthesis for precidence dictation.
983 rpar: Sequence[RightParen] = ()
985 #: Whitespace between the ``left`` and ``right`` substrings.
986 whitespace_between: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
988 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
989 if super(ConcatenatedString, self)._safe_to_use_with_word_operator(position):
990 # if we have parenthesis, we're safe.
991 return True
992 return self._check_left_right_word_concatenation_safety(
993 position, self.left, self.right
994 )
996 def _validate(self) -> None:
997 super(ConcatenatedString, self)._validate()
999 # Strings that are concatenated cannot have parens.
1000 if bool(self.left.lpar) or bool(self.left.rpar):
1001 raise CSTValidationError("Cannot concatenate parenthesized strings.")
1002 if bool(self.right.lpar) or bool(self.right.rpar):
1003 raise CSTValidationError("Cannot concatenate parenthesized strings.")
1005 # Cannot concatenate str and bytes
1006 leftbytes = "b" in self.left.prefix
1007 right = self.right
1008 if isinstance(right, ConcatenatedString):
1009 rightbytes = "b" in right.left.prefix
1010 elif isinstance(right, SimpleString):
1011 rightbytes = "b" in right.prefix
1012 elif isinstance(right, FormattedString):
1013 rightbytes = "b" in right.prefix
1014 else:
1015 raise Exception("Logic error!")
1016 if leftbytes != rightbytes:
1017 raise CSTValidationError("Cannot concatenate string and bytes.")
1019 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ConcatenatedString":
1020 return ConcatenatedString(
1021 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1022 left=visit_required(self, "left", self.left, visitor),
1023 whitespace_between=visit_required(
1024 self, "whitespace_between", self.whitespace_between, visitor
1025 ),
1026 right=visit_required(self, "right", self.right, visitor),
1027 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1028 )
1030 def _codegen_impl(self, state: CodegenState) -> None:
1031 with self._parenthesize(state):
1032 self.left._codegen(state)
1033 self.whitespace_between._codegen(state)
1034 self.right._codegen(state)
1036 @property
1037 def evaluated_value(self) -> Union[str, bytes, None]:
1038 """
1039 Return an :func:`ast.literal_eval` evaluated str of recursively concatenated :py:attr:`left` and :py:attr:`right`
1040 if and only if both :py:attr:`left` and :py:attr:`right` are composed by :class:`SimpleString` or :class:`ConcatenatedString`
1041 (:class:`FormattedString` cannot be evaluated).
1042 """
1043 left = self.left
1044 right = self.right
1045 if isinstance(left, FormattedString) or isinstance(right, FormattedString):
1046 return None
1047 left_val = left.evaluated_value
1048 right_val = right.evaluated_value
1049 if right_val is None:
1050 return None
1051 if isinstance(left_val, bytes) and isinstance(right_val, bytes):
1052 return left_val + right_val
1053 if isinstance(left_val, str) and isinstance(right_val, str):
1054 return left_val + right_val
1055 return None
1058@add_slots
1059@dataclass(frozen=True)
1060class ComparisonTarget(CSTNode):
1061 """
1062 A target for a :class:`Comparison`. Owns the comparison operator and the value to
1063 the right of the operator.
1064 """
1066 #: A comparison operator such as ``<``, ``>=``, ``==``, ``is``, or ``in``.
1067 operator: BaseCompOp
1069 #: The right hand side of the comparison operation.
1070 comparator: BaseExpression
1072 def _validate(self) -> None:
1073 # Validate operator spacing rules
1074 operator = self.operator
1075 if (
1076 isinstance(operator, (In, NotIn, Is, IsNot))
1077 and operator.whitespace_after.empty
1078 and not self.comparator._safe_to_use_with_word_operator(
1079 ExpressionPosition.RIGHT
1080 )
1081 ):
1082 raise CSTValidationError(
1083 "Must have at least one space around comparison operator."
1084 )
1086 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ComparisonTarget":
1087 return ComparisonTarget(
1088 operator=visit_required(self, "operator", self.operator, visitor),
1089 comparator=visit_required(self, "comparator", self.comparator, visitor),
1090 )
1092 def _codegen_impl(self, state: CodegenState) -> None:
1093 self.operator._codegen(state)
1094 self.comparator._codegen(state)
1097@add_slots
1098@dataclass(frozen=True)
1099class Comparison(BaseExpression):
1100 """
1101 A comparison between multiple values such as ``x < y``, ``x < y < z``, or
1102 ``x in [y, z]``. These comparisions typically result in boolean values.
1104 Unlike :class:`BinaryOperation` and :class:`BooleanOperation`, comparisons are not
1105 restricted to a left and right child. Instead they can contain an arbitrary number
1106 of :class:`ComparisonTarget` children.
1108 ``x < y < z`` is not equivalent to ``(x < y) < z`` or ``x < (y < z)``. Instead,
1109 it's roughly equivalent to ``x < y and y < z``.
1111 For more details, see `Python's documentation on comparisons
1112 <https://docs.python.org/3/reference/expressions.html#comparisons>`_.
1114 ::
1116 # x < y < z
1118 Comparison(
1119 Name("x"),
1120 [
1121 ComparisonTarget(LessThan(), Name("y")),
1122 ComparisonTarget(LessThan(), Name("z")),
1123 ],
1124 )
1125 """
1127 #: The first value in the full sequence of values to compare. This value will be
1128 #: compared against the first value in ``comparisions``.
1129 left: BaseExpression
1131 #: Pairs of :class:`BaseCompOp` operators and expression values to compare. These
1132 #: come after ``left``. Each value is compared against the value before and after
1133 #: itself in the sequence.
1134 comparisons: Sequence[ComparisonTarget]
1136 lpar: Sequence[LeftParen] = ()
1137 #: Sequence of parenthesis for precedence dictation.
1138 rpar: Sequence[RightParen] = ()
1140 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1141 if super(Comparison, self)._safe_to_use_with_word_operator(position):
1142 # we have parenthesis
1143 return True
1144 return self._check_left_right_word_concatenation_safety(
1145 position, self.left, self.comparisons[-1].comparator
1146 )
1148 def _validate(self) -> None:
1149 # Perform any validation on base type
1150 super(Comparison, self)._validate()
1152 if len(self.comparisons) == 0:
1153 raise CSTValidationError("Must have at least one ComparisonTarget.")
1155 # Validate operator spacing rules
1156 previous_comparator = self.left
1157 for target in self.comparisons:
1158 operator = target.operator
1159 if (
1160 isinstance(operator, (In, NotIn, Is, IsNot))
1161 and operator.whitespace_before.empty
1162 and not previous_comparator._safe_to_use_with_word_operator(
1163 ExpressionPosition.LEFT
1164 )
1165 ):
1166 raise CSTValidationError(
1167 "Must have at least one space around comparison operator."
1168 )
1169 previous_comparator = target.comparator
1171 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Comparison":
1172 return Comparison(
1173 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1174 left=visit_required(self, "left", self.left, visitor),
1175 comparisons=visit_sequence(self, "comparisons", self.comparisons, visitor),
1176 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1177 )
1179 def _codegen_impl(self, state: CodegenState) -> None:
1180 with self._parenthesize(state):
1181 self.left._codegen(state)
1182 for comp in self.comparisons:
1183 comp._codegen(state)
1186@add_slots
1187@dataclass(frozen=True)
1188class UnaryOperation(BaseExpression):
1189 """
1190 Any generic unary expression, such as ``not x`` or ``-x``. :class:`UnaryOperation`
1191 nodes apply a :class:`BaseUnaryOp` to an expression.
1192 """
1194 #: The unary operator that applies some operation (e.g. negation) to the
1195 #: ``expression``.
1196 operator: BaseUnaryOp
1198 #: The expression that should be transformed (e.g. negated) by the operator to
1199 #: create a new value.
1200 expression: BaseExpression
1202 lpar: Sequence[LeftParen] = ()
1203 #: Sequence of parenthesis for precedence dictation.
1204 rpar: Sequence[RightParen] = ()
1206 def _validate(self) -> None:
1207 # Perform any validation on base type
1208 super(UnaryOperation, self)._validate()
1210 if (
1211 isinstance(self.operator, Not)
1212 and self.operator.whitespace_after.empty
1213 and not self.expression._safe_to_use_with_word_operator(
1214 ExpressionPosition.RIGHT
1215 )
1216 ):
1217 raise CSTValidationError("Must have at least one space after not operator.")
1219 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "UnaryOperation":
1220 return UnaryOperation(
1221 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1222 operator=visit_required(self, "operator", self.operator, visitor),
1223 expression=visit_required(self, "expression", self.expression, visitor),
1224 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1225 )
1227 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1228 """
1229 As long as we aren't comprised of the Not unary operator, we are safe to use
1230 without space.
1231 """
1232 if super(UnaryOperation, self)._safe_to_use_with_word_operator(position):
1233 return True
1234 if position == ExpressionPosition.RIGHT:
1235 return not isinstance(self.operator, Not)
1236 if position == ExpressionPosition.LEFT:
1237 return self.expression._safe_to_use_with_word_operator(
1238 ExpressionPosition.LEFT
1239 )
1240 return False
1242 def _codegen_impl(self, state: CodegenState) -> None:
1243 with self._parenthesize(state):
1244 self.operator._codegen(state)
1245 self.expression._codegen(state)
1248@add_slots
1249@dataclass(frozen=True)
1250class BinaryOperation(BaseExpression):
1251 """
1252 An operation that combines two expression such as ``x << y`` or ``y + z``.
1253 :class:`BinaryOperation` nodes apply a :class:`BaseBinaryOp` to an expression.
1255 Binary operations do not include operations performed with :class:`BaseBooleanOp`
1256 nodes, such as ``and`` or ``or``. Instead, those operations are provided by
1257 :class:`BooleanOperation`.
1259 It also does not include support for comparision operators performed with
1260 :class:`BaseCompOp`, such as ``<``, ``>=``, ``==``, ``is``, or ``in``. Instead,
1261 those operations are provided by :class:`Comparison`.
1262 """
1264 #: The left hand side of the operation.
1265 left: BaseExpression
1267 #: The actual operator such as ``<<`` or ``+`` that combines the ``left`` and
1268 #: ``right`` expressions.
1269 operator: BaseBinaryOp
1271 #: The right hand side of the operation.
1272 right: BaseExpression
1274 lpar: Sequence[LeftParen] = ()
1275 #: Sequence of parenthesis for precedence dictation.
1276 rpar: Sequence[RightParen] = ()
1278 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "BinaryOperation":
1279 return BinaryOperation(
1280 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1281 left=visit_required(self, "left", self.left, visitor),
1282 operator=visit_required(self, "operator", self.operator, visitor),
1283 right=visit_required(self, "right", self.right, visitor),
1284 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1285 )
1287 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1288 if super(BinaryOperation, self)._safe_to_use_with_word_operator(position):
1289 return True
1290 return self._check_left_right_word_concatenation_safety(
1291 position, self.left, self.right
1292 )
1294 def _codegen_impl(self, state: CodegenState) -> None:
1295 with self._parenthesize(state):
1296 self.left._codegen(state)
1297 self.operator._codegen(state)
1298 self.right._codegen(state)
1301@add_slots
1302@dataclass(frozen=True)
1303class BooleanOperation(BaseExpression):
1304 """
1305 An operation that combines two booleans such as ``x or y`` or ``z and w``
1306 :class:`BooleanOperation` nodes apply a :class:`BaseBooleanOp` to an expression.
1308 Boolean operations do not include operations performed with :class:`BaseBinaryOp`
1309 nodes, such as ``+`` or ``<<``. Instead, those operations are provided by
1310 :class:`BinaryOperation`.
1312 It also does not include support for comparision operators performed with
1313 :class:`BaseCompOp`, such as ``<``, ``>=``, ``==``, ``is``, or ``in``. Instead,
1314 those operations are provided by :class:`Comparison`.
1315 """
1317 #: The left hand side of the operation.
1318 left: BaseExpression
1320 #: The actual operator such as ``and`` or ``or`` that combines the ``left`` and
1321 #: ``right`` expressions.
1322 operator: BaseBooleanOp
1324 #: The right hand side of the operation.
1325 right: BaseExpression
1327 lpar: Sequence[LeftParen] = ()
1328 #: Sequence of parenthesis for precedence dictation.
1329 rpar: Sequence[RightParen] = ()
1331 def _validate(self) -> None:
1332 # Paren validation and such
1333 super(BooleanOperation, self)._validate()
1334 # Validate spacing rules
1335 if (
1336 self.operator.whitespace_before.empty
1337 and not self.left._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
1338 ):
1339 raise CSTValidationError(
1340 "Must have at least one space around boolean operator."
1341 )
1342 if (
1343 self.operator.whitespace_after.empty
1344 and not self.right._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
1345 ):
1346 raise CSTValidationError(
1347 "Must have at least one space around boolean operator."
1348 )
1350 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "BooleanOperation":
1351 return BooleanOperation(
1352 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1353 left=visit_required(self, "left", self.left, visitor),
1354 operator=visit_required(self, "operator", self.operator, visitor),
1355 right=visit_required(self, "right", self.right, visitor),
1356 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1357 )
1359 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1360 if super(BooleanOperation, self)._safe_to_use_with_word_operator(position):
1361 return True
1362 return self._check_left_right_word_concatenation_safety(
1363 position, self.left, self.right
1364 )
1366 def _codegen_impl(self, state: CodegenState) -> None:
1367 with self._parenthesize(state):
1368 self.left._codegen(state)
1369 self.operator._codegen(state)
1370 self.right._codegen(state)
1373@add_slots
1374@dataclass(frozen=True)
1375class Attribute(BaseAssignTargetExpression, BaseDelTargetExpression):
1376 """
1377 An attribute reference, such as ``x.y``.
1379 Note that in the case of ``x.y.z``, the outer attribute will have an attr of ``z``
1380 and the value will be another :class:`Attribute` referencing the ``y`` attribute on
1381 ``x``::
1383 Attribute(
1384 value=Attribute(
1385 value=Name("x")
1386 attr=Name("y")
1387 ),
1388 attr=Name("z"),
1389 )
1390 """
1392 #: An expression which, when evaluated, will produce an object with ``attr`` as an
1393 #: attribute.
1394 value: BaseExpression
1396 #: The name of the attribute being accessed on the ``value`` object.
1397 attr: Name
1399 #: A separating dot. If there's whitespace between the ``value`` and ``attr``, this
1400 #: dot owns it.
1401 dot: Dot = Dot()
1403 lpar: Sequence[LeftParen] = ()
1404 #: Sequence of parenthesis for precedence dictation.
1405 rpar: Sequence[RightParen] = ()
1407 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Attribute":
1408 return Attribute(
1409 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1410 value=visit_required(self, "value", self.value, visitor),
1411 dot=visit_required(self, "dot", self.dot, visitor),
1412 attr=visit_required(self, "attr", self.attr, visitor),
1413 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1414 )
1416 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1417 if super(Attribute, self)._safe_to_use_with_word_operator(position):
1418 return True
1419 return self._check_left_right_word_concatenation_safety(
1420 position, self.value, self.attr
1421 )
1423 def _codegen_impl(self, state: CodegenState) -> None:
1424 with self._parenthesize(state):
1425 self.value._codegen(state)
1426 self.dot._codegen(state)
1427 self.attr._codegen(state)
1430class BaseSlice(CSTNode, ABC):
1431 """
1432 Any slice type that can slot into a :class:`SubscriptElement`.
1433 This node is purely for typing.
1434 """
1436 __slots__ = ()
1439@add_slots
1440@dataclass(frozen=True)
1441class Index(BaseSlice):
1442 """
1443 Any index as passed to a :class:`Subscript`. In ``x[2]``, this would be the ``2``
1444 value.
1445 """
1447 #: The index value itself.
1448 value: BaseExpression
1450 #: An optional string with an asterisk appearing before the name. This is
1451 #: expanded into variable number of positional arguments. See PEP-646
1452 star: Optional[Literal["*"]] = None
1454 #: Whitespace after the ``star`` (if it exists), but before the ``value``.
1455 whitespace_after_star: Optional[BaseParenthesizableWhitespace] = None
1457 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Index":
1458 return Index(
1459 star=self.star,
1460 whitespace_after_star=visit_optional(
1461 self, "whitespace_after_star", self.whitespace_after_star, visitor
1462 ),
1463 value=visit_required(self, "value", self.value, visitor),
1464 )
1466 def _codegen_impl(self, state: CodegenState) -> None:
1467 star = self.star
1468 if star is not None:
1469 state.add_token(star)
1470 ws = self.whitespace_after_star
1471 if ws is not None:
1472 ws._codegen(state)
1473 self.value._codegen(state)
1476@add_slots
1477@dataclass(frozen=True)
1478class Slice(BaseSlice):
1479 """
1480 Any slice operation in a :class:`Subscript`, such as ``1:``, ``2:3:4``, etc.
1482 Note that the grammar does NOT allow parenthesis around a slice so they are not
1483 supported here.
1484 """
1486 #: The lower bound in the slice, if present
1487 lower: Optional[BaseExpression]
1489 #: The upper bound in the slice, if present
1490 upper: Optional[BaseExpression]
1492 #: The step in the slice, if present
1493 step: Optional[BaseExpression] = None
1495 #: The first slice operator
1496 first_colon: Colon = Colon.field()
1498 #: The second slice operator, usually omitted
1499 second_colon: Union[Colon, MaybeSentinel] = MaybeSentinel.DEFAULT
1501 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Slice":
1502 return Slice(
1503 lower=visit_optional(self, "lower", self.lower, visitor),
1504 first_colon=visit_required(self, "first_colon", self.first_colon, visitor),
1505 upper=visit_optional(self, "upper", self.upper, visitor),
1506 second_colon=visit_sentinel(
1507 self, "second_colon", self.second_colon, visitor
1508 ),
1509 step=visit_optional(self, "step", self.step, visitor),
1510 )
1512 def _codegen_impl(self, state: CodegenState) -> None:
1513 lower = self.lower
1514 if lower is not None:
1515 lower._codegen(state)
1516 self.first_colon._codegen(state)
1517 upper = self.upper
1518 if upper is not None:
1519 upper._codegen(state)
1520 second_colon = self.second_colon
1521 if second_colon is MaybeSentinel.DEFAULT and self.step is not None:
1522 state.add_token(":")
1523 elif isinstance(second_colon, Colon):
1524 second_colon._codegen(state)
1525 step = self.step
1526 if step is not None:
1527 step._codegen(state)
1530@add_slots
1531@dataclass(frozen=True)
1532class SubscriptElement(CSTNode):
1533 """
1534 Part of a sequence of slices in a :class:`Subscript`, such as ``1:2, 3``. This is
1535 not used in Python's standard library, but it is used in some third-party
1536 libraries. For example, `NumPy uses it to select values and ranges from
1537 multi-dimensional arrays
1538 <https://docs.scipy.org/doc/numpy-1.10.1/user/basics.indexing.html>`_.
1539 """
1541 #: A slice or index that is part of a subscript.
1542 slice: BaseSlice
1544 #: A separating comma, with any whitespace it owns.
1545 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
1547 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "SubscriptElement":
1548 return SubscriptElement(
1549 slice=visit_required(self, "slice", self.slice, visitor),
1550 comma=visit_sentinel(self, "comma", self.comma, visitor),
1551 )
1553 def _codegen_impl(self, state: CodegenState, default_comma: bool = False) -> None:
1554 with state.record_syntactic_position(self):
1555 self.slice._codegen(state)
1557 comma = self.comma
1558 if comma is MaybeSentinel.DEFAULT and default_comma:
1559 state.add_token(", ")
1560 elif isinstance(comma, Comma):
1561 comma._codegen(state)
1564@add_slots
1565@dataclass(frozen=True)
1566class Subscript(BaseAssignTargetExpression, BaseDelTargetExpression):
1567 """
1568 A indexed subscript reference (:class:`Index`) such as ``x[2]``, a :class:`Slice`
1569 such as ``x[1:-1]``, or an extended slice (:class:`SubscriptElement`) such as ``x[1:2, 3]``.
1570 """
1572 #: The left-hand expression which, when evaluated, will be subscripted, such as
1573 #: ``x`` in ``x[2]``.
1574 value: BaseExpression
1576 #: The :class:`SubscriptElement` to extract from the ``value``.
1577 slice: Sequence[SubscriptElement]
1579 lbracket: LeftSquareBracket = LeftSquareBracket.field()
1580 #: Brackets after the ``value`` surrounding the ``slice``.
1581 rbracket: RightSquareBracket = RightSquareBracket.field()
1583 lpar: Sequence[LeftParen] = ()
1584 #: Sequence of parenthesis for precedence dictation.
1585 rpar: Sequence[RightParen] = ()
1587 #: Whitespace after the ``value``, but before the ``lbracket``.
1588 whitespace_after_value: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1590 def _validate(self) -> None:
1591 super(Subscript, self)._validate()
1592 # Validate valid commas
1593 if len(self.slice) < 1:
1594 raise CSTValidationError("Cannot have empty SubscriptElement.")
1596 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Subscript":
1597 return Subscript(
1598 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
1599 value=visit_required(self, "value", self.value, visitor),
1600 whitespace_after_value=visit_required(
1601 self, "whitespace_after_value", self.whitespace_after_value, visitor
1602 ),
1603 lbracket=visit_required(self, "lbracket", self.lbracket, visitor),
1604 slice=visit_sequence(self, "slice", self.slice, visitor),
1605 rbracket=visit_required(self, "rbracket", self.rbracket, visitor),
1606 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
1607 )
1609 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
1610 if position == ExpressionPosition.LEFT:
1611 return True
1612 if super(Subscript, self)._safe_to_use_with_word_operator(position):
1613 return True
1614 if position == ExpressionPosition.RIGHT:
1615 return self.value._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
1616 return False
1618 def _codegen_impl(self, state: CodegenState) -> None:
1619 with self._parenthesize(state):
1620 self.value._codegen(state)
1621 self.whitespace_after_value._codegen(state)
1622 self.lbracket._codegen(state)
1623 lastslice = len(self.slice) - 1
1624 for i, slice in enumerate(self.slice):
1625 slice._codegen(state, default_comma=(i != lastslice))
1626 self.rbracket._codegen(state)
1629@add_slots
1630@dataclass(frozen=True)
1631class Annotation(CSTNode):
1632 """
1633 An annotation for a function (`PEP 3107`_) or on a variable (`PEP 526`_). Typically
1634 these are used in the context of type hints (`PEP 484`_), such as::
1636 # a variable with a type
1637 good_ideas: List[str] = []
1639 # a function with type annotations
1640 def concat(substrings: Sequence[str]) -> str:
1641 ...
1643 .. _PEP 3107: https://www.python.org/dev/peps/pep-3107/
1644 .. _PEP 526: https://www.python.org/dev/peps/pep-0526/
1645 .. _PEP 484: https://www.python.org/dev/peps/pep-0484/
1646 """
1648 #: The annotation's value itself. This is the part of the annotation after the
1649 #: colon or arrow.
1650 annotation: BaseExpression
1652 whitespace_before_indicator: Union[
1653 BaseParenthesizableWhitespace, MaybeSentinel
1654 ] = MaybeSentinel.DEFAULT
1655 whitespace_after_indicator: BaseParenthesizableWhitespace = SimpleWhitespace.field(
1656 " "
1657 )
1659 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Annotation":
1660 return Annotation(
1661 whitespace_before_indicator=visit_sentinel(
1662 self,
1663 "whitespace_before_indicator",
1664 self.whitespace_before_indicator,
1665 visitor,
1666 ),
1667 whitespace_after_indicator=visit_required(
1668 self,
1669 "whitespace_after_indicator",
1670 self.whitespace_after_indicator,
1671 visitor,
1672 ),
1673 annotation=visit_required(self, "annotation", self.annotation, visitor),
1674 )
1676 def _codegen_impl(
1677 self, state: CodegenState, default_indicator: Optional[str] = None
1678 ) -> None:
1679 # First, figure out the indicator which tells us default whitespace.
1680 if default_indicator is None:
1681 raise CSTCodegenError(
1682 "Must specify a concrete default_indicator if default used on indicator."
1683 )
1685 # Now, output the whitespace
1686 whitespace_before_indicator = self.whitespace_before_indicator
1687 if isinstance(whitespace_before_indicator, BaseParenthesizableWhitespace):
1688 whitespace_before_indicator._codegen(state)
1689 elif isinstance(whitespace_before_indicator, MaybeSentinel):
1690 if default_indicator == "->":
1691 state.add_token(" ")
1692 else:
1693 raise Exception("Logic error!")
1695 # Now, output the indicator and the rest of the annotation
1696 state.add_token(default_indicator)
1697 self.whitespace_after_indicator._codegen(state)
1699 with state.record_syntactic_position(self):
1700 self.annotation._codegen(state)
1703@add_slots
1704@dataclass(frozen=True)
1705class ParamStar(CSTNode):
1706 """
1707 A sentinel indicator on a :class:`Parameters` list to denote that the subsequent
1708 params are keyword-only args.
1710 This syntax is described in `PEP 3102`_.
1712 .. _PEP 3102: https://www.python.org/dev/peps/pep-3102/#specification
1713 """
1715 # Comma that comes after the star.
1716 comma: Comma = Comma.field(whitespace_after=SimpleWhitespace(" "))
1718 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ParamStar":
1719 return ParamStar(comma=visit_required(self, "comma", self.comma, visitor))
1721 def _codegen_impl(self, state: CodegenState) -> None:
1722 state.add_token("*")
1723 self.comma._codegen(state)
1726@add_slots
1727@dataclass(frozen=True)
1728class ParamSlash(CSTNode):
1729 """
1730 A sentinel indicator on a :class:`Parameters` list to denote that the previous
1731 params are positional-only args.
1733 This syntax is described in `PEP 570`_.
1735 .. _PEP 570: https://www.python.org/dev/peps/pep-0570/#specification
1736 """
1738 #: Optional comma that comes after the slash. This comma doesn't own the whitespace
1739 #: between ``/`` and ``,``.
1740 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
1742 #: Whitespace after the ``/`` character. This is captured here in case there is a
1743 #: comma.
1744 whitespace_after: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1746 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ParamSlash":
1747 return ParamSlash(
1748 comma=visit_sentinel(self, "comma", self.comma, visitor),
1749 whitespace_after=visit_required(
1750 self, "whitespace_after", self.whitespace_after, visitor
1751 ),
1752 )
1754 def _codegen_impl(self, state: CodegenState, default_comma: bool = False) -> None:
1755 state.add_token("/")
1757 self.whitespace_after._codegen(state)
1758 comma = self.comma
1759 if comma is MaybeSentinel.DEFAULT and default_comma:
1760 state.add_token(", ")
1761 elif isinstance(comma, Comma):
1762 comma._codegen(state)
1765@add_slots
1766@dataclass(frozen=True)
1767class Param(CSTNode):
1768 """
1769 A positional or keyword argument in a :class:`Parameters` list. May contain an
1770 :class:`Annotation` and, in some cases, a ``default``.
1771 """
1773 #: The parameter name itself.
1774 name: Name
1776 #: Any optional :class:`Annotation`. These annotations are usually used as type
1777 #: hints.
1778 annotation: Optional[Annotation] = None
1780 #: The equal sign used to denote assignment if there is a default.
1781 equal: Union[AssignEqual, MaybeSentinel] = MaybeSentinel.DEFAULT
1783 #: Any optional default value, used when the argument is not supplied.
1784 default: Optional[BaseExpression] = None
1786 #: A trailing comma. If one is not provided, :class:`MaybeSentinel` will be
1787 #: replaced with a comma only if a comma is required.
1788 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
1790 #: Zero, one, or two asterisks appearing before name for :class:`Param`'s
1791 #: ``star_arg`` and ``star_kwarg``.
1792 star: Union[str, MaybeSentinel] = MaybeSentinel.DEFAULT
1794 #: The whitespace before ``name``. It will appear after ``star`` when a star
1795 #: exists.
1796 whitespace_after_star: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1798 #: The whitespace after this entire node.
1799 whitespace_after_param: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
1801 def _validate(self) -> None:
1802 if self.default is None and isinstance(self.equal, AssignEqual):
1803 raise CSTValidationError(
1804 "Must have a default when specifying an AssignEqual."
1805 )
1806 if isinstance(self.star, str) and self.star not in ("", "*", "**"):
1807 raise CSTValidationError("Must specify either '', '*' or '**' for star.")
1809 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Param":
1810 return Param(
1811 star=self.star,
1812 whitespace_after_star=visit_required(
1813 self, "whitespace_after_star", self.whitespace_after_star, visitor
1814 ),
1815 name=visit_required(self, "name", self.name, visitor),
1816 annotation=visit_optional(self, "annotation", self.annotation, visitor),
1817 equal=visit_sentinel(self, "equal", self.equal, visitor),
1818 default=visit_optional(self, "default", self.default, visitor),
1819 comma=visit_sentinel(self, "comma", self.comma, visitor),
1820 whitespace_after_param=visit_required(
1821 self, "whitespace_after_param", self.whitespace_after_param, visitor
1822 ),
1823 )
1825 def _codegen_impl(
1826 self,
1827 state: CodegenState,
1828 default_star: Optional[str] = None,
1829 default_comma: bool = False,
1830 ) -> None:
1831 with state.record_syntactic_position(self):
1832 star = self.star
1833 if isinstance(star, MaybeSentinel):
1834 if default_star is None:
1835 raise CSTCodegenError(
1836 "Must specify a concrete default_star if default used on star."
1837 )
1838 star = default_star
1839 if isinstance(star, str):
1840 state.add_token(star)
1841 self.whitespace_after_star._codegen(state)
1842 self.name._codegen(state)
1844 annotation = self.annotation
1845 if annotation is not None:
1846 annotation._codegen(state, default_indicator=":")
1847 equal = self.equal
1848 if equal is MaybeSentinel.DEFAULT and self.default is not None:
1849 state.add_token(" = ")
1850 elif isinstance(equal, AssignEqual):
1851 equal._codegen(state)
1852 default = self.default
1853 if default is not None:
1854 default._codegen(state)
1855 comma = self.comma
1856 if comma is MaybeSentinel.DEFAULT and default_comma:
1857 state.add_token(", ")
1858 elif isinstance(comma, Comma):
1859 comma._codegen(state)
1861 self.whitespace_after_param._codegen(state)
1864@add_slots
1865@dataclass(frozen=True)
1866class Parameters(CSTNode):
1867 """
1868 A function or lambda parameter list.
1869 """
1871 #: Positional parameters, with or without defaults. Positional parameters
1872 #: with defaults must all be after those without defaults.
1873 params: Sequence[Param] = ()
1875 # Optional parameter that captures unspecified positional arguments or a sentinel
1876 # star that dictates parameters following are kwonly args.
1877 star_arg: Union[Param, ParamStar, MaybeSentinel] = MaybeSentinel.DEFAULT
1879 #: Keyword-only params that may or may not have defaults.
1880 kwonly_params: Sequence[Param] = ()
1882 #: Optional parameter that captures unspecified kwargs.
1883 star_kwarg: Optional[Param] = None
1885 #: Positional-only parameters, with or without defaults. Positional-only
1886 #: parameters with defaults must all be after those without defaults.
1887 posonly_params: Sequence[Param] = ()
1889 #: Optional sentinel that dictates parameters preceeding are positional-only
1890 #: args.
1891 posonly_ind: Union[ParamSlash, MaybeSentinel] = MaybeSentinel.DEFAULT
1893 def _validate_stars_sequence(self, vals: Sequence[Param], *, section: str) -> None:
1894 if len(vals) == 0:
1895 return
1896 for val in vals:
1897 if isinstance(val.star, str) and val.star != "":
1898 raise CSTValidationError(
1899 f"Expecting a star prefix of '' for {section} Param."
1900 )
1902 def _validate_posonly_ind(self) -> None:
1903 if isinstance(self.posonly_ind, ParamSlash) and len(self.posonly_params) == 0:
1904 raise CSTValidationError(
1905 "Must have at least one posonly param if ParamSlash is used."
1906 )
1908 def _validate_kwonly_star(self) -> None:
1909 if isinstance(self.star_arg, ParamStar) and len(self.kwonly_params) == 0:
1910 raise CSTValidationError(
1911 "Must have at least one kwonly param if ParamStar is used."
1912 )
1914 def _validate_defaults(self) -> None:
1915 seen_default = False
1916 # pyre-fixme[60]: Concatenation not yet support for multiple variadic
1917 # tuples: `*self.posonly_params, *self.params`.
1918 for param in (*self.posonly_params, *self.params):
1919 if param.default:
1920 # Mark that we've moved onto defaults
1921 if not seen_default:
1922 seen_default = True
1923 else:
1924 if seen_default:
1925 # We accidentally included a non-default after a default arg!
1926 raise CSTValidationError(
1927 "Cannot have param without defaults following a param with defaults."
1928 )
1929 star_arg = self.star_arg
1930 if isinstance(star_arg, Param) and star_arg.default is not None:
1931 raise CSTValidationError("Cannot have default for star_arg.")
1932 star_kwarg = self.star_kwarg
1933 if star_kwarg is not None and star_kwarg.default is not None:
1934 raise CSTValidationError("Cannot have default for star_kwarg.")
1936 def _validate_stars(self) -> None:
1937 if len(self.params) > 0:
1938 self._validate_stars_sequence(self.params, section="params")
1939 if len(self.posonly_params) > 0:
1940 self._validate_stars_sequence(self.posonly_params, section="posonly_params")
1941 star_arg = self.star_arg
1942 if (
1943 isinstance(star_arg, Param)
1944 and isinstance(star_arg.star, str)
1945 and star_arg.star != "*"
1946 ):
1947 raise CSTValidationError(
1948 "Expecting a star prefix of '*' for star_arg Param."
1949 )
1950 if len(self.kwonly_params) > 0:
1951 self._validate_stars_sequence(self.kwonly_params, section="kwonly_params")
1952 star_kwarg = self.star_kwarg
1953 if (
1954 star_kwarg is not None
1955 and isinstance(star_kwarg.star, str)
1956 and star_kwarg.star != "**"
1957 ):
1958 raise CSTValidationError(
1959 "Expecting a star prefix of '**' for star_kwarg Param."
1960 )
1962 def _validate(self) -> None:
1963 # Validate posonly_params slash placement semantics.
1964 self._validate_posonly_ind()
1965 # Validate kwonly_param star placement semantics.
1966 self._validate_kwonly_star()
1967 # Validate defaults semantics for params and star_arg/star_kwarg.
1968 self._validate_defaults()
1969 # Validate that we don't have random stars on non star_kwarg.
1970 self._validate_stars()
1972 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Parameters":
1973 return Parameters(
1974 posonly_params=visit_sequence(
1975 self, "posonly_params", self.posonly_params, visitor
1976 ),
1977 posonly_ind=visit_sentinel(self, "posonly_ind", self.posonly_ind, visitor),
1978 params=visit_sequence(self, "params", self.params, visitor),
1979 star_arg=visit_sentinel(self, "star_arg", self.star_arg, visitor),
1980 kwonly_params=visit_sequence(
1981 self, "kwonly_params", self.kwonly_params, visitor
1982 ),
1983 star_kwarg=visit_optional(self, "star_kwarg", self.star_kwarg, visitor),
1984 )
1986 def _safe_to_join_with_lambda(self) -> bool:
1987 """
1988 Determine if Parameters need a space after the `lambda` keyword. Returns True
1989 iff it's safe to omit the space between `lambda` and these Parameters.
1991 See also `BaseExpression._safe_to_use_with_word_operator`.
1993 For example: `lambda*_: pass`
1994 """
1995 if len(self.posonly_params) != 0:
1996 return False
1998 # posonly_ind can't appear if above condition is false
2000 if len(self.params) > 0 and self.params[0].star not in {"*", "**"}:
2001 return False
2003 return True
2005 def _codegen_impl(self, state: CodegenState) -> None: # noqa: C901
2006 # Compute the star existence first so we can ask about whether
2007 # each element is the last in the list or not.
2008 star_arg = self.star_arg
2009 if isinstance(star_arg, MaybeSentinel):
2010 starincluded = len(self.kwonly_params) > 0
2011 elif isinstance(star_arg, (Param, ParamStar)):
2012 starincluded = True
2013 else:
2014 starincluded = False
2015 # Render out the positional-only params first. They will always have trailing
2016 # commas because in order to have positional-only params, there must be a
2017 # slash afterwards.
2018 for i, param in enumerate(self.posonly_params):
2019 param._codegen(state, default_star="", default_comma=True)
2020 # Render out the positional-only indicator if necessary.
2021 more_values = (
2022 starincluded
2023 or len(self.params) > 0
2024 or len(self.kwonly_params) > 0
2025 or self.star_kwarg is not None
2026 )
2027 posonly_ind = self.posonly_ind
2028 if isinstance(posonly_ind, ParamSlash):
2029 # Its explicitly included, so render the version we have here which
2030 # might have spacing applied to its comma.
2031 posonly_ind._codegen(state, default_comma=more_values)
2032 elif len(self.posonly_params) > 0:
2033 if more_values:
2034 state.add_token("/, ")
2035 else:
2036 state.add_token("/")
2037 # Render out the params next, computing necessary trailing commas.
2038 lastparam = len(self.params) - 1
2039 more_values = (
2040 starincluded or len(self.kwonly_params) > 0 or self.star_kwarg is not None
2041 )
2042 for i, param in enumerate(self.params):
2043 param._codegen(
2044 state, default_star="", default_comma=(i < lastparam or more_values)
2045 )
2046 # Render out optional star sentinel if its explicitly included or
2047 # if we are inferring it from kwonly_params. Otherwise, render out the
2048 # optional star_arg.
2049 if isinstance(star_arg, MaybeSentinel):
2050 if starincluded:
2051 state.add_token("*, ")
2052 elif isinstance(star_arg, Param):
2053 more_values = len(self.kwonly_params) > 0 or self.star_kwarg is not None
2054 star_arg._codegen(state, default_star="*", default_comma=more_values)
2055 elif isinstance(star_arg, ParamStar):
2056 star_arg._codegen(state)
2057 # Render out the kwonly_args next, computing necessary trailing commas.
2058 lastparam = len(self.kwonly_params) - 1
2059 more_values = self.star_kwarg is not None
2060 for i, param in enumerate(self.kwonly_params):
2061 param._codegen(
2062 state, default_star="", default_comma=(i < lastparam or more_values)
2063 )
2064 # Finally, render out any optional star_kwarg
2065 star_kwarg = self.star_kwarg
2066 if star_kwarg is not None:
2067 star_kwarg._codegen(state, default_star="**", default_comma=False)
2070@add_slots
2071@dataclass(frozen=True)
2072class Lambda(BaseExpression):
2073 """
2074 A lambda expression that creates an anonymous function.
2076 ::
2078 Lambda(
2079 params=Parameters([Param(Name("arg"))]),
2080 body=Ellipsis(),
2081 )
2083 Represents the following code::
2085 lambda arg: ...
2087 Named functions statements are provided by :class:`FunctionDef`.
2088 """
2090 #: The arguments to the lambda. This is similar to the arguments on a
2091 #: :class:`FunctionDef`, however lambda arguments are not allowed to have an
2092 #: :class:`Annotation`.
2093 params: Parameters
2095 #: The value that the lambda computes and returns when called.
2096 body: BaseExpression
2098 #: The colon separating the parameters from the body.
2099 colon: Colon = Colon.field(whitespace_after=SimpleWhitespace(" "))
2101 lpar: Sequence[LeftParen] = ()
2102 #: Sequence of parenthesis for precedence dictation.
2103 rpar: Sequence[RightParen] = ()
2105 #: Whitespace after the lambda keyword, but before any argument or the colon.
2106 whitespace_after_lambda: Union[
2107 BaseParenthesizableWhitespace, MaybeSentinel
2108 ] = MaybeSentinel.DEFAULT
2110 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2111 if position == ExpressionPosition.LEFT:
2112 return len(self.rpar) > 0 or self.body._safe_to_use_with_word_operator(
2113 position
2114 )
2115 return super()._safe_to_use_with_word_operator(position)
2117 def _validate(self) -> None:
2118 # Validate parents
2119 super(Lambda, self)._validate()
2120 # Sum up all parameters
2121 all_params = [
2122 *self.params.posonly_params,
2123 *self.params.params,
2124 *self.params.kwonly_params,
2125 ]
2126 star_arg = self.params.star_arg
2127 if isinstance(star_arg, Param):
2128 all_params.append(star_arg)
2129 star_kwarg = self.params.star_kwarg
2130 if star_kwarg is not None:
2131 all_params.append(star_kwarg)
2132 # Check for nonzero parameters because several checks care
2133 # about this.
2134 if len(all_params) > 0:
2135 for param in all_params:
2136 if param.annotation is not None:
2137 raise CSTValidationError(
2138 "Lambda params cannot have type annotations."
2139 )
2140 whitespace_after_lambda = self.whitespace_after_lambda
2141 if (
2142 isinstance(whitespace_after_lambda, BaseParenthesizableWhitespace)
2143 and whitespace_after_lambda.empty
2144 and not self.params._safe_to_join_with_lambda()
2145 ):
2146 raise CSTValidationError(
2147 "Must have at least one space after lambda when specifying params"
2148 )
2150 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Lambda":
2151 return Lambda(
2152 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2153 whitespace_after_lambda=visit_sentinel(
2154 self, "whitespace_after_lambda", self.whitespace_after_lambda, visitor
2155 ),
2156 params=visit_required(self, "params", self.params, visitor),
2157 colon=visit_required(self, "colon", self.colon, visitor),
2158 body=visit_required(self, "body", self.body, visitor),
2159 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2160 )
2162 def _codegen_impl(self, state: CodegenState) -> None:
2163 with self._parenthesize(state):
2164 state.add_token("lambda")
2165 whitespace_after_lambda = self.whitespace_after_lambda
2166 if isinstance(whitespace_after_lambda, MaybeSentinel):
2167 if not (
2168 len(self.params.posonly_params) == 0
2169 and len(self.params.params) == 0
2170 and not isinstance(self.params.star_arg, Param)
2171 and len(self.params.kwonly_params) == 0
2172 and self.params.star_kwarg is None
2173 ):
2174 # We have one or more params, provide a space
2175 state.add_token(" ")
2176 elif isinstance(whitespace_after_lambda, BaseParenthesizableWhitespace):
2177 whitespace_after_lambda._codegen(state)
2178 self.params._codegen(state)
2179 self.colon._codegen(state)
2180 self.body._codegen(state)
2183@add_slots
2184@dataclass(frozen=True)
2185class Arg(CSTNode):
2186 """
2187 A single argument to a :class:`Call`.
2189 This supports named keyword arguments in the form of ``keyword=value`` and variable
2190 argument expansion using ``*args`` or ``**kwargs`` syntax.
2191 """
2193 #: The argument expression itself, not including a preceding keyword, or any of
2194 #: the surrounding the value, like a comma or asterisks.
2195 value: BaseExpression
2197 #: Optional keyword for the argument.
2198 keyword: Optional[Name] = None
2200 #: The equal sign used to denote assignment if there is a keyword.
2201 equal: Union[AssignEqual, MaybeSentinel] = MaybeSentinel.DEFAULT
2203 #: Any trailing comma.
2204 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2206 #: A string with zero, one, or two asterisks appearing before the name. These are
2207 #: expanded into variable number of positional or keyword arguments.
2208 star: Literal["", "*", "**"] = ""
2210 #: Whitespace after the ``star`` (if it exists), but before the ``keyword`` or
2211 #: ``value`` (if no keyword is provided).
2212 whitespace_after_star: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2213 #: Whitespace after this entire node. The :class:`Comma` node (if it exists) may
2214 #: also store some trailing whitespace.
2215 whitespace_after_arg: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2217 def _validate(self) -> None:
2218 if self.keyword is None and isinstance(self.equal, AssignEqual):
2219 raise CSTValidationError(
2220 "Must have a keyword when specifying an AssignEqual."
2221 )
2222 if self.star not in ("", "*", "**"):
2223 raise CSTValidationError("Must specify either '', '*' or '**' for star.")
2224 if self.star in ("*", "**") and self.keyword is not None:
2225 raise CSTValidationError("Cannot specify a star and a keyword together.")
2227 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Arg":
2228 return Arg(
2229 star=self.star,
2230 whitespace_after_star=visit_required(
2231 self, "whitespace_after_star", self.whitespace_after_star, visitor
2232 ),
2233 keyword=visit_optional(self, "keyword", self.keyword, visitor),
2234 equal=visit_sentinel(self, "equal", self.equal, visitor),
2235 value=visit_required(self, "value", self.value, visitor),
2236 comma=visit_sentinel(self, "comma", self.comma, visitor),
2237 whitespace_after_arg=visit_required(
2238 self, "whitespace_after_arg", self.whitespace_after_arg, visitor
2239 ),
2240 )
2242 def _codegen_impl(self, state: CodegenState, default_comma: bool = False) -> None:
2243 with state.record_syntactic_position(self):
2244 state.add_token(self.star)
2245 self.whitespace_after_star._codegen(state)
2246 keyword = self.keyword
2247 if keyword is not None:
2248 keyword._codegen(state)
2249 equal = self.equal
2250 if equal is MaybeSentinel.DEFAULT and self.keyword is not None:
2251 state.add_token(" = ")
2252 elif isinstance(equal, AssignEqual):
2253 equal._codegen(state)
2254 self.value._codegen(state)
2256 comma = self.comma
2257 if comma is MaybeSentinel.DEFAULT and default_comma:
2258 state.add_token(", ")
2259 elif isinstance(comma, Comma):
2260 comma._codegen(state)
2261 self.whitespace_after_arg._codegen(state)
2264class _BaseExpressionWithArgs(BaseExpression, ABC):
2265 """
2266 Arguments are complicated enough that we can't represent them easily
2267 in typing. So, we have common validation functions here.
2268 """
2270 __slots__ = ()
2272 #: Sequence of arguments that will be passed to the function call.
2273 args: Sequence[Arg] = ()
2275 def _check_kwargs_or_keywords(self, arg: Arg) -> None:
2276 """
2277 Validates that we only have a mix of "keyword=arg" and "**arg" expansion.
2278 """
2280 if arg.keyword is not None:
2281 # Valid, keyword argument
2282 return None
2283 elif arg.star == "**":
2284 # Valid, kwargs
2285 return None
2286 elif arg.star == "*":
2287 # Invalid, cannot have "*" follow "**"
2288 raise CSTValidationError(
2289 "Cannot have iterable argument unpacking after keyword argument unpacking."
2290 )
2291 else:
2292 # Invalid, cannot have positional argument follow **/keyword
2293 raise CSTValidationError(
2294 "Cannot have positional argument after keyword argument unpacking."
2295 )
2297 def _check_starred_or_keywords(
2298 self, arg: Arg
2299 ) -> Optional[Callable[[Arg], Callable[[Arg], None]]]:
2300 """
2301 Validates that we only have a mix of "*arg" expansion and "keyword=arg".
2302 """
2304 if arg.keyword is not None:
2305 # Valid, keyword argument
2306 return None
2307 elif arg.star == "**":
2308 # Valid, but we now no longer allow "*" args
2309 # pyre-fixme[7]: Expected `Optional[Callable[[Arg], Callable[...,
2310 # Any]]]` but got `Callable[[Arg], Optional[Callable[[Arg], Callable[...,
2311 # Any]]]]`.
2312 return self._check_kwargs_or_keywords
2313 elif arg.star == "*":
2314 # Valid, iterable unpacking
2315 return None
2316 else:
2317 # Invalid, cannot have positional argument follow **/keyword
2318 raise CSTValidationError(
2319 "Cannot have positional argument after keyword argument."
2320 )
2322 def _check_positional(
2323 self, arg: Arg
2324 ) -> Optional[Callable[[Arg], Callable[[Arg], Callable[[Arg], None]]]]:
2325 """
2326 Validates that we only have a mix of positional args and "*arg" expansion.
2327 """
2329 if arg.keyword is not None:
2330 # Valid, but this puts us into starred/keyword state
2331 # pyre-fixme[7]: Expected `Optional[Callable[[Arg], Callable[...,
2332 # Any]]]` but got `Callable[[Arg], Optional[Callable[[Arg], Callable[...,
2333 # Any]]]]`.
2334 return self._check_starred_or_keywords
2335 elif arg.star == "**":
2336 # Valid, but we skip states to kwargs/keywords
2337 # pyre-fixme[7]: Expected `Optional[Callable[[Arg], Callable[...,
2338 # Any]]]` but got `Callable[[Arg], Optional[Callable[[Arg], Callable[...,
2339 # Any]]]]`.
2340 return self._check_kwargs_or_keywords
2341 elif arg.star == "*":
2342 # Valid, iterator expansion
2343 return None
2344 else:
2345 # Valid, allowed to have positional arguments here
2346 return None
2348 # pyre-fixme[30]: Pyre gave up inferring some types - function `_validate` was
2349 # too complex.
2350 def _validate(self) -> None:
2351 # Validate any super-class stuff, whatever it may be.
2352 super()._validate()
2353 # Now, validate the weird intermingling rules for arguments by running
2354 # a small validator state machine. This works by passing each argument
2355 # to a validator function which can either raise an exception if it
2356 # detects an invalid sequence, return a new validator to be used for the
2357 # next arg, or return None to use the same validator. We could enforce
2358 # always returning ourselves instead of None but it ends up making the
2359 # functions themselves less readable. In this way, the current validator
2360 # function encodes the state we're in (positional state, iterable
2361 # expansion state, or dictionary expansion state).
2362 validator = self._check_positional
2363 for arg in self.args:
2364 validator = validator(arg) or validator
2367@add_slots
2368@dataclass(frozen=True)
2369class Call(_BaseExpressionWithArgs):
2370 """
2371 An expression representing a function call, such as ``do_math(1, 2)`` or
2372 ``picture.post_on_instagram()``.
2374 Function calls consist of a function name and a sequence of arguments wrapped in
2375 :class:`Arg` nodes.
2376 """
2378 #: The expression resulting in a callable that we are to call. Often a :class:`Name`
2379 #: or :class:`Attribute`.
2380 func: BaseExpression
2382 #: The arguments to pass to the resulting callable. These may be a mix of
2383 #: positional arguments, keyword arguments, or "starred" arguments.
2384 args: Sequence[Arg] = ()
2386 lpar: Sequence[LeftParen] = ()
2387 #: Sequence of parenthesis for precedence dictation. These are not the parenthesis
2388 #: before and after the list of ``args``, but rather arguments around the entire
2389 #: call expression, such as ``(( do_math(1, 2) ))``.
2390 rpar: Sequence[RightParen] = ()
2392 #: Whitespace after the ``func`` name, but before the opening parenthesis.
2393 whitespace_after_func: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2394 #: Whitespace after the opening parenthesis but before the first argument (if there
2395 #: are any). Whitespace after the last argument but before the closing parenthesis
2396 #: is owned by the last :class:`Arg` if it exists.
2397 whitespace_before_args: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2399 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2400 """
2401 Calls have a close paren on the right side regardless of whether they're
2402 parenthesized as a whole. As a result, they are safe to use directly against
2403 an adjacent node to the right.
2404 """
2405 if position == ExpressionPosition.LEFT:
2406 return True
2407 if super(Call, self)._safe_to_use_with_word_operator(position):
2408 return True
2409 if position == ExpressionPosition.RIGHT:
2410 return self.func._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
2411 return False
2413 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Call":
2414 return Call(
2415 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2416 func=visit_required(self, "func", self.func, visitor),
2417 whitespace_after_func=visit_required(
2418 self, "whitespace_after_func", self.whitespace_after_func, visitor
2419 ),
2420 whitespace_before_args=visit_required(
2421 self, "whitespace_before_args", self.whitespace_before_args, visitor
2422 ),
2423 args=visit_sequence(self, "args", self.args, visitor),
2424 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2425 )
2427 def _codegen_impl(self, state: CodegenState) -> None:
2428 with self._parenthesize(state):
2429 self.func._codegen(state)
2430 self.whitespace_after_func._codegen(state)
2431 state.add_token("(")
2432 self.whitespace_before_args._codegen(state)
2433 lastarg = len(self.args) - 1
2434 for i, arg in enumerate(self.args):
2435 arg._codegen(state, default_comma=(i != lastarg))
2436 state.add_token(")")
2439@add_slots
2440@dataclass(frozen=True)
2441class Await(BaseExpression):
2442 """
2443 An await expression. Await expressions are only valid inside the body of an
2444 asynchronous :class:`FunctionDef` or (as of Python 3.7) inside of an asynchronous
2445 :class:`GeneratorExp` nodes.
2446 """
2448 #: The actual expression we need to wait for.
2449 expression: BaseExpression
2451 lpar: Sequence[LeftParen] = ()
2452 #: Sequence of parenthesis for precedence dictation.
2453 rpar: Sequence[RightParen] = ()
2455 #: Whitespace that appears after the ``async`` keyword, but before the inner
2456 #: ``expression``.
2457 whitespace_after_await: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2459 def _validate(self) -> None:
2460 # Validate any super-class stuff, whatever it may be.
2461 super(Await, self)._validate()
2462 # Make sure we don't run identifiers together.
2463 if (
2464 self.whitespace_after_await.empty
2465 and not self.expression._safe_to_use_with_word_operator(
2466 ExpressionPosition.RIGHT
2467 )
2468 ):
2469 raise CSTValidationError("Must have at least one space after await")
2471 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Await":
2472 return Await(
2473 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2474 whitespace_after_await=visit_required(
2475 self, "whitespace_after_await", self.whitespace_after_await, visitor
2476 ),
2477 expression=visit_required(self, "expression", self.expression, visitor),
2478 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2479 )
2481 def _codegen_impl(self, state: CodegenState) -> None:
2482 with self._parenthesize(state):
2483 state.add_token("await")
2484 self.whitespace_after_await._codegen(state)
2485 self.expression._codegen(state)
2488@add_slots
2489@dataclass(frozen=True)
2490class IfExp(BaseExpression):
2491 """
2492 An if expression of the form ``body if test else orelse``.
2494 If statements are provided by :class:`If` and :class:`Else` nodes.
2495 """
2497 #: The test to perform.
2498 test: BaseExpression
2500 #: The expression to evaluate when the test is true.
2501 body: BaseExpression
2503 #: The expression to evaluate when the test is false.
2504 orelse: BaseExpression
2506 lpar: Sequence[LeftParen] = ()
2507 #: Sequence of parenthesis for precedence dictation.
2508 rpar: Sequence[RightParen] = ()
2510 #: Whitespace after the ``body`` expression, but before the ``if`` keyword.
2511 whitespace_before_if: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2513 #: Whitespace after the ``if`` keyword, but before the ``test`` clause.
2514 whitespace_after_if: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2516 #: Whitespace after the ``test`` expression, but before the ``else`` keyword.
2517 whitespace_before_else: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2519 #: Whitespace after the ``else`` keyword, but before the ``orelse`` expression.
2520 whitespace_after_else: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2522 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2523 if position == ExpressionPosition.RIGHT:
2524 return self.body._safe_to_use_with_word_operator(position)
2525 else:
2526 return self.orelse._safe_to_use_with_word_operator(position)
2528 def _validate(self) -> None:
2529 # Paren validation and such
2530 super(IfExp, self)._validate()
2531 # Validate spacing rules
2532 if (
2533 self.whitespace_before_if.empty
2534 and not self.body._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
2535 ):
2536 raise CSTValidationError(
2537 "Must have at least one space before 'if' keyword."
2538 )
2539 if (
2540 self.whitespace_after_if.empty
2541 and not self.test._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
2542 ):
2543 raise CSTValidationError("Must have at least one space after 'if' keyword.")
2544 if (
2545 self.whitespace_before_else.empty
2546 and not self.test._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
2547 ):
2548 raise CSTValidationError(
2549 "Must have at least one space before 'else' keyword."
2550 )
2551 if (
2552 self.whitespace_after_else.empty
2553 and not self.orelse._safe_to_use_with_word_operator(
2554 ExpressionPosition.RIGHT
2555 )
2556 ):
2557 raise CSTValidationError(
2558 "Must have at least one space after 'else' keyword."
2559 )
2561 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "IfExp":
2562 return IfExp(
2563 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2564 body=visit_required(self, "body", self.body, visitor),
2565 whitespace_before_if=visit_required(
2566 self, "whitespace_before_if", self.whitespace_before_if, visitor
2567 ),
2568 whitespace_after_if=visit_required(
2569 self, "whitespace_after_if", self.whitespace_after_if, visitor
2570 ),
2571 test=visit_required(self, "test", self.test, visitor),
2572 whitespace_before_else=visit_required(
2573 self, "whitespace_before_else", self.whitespace_before_else, visitor
2574 ),
2575 whitespace_after_else=visit_required(
2576 self, "whitespace_after_else", self.whitespace_after_else, visitor
2577 ),
2578 orelse=visit_required(self, "orelse", self.orelse, visitor),
2579 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2580 )
2582 def _codegen_impl(self, state: CodegenState) -> None:
2583 with self._parenthesize(state):
2584 self.body._codegen(state)
2585 self.whitespace_before_if._codegen(state)
2586 state.add_token("if")
2587 self.whitespace_after_if._codegen(state)
2588 self.test._codegen(state)
2589 self.whitespace_before_else._codegen(state)
2590 state.add_token("else")
2591 self.whitespace_after_else._codegen(state)
2592 self.orelse._codegen(state)
2595@add_slots
2596@dataclass(frozen=True)
2597class From(CSTNode):
2598 """
2599 A ``from x`` stanza in a :class:`Yield` or :class:`Raise`.
2600 """
2602 #: The expression that we are yielding/raising from.
2603 item: BaseExpression
2605 #: The whitespace at the very start of this node.
2606 whitespace_before_from: Union[
2607 BaseParenthesizableWhitespace, MaybeSentinel
2608 ] = MaybeSentinel.DEFAULT
2610 #: The whitespace after the ``from`` keyword, but before the ``item``.
2611 whitespace_after_from: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2613 def _validate(self) -> None:
2614 if (
2615 isinstance(self.whitespace_after_from, BaseParenthesizableWhitespace)
2616 and self.whitespace_after_from.empty
2617 and not self.item._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
2618 ):
2619 raise CSTValidationError(
2620 "Must have at least one space after 'from' keyword."
2621 )
2623 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "From":
2624 return From(
2625 whitespace_before_from=visit_sentinel(
2626 self, "whitespace_before_from", self.whitespace_before_from, visitor
2627 ),
2628 whitespace_after_from=visit_required(
2629 self, "whitespace_after_from", self.whitespace_after_from, visitor
2630 ),
2631 item=visit_required(self, "item", self.item, visitor),
2632 )
2634 def _codegen_impl(self, state: CodegenState, default_space: str = "") -> None:
2635 whitespace_before_from = self.whitespace_before_from
2636 if isinstance(whitespace_before_from, BaseParenthesizableWhitespace):
2637 whitespace_before_from._codegen(state)
2638 else:
2639 state.add_token(default_space)
2641 with state.record_syntactic_position(self):
2642 state.add_token("from")
2643 self.whitespace_after_from._codegen(state)
2644 self.item._codegen(state)
2647@add_slots
2648@dataclass(frozen=True)
2649class Yield(BaseExpression):
2650 """
2651 A yield expression similar to ``yield x`` or ``yield from fun()``.
2653 To learn more about the ways that yield can be used in generators, refer to
2654 `Python's language reference
2655 <https://docs.python.org/3/reference/expressions.html#yieldexpr>`__.
2656 """
2658 #: The value yielded from the generator, in the case of a :class:`From` clause, a
2659 #: sub-generator to iterate over.
2660 value: Optional[Union[BaseExpression, From]] = None
2662 lpar: Sequence[LeftParen] = ()
2663 #: Sequence of parenthesis for precedence dictation.
2664 rpar: Sequence[RightParen] = ()
2666 #: Whitespace after the ``yield`` keyword, but before the ``value``.
2667 whitespace_after_yield: Union[
2668 BaseParenthesizableWhitespace, MaybeSentinel
2669 ] = MaybeSentinel.DEFAULT
2671 def _validate(self) -> None:
2672 # Paren rules and such
2673 super(Yield, self)._validate()
2674 # Our own rules
2675 whitespace_after_yield = self.whitespace_after_yield
2676 if (
2677 isinstance(whitespace_after_yield, BaseParenthesizableWhitespace)
2678 and whitespace_after_yield.empty
2679 ):
2680 value = self.value
2681 if isinstance(value, From):
2682 raise CSTValidationError(
2683 "Must have at least one space after 'yield' keyword."
2684 )
2685 if isinstance(
2686 value, BaseExpression
2687 ) and not value._safe_to_use_with_word_operator(ExpressionPosition.RIGHT):
2688 raise CSTValidationError(
2689 "Must have at least one space after 'yield' keyword."
2690 )
2692 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Yield":
2693 return Yield(
2694 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2695 whitespace_after_yield=visit_sentinel(
2696 self, "whitespace_after_yield", self.whitespace_after_yield, visitor
2697 ),
2698 value=visit_optional(self, "value", self.value, visitor),
2699 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2700 )
2702 def _codegen_impl(self, state: CodegenState) -> None:
2703 with self._parenthesize(state):
2704 state.add_token("yield")
2705 whitespace_after_yield = self.whitespace_after_yield
2706 if isinstance(whitespace_after_yield, BaseParenthesizableWhitespace):
2707 whitespace_after_yield._codegen(state)
2708 else:
2709 # Only need a space after yield if there is a value to yield.
2710 if self.value is not None:
2711 state.add_token(" ")
2712 value = self.value
2713 if isinstance(value, From):
2714 value._codegen(state, default_space="")
2715 elif value is not None:
2716 value._codegen(state)
2719class _BaseElementImpl(CSTNode, ABC):
2720 """
2721 An internal base class for :class:`Element` and :class:`DictElement`.
2722 """
2724 __slots__ = ()
2726 value: BaseExpression
2727 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2729 def _codegen_comma(
2730 self,
2731 state: CodegenState,
2732 default_comma: bool = False,
2733 default_comma_whitespace: bool = False, # False for a single-item collection
2734 ) -> None:
2735 """
2736 Called by `_codegen_impl` in subclasses to generate the comma.
2737 """
2738 comma = self.comma
2739 if comma is MaybeSentinel.DEFAULT and default_comma:
2740 if default_comma_whitespace:
2741 state.add_token(", ")
2742 else:
2743 state.add_token(",")
2744 elif isinstance(comma, Comma):
2745 comma._codegen(state)
2747 @abstractmethod
2748 def _codegen_impl(
2749 self,
2750 state: CodegenState,
2751 default_comma: bool = False,
2752 default_comma_whitespace: bool = False, # False for a single-item collection
2753 ) -> None:
2754 ...
2757class BaseElement(_BaseElementImpl, ABC):
2758 """
2759 An element of a literal list, tuple, or set. For elements of a literal dict, see
2760 BaseDictElement.
2761 """
2763 __slots__ = ()
2766class BaseDictElement(_BaseElementImpl, ABC):
2767 """
2768 An element of a literal dict. For elements of a list, tuple, or set, see
2769 BaseElement.
2770 """
2772 __slots__ = ()
2775@add_slots
2776@dataclass(frozen=True)
2777class Element(BaseElement):
2778 """
2779 A simple value in a literal :class:`List`, :class:`Tuple`, or :class:`Set`.
2780 These a literal collection may also contain a :class:`StarredElement`.
2782 If you're using a literal :class:`Dict`, see :class:`DictElement` instead.
2783 """
2785 value: BaseExpression
2787 #: A trailing comma. By default, we'll only insert a comma if one is required.
2788 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2790 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Element":
2791 return Element(
2792 value=visit_required(self, "value", self.value, visitor),
2793 comma=visit_sentinel(self, "comma", self.comma, visitor),
2794 )
2796 def _codegen_impl(
2797 self,
2798 state: CodegenState,
2799 default_comma: bool = False,
2800 default_comma_whitespace: bool = False,
2801 ) -> None:
2802 with state.record_syntactic_position(self):
2803 self.value._codegen(state)
2804 self._codegen_comma(state, default_comma, default_comma_whitespace)
2807@add_slots
2808@dataclass(frozen=True)
2809class DictElement(BaseDictElement):
2810 """
2811 A simple ``key: value`` pair that represents a single entry in a literal
2812 :class:`Dict`. :class:`Dict` nodes may also contain a
2813 :class:`StarredDictElement`.
2815 If you're using a literal :class:`List`, :class:`Tuple`, or :class:`Set`,
2816 see :class:`Element` instead.
2817 """
2819 key: BaseExpression
2820 value: BaseExpression
2822 #: A trailing comma. By default, we'll only insert a comma if one is required.
2823 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2825 #: Whitespace after the key, but before the colon in ``key : value``.
2826 whitespace_before_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2827 #: Whitespace after the colon, but before the value in ``key : value``.
2828 whitespace_after_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
2830 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "DictElement":
2831 return DictElement(
2832 key=visit_required(self, "key", self.key, visitor),
2833 whitespace_before_colon=visit_required(
2834 self, "whitespace_before_colon", self.whitespace_before_colon, visitor
2835 ),
2836 whitespace_after_colon=visit_required(
2837 self, "whitespace_after_colon", self.whitespace_after_colon, visitor
2838 ),
2839 value=visit_required(self, "value", self.value, visitor),
2840 comma=visit_sentinel(self, "comma", self.comma, visitor),
2841 )
2843 def _codegen_impl(
2844 self,
2845 state: CodegenState,
2846 default_comma: bool = False,
2847 default_comma_whitespace: bool = False,
2848 ) -> None:
2849 with state.record_syntactic_position(self):
2850 self.key._codegen(state)
2851 self.whitespace_before_colon._codegen(state)
2852 state.add_token(":")
2853 self.whitespace_after_colon._codegen(state)
2854 self.value._codegen(state)
2855 self._codegen_comma(state, default_comma, default_comma_whitespace)
2858@add_slots
2859@dataclass(frozen=True)
2860class StarredElement(BaseElement, BaseExpression, _BaseParenthesizedNode):
2861 """
2862 A starred ``*value`` element that expands to represent multiple values in a literal
2863 :class:`List`, :class:`Tuple`, or :class:`Set`.
2865 If you're using a literal :class:`Dict`, see :class:`StarredDictElement` instead.
2867 If this node owns parenthesis, those parenthesis wrap the leading asterisk, but not
2868 the trailing comma. For example::
2870 StarredElement(
2871 cst.Name("el"),
2872 comma=cst.Comma(),
2873 lpar=[cst.LeftParen()],
2874 rpar=[cst.RightParen()],
2875 )
2877 will generate::
2879 (*el),
2880 """
2882 value: BaseExpression
2884 #: A trailing comma. By default, we'll only insert a comma if one is required.
2885 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2887 #: Parenthesis at the beginning of the node, before the leading asterisk.
2888 lpar: Sequence[LeftParen] = ()
2889 #: Parentheses after the value, but before a comma (if there is one).
2890 rpar: Sequence[RightParen] = ()
2892 #: Whitespace between the leading asterisk and the value expression.
2893 whitespace_before_value: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2895 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "StarredElement":
2896 return StarredElement(
2897 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
2898 whitespace_before_value=visit_required(
2899 self, "whitespace_before_value", self.whitespace_before_value, visitor
2900 ),
2901 value=visit_required(self, "value", self.value, visitor),
2902 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
2903 comma=visit_sentinel(self, "comma", self.comma, visitor),
2904 )
2906 def _codegen_impl(
2907 self,
2908 state: CodegenState,
2909 default_comma: bool = False,
2910 default_comma_whitespace: bool = False,
2911 ) -> None:
2912 with self._parenthesize(state):
2913 state.add_token("*")
2914 self.whitespace_before_value._codegen(state)
2915 self.value._codegen(state)
2916 self._codegen_comma(state, default_comma, default_comma_whitespace)
2919@add_slots
2920@dataclass(frozen=True)
2921class StarredDictElement(BaseDictElement):
2922 """
2923 A starred ``**value`` element that expands to represent multiple values in a literal
2924 :class:`Dict`.
2926 If you're using a literal :class:`List`, :class:`Tuple`, or :class:`Set`,
2927 see :class:`StarredElement` instead.
2929 Unlike :class:`StarredElement`, this node does not own left or right parenthesis,
2930 but the ``value`` field may still contain parenthesis. This is due to some
2931 asymmetry in Python's grammar.
2932 """
2934 value: BaseExpression
2936 #: A trailing comma. By default, we'll only insert a comma if one is required.
2937 comma: Union[Comma, MaybeSentinel] = MaybeSentinel.DEFAULT
2939 #: Whitespace between the leading asterisks and the value expression.
2940 whitespace_before_value: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
2942 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "StarredDictElement":
2943 return StarredDictElement(
2944 whitespace_before_value=visit_required(
2945 self, "whitespace_before_value", self.whitespace_before_value, visitor
2946 ),
2947 value=visit_required(self, "value", self.value, visitor),
2948 comma=visit_sentinel(self, "comma", self.comma, visitor),
2949 )
2951 def _codegen_impl(
2952 self,
2953 state: CodegenState,
2954 default_comma: bool = False,
2955 default_comma_whitespace: bool = False,
2956 ) -> None:
2957 with state.record_syntactic_position(self):
2958 state.add_token("**")
2959 self.whitespace_before_value._codegen(state)
2960 self.value._codegen(state)
2961 self._codegen_comma(state, default_comma, default_comma_whitespace)
2964@add_slots
2965@dataclass(frozen=True)
2966class Tuple(BaseAssignTargetExpression, BaseDelTargetExpression):
2967 """
2968 An immutable literal tuple. Tuples are often (but not always) parenthesized.
2970 ::
2972 Tuple([
2973 Element(Integer("1")),
2974 Element(Integer("2")),
2975 StarredElement(Name("others")),
2976 ])
2978 generates the following code::
2980 (1, 2, *others)
2981 """
2983 #: A sequence containing all the :class:`Element` and :class:`StarredElement` nodes
2984 #: in the tuple.
2985 elements: Sequence[BaseElement]
2987 lpar: Sequence[LeftParen] = field(default_factory=lambda: (LeftParen(),))
2988 #: Sequence of parenthesis for precedence dictation.
2989 rpar: Sequence[RightParen] = field(default_factory=lambda: (RightParen(),))
2991 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
2992 if super(Tuple, self)._safe_to_use_with_word_operator(position):
2993 # if we have parenthesis, we're safe.
2994 return True
2995 # elements[-1] and elements[0] must exist past this point, because
2996 # we're not parenthesized, meaning we must have at least one element.
2997 elements = self.elements
2998 if position == ExpressionPosition.LEFT:
2999 last_element = elements[-1]
3000 return (
3001 isinstance(last_element.comma, Comma)
3002 or (
3003 isinstance(last_element, StarredElement)
3004 and len(last_element.rpar) > 0
3005 )
3006 or last_element.value._safe_to_use_with_word_operator(position)
3007 )
3008 else: # ExpressionPosition.RIGHT
3009 first_element = elements[0]
3010 # starred elements are always safe because they begin with ( or *
3011 return isinstance(
3012 first_element, StarredElement
3013 ) or first_element.value._safe_to_use_with_word_operator(position)
3015 def _validate(self) -> None:
3016 # Paren validation and such
3017 super(Tuple, self)._validate()
3019 if len(self.elements) == 0:
3020 if len(self.lpar) == 0: # assumes len(lpar) == len(rpar), via superclass
3021 raise CSTValidationError(
3022 "A zero-length tuple must be wrapped in parentheses."
3023 )
3024 # Invalid commas aren't possible, because MaybeSentinel will ensure that there
3025 # is a comma where required.
3027 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Tuple":
3028 return Tuple(
3029 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3030 elements=visit_sequence(self, "elements", self.elements, visitor),
3031 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3032 )
3034 def _codegen_impl(self, state: CodegenState) -> None:
3035 with self._parenthesize(state):
3036 elements = self.elements
3037 if len(elements) == 1:
3038 elements[0]._codegen(
3039 state, default_comma=True, default_comma_whitespace=False
3040 )
3041 else:
3042 for idx, el in enumerate(elements):
3043 el._codegen(
3044 state,
3045 default_comma=(idx < len(elements) - 1),
3046 default_comma_whitespace=True,
3047 )
3050class BaseList(BaseExpression, ABC):
3051 """
3052 A base class for :class:`List` and :class:`ListComp`, which both result in a list
3053 object when evaluated.
3054 """
3056 __slots__ = ()
3058 lbracket: LeftSquareBracket = LeftSquareBracket.field()
3059 #: Brackets surrounding the list.
3060 rbracket: RightSquareBracket = RightSquareBracket.field()
3062 lpar: Sequence[LeftParen] = ()
3063 #: Sequence of parenthesis for precedence dictation.
3064 rpar: Sequence[RightParen] = ()
3066 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3067 return True
3069 @contextmanager
3070 def _bracketize(self, state: CodegenState) -> Generator[None, None, None]:
3071 self.lbracket._codegen(state)
3072 yield
3073 self.rbracket._codegen(state)
3076@add_slots
3077@dataclass(frozen=True)
3078class List(BaseList, BaseAssignTargetExpression, BaseDelTargetExpression):
3079 """
3080 A mutable literal list.
3082 ::
3084 List([
3085 Element(Integer("1")),
3086 Element(Integer("2")),
3087 StarredElement(Name("others")),
3088 ])
3090 generates the following code::
3092 [1, 2, *others]
3094 List comprehensions are represented with a :class:`ListComp` node.
3095 """
3097 #: A sequence containing all the :class:`Element` and :class:`StarredElement` nodes
3098 #: in the list.
3099 elements: Sequence[BaseElement]
3101 lbracket: LeftSquareBracket = LeftSquareBracket.field()
3102 #: Brackets surrounding the list.
3103 rbracket: RightSquareBracket = RightSquareBracket.field()
3105 lpar: Sequence[LeftParen] = ()
3106 #: Sequence of parenthesis for precedence dictation.
3107 rpar: Sequence[RightParen] = ()
3109 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "List":
3110 return List(
3111 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3112 lbracket=visit_required(self, "lbracket", self.lbracket, visitor),
3113 elements=visit_sequence(self, "elements", self.elements, visitor),
3114 rbracket=visit_required(self, "rbracket", self.rbracket, visitor),
3115 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3116 )
3118 def _codegen_impl(self, state: CodegenState) -> None:
3119 with self._parenthesize(state), self._bracketize(state):
3120 elements = self.elements
3121 for idx, el in enumerate(elements):
3122 el._codegen(
3123 state,
3124 default_comma=(idx < len(elements) - 1),
3125 default_comma_whitespace=True,
3126 )
3129class _BaseSetOrDict(BaseExpression, ABC):
3130 """
3131 An abstract base class for :class:`BaseSet` and :class:`BaseDict`.
3133 Literal sets and dicts are syntactically similar (hence this shared base class), but
3134 are semantically different. This base class is an implementation detail and
3135 shouldn't be exported.
3136 """
3138 __slots__ = ()
3140 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3141 #: Braces surrounding the set or dict.
3142 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3144 lpar: Sequence[LeftParen] = ()
3145 #: Sequence of parenthesis for precedence dictation.
3146 rpar: Sequence[RightParen] = ()
3148 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3149 return True
3151 # brace-ize seems like a very made-up word. And it is!
3152 @contextmanager
3153 def _braceize(self, state: CodegenState) -> Generator[None, None, None]:
3154 self.lbrace._codegen(state)
3155 yield
3156 self.rbrace._codegen(state)
3159class BaseSet(_BaseSetOrDict, ABC):
3160 """
3161 An abstract base class for :class:`Set` and :class:`SetComp`, which both result in
3162 a set object when evaluated.
3163 """
3165 __slots__ = ()
3168@add_slots
3169@dataclass(frozen=True)
3170class Set(BaseSet):
3171 """
3172 A mutable literal set.
3174 ::
3176 Set([
3177 Element(Integer("1")),
3178 Element(Integer("2")),
3179 StarredElement(Name("others")),
3180 ])
3182 generates the following code::
3184 {1, 2, *others}
3186 Set comprehensions are represented with a :class:`SetComp` node.
3187 """
3189 #: A sequence containing all the :class:`Element` and :class:`StarredElement` nodes
3190 #: in the set.
3191 elements: Sequence[BaseElement]
3193 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3194 #: Braces surrounding the set.
3195 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3197 lpar: Sequence[LeftParen] = ()
3198 #: Sequence of parenthesis for precedence dictation.
3199 rpar: Sequence[RightParen] = ()
3201 def _validate(self) -> None:
3202 super(Set, self)._validate()
3204 if len(self.elements) == 0:
3205 raise CSTValidationError(
3206 "A literal set must have at least one element. A zero-element set "
3207 + "would be syntatically ambiguous with an empty dict, `{}`."
3208 )
3210 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Set":
3211 return Set(
3212 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3213 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3214 elements=visit_sequence(self, "elements", self.elements, visitor),
3215 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3216 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3217 )
3219 def _codegen_impl(self, state: CodegenState) -> None:
3220 with self._parenthesize(state), self._braceize(state):
3221 elements = self.elements
3222 for idx, el in enumerate(elements):
3223 el._codegen(
3224 state,
3225 default_comma=(idx < len(elements) - 1),
3226 default_comma_whitespace=True,
3227 )
3230class BaseDict(_BaseSetOrDict, ABC):
3231 """
3232 An abstract base class for :class:`Dict` and :class:`DictComp`, which both result in
3233 a dict object when evaluated.
3234 """
3236 __slots__ = ()
3239@add_slots
3240@dataclass(frozen=True)
3241class Dict(BaseDict):
3242 """
3243 A literal dictionary. Key-value pairs are stored in ``elements`` using
3244 :class:`DictElement` nodes.
3246 It's possible to expand one dictionary into another, as in ``{k: v, **expanded}``.
3247 Expanded elements are stored as :class:`StarredDictElement` nodes.
3249 ::
3251 Dict([
3252 DictElement(Name("k1"), Name("v1")),
3253 DictElement(Name("k2"), Name("v2")),
3254 StarredDictElement(Name("expanded")),
3255 ])
3257 generates the following code::
3259 {k1: v1, k2: v2, **expanded}
3260 """
3262 elements: Sequence[BaseDictElement]
3263 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3264 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3265 lpar: Sequence[LeftParen] = ()
3266 rpar: Sequence[RightParen] = ()
3268 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "Dict":
3269 return Dict(
3270 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3271 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3272 elements=visit_sequence(self, "elements", self.elements, visitor),
3273 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3274 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3275 )
3277 def _codegen_impl(self, state: CodegenState) -> None:
3278 with self._parenthesize(state), self._braceize(state):
3279 elements = self.elements
3280 for idx, el in enumerate(elements):
3281 el._codegen(
3282 state,
3283 default_comma=(idx < len(elements) - 1),
3284 default_comma_whitespace=True,
3285 )
3288@add_slots
3289@dataclass(frozen=True)
3290class CompFor(CSTNode):
3291 """
3292 One ``for`` clause in a :class:`BaseComp`, or a nested hierarchy of
3293 ``for`` clauses.
3295 Nested loops in comprehensions are difficult to get right, but they can be thought
3296 of as a flat representation of nested clauses.
3298 ``elt for a in b for c in d if e`` can be thought of as::
3300 for a in b:
3301 for c in d:
3302 if e:
3303 yield elt
3305 And that would form the following CST::
3307 ListComp(
3308 elt=Name("elt"),
3309 for_in=CompFor(
3310 target=Name("a"),
3311 iter=Name("b"),
3312 ifs=[],
3313 inner_comp_for=CompFor(
3314 target=Name("c"),
3315 iter=Name("d"),
3316 ifs=[
3317 CompIf(
3318 test=Name("e"),
3319 ),
3320 ],
3321 ),
3322 ),
3323 )
3325 Normal ``for`` statements are provided by :class:`For`.
3326 """
3328 #: The target to assign a value to in each iteration of the loop. This is different
3329 #: from :attr:`GeneratorExp.elt`, :attr:`ListComp.elt`, :attr:`SetComp.elt`, and
3330 #: ``key`` and ``value`` in :class:`DictComp`, because it doesn't directly effect
3331 #: the value of resulting generator, list, set, or dict.
3332 target: BaseAssignTargetExpression
3334 #: The value to iterate over. Every value in ``iter`` is stored in ``target``.
3335 iter: BaseExpression
3337 #: Zero or more conditional clauses that control this loop. If any of these tests
3338 #: fail, the ``target`` item is skipped.
3339 #:
3340 #: ::
3341 #:
3342 #: if a if b if c
3343 #:
3344 #: has similar semantics to::
3345 #:
3346 #: if a and b and c
3347 ifs: Sequence["CompIf"] = ()
3349 #: Another :class:`CompFor` node used to form nested loops. Nested comprehensions
3350 #: can be useful, but they tend to be difficult to read and write. As a result they
3351 #: are uncommon.
3352 inner_for_in: Optional["CompFor"] = None
3354 #: An optional async modifier that appears before the ``for`` keyword.
3355 asynchronous: Optional[Asynchronous] = None
3357 #: Whitespace that appears at the beginning of this node, before the ``for`` and
3358 #: ``async`` keywords.
3359 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3361 #: Whitespace appearing after the ``for`` keyword, but before the ``target``.
3362 whitespace_after_for: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3364 #: Whitespace appearing after the ``target``, but before the ``in`` keyword.
3365 whitespace_before_in: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3367 #: Whitespace appearing after the ``in`` keyword, but before the ``iter``.
3368 whitespace_after_in: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3370 def _validate(self) -> None:
3371 if (
3372 self.whitespace_after_for.empty
3373 and not self.target._safe_to_use_with_word_operator(
3374 ExpressionPosition.RIGHT
3375 )
3376 ):
3377 raise CSTValidationError(
3378 "Must have at least one space after 'for' keyword."
3379 )
3381 if (
3382 self.whitespace_before_in.empty
3383 and not self.target._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3384 ):
3385 raise CSTValidationError(
3386 "Must have at least one space before 'in' keyword."
3387 )
3389 if (
3390 self.whitespace_after_in.empty
3391 and not self.iter._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
3392 ):
3393 raise CSTValidationError("Must have at least one space after 'in' keyword.")
3395 prev_expr = self.iter
3396 for if_clause in self.ifs:
3397 if (
3398 if_clause.whitespace_before.empty
3399 and not prev_expr._safe_to_use_with_word_operator(
3400 ExpressionPosition.LEFT
3401 )
3402 ):
3403 raise CSTValidationError(
3404 "Must have at least one space before 'if' keyword."
3405 )
3406 prev_expr = if_clause.test
3408 inner_for_in = self.inner_for_in
3409 if (
3410 inner_for_in is not None
3411 and inner_for_in.whitespace_before.empty
3412 and not prev_expr._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3413 ):
3414 keyword = "async" if inner_for_in.asynchronous else "for"
3415 raise CSTValidationError(
3416 f"Must have at least one space before '{keyword}' keyword."
3417 )
3419 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "CompFor":
3420 return CompFor(
3421 whitespace_before=visit_required(
3422 self, "whitespace_before", self.whitespace_before, visitor
3423 ),
3424 asynchronous=visit_optional(
3425 self, "asynchronous", self.asynchronous, visitor
3426 ),
3427 whitespace_after_for=visit_required(
3428 self, "whitespace_after_for", self.whitespace_after_for, visitor
3429 ),
3430 target=visit_required(self, "target", self.target, visitor),
3431 whitespace_before_in=visit_required(
3432 self, "whitespace_before_in", self.whitespace_before_in, visitor
3433 ),
3434 whitespace_after_in=visit_required(
3435 self, "whitespace_after_in", self.whitespace_after_in, visitor
3436 ),
3437 iter=visit_required(self, "iter", self.iter, visitor),
3438 ifs=visit_sequence(self, "ifs", self.ifs, visitor),
3439 inner_for_in=visit_optional(
3440 self, "inner_for_in", self.inner_for_in, visitor
3441 ),
3442 )
3444 def _codegen_impl(self, state: CodegenState) -> None:
3445 self.whitespace_before._codegen(state)
3446 asynchronous = self.asynchronous
3447 if asynchronous is not None:
3448 asynchronous._codegen(state)
3449 state.add_token("for")
3450 self.whitespace_after_for._codegen(state)
3451 self.target._codegen(state)
3452 self.whitespace_before_in._codegen(state)
3453 state.add_token("in")
3454 self.whitespace_after_in._codegen(state)
3455 self.iter._codegen(state)
3456 ifs = self.ifs
3457 for if_clause in ifs:
3458 if_clause._codegen(state)
3459 inner_for_in = self.inner_for_in
3460 if inner_for_in is not None:
3461 inner_for_in._codegen(state)
3464@add_slots
3465@dataclass(frozen=True)
3466class CompIf(CSTNode):
3467 """
3468 A conditional clause in a :class:`CompFor`, used as part of a generator or
3469 comprehension expression.
3471 If the ``test`` fails, the current element in the :class:`CompFor` will be skipped.
3472 """
3474 #: An expression to evaluate. When interpreted, Python will coerce it to a boolean.
3475 test: BaseExpression
3477 #: Whitespace before the ``if`` keyword.
3478 whitespace_before: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3480 #: Whitespace after the ``if`` keyword, but before the ``test`` expression.
3481 whitespace_before_test: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3483 def _validate(self) -> None:
3484 if (
3485 self.whitespace_before_test.empty
3486 and not self.test._safe_to_use_with_word_operator(ExpressionPosition.RIGHT)
3487 ):
3488 raise CSTValidationError("Must have at least one space after 'if' keyword.")
3490 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "CompIf":
3491 return CompIf(
3492 whitespace_before=visit_required(
3493 self, "whitespace_before", self.whitespace_before, visitor
3494 ),
3495 whitespace_before_test=visit_required(
3496 self, "whitespace_before_test", self.whitespace_before_test, visitor
3497 ),
3498 test=visit_required(self, "test", self.test, visitor),
3499 )
3501 def _codegen_impl(self, state: CodegenState) -> None:
3502 self.whitespace_before._codegen(state)
3503 state.add_token("if")
3504 self.whitespace_before_test._codegen(state)
3505 self.test._codegen(state)
3508class BaseComp(BaseExpression, ABC):
3509 """
3510 A base class for all comprehension and generator expressions, including
3511 :class:`GeneratorExp`, :class:`ListComp`, :class:`SetComp`, and :class:`DictComp`.
3512 """
3514 __slots__ = ()
3516 for_in: CompFor
3519class BaseSimpleComp(BaseComp, ABC):
3520 """
3521 The base class for :class:`ListComp`, :class:`SetComp`, and :class:`GeneratorExp`.
3522 :class:`DictComp` is not a :class:`BaseSimpleComp`, because it uses ``key`` and
3523 ``value``.
3524 """
3526 __slots__ = ()
3528 #: The expression evaluated during each iteration of the comprehension. This
3529 #: lexically comes before the ``for_in`` clause, but it is semantically the
3530 #: inner-most element, evaluated inside the ``for_in`` clause.
3531 elt: BaseExpression
3533 #: The ``for ... in ... if ...`` clause that lexically comes after ``elt``. This may
3534 #: be a nested structure for nested comprehensions. See :class:`CompFor` for
3535 #: details.
3536 for_in: CompFor
3538 def _validate(self) -> None:
3539 super(BaseSimpleComp, self)._validate()
3541 for_in = self.for_in
3542 if (
3543 for_in.whitespace_before.empty
3544 and not self.elt._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3545 ):
3546 keyword = "async" if for_in.asynchronous else "for"
3547 raise CSTValidationError(
3548 f"Must have at least one space before '{keyword}' keyword."
3549 )
3552@add_slots
3553@dataclass(frozen=True)
3554class GeneratorExp(BaseSimpleComp):
3555 """
3556 A generator expression. ``elt`` represents the value yielded for each item in
3557 :attr:`CompFor.iter`.
3559 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3560 :class:`CompFor` data structure inside ``for_in``.
3561 """
3563 #: The expression evaluated and yielded during each iteration of the generator.
3564 elt: BaseExpression
3566 #: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
3567 #: nested structure for nested comprehensions. See :class:`CompFor` for details.
3568 for_in: CompFor
3570 lpar: Sequence[LeftParen] = field(default_factory=lambda: (LeftParen(),))
3571 #: Sequence of parentheses for precedence dictation. Generator expressions must
3572 #: always be parenthesized. However, if a generator expression is the only argument
3573 #: inside a function call, the enclosing :class:`Call` node may own the parentheses
3574 #: instead.
3575 rpar: Sequence[RightParen] = field(default_factory=lambda: (RightParen(),))
3577 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3578 # Generators are always parenthesized
3579 return True
3581 # A note about validation: Generators must always be parenthesized, but it's
3582 # possible that this Generator node doesn't own those parenthesis (in the case of a
3583 # function call with a single generator argument).
3584 #
3585 # Therefore, there's no useful validation we can do here. In theory, our parent
3586 # could do the validation, but there's a ton of potential parents to a Generator, so
3587 # it's not worth the effort.
3589 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "GeneratorExp":
3590 return GeneratorExp(
3591 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3592 elt=visit_required(self, "elt", self.elt, visitor),
3593 for_in=visit_required(self, "for_in", self.for_in, visitor),
3594 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3595 )
3597 def _codegen_impl(self, state: CodegenState) -> None:
3598 with self._parenthesize(state):
3599 self.elt._codegen(state)
3600 self.for_in._codegen(state)
3603@add_slots
3604@dataclass(frozen=True)
3605class ListComp(BaseList, BaseSimpleComp):
3606 """
3607 A list comprehension. ``elt`` represents the value stored for each item in
3608 :attr:`CompFor.iter`.
3610 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3611 :class:`CompFor` data structure inside ``for_in``.
3612 """
3614 #: The expression evaluated and stored during each iteration of the comprehension.
3615 elt: BaseExpression
3617 #: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
3618 #: nested structure for nested comprehensions. See :class:`CompFor` for details.
3619 for_in: CompFor
3621 lbracket: LeftSquareBracket = LeftSquareBracket.field()
3622 #: Brackets surrounding the list comprehension.
3623 rbracket: RightSquareBracket = RightSquareBracket.field()
3625 lpar: Sequence[LeftParen] = ()
3626 #: Sequence of parenthesis for precedence dictation.
3627 rpar: Sequence[RightParen] = ()
3629 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "ListComp":
3630 return ListComp(
3631 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3632 lbracket=visit_required(self, "lbracket", self.lbracket, visitor),
3633 elt=visit_required(self, "elt", self.elt, visitor),
3634 for_in=visit_required(self, "for_in", self.for_in, visitor),
3635 rbracket=visit_required(self, "rbracket", self.rbracket, visitor),
3636 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3637 )
3639 def _codegen_impl(self, state: CodegenState) -> None:
3640 with self._parenthesize(state), self._bracketize(state):
3641 self.elt._codegen(state)
3642 self.for_in._codegen(state)
3645@add_slots
3646@dataclass(frozen=True)
3647class SetComp(BaseSet, BaseSimpleComp):
3648 """
3649 A set comprehension. ``elt`` represents the value stored for each item in
3650 :attr:`CompFor.iter`.
3652 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3653 :class:`CompFor` data structure inside ``for_in``.
3654 """
3656 #: The expression evaluated and stored during each iteration of the comprehension.
3657 elt: BaseExpression
3659 #: The ``for ... in ... if ...`` clause that comes after ``elt``. This may be a
3660 #: nested structure for nested comprehensions. See :class:`CompFor` for details.
3661 for_in: CompFor
3663 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3664 #: Braces surrounding the set comprehension.
3665 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3667 lpar: Sequence[LeftParen] = ()
3668 #: Sequence of parenthesis for precedence dictation.
3669 rpar: Sequence[RightParen] = ()
3671 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "SetComp":
3672 return SetComp(
3673 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3674 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3675 elt=visit_required(self, "elt", self.elt, visitor),
3676 for_in=visit_required(self, "for_in", self.for_in, visitor),
3677 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3678 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3679 )
3681 def _codegen_impl(self, state: CodegenState) -> None:
3682 with self._parenthesize(state), self._braceize(state):
3683 self.elt._codegen(state)
3684 self.for_in._codegen(state)
3687@add_slots
3688@dataclass(frozen=True)
3689class DictComp(BaseDict, BaseComp):
3690 """
3691 A dictionary comprehension. ``key`` and ``value`` represent the dictionary entry
3692 evaluated for each item.
3694 All ``for ... in ...`` and ``if ...`` clauses are stored as a recursive
3695 :class:`CompFor` data structure inside ``for_in``.
3696 """
3698 #: The key inserted into the dictionary during each iteration of the comprehension.
3699 key: BaseExpression
3700 #: The value associated with the ``key`` inserted into the dictionary during each
3701 #: iteration of the comprehension.
3702 value: BaseExpression
3704 #: The ``for ... in ... if ...`` clause that lexically comes after ``key`` and
3705 #: ``value``. This may be a nested structure for nested comprehensions. See
3706 #: :class:`CompFor` for details.
3707 for_in: CompFor
3709 lbrace: LeftCurlyBrace = LeftCurlyBrace.field()
3710 #: Braces surrounding the dict comprehension.
3711 rbrace: RightCurlyBrace = RightCurlyBrace.field()
3713 lpar: Sequence[LeftParen] = ()
3714 #: Sequence of parenthesis for precedence dictation.
3715 rpar: Sequence[RightParen] = ()
3717 #: Whitespace after the key, but before the colon in ``key : value``.
3718 whitespace_before_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field("")
3719 #: Whitespace after the colon, but before the value in ``key : value``.
3720 whitespace_after_colon: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3722 def _validate(self) -> None:
3723 super(DictComp, self)._validate()
3725 for_in = self.for_in
3726 if (
3727 for_in.whitespace_before.empty
3728 and not self.value._safe_to_use_with_word_operator(ExpressionPosition.LEFT)
3729 ):
3730 keyword = "async" if for_in.asynchronous else "for"
3731 raise CSTValidationError(
3732 f"Must have at least one space before '{keyword}' keyword."
3733 )
3735 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "DictComp":
3736 return DictComp(
3737 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3738 lbrace=visit_required(self, "lbrace", self.lbrace, visitor),
3739 key=visit_required(self, "key", self.key, visitor),
3740 whitespace_before_colon=visit_required(
3741 self, "whitespace_before_colon", self.whitespace_before_colon, visitor
3742 ),
3743 whitespace_after_colon=visit_required(
3744 self, "whitespace_after_colon", self.whitespace_after_colon, visitor
3745 ),
3746 value=visit_required(self, "value", self.value, visitor),
3747 for_in=visit_required(self, "for_in", self.for_in, visitor),
3748 rbrace=visit_required(self, "rbrace", self.rbrace, visitor),
3749 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3750 )
3752 def _codegen_impl(self, state: CodegenState) -> None:
3753 with self._parenthesize(state), self._braceize(state):
3754 self.key._codegen(state)
3755 self.whitespace_before_colon._codegen(state)
3756 state.add_token(":")
3757 self.whitespace_after_colon._codegen(state)
3758 self.value._codegen(state)
3759 self.for_in._codegen(state)
3762@add_slots
3763@dataclass(frozen=True)
3764class NamedExpr(BaseExpression):
3765 """
3766 An expression that is also an assignment, such as ``x := y + z``. Affectionately
3767 known as the walrus operator, this expression allows you to make an assignment
3768 inside an expression. This greatly simplifies loops::
3770 while line := read_some_line_or_none():
3771 do_thing_with_line(line)
3772 """
3774 #: The target that is being assigned to.
3775 target: BaseExpression
3777 #: The expression being assigned to the target.
3778 value: BaseExpression
3780 #: Sequence of parenthesis for precedence dictation.
3781 lpar: Sequence[LeftParen] = ()
3782 #: Sequence of parenthesis for precedence dictation.
3783 rpar: Sequence[RightParen] = ()
3785 #: Whitespace after the target, but before the walrus operator.
3786 whitespace_before_walrus: BaseParenthesizableWhitespace = SimpleWhitespace.field(
3787 " "
3788 )
3789 #: Whitespace after the walrus operator, but before the value.
3790 whitespace_after_walrus: BaseParenthesizableWhitespace = SimpleWhitespace.field(" ")
3792 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "NamedExpr":
3793 return NamedExpr(
3794 lpar=visit_sequence(self, "lpar", self.lpar, visitor),
3795 target=visit_required(self, "target", self.target, visitor),
3796 whitespace_before_walrus=visit_required(
3797 self, "whitespace_before_walrus", self.whitespace_before_walrus, visitor
3798 ),
3799 whitespace_after_walrus=visit_required(
3800 self, "whitespace_after_walrus", self.whitespace_after_walrus, visitor
3801 ),
3802 value=visit_required(self, "value", self.value, visitor),
3803 rpar=visit_sequence(self, "rpar", self.rpar, visitor),
3804 )
3806 def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool:
3807 if position == ExpressionPosition.LEFT:
3808 return len(self.rpar) > 0 or self.value._safe_to_use_with_word_operator(
3809 position
3810 )
3811 return len(self.lpar) > 0 or self.target._safe_to_use_with_word_operator(
3812 position
3813 )
3815 def _codegen_impl(self, state: CodegenState) -> None:
3816 with self._parenthesize(state):
3817 self.target._codegen(state)
3818 self.whitespace_before_walrus._codegen(state)
3819 state.add_token(":=")
3820 self.whitespace_after_walrus._codegen(state)
3821 self.value._codegen(state)