Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/_parser/conversions/expression.py: 30%
473 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:43 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
5# pyre-unsafe
7import re
8import typing
9from tokenize import (
10 Floatnumber as FLOATNUMBER_RE,
11 Imagnumber as IMAGNUMBER_RE,
12 Intnumber as INTNUMBER_RE,
13)
15from libcst._exceptions import PartialParserSyntaxError
16from libcst._maybe_sentinel import MaybeSentinel
17from libcst._nodes.expression import (
18 Arg,
19 Asynchronous,
20 Attribute,
21 Await,
22 BinaryOperation,
23 BooleanOperation,
24 Call,
25 Comparison,
26 ComparisonTarget,
27 CompFor,
28 CompIf,
29 ConcatenatedString,
30 Dict,
31 DictComp,
32 DictElement,
33 Element,
34 Ellipsis,
35 Float,
36 FormattedString,
37 FormattedStringExpression,
38 FormattedStringText,
39 From,
40 GeneratorExp,
41 IfExp,
42 Imaginary,
43 Index,
44 Integer,
45 Lambda,
46 LeftCurlyBrace,
47 LeftParen,
48 LeftSquareBracket,
49 List,
50 ListComp,
51 Name,
52 NamedExpr,
53 Param,
54 Parameters,
55 RightCurlyBrace,
56 RightParen,
57 RightSquareBracket,
58 Set,
59 SetComp,
60 Slice,
61 StarredDictElement,
62 StarredElement,
63 Subscript,
64 SubscriptElement,
65 Tuple,
66 UnaryOperation,
67 Yield,
68)
69from libcst._nodes.op import (
70 Add,
71 And,
72 AssignEqual,
73 BaseBinaryOp,
74 BaseBooleanOp,
75 BaseCompOp,
76 BitAnd,
77 BitInvert,
78 BitOr,
79 BitXor,
80 Colon,
81 Comma,
82 Divide,
83 Dot,
84 Equal,
85 FloorDivide,
86 GreaterThan,
87 GreaterThanEqual,
88 In,
89 Is,
90 IsNot,
91 LeftShift,
92 LessThan,
93 LessThanEqual,
94 MatrixMultiply,
95 Minus,
96 Modulo,
97 Multiply,
98 Not,
99 NotEqual,
100 NotIn,
101 Or,
102 Plus,
103 Power,
104 RightShift,
105 Subtract,
106)
107from libcst._nodes.whitespace import SimpleWhitespace
108from libcst._parser.custom_itertools import grouper
109from libcst._parser.production_decorator import with_production
110from libcst._parser.types.config import ParserConfig
111from libcst._parser.types.partials import (
112 ArglistPartial,
113 AttributePartial,
114 CallPartial,
115 FormattedStringConversionPartial,
116 FormattedStringFormatSpecPartial,
117 SlicePartial,
118 SubscriptPartial,
119 WithLeadingWhitespace,
120)
121from libcst._parser.types.token import Token
122from libcst._parser.whitespace_parser import parse_parenthesizable_whitespace
124BINOP_TOKEN_LUT: typing.Dict[str, typing.Type[BaseBinaryOp]] = {
125 "*": Multiply,
126 "@": MatrixMultiply,
127 "/": Divide,
128 "%": Modulo,
129 "//": FloorDivide,
130 "+": Add,
131 "-": Subtract,
132 "<<": LeftShift,
133 ">>": RightShift,
134 "&": BitAnd,
135 "^": BitXor,
136 "|": BitOr,
137}
140BOOLOP_TOKEN_LUT: typing.Dict[str, typing.Type[BaseBooleanOp]] = {"and": And, "or": Or}
143COMPOP_TOKEN_LUT: typing.Dict[str, typing.Type[BaseCompOp]] = {
144 "<": LessThan,
145 ">": GreaterThan,
146 "==": Equal,
147 "<=": LessThanEqual,
148 ">=": GreaterThanEqual,
149 "in": In,
150 "is": Is,
151}
154# N.B. This uses a `testlist | star_expr`, not a `testlist_star_expr` because
155# `testlist_star_expr` may not always be representable by a non-partial node, since it's
156# only used as part of `expr_stmt`.
157@with_production("expression_input", "(testlist | star_expr) ENDMARKER")
158def convert_expression_input(
159 config: ParserConfig, children: typing.Sequence[typing.Any]
160) -> typing.Any:
161 (child, endmarker) = children
162 # HACK: UGLY! REMOVE THIS SOON!
163 # Unwrap WithLeadingWhitespace if it exists. It shouldn't exist by this point, but
164 # testlist isn't fully implemented, and we currently leak these partial objects.
165 if isinstance(child, WithLeadingWhitespace):
166 child = child.value
167 return child
170@with_production("namedexpr_test", "test [':=' test]", version=">=3.8")
171def convert_namedexpr_test(
172 config: ParserConfig, children: typing.Sequence[typing.Any]
173) -> typing.Any:
174 test, *assignment = children
175 if len(assignment) == 0:
176 return test
178 # Convert all of the operations that have no precedence in a loop
179 (walrus, value) = assignment
180 return WithLeadingWhitespace(
181 NamedExpr(
182 target=test.value,
183 whitespace_before_walrus=parse_parenthesizable_whitespace(
184 config, walrus.whitespace_before
185 ),
186 whitespace_after_walrus=parse_parenthesizable_whitespace(
187 config, walrus.whitespace_after
188 ),
189 value=value.value,
190 ),
191 test.whitespace_before,
192 )
195@with_production("test", "or_test ['if' or_test 'else' test] | lambdef")
196def convert_test(
197 config: ParserConfig, children: typing.Sequence[typing.Any]
198) -> typing.Any:
199 if len(children) == 1:
200 (child,) = children
201 return child
202 else:
203 (body, if_token, test, else_token, orelse) = children
204 return WithLeadingWhitespace(
205 IfExp(
206 body=body.value,
207 test=test.value,
208 orelse=orelse.value,
209 whitespace_before_if=parse_parenthesizable_whitespace(
210 config, if_token.whitespace_before
211 ),
212 whitespace_after_if=parse_parenthesizable_whitespace(
213 config, if_token.whitespace_after
214 ),
215 whitespace_before_else=parse_parenthesizable_whitespace(
216 config, else_token.whitespace_before
217 ),
218 whitespace_after_else=parse_parenthesizable_whitespace(
219 config, else_token.whitespace_after
220 ),
221 ),
222 body.whitespace_before,
223 )
226@with_production("test_nocond", "or_test | lambdef_nocond")
227def convert_test_nocond(
228 config: ParserConfig, children: typing.Sequence[typing.Any]
229) -> typing.Any:
230 (child,) = children
231 return child
234@with_production("lambdef", "'lambda' [varargslist] ':' test")
235@with_production("lambdef_nocond", "'lambda' [varargslist] ':' test_nocond")
236def convert_lambda(
237 config: ParserConfig, children: typing.Sequence[typing.Any]
238) -> typing.Any:
239 lambdatoken, *params, colontoken, test = children
241 # Grab the whitespace around the colon. If there are no params, then
242 # the colon owns the whitespace before and after it. If there are
243 # any params, then the last param owns the whitespace before the colon.
244 # We handle the parameter movement below.
245 colon = Colon(
246 whitespace_before=parse_parenthesizable_whitespace(
247 config, colontoken.whitespace_before
248 ),
249 whitespace_after=parse_parenthesizable_whitespace(
250 config, colontoken.whitespace_after
251 ),
252 )
254 # Unpack optional parameters
255 if len(params) == 0:
256 parameters = Parameters()
257 whitespace_after_lambda = MaybeSentinel.DEFAULT
258 else:
259 (parameters,) = params
260 whitespace_after_lambda = parse_parenthesizable_whitespace(
261 config, lambdatoken.whitespace_after
262 )
264 # Handle pre-colon whitespace
265 if parameters.star_kwarg is not None:
266 if parameters.star_kwarg.comma == MaybeSentinel.DEFAULT:
267 parameters = parameters.with_changes(
268 star_kwarg=parameters.star_kwarg.with_changes(
269 whitespace_after_param=colon.whitespace_before
270 )
271 )
272 elif parameters.kwonly_params:
273 if parameters.kwonly_params[-1].comma == MaybeSentinel.DEFAULT:
274 parameters = parameters.with_changes(
275 kwonly_params=(
276 *parameters.kwonly_params[:-1],
277 parameters.kwonly_params[-1].with_changes(
278 whitespace_after_param=colon.whitespace_before
279 ),
280 )
281 )
282 elif isinstance(parameters.star_arg, Param):
283 if parameters.star_arg.comma == MaybeSentinel.DEFAULT:
284 parameters = parameters.with_changes(
285 star_arg=parameters.star_arg.with_changes(
286 whitespace_after_param=colon.whitespace_before
287 )
288 )
289 elif parameters.params:
290 if parameters.params[-1].comma == MaybeSentinel.DEFAULT:
291 parameters = parameters.with_changes(
292 params=(
293 *parameters.params[:-1],
294 parameters.params[-1].with_changes(
295 whitespace_after_param=colon.whitespace_before
296 ),
297 )
298 )
300 # Colon doesn't own its own pre-whitespace now.
301 colon = colon.with_changes(whitespace_before=SimpleWhitespace(""))
303 # Return a lambda
304 return WithLeadingWhitespace(
305 Lambda(
306 whitespace_after_lambda=whitespace_after_lambda,
307 params=parameters,
308 body=test.value,
309 colon=colon,
310 ),
311 lambdatoken.whitespace_before,
312 )
315@with_production("or_test", "and_test ('or' and_test)*")
316@with_production("and_test", "not_test ('and' not_test)*")
317def convert_boolop(
318 config: ParserConfig, children: typing.Sequence[typing.Any]
319) -> typing.Any:
320 leftexpr, *rightexprs = children
321 if len(rightexprs) == 0:
322 return leftexpr
324 whitespace_before = leftexpr.whitespace_before
325 leftexpr = leftexpr.value
327 # Convert all of the operations that have no precedence in a loop
328 for op, rightexpr in grouper(rightexprs, 2):
329 if op.string not in BOOLOP_TOKEN_LUT:
330 raise Exception(f"Unexpected token '{op.string}'!")
331 leftexpr = BooleanOperation(
332 left=leftexpr,
333 # pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
334 operator=BOOLOP_TOKEN_LUT[op.string](
335 whitespace_before=parse_parenthesizable_whitespace(
336 config, op.whitespace_before
337 ),
338 whitespace_after=parse_parenthesizable_whitespace(
339 config, op.whitespace_after
340 ),
341 ),
342 right=rightexpr.value,
343 )
344 return WithLeadingWhitespace(leftexpr, whitespace_before)
347@with_production("not_test", "'not' not_test | comparison")
348def convert_not_test(
349 config: ParserConfig, children: typing.Sequence[typing.Any]
350) -> typing.Any:
351 if len(children) == 1:
352 (child,) = children
353 return child
354 else:
355 nottoken, nottest = children
356 return WithLeadingWhitespace(
357 UnaryOperation(
358 operator=Not(
359 whitespace_after=parse_parenthesizable_whitespace(
360 config, nottoken.whitespace_after
361 )
362 ),
363 expression=nottest.value,
364 ),
365 nottoken.whitespace_before,
366 )
369@with_production("comparison", "expr (comp_op expr)*")
370def convert_comparison(
371 config: ParserConfig, children: typing.Sequence[typing.Any]
372) -> typing.Any:
373 if len(children) == 1:
374 (child,) = children
375 return child
377 lhs, *rest = children
379 comparisons: typing.List[ComparisonTarget] = []
380 for operator, comparator in grouper(rest, 2):
381 comparisons.append(
382 ComparisonTarget(operator=operator, comparator=comparator.value)
383 )
385 return WithLeadingWhitespace(
386 Comparison(left=lhs.value, comparisons=tuple(comparisons)),
387 lhs.whitespace_before,
388 )
391@with_production(
392 "comp_op", "('<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not')"
393)
394def convert_comp_op(
395 config: ParserConfig, children: typing.Sequence[typing.Any]
396) -> typing.Any:
397 if len(children) == 1:
398 (op,) = children
399 if op.string in COMPOP_TOKEN_LUT:
400 # A regular comparison containing one token
401 # pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
402 return COMPOP_TOKEN_LUT[op.string](
403 whitespace_before=parse_parenthesizable_whitespace(
404 config, op.whitespace_before
405 ),
406 whitespace_after=parse_parenthesizable_whitespace(
407 config, op.whitespace_after
408 ),
409 )
410 elif op.string in ["!=", "<>"]:
411 # Not equal, which can take two forms in some cases
412 return NotEqual(
413 whitespace_before=parse_parenthesizable_whitespace(
414 config, op.whitespace_before
415 ),
416 value=op.string,
417 whitespace_after=parse_parenthesizable_whitespace(
418 config, op.whitespace_after
419 ),
420 )
421 else:
422 # this should be unreachable
423 raise Exception(f"Unexpected token '{op.string}'!")
424 else:
425 # A two-token comparison
426 leftcomp, rightcomp = children
428 if leftcomp.string == "not" and rightcomp.string == "in":
429 return NotIn(
430 whitespace_before=parse_parenthesizable_whitespace(
431 config, leftcomp.whitespace_before
432 ),
433 whitespace_between=parse_parenthesizable_whitespace(
434 config, leftcomp.whitespace_after
435 ),
436 whitespace_after=parse_parenthesizable_whitespace(
437 config, rightcomp.whitespace_after
438 ),
439 )
440 elif leftcomp.string == "is" and rightcomp.string == "not":
441 return IsNot(
442 whitespace_before=parse_parenthesizable_whitespace(
443 config, leftcomp.whitespace_before
444 ),
445 whitespace_between=parse_parenthesizable_whitespace(
446 config, leftcomp.whitespace_after
447 ),
448 whitespace_after=parse_parenthesizable_whitespace(
449 config, rightcomp.whitespace_after
450 ),
451 )
452 else:
453 # this should be unreachable
454 raise Exception(f"Unexpected token '{leftcomp.string} {rightcomp.string}'!")
457@with_production("star_expr", "'*' expr")
458def convert_star_expr(
459 config: ParserConfig, children: typing.Sequence[typing.Any]
460) -> typing.Any:
461 star, expr = children
462 return WithLeadingWhitespace(
463 StarredElement(
464 expr.value,
465 whitespace_before_value=parse_parenthesizable_whitespace(
466 config, expr.whitespace_before
467 ),
468 # atom is responsible for parenthesis and trailing_whitespace if they exist
469 # testlist_comp, exprlist, dictorsetmaker, etc are responsible for the comma
470 # if it exists.
471 ),
472 whitespace_before=star.whitespace_before,
473 )
476@with_production("expr", "xor_expr ('|' xor_expr)*")
477@with_production("xor_expr", "and_expr ('^' and_expr)*")
478@with_production("and_expr", "shift_expr ('&' shift_expr)*")
479@with_production("shift_expr", "arith_expr (('<<'|'>>') arith_expr)*")
480@with_production("arith_expr", "term (('+'|'-') term)*")
481@with_production("term", "factor (('*'|'@'|'/'|'%'|'//') factor)*", version=">=3.5")
482@with_production("term", "factor (('*'|'/'|'%'|'//') factor)*", version="<3.5")
483def convert_binop(
484 config: ParserConfig, children: typing.Sequence[typing.Any]
485) -> typing.Any:
486 leftexpr, *rightexprs = children
487 if len(rightexprs) == 0:
488 return leftexpr
490 whitespace_before = leftexpr.whitespace_before
491 leftexpr = leftexpr.value
493 # Convert all of the operations that have no precedence in a loop
494 for op, rightexpr in grouper(rightexprs, 2):
495 if op.string not in BINOP_TOKEN_LUT:
496 raise Exception(f"Unexpected token '{op.string}'!")
497 leftexpr = BinaryOperation(
498 left=leftexpr,
499 # pyre-ignore Pyre thinks that the type of the LUT is CSTNode.
500 operator=BINOP_TOKEN_LUT[op.string](
501 whitespace_before=parse_parenthesizable_whitespace(
502 config, op.whitespace_before
503 ),
504 whitespace_after=parse_parenthesizable_whitespace(
505 config, op.whitespace_after
506 ),
507 ),
508 right=rightexpr.value,
509 )
510 return WithLeadingWhitespace(leftexpr, whitespace_before)
513@with_production("factor", "('+'|'-'|'~') factor | power")
514def convert_factor(
515 config: ParserConfig, children: typing.Sequence[typing.Any]
516) -> typing.Any:
517 if len(children) == 1:
518 (child,) = children
519 return child
521 op, factor = children
523 # First, tokenize the unary operator
524 if op.string == "+":
525 opnode = Plus(
526 whitespace_after=parse_parenthesizable_whitespace(
527 config, op.whitespace_after
528 )
529 )
530 elif op.string == "-":
531 opnode = Minus(
532 whitespace_after=parse_parenthesizable_whitespace(
533 config, op.whitespace_after
534 )
535 )
536 elif op.string == "~":
537 opnode = BitInvert(
538 whitespace_after=parse_parenthesizable_whitespace(
539 config, op.whitespace_after
540 )
541 )
542 else:
543 raise Exception(f"Unexpected token '{op.string}'!")
545 return WithLeadingWhitespace(
546 UnaryOperation(operator=opnode, expression=factor.value), op.whitespace_before
547 )
550@with_production("power", "atom_expr ['**' factor]")
551def convert_power(
552 config: ParserConfig, children: typing.Sequence[typing.Any]
553) -> typing.Any:
554 if len(children) == 1:
555 (child,) = children
556 return child
558 left, power, right = children
559 return WithLeadingWhitespace(
560 BinaryOperation(
561 left=left.value,
562 operator=Power(
563 whitespace_before=parse_parenthesizable_whitespace(
564 config, power.whitespace_before
565 ),
566 whitespace_after=parse_parenthesizable_whitespace(
567 config, power.whitespace_after
568 ),
569 ),
570 right=right.value,
571 ),
572 left.whitespace_before,
573 )
576@with_production("atom_expr", "atom_expr_await | atom_expr_trailer")
577def convert_atom_expr(
578 config: ParserConfig, children: typing.Sequence[typing.Any]
579) -> typing.Any:
580 (child,) = children
581 return child
584@with_production("atom_expr_await", "AWAIT atom_expr_trailer")
585def convert_atom_expr_await(
586 config: ParserConfig, children: typing.Sequence[typing.Any]
587) -> typing.Any:
588 keyword, expr = children
589 return WithLeadingWhitespace(
590 Await(
591 whitespace_after_await=parse_parenthesizable_whitespace(
592 config, keyword.whitespace_after
593 ),
594 expression=expr.value,
595 ),
596 keyword.whitespace_before,
597 )
600@with_production("atom_expr_trailer", "atom trailer*")
601def convert_atom_expr_trailer(
602 config: ParserConfig, children: typing.Sequence[typing.Any]
603) -> typing.Any:
604 atom, *trailers = children
605 whitespace_before = atom.whitespace_before
606 atom = atom.value
608 # Need to walk through all trailers from left to right and construct
609 # a series of nodes based on each partial type. We can't do this with
610 # left recursion due to limits in the parser.
611 for trailer in trailers:
612 if isinstance(trailer, SubscriptPartial):
613 atom = Subscript(
614 value=atom,
615 whitespace_after_value=parse_parenthesizable_whitespace(
616 config, trailer.whitespace_before
617 ),
618 lbracket=trailer.lbracket,
619 # pyre-fixme[6]: Expected `Sequence[SubscriptElement]` for 4th param
620 # but got `Union[typing.Sequence[SubscriptElement], Index, Slice]`.
621 slice=trailer.slice,
622 rbracket=trailer.rbracket,
623 )
624 elif isinstance(trailer, AttributePartial):
625 atom = Attribute(value=atom, dot=trailer.dot, attr=trailer.attr)
626 elif isinstance(trailer, CallPartial):
627 # If the trailing argument doesn't have a comma, then it owns the
628 # trailing whitespace before the rpar. Otherwise, the comma owns
629 # it.
630 if (
631 len(trailer.args) > 0
632 and trailer.args[-1].comma == MaybeSentinel.DEFAULT
633 ):
634 args = (
635 *trailer.args[:-1],
636 trailer.args[-1].with_changes(
637 whitespace_after_arg=trailer.rpar.whitespace_before
638 ),
639 )
640 else:
641 args = trailer.args
642 atom = Call(
643 func=atom,
644 whitespace_after_func=parse_parenthesizable_whitespace(
645 config, trailer.lpar.whitespace_before
646 ),
647 whitespace_before_args=trailer.lpar.value.whitespace_after,
648 # pyre-fixme[6]: Expected `Sequence[Arg]` for 4th param but got
649 # `Tuple[object, ...]`.
650 args=tuple(args),
651 )
652 else:
653 # This is an invalid trailer, so lets give up
654 raise Exception("Logic error!")
655 return WithLeadingWhitespace(atom, whitespace_before)
658@with_production(
659 "trailer", "trailer_arglist | trailer_subscriptlist | trailer_attribute"
660)
661def convert_trailer(
662 config: ParserConfig, children: typing.Sequence[typing.Any]
663) -> typing.Any:
664 (child,) = children
665 return child
668@with_production("trailer_arglist", "'(' [arglist] ')'")
669def convert_trailer_arglist(
670 config: ParserConfig, children: typing.Sequence[typing.Any]
671) -> typing.Any:
672 lpar, *arglist, rpar = children
673 return CallPartial(
674 lpar=WithLeadingWhitespace(
675 LeftParen(
676 whitespace_after=parse_parenthesizable_whitespace(
677 config, lpar.whitespace_after
678 )
679 ),
680 lpar.whitespace_before,
681 ),
682 args=() if not arglist else arglist[0].args,
683 rpar=RightParen(
684 whitespace_before=parse_parenthesizable_whitespace(
685 config, rpar.whitespace_before
686 )
687 ),
688 )
691@with_production("trailer_subscriptlist", "'[' subscriptlist ']'")
692def convert_trailer_subscriptlist(
693 config: ParserConfig, children: typing.Sequence[typing.Any]
694) -> typing.Any:
695 (lbracket, subscriptlist, rbracket) = children
696 return SubscriptPartial(
697 lbracket=LeftSquareBracket(
698 whitespace_after=parse_parenthesizable_whitespace(
699 config, lbracket.whitespace_after
700 )
701 ),
702 slice=subscriptlist.value,
703 rbracket=RightSquareBracket(
704 whitespace_before=parse_parenthesizable_whitespace(
705 config, rbracket.whitespace_before
706 )
707 ),
708 whitespace_before=lbracket.whitespace_before,
709 )
712@with_production("subscriptlist", "subscript (',' subscript)* [',']")
713def convert_subscriptlist(
714 config: ParserConfig, children: typing.Sequence[typing.Any]
715) -> typing.Any:
716 # This is a list of SubscriptElement, so construct as such by grouping every
717 # subscript with an optional comma and adding to a list.
718 elements = []
719 for slice, comma in grouper(children, 2):
720 if comma is None:
721 elements.append(SubscriptElement(slice=slice.value))
722 else:
723 elements.append(
724 SubscriptElement(
725 slice=slice.value,
726 comma=Comma(
727 whitespace_before=parse_parenthesizable_whitespace(
728 config, comma.whitespace_before
729 ),
730 whitespace_after=parse_parenthesizable_whitespace(
731 config, comma.whitespace_after
732 ),
733 ),
734 )
735 )
736 return WithLeadingWhitespace(elements, children[0].whitespace_before)
739@with_production("subscript", "test | [test] ':' [test] [sliceop]")
740def convert_subscript(
741 config: ParserConfig, children: typing.Sequence[typing.Any]
742) -> typing.Any:
743 if len(children) == 1 and not isinstance(children[0], Token):
744 # This is just an index node
745 (test,) = children
746 return WithLeadingWhitespace(Index(test.value), test.whitespace_before)
748 if isinstance(children[-1], SlicePartial):
749 # We got a partial slice as the final param. Extract the final
750 # bits of the full subscript.
751 *others, sliceop = children
752 whitespace_before = others[0].whitespace_before
753 second_colon = sliceop.second_colon
754 step = sliceop.step
755 else:
756 # We can just parse this below, without taking extras from the
757 # partial child.
758 others = children
759 whitespace_before = others[0].whitespace_before
760 second_colon = MaybeSentinel.DEFAULT
761 step = None
763 # We need to create a partial slice to pass up. So, align so we have
764 # a list that's always [Optional[Test], Colon, Optional[Test]].
765 if isinstance(others[0], Token):
766 # First token is a colon, so insert an empty test on the LHS. We
767 # know the RHS is a test since it's not a sliceop.
768 slicechildren = [None, *others]
769 else:
770 # First token is non-colon, so its a test.
771 slicechildren = [*others]
773 if len(slicechildren) < 3:
774 # Now, we have to fill in the RHS. We know its two long
775 # at this point if its not already 3.
776 slicechildren = [*slicechildren, None]
778 lower, first_colon, upper = slicechildren
779 return WithLeadingWhitespace(
780 Slice(
781 lower=lower.value if lower is not None else None,
782 first_colon=Colon(
783 whitespace_before=parse_parenthesizable_whitespace(
784 config,
785 first_colon.whitespace_before,
786 ),
787 whitespace_after=parse_parenthesizable_whitespace(
788 config,
789 first_colon.whitespace_after,
790 ),
791 ),
792 upper=upper.value if upper is not None else None,
793 second_colon=second_colon,
794 step=step,
795 ),
796 whitespace_before=whitespace_before,
797 )
800@with_production("sliceop", "':' [test]")
801def convert_sliceop(
802 config: ParserConfig, children: typing.Sequence[typing.Any]
803) -> typing.Any:
804 if len(children) == 2:
805 colon, test = children
806 step = test.value
807 else:
808 (colon,) = children
809 step = None
810 return SlicePartial(
811 second_colon=Colon(
812 whitespace_before=parse_parenthesizable_whitespace(
813 config, colon.whitespace_before
814 ),
815 whitespace_after=parse_parenthesizable_whitespace(
816 config, colon.whitespace_after
817 ),
818 ),
819 step=step,
820 )
823@with_production("trailer_attribute", "'.' NAME")
824def convert_trailer_attribute(
825 config: ParserConfig, children: typing.Sequence[typing.Any]
826) -> typing.Any:
827 dot, name = children
828 return AttributePartial(
829 dot=Dot(
830 whitespace_before=parse_parenthesizable_whitespace(
831 config, dot.whitespace_before
832 ),
833 whitespace_after=parse_parenthesizable_whitespace(
834 config, dot.whitespace_after
835 ),
836 ),
837 attr=Name(name.string),
838 )
841@with_production(
842 "atom",
843 "atom_parens | atom_squarebrackets | atom_curlybraces | atom_string | atom_basic | atom_ellipses",
844)
845def convert_atom(
846 config: ParserConfig, children: typing.Sequence[typing.Any]
847) -> typing.Any:
848 (child,) = children
849 return child
852@with_production("atom_basic", "NAME | NUMBER | 'None' | 'True' | 'False'")
853def convert_atom_basic(
854 config: ParserConfig, children: typing.Sequence[typing.Any]
855) -> typing.Any:
856 (child,) = children
857 if child.type.name == "NAME":
858 # This also handles 'None', 'True', and 'False' directly, but we
859 # keep it in the grammar to be more correct.
860 return WithLeadingWhitespace(Name(child.string), child.whitespace_before)
861 elif child.type.name == "NUMBER":
862 # We must determine what type of number it is since we split node
863 # types up this way.
864 if re.fullmatch(INTNUMBER_RE, child.string):
865 return WithLeadingWhitespace(Integer(child.string), child.whitespace_before)
866 elif re.fullmatch(FLOATNUMBER_RE, child.string):
867 return WithLeadingWhitespace(Float(child.string), child.whitespace_before)
868 elif re.fullmatch(IMAGNUMBER_RE, child.string):
869 return WithLeadingWhitespace(
870 Imaginary(child.string), child.whitespace_before
871 )
872 else:
873 raise Exception(f"Unparseable number {child.string}")
874 else:
875 raise Exception(f"Logic error, unexpected token {child.type.name}")
878@with_production("atom_squarebrackets", "'[' [testlist_comp_list] ']'")
879def convert_atom_squarebrackets(
880 config: ParserConfig, children: typing.Sequence[typing.Any]
881) -> typing.Any:
882 lbracket_tok, *body, rbracket_tok = children
883 lbracket = LeftSquareBracket(
884 whitespace_after=parse_parenthesizable_whitespace(
885 config, lbracket_tok.whitespace_after
886 )
887 )
889 rbracket = RightSquareBracket(
890 whitespace_before=parse_parenthesizable_whitespace(
891 config, rbracket_tok.whitespace_before
892 )
893 )
895 if len(body) == 0:
896 list_node = List((), lbracket=lbracket, rbracket=rbracket)
897 else: # len(body) == 1
898 # body[0] is a List or ListComp
899 list_node = body[0].value.with_changes(lbracket=lbracket, rbracket=rbracket)
901 return WithLeadingWhitespace(list_node, lbracket_tok.whitespace_before)
904@with_production("atom_curlybraces", "'{' [dictorsetmaker] '}'")
905def convert_atom_curlybraces(
906 config: ParserConfig, children: typing.Sequence[typing.Any]
907) -> typing.Any:
908 lbrace_tok, *body, rbrace_tok = children
909 lbrace = LeftCurlyBrace(
910 whitespace_after=parse_parenthesizable_whitespace(
911 config, lbrace_tok.whitespace_after
912 )
913 )
915 rbrace = RightCurlyBrace(
916 whitespace_before=parse_parenthesizable_whitespace(
917 config, rbrace_tok.whitespace_before
918 )
919 )
921 if len(body) == 0:
922 dict_or_set_node = Dict((), lbrace=lbrace, rbrace=rbrace)
923 else: # len(body) == 1
924 dict_or_set_node = body[0].value.with_changes(lbrace=lbrace, rbrace=rbrace)
926 return WithLeadingWhitespace(dict_or_set_node, lbrace_tok.whitespace_before)
929@with_production("atom_parens", "'(' [yield_expr|testlist_comp_tuple] ')'")
930def convert_atom_parens(
931 config: ParserConfig, children: typing.Sequence[typing.Any]
932) -> typing.Any:
933 lpar_tok, *atoms, rpar_tok = children
935 lpar = LeftParen(
936 whitespace_after=parse_parenthesizable_whitespace(
937 config, lpar_tok.whitespace_after
938 )
939 )
941 rpar = RightParen(
942 whitespace_before=parse_parenthesizable_whitespace(
943 config, rpar_tok.whitespace_before
944 )
945 )
947 if len(atoms) == 1:
948 # inner_atom is a _BaseParenthesizedNode
949 inner_atom = atoms[0].value
950 return WithLeadingWhitespace(
951 inner_atom.with_changes(
952 # pyre-fixme[60]: Expected to unpack an iterable, but got `unknown`.
953 lpar=(lpar, *inner_atom.lpar),
954 # pyre-fixme[60]: Expected to unpack an iterable, but got `unknown`.
955 rpar=(*inner_atom.rpar, rpar),
956 ),
957 lpar_tok.whitespace_before,
958 )
959 else:
960 return WithLeadingWhitespace(
961 Tuple((), lpar=(lpar,), rpar=(rpar,)), lpar_tok.whitespace_before
962 )
965@with_production("atom_ellipses", "'...'")
966def convert_atom_ellipses(
967 config: ParserConfig, children: typing.Sequence[typing.Any]
968) -> typing.Any:
969 (token,) = children
970 return WithLeadingWhitespace(Ellipsis(), token.whitespace_before)
973@with_production("atom_string", "(STRING | fstring) [atom_string]")
974def convert_atom_string(
975 config: ParserConfig, children: typing.Sequence[typing.Any]
976) -> typing.Any:
977 if len(children) == 1:
978 return children[0]
979 else:
980 left, right = children
981 return WithLeadingWhitespace(
982 ConcatenatedString(
983 left=left.value,
984 whitespace_between=parse_parenthesizable_whitespace(
985 config, right.whitespace_before
986 ),
987 right=right.value,
988 ),
989 left.whitespace_before,
990 )
993@with_production("fstring", "FSTRING_START fstring_content* FSTRING_END")
994def convert_fstring(
995 config: ParserConfig, children: typing.Sequence[typing.Any]
996) -> typing.Any:
997 start, *content, end = children
998 return WithLeadingWhitespace(
999 FormattedString(start=start.string, parts=tuple(content), end=end.string),
1000 start.whitespace_before,
1001 )
1004@with_production("fstring_content", "FSTRING_STRING | fstring_expr")
1005def convert_fstring_content(
1006 config: ParserConfig, children: typing.Sequence[typing.Any]
1007) -> typing.Any:
1008 (child,) = children
1009 if isinstance(child, Token):
1010 # Construct and return a raw string portion.
1011 return FormattedStringText(child.string)
1012 else:
1013 # Pass the expression up one production.
1014 return child
1017@with_production("fstring_conversion", "'!' NAME")
1018def convert_fstring_conversion(
1019 config: ParserConfig, children: typing.Sequence[typing.Any]
1020) -> typing.Any:
1021 exclaim, name = children
1022 # There cannot be a space between the two tokens, so no need to preserve this.
1023 return FormattedStringConversionPartial(name.string, exclaim.whitespace_before)
1026@with_production("fstring_equality", "'='", version=">=3.8")
1027def convert_fstring_equality(
1028 config: ParserConfig, children: typing.Sequence[typing.Any]
1029) -> typing.Any:
1030 (equal,) = children
1031 return AssignEqual(
1032 whitespace_before=parse_parenthesizable_whitespace(
1033 config, equal.whitespace_before
1034 ),
1035 whitespace_after=parse_parenthesizable_whitespace(
1036 config, equal.whitespace_after
1037 ),
1038 )
1041@with_production(
1042 "fstring_expr",
1043 "'{' (testlist_comp_tuple | yield_expr) [ fstring_equality ] [ fstring_conversion ] [ fstring_format_spec ] '}'",
1044 version=">=3.8",
1045)
1046@with_production(
1047 "fstring_expr",
1048 "'{' (testlist_comp_tuple | yield_expr) [ fstring_conversion ] [ fstring_format_spec ] '}'",
1049 version="<3.8",
1050)
1051def convert_fstring_expr(
1052 config: ParserConfig, children: typing.Sequence[typing.Any]
1053) -> typing.Any:
1054 openbrkt, testlist, *conversions, closebrkt = children
1056 # Extract any optional equality (self-debugging expressions)
1057 if len(conversions) > 0 and isinstance(conversions[0], AssignEqual):
1058 equal = conversions[0]
1059 conversions = conversions[1:]
1060 else:
1061 equal = None
1063 # Extract any optional conversion
1064 if len(conversions) > 0 and isinstance(
1065 conversions[0], FormattedStringConversionPartial
1066 ):
1067 conversion = conversions[0].value
1068 conversions = conversions[1:]
1069 else:
1070 conversion = None
1072 # Extract any optional format spec
1073 if len(conversions) > 0:
1074 format_spec = conversions[0].values
1075 else:
1076 format_spec = None
1078 # Fix up any spacing issue we find due to the fact that the equal can
1079 # have whitespace and is also at the end of the expression.
1080 if equal is not None:
1081 whitespace_after_expression = SimpleWhitespace("")
1082 else:
1083 whitespace_after_expression = parse_parenthesizable_whitespace(
1084 config, children[2].whitespace_before
1085 )
1087 return FormattedStringExpression(
1088 whitespace_before_expression=parse_parenthesizable_whitespace(
1089 config, testlist.whitespace_before
1090 ),
1091 expression=testlist.value,
1092 equal=equal,
1093 whitespace_after_expression=whitespace_after_expression,
1094 conversion=conversion,
1095 format_spec=format_spec,
1096 )
1099@with_production("fstring_format_spec", "':' fstring_content*")
1100def convert_fstring_format_spec(
1101 config: ParserConfig, children: typing.Sequence[typing.Any]
1102) -> typing.Any:
1103 colon, *content = children
1104 return FormattedStringFormatSpecPartial(tuple(content), colon.whitespace_before)
1107@with_production(
1108 "testlist_comp_tuple",
1109 "(namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )",
1110 version=">=3.8",
1111)
1112@with_production(
1113 "testlist_comp_tuple",
1114 "(test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )",
1115 version=">=3.5,<3.8",
1116)
1117@with_production(
1118 "testlist_comp_tuple",
1119 "(test) ( comp_for | (',' (test))* [','] )",
1120 version="<3.5",
1121)
1122def convert_testlist_comp_tuple(
1123 config: ParserConfig, children: typing.Sequence[typing.Any]
1124) -> typing.Any:
1125 return _convert_testlist_comp(
1126 config,
1127 children,
1128 single_child_is_sequence=False,
1129 sequence_type=Tuple,
1130 comprehension_type=GeneratorExp,
1131 )
1134@with_production(
1135 "testlist_comp_list",
1136 "(namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )",
1137 version=">=3.8",
1138)
1139@with_production(
1140 "testlist_comp_list",
1141 "(test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )",
1142 version=">=3.5,<3.8",
1143)
1144@with_production(
1145 "testlist_comp_list",
1146 "(test) ( comp_for | (',' (test))* [','] )",
1147 version="<3.5",
1148)
1149def convert_testlist_comp_list(
1150 config: ParserConfig, children: typing.Sequence[typing.Any]
1151) -> typing.Any:
1152 return _convert_testlist_comp(
1153 config,
1154 children,
1155 single_child_is_sequence=True,
1156 sequence_type=List,
1157 comprehension_type=ListComp,
1158 )
1161def _convert_testlist_comp(
1162 config: ParserConfig,
1163 children: typing.Sequence[typing.Any],
1164 single_child_is_sequence: bool,
1165 sequence_type: typing.Union[
1166 typing.Type[Tuple], typing.Type[List], typing.Type[Set]
1167 ],
1168 comprehension_type: typing.Union[
1169 typing.Type[GeneratorExp], typing.Type[ListComp], typing.Type[SetComp]
1170 ],
1171) -> typing.Any:
1172 # This is either a single-element list, or the second token is a comma, so we're not
1173 # in a generator.
1174 if len(children) == 1 or isinstance(children[1], Token):
1175 return _convert_sequencelike(
1176 config, children, single_child_is_sequence, sequence_type
1177 )
1178 else:
1179 # N.B. The parent node (e.g. atom) is responsible for computing and attaching
1180 # whitespace information on any parenthesis, square brackets, or curly braces
1181 elt, for_in = children
1182 return WithLeadingWhitespace(
1183 comprehension_type(elt=elt.value, for_in=for_in, lpar=(), rpar=()),
1184 elt.whitespace_before,
1185 )
1188@with_production("testlist_star_expr", "(test|star_expr) (',' (test|star_expr))* [',']")
1189@with_production("testlist", "test (',' test)* [',']")
1190@with_production("exprlist", "(expr|star_expr) (',' (expr|star_expr))* [',']")
1191def convert_test_or_expr_list(
1192 config: ParserConfig, children: typing.Sequence[typing.Any]
1193) -> typing.Any:
1194 # Used by expression statements and assignments. Neither of these cases want to
1195 # treat a single child as a sequence.
1196 return _convert_sequencelike(
1197 config, children, single_child_is_sequence=False, sequence_type=Tuple
1198 )
1201def _convert_sequencelike(
1202 config: ParserConfig,
1203 children: typing.Sequence[typing.Any],
1204 single_child_is_sequence: bool,
1205 sequence_type: typing.Union[
1206 typing.Type[Tuple], typing.Type[List], typing.Type[Set]
1207 ],
1208) -> typing.Any:
1209 if not single_child_is_sequence and len(children) == 1:
1210 return children[0]
1211 # N.B. The parent node (e.g. atom) is responsible for computing and attaching
1212 # whitespace information on any parenthesis, square brackets, or curly braces
1213 elements = []
1214 for wrapped_expr_or_starred_element, comma_token in grouper(children, 2):
1215 expr_or_starred_element = wrapped_expr_or_starred_element.value
1216 if comma_token is None:
1217 comma = MaybeSentinel.DEFAULT
1218 else:
1219 comma = Comma(
1220 whitespace_before=parse_parenthesizable_whitespace(
1221 config, comma_token.whitespace_before
1222 ),
1223 # Only compute whitespace_after if we're not a trailing comma.
1224 # If we're a trailing comma, that whitespace should be consumed by the
1225 # TrailingWhitespace, parenthesis, etc.
1226 whitespace_after=(
1227 parse_parenthesizable_whitespace(
1228 config, comma_token.whitespace_after
1229 )
1230 if comma_token is not children[-1]
1231 else SimpleWhitespace("")
1232 ),
1233 )
1235 if isinstance(expr_or_starred_element, StarredElement):
1236 starred_element = expr_or_starred_element
1237 elements.append(starred_element.with_changes(comma=comma))
1238 else:
1239 expr = expr_or_starred_element
1240 elements.append(Element(value=expr, comma=comma))
1242 # lpar/rpar are the responsibility of our parent
1243 return WithLeadingWhitespace(
1244 sequence_type(elements, lpar=(), rpar=()),
1245 children[0].whitespace_before,
1246 )
1249@with_production(
1250 "dictorsetmaker",
1251 (
1252 "( ((test ':' test | '**' expr)"
1253 + " (comp_for | (',' (test ':' test | '**' expr))* [','])) |"
1254 + "((test | star_expr) "
1255 + " (comp_for | (',' (test | star_expr))* [','])) )"
1256 ),
1257 version=">=3.5",
1258)
1259@with_production(
1260 "dictorsetmaker",
1261 (
1262 "( ((test ':' test)"
1263 + " (comp_for | (',' (test ':' test))* [','])) |"
1264 + "((test) "
1265 + " (comp_for | (',' (test))* [','])) )"
1266 ),
1267 version="<3.5",
1268)
1269def convert_dictorsetmaker(
1270 config: ParserConfig, children: typing.Sequence[typing.Any]
1271) -> typing.Any:
1272 # We'll always have at least one child. `atom_curlybraces` handles empty
1273 # dicts.
1274 if len(children) > 1 and (
1275 (isinstance(children[1], Token) and children[1].string == ":")
1276 or (isinstance(children[0], Token) and children[0].string == "**")
1277 ):
1278 return _convert_dict(config, children)
1279 else:
1280 return _convert_set(config, children)
1283def _convert_dict_element(
1284 config: ParserConfig,
1285 children_iter: typing.Iterator[typing.Any],
1286 last_child: typing.Any,
1287) -> typing.Union[DictElement, StarredDictElement]:
1288 first = next(children_iter)
1289 if isinstance(first, Token) and first.string == "**":
1290 expr = next(children_iter)
1291 element = StarredDictElement(
1292 expr.value,
1293 whitespace_before_value=parse_parenthesizable_whitespace(
1294 config, expr.whitespace_before
1295 ),
1296 )
1297 else:
1298 key = first
1299 colon_tok = next(children_iter)
1300 value = next(children_iter)
1301 element = DictElement(
1302 key.value,
1303 value.value,
1304 whitespace_before_colon=parse_parenthesizable_whitespace(
1305 config, colon_tok.whitespace_before
1306 ),
1307 whitespace_after_colon=parse_parenthesizable_whitespace(
1308 config, colon_tok.whitespace_after
1309 ),
1310 )
1311 # Handle the trailing comma (if there is one)
1312 try:
1313 comma_token = next(children_iter)
1314 element = element.with_changes(
1315 comma=Comma(
1316 whitespace_before=parse_parenthesizable_whitespace(
1317 config, comma_token.whitespace_before
1318 ),
1319 # Only compute whitespace_after if we're not a trailing comma.
1320 # If we're a trailing comma, that whitespace should be consumed by the
1321 # RightBracket.
1322 whitespace_after=(
1323 parse_parenthesizable_whitespace(
1324 config, comma_token.whitespace_after
1325 )
1326 if comma_token is not last_child
1327 else SimpleWhitespace("")
1328 ),
1329 )
1330 )
1331 except StopIteration:
1332 pass
1333 return element
1336def _convert_dict(
1337 config: ParserConfig, children: typing.Sequence[typing.Any]
1338) -> typing.Any:
1339 is_first_starred = isinstance(children[0], Token) and children[0].string == "**"
1340 if is_first_starred:
1341 possible_comp_for = None if len(children) < 3 else children[2]
1342 else:
1343 possible_comp_for = None if len(children) < 4 else children[3]
1344 if isinstance(possible_comp_for, CompFor):
1345 if is_first_starred:
1346 raise PartialParserSyntaxError(
1347 "dict unpacking cannot be used in dict comprehension"
1348 )
1349 return _convert_dict_comp(config, children)
1351 children_iter = iter(children)
1352 last_child = children[-1]
1353 elements = []
1354 while True:
1355 try:
1356 elements.append(_convert_dict_element(config, children_iter, last_child))
1357 except StopIteration:
1358 break
1359 # lbrace, rbrace, lpar, and rpar will be attached as-needed by the atom grammar
1360 return WithLeadingWhitespace(Dict(tuple(elements)), children[0].whitespace_before)
1363def _convert_dict_comp(config, children: typing.Sequence[typing.Any]) -> typing.Any:
1364 key, colon_token, value, comp_for = children
1365 return WithLeadingWhitespace(
1366 DictComp(
1367 key.value,
1368 value.value,
1369 comp_for,
1370 # lbrace, rbrace, lpar, and rpar will be attached as-needed by the atom grammar
1371 whitespace_before_colon=parse_parenthesizable_whitespace(
1372 config, colon_token.whitespace_before
1373 ),
1374 whitespace_after_colon=parse_parenthesizable_whitespace(
1375 config, colon_token.whitespace_after
1376 ),
1377 ),
1378 key.whitespace_before,
1379 )
1382def _convert_set(
1383 config: ParserConfig, children: typing.Sequence[typing.Any]
1384) -> typing.Any:
1385 return _convert_testlist_comp(
1386 config,
1387 children,
1388 single_child_is_sequence=True,
1389 sequence_type=Set,
1390 comprehension_type=SetComp,
1391 )
1394@with_production("arglist", "argument (',' argument)* [',']")
1395def convert_arglist(
1396 config: ParserConfig, children: typing.Sequence[typing.Any]
1397) -> typing.Any:
1398 args = []
1399 for argument, comma in grouper(children, 2):
1400 if comma is None:
1401 args.append(argument)
1402 else:
1403 args.append(
1404 argument.with_changes(
1405 comma=Comma(
1406 whitespace_before=parse_parenthesizable_whitespace(
1407 config, comma.whitespace_before
1408 ),
1409 whitespace_after=parse_parenthesizable_whitespace(
1410 config, comma.whitespace_after
1411 ),
1412 )
1413 )
1414 )
1415 return ArglistPartial(args)
1418@with_production("argument", "arg_assign_comp_for | star_arg")
1419def convert_argument(
1420 config: ParserConfig, children: typing.Sequence[typing.Any]
1421) -> typing.Any:
1422 (child,) = children
1423 return child
1426@with_production(
1427 "arg_assign_comp_for", "test [comp_for] | test '=' test", version="<=3.7"
1428)
1429@with_production(
1430 "arg_assign_comp_for",
1431 "test [comp_for] | test ':=' test | test '=' test",
1432 version=">=3.8",
1433)
1434def convert_arg_assign_comp_for(
1435 config: ParserConfig, children: typing.Sequence[typing.Any]
1436) -> typing.Any:
1437 if len(children) == 1:
1438 # Simple test
1439 (child,) = children
1440 return Arg(value=child.value)
1441 elif len(children) == 2:
1442 elt, for_in = children
1443 return Arg(value=GeneratorExp(elt.value, for_in, lpar=(), rpar=()))
1444 else:
1445 lhs, equal, rhs = children
1446 # "key := value" assignment; positional
1447 if equal.string == ":=":
1448 val = convert_namedexpr_test(config, children)
1449 if not isinstance(val, WithLeadingWhitespace):
1450 raise Exception(
1451 f"convert_namedexpr_test returned {val!r}, not WithLeadingWhitespace"
1452 )
1453 return Arg(value=val.value)
1454 # "key = value" assignment; keyword argument
1455 return Arg(
1456 keyword=lhs.value,
1457 equal=AssignEqual(
1458 whitespace_before=parse_parenthesizable_whitespace(
1459 config, equal.whitespace_before
1460 ),
1461 whitespace_after=parse_parenthesizable_whitespace(
1462 config, equal.whitespace_after
1463 ),
1464 ),
1465 value=rhs.value,
1466 )
1469@with_production("star_arg", "'**' test | '*' test")
1470def convert_star_arg(
1471 config: ParserConfig, children: typing.Sequence[typing.Any]
1472) -> typing.Any:
1473 star, test = children
1474 return Arg(
1475 star=star.string,
1476 whitespace_after_star=parse_parenthesizable_whitespace(
1477 config, star.whitespace_after
1478 ),
1479 value=test.value,
1480 )
1483@with_production("sync_comp_for", "'for' exprlist 'in' or_test comp_if* [comp_for]")
1484def convert_sync_comp_for(
1485 config: ParserConfig, children: typing.Sequence[typing.Any]
1486) -> typing.Any:
1487 # unpack
1488 for_tok, target, in_tok, iter, *trailing = children
1489 if len(trailing) and isinstance(trailing[-1], CompFor):
1490 *ifs, inner_for_in = trailing
1491 else:
1492 ifs, inner_for_in = trailing, None
1494 return CompFor(
1495 target=target.value,
1496 iter=iter.value,
1497 ifs=ifs,
1498 inner_for_in=inner_for_in,
1499 whitespace_before=parse_parenthesizable_whitespace(
1500 config, for_tok.whitespace_before
1501 ),
1502 whitespace_after_for=parse_parenthesizable_whitespace(
1503 config, for_tok.whitespace_after
1504 ),
1505 whitespace_before_in=parse_parenthesizable_whitespace(
1506 config, in_tok.whitespace_before
1507 ),
1508 whitespace_after_in=parse_parenthesizable_whitespace(
1509 config, in_tok.whitespace_after
1510 ),
1511 )
1514@with_production("comp_for", "[ASYNC] sync_comp_for", version=">=3.6")
1515@with_production("comp_for", "sync_comp_for", version="<=3.5")
1516def convert_comp_for(
1517 config: ParserConfig, children: typing.Sequence[typing.Any]
1518) -> typing.Any:
1519 if len(children) == 1:
1520 (sync_comp_for,) = children
1521 return sync_comp_for
1522 else:
1523 (async_tok, sync_comp_for) = children
1524 return sync_comp_for.with_changes(
1525 # asynchronous steals the `CompFor`'s `whitespace_before`.
1526 asynchronous=Asynchronous(whitespace_after=sync_comp_for.whitespace_before),
1527 # But, in exchange, `CompFor` gets to keep `async_tok`'s leading
1528 # whitespace, because that's now the beginning of the `CompFor`.
1529 whitespace_before=parse_parenthesizable_whitespace(
1530 config, async_tok.whitespace_before
1531 ),
1532 )
1535@with_production("comp_if", "'if' test_nocond")
1536def convert_comp_if(
1537 config: ParserConfig, children: typing.Sequence[typing.Any]
1538) -> typing.Any:
1539 if_tok, test = children
1540 return CompIf(
1541 test.value,
1542 whitespace_before=parse_parenthesizable_whitespace(
1543 config, if_tok.whitespace_before
1544 ),
1545 whitespace_before_test=parse_parenthesizable_whitespace(
1546 config, test.whitespace_before
1547 ),
1548 )
1551@with_production("yield_expr", "'yield' [yield_arg]")
1552def convert_yield_expr(
1553 config: ParserConfig, children: typing.Sequence[typing.Any]
1554) -> typing.Any:
1555 if len(children) == 1:
1556 # Yielding implicit none
1557 (yield_token,) = children
1558 yield_node = Yield(value=None)
1559 else:
1560 # Yielding explicit value
1561 (yield_token, yield_arg) = children
1562 yield_node = Yield(
1563 value=yield_arg.value,
1564 whitespace_after_yield=parse_parenthesizable_whitespace(
1565 config, yield_arg.whitespace_before
1566 ),
1567 )
1569 return WithLeadingWhitespace(yield_node, yield_token.whitespace_before)
1572@with_production("yield_arg", "testlist", version="<3.3")
1573@with_production("yield_arg", "'from' test | testlist", version=">=3.3,<3.8")
1574@with_production("yield_arg", "'from' test | testlist_star_expr", version=">=3.8")
1575def convert_yield_arg(
1576 config: ParserConfig, children: typing.Sequence[typing.Any]
1577) -> typing.Any:
1578 if len(children) == 1:
1579 # Just a regular testlist, pass it up
1580 (child,) = children
1581 return child
1582 else:
1583 # Its a yield from
1584 (from_token, test) = children
1586 return WithLeadingWhitespace(
1587 From(
1588 item=test.value,
1589 whitespace_after_from=parse_parenthesizable_whitespace(
1590 config, test.whitespace_before
1591 ),
1592 ),
1593 from_token.whitespace_before,
1594 )