Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/_parser/conversions/statement.py: 30%
427 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:43 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
5# pyre-unsafe
7from typing import Any, Dict, List, Optional, Sequence, Tuple, Type
9from libcst._exceptions import PartialParserSyntaxError
10from libcst._maybe_sentinel import MaybeSentinel
11from libcst._nodes.expression import (
12 Annotation,
13 Arg,
14 Asynchronous,
15 Attribute,
16 Call,
17 From,
18 LeftParen,
19 Name,
20 Param,
21 Parameters,
22 RightParen,
23)
24from libcst._nodes.op import (
25 AddAssign,
26 AssignEqual,
27 BaseAugOp,
28 BitAndAssign,
29 BitOrAssign,
30 BitXorAssign,
31 Comma,
32 DivideAssign,
33 Dot,
34 FloorDivideAssign,
35 ImportStar,
36 LeftShiftAssign,
37 MatrixMultiplyAssign,
38 ModuloAssign,
39 MultiplyAssign,
40 PowerAssign,
41 RightShiftAssign,
42 Semicolon,
43 SubtractAssign,
44)
45from libcst._nodes.statement import (
46 AnnAssign,
47 AsName,
48 Assert,
49 Assign,
50 AssignTarget,
51 AugAssign,
52 Break,
53 ClassDef,
54 Continue,
55 Decorator,
56 Del,
57 Else,
58 ExceptHandler,
59 Expr,
60 Finally,
61 For,
62 FunctionDef,
63 Global,
64 If,
65 Import,
66 ImportAlias,
67 ImportFrom,
68 IndentedBlock,
69 NameItem,
70 Nonlocal,
71 Pass,
72 Raise,
73 Return,
74 SimpleStatementLine,
75 SimpleStatementSuite,
76 Try,
77 While,
78 With,
79 WithItem,
80)
81from libcst._nodes.whitespace import EmptyLine, SimpleWhitespace
82from libcst._parser.custom_itertools import grouper
83from libcst._parser.production_decorator import with_production
84from libcst._parser.types.config import ParserConfig
85from libcst._parser.types.partials import (
86 AnnAssignPartial,
87 AssignPartial,
88 AugAssignPartial,
89 DecoratorPartial,
90 ExceptClausePartial,
91 FuncdefPartial,
92 ImportPartial,
93 ImportRelativePartial,
94 SimpleStatementPartial,
95 WithLeadingWhitespace,
96)
97from libcst._parser.types.token import Token
98from libcst._parser.whitespace_parser import (
99 parse_empty_lines,
100 parse_parenthesizable_whitespace,
101 parse_simple_whitespace,
102)
104AUGOP_TOKEN_LUT: Dict[str, Type[BaseAugOp]] = {
105 "+=": AddAssign,
106 "-=": SubtractAssign,
107 "*=": MultiplyAssign,
108 "@=": MatrixMultiplyAssign,
109 "/=": DivideAssign,
110 "%=": ModuloAssign,
111 "&=": BitAndAssign,
112 "|=": BitOrAssign,
113 "^=": BitXorAssign,
114 "<<=": LeftShiftAssign,
115 ">>=": RightShiftAssign,
116 "**=": PowerAssign,
117 "//=": FloorDivideAssign,
118}
121@with_production("stmt_input", "stmt ENDMARKER")
122def convert_stmt_input(config: ParserConfig, children: Sequence[Any]) -> Any:
123 (child, endmarker) = children
124 return child
127@with_production("stmt", "simple_stmt_line | compound_stmt")
128def convert_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
129 (child,) = children
130 return child
133@with_production("simple_stmt_partial", "small_stmt (';' small_stmt)* [';'] NEWLINE")
134def convert_simple_stmt_partial(config: ParserConfig, children: Sequence[Any]) -> Any:
135 *statements, trailing_whitespace = children
137 last_stmt = len(statements) / 2
138 body = []
139 for i, (stmt_body, semi) in enumerate(grouper(statements, 2)):
140 if semi is not None:
141 if i == (last_stmt - 1):
142 # Trailing semicolons only own the whitespace before.
143 semi = Semicolon(
144 whitespace_before=parse_simple_whitespace(
145 config, semi.whitespace_before
146 ),
147 whitespace_after=SimpleWhitespace(""),
148 )
149 else:
150 # Middle semicolons own the whitespace before and after.
151 semi = Semicolon(
152 whitespace_before=parse_simple_whitespace(
153 config, semi.whitespace_before
154 ),
155 whitespace_after=parse_simple_whitespace(
156 config, semi.whitespace_after
157 ),
158 )
159 else:
160 semi = MaybeSentinel.DEFAULT
161 body.append(stmt_body.value.with_changes(semicolon=semi))
162 return SimpleStatementPartial(
163 body,
164 whitespace_before=statements[0].whitespace_before,
165 trailing_whitespace=trailing_whitespace,
166 )
169@with_production("simple_stmt_line", "simple_stmt_partial")
170def convert_simple_stmt_line(config: ParserConfig, children: Sequence[Any]) -> Any:
171 """
172 This function is similar to convert_simple_stmt_suite, but yields a different type
173 """
174 (partial,) = children
175 return SimpleStatementLine(
176 partial.body,
177 leading_lines=parse_empty_lines(config, partial.whitespace_before),
178 trailing_whitespace=partial.trailing_whitespace,
179 )
182@with_production("simple_stmt_suite", "simple_stmt_partial")
183def convert_simple_stmt_suite(config: ParserConfig, children: Sequence[Any]) -> Any:
184 """
185 This function is similar to convert_simple_stmt_line, but yields a different type
186 """
187 (partial,) = children
188 return SimpleStatementSuite(
189 partial.body,
190 leading_whitespace=parse_simple_whitespace(config, partial.whitespace_before),
191 trailing_whitespace=partial.trailing_whitespace,
192 )
195@with_production(
196 "small_stmt",
197 (
198 "expr_stmt | del_stmt | pass_stmt | break_stmt | continue_stmt | return_stmt"
199 + "| raise_stmt | yield_stmt | import_stmt | global_stmt | nonlocal_stmt"
200 + "| assert_stmt"
201 ),
202)
203def convert_small_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
204 # Doesn't construct SmallStatement, because we don't know about semicolons yet.
205 # convert_simple_stmt will construct the SmallStatement nodes.
206 (small_stmt_body,) = children
207 return small_stmt_body
210@with_production(
211 "expr_stmt",
212 "testlist_star_expr (annassign | augassign | assign* )",
213 version=">=3.6",
214)
215@with_production(
216 "expr_stmt", "testlist_star_expr (augassign | assign* )", version="<=3.5"
217)
218@with_production("yield_stmt", "yield_expr")
219def convert_expr_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
220 if len(children) == 1:
221 # This is an unassigned expr statement (like a function call)
222 (test_node,) = children
223 return WithLeadingWhitespace(
224 Expr(value=test_node.value), test_node.whitespace_before
225 )
226 elif len(children) == 2:
227 lhs, rhs = children
228 if isinstance(rhs, AnnAssignPartial):
229 return WithLeadingWhitespace(
230 AnnAssign(
231 target=lhs.value,
232 annotation=rhs.annotation,
233 equal=MaybeSentinel.DEFAULT if rhs.equal is None else rhs.equal,
234 value=rhs.value,
235 ),
236 lhs.whitespace_before,
237 )
238 elif isinstance(rhs, AugAssignPartial):
239 return WithLeadingWhitespace(
240 AugAssign(target=lhs.value, operator=rhs.operator, value=rhs.value),
241 lhs.whitespace_before,
242 )
243 # The only thing it could be at this point is an assign with one or more targets.
244 # So, walk the children moving the equals ownership back one and constructing a
245 # list of AssignTargets.
246 targets = []
247 for i in range(len(children) - 1):
248 target = children[i].value
249 equal = children[i + 1].equal
251 targets.append(
252 AssignTarget(
253 target=target,
254 whitespace_before_equal=equal.whitespace_before,
255 whitespace_after_equal=equal.whitespace_after,
256 )
257 )
259 return WithLeadingWhitespace(
260 Assign(targets=tuple(targets), value=children[-1].value),
261 children[0].whitespace_before,
262 )
265@with_production("annassign", "':' test ['=' test]", version=">=3.6,<3.8")
266@with_production(
267 "annassign", "':' test ['=' (yield_expr|testlist_star_expr)]", version=">=3.8"
268)
269def convert_annassign(config: ParserConfig, children: Sequence[Any]) -> Any:
270 if len(children) == 2:
271 # Variable annotation only
272 colon, annotation = children
273 annotation = annotation.value
274 equal = None
275 value = None
276 elif len(children) == 4:
277 # Variable annotation and assignment
278 colon, annotation, equal, value = children
279 annotation = annotation.value
280 value = value.value
281 equal = AssignEqual(
282 whitespace_before=parse_simple_whitespace(config, equal.whitespace_before),
283 whitespace_after=parse_simple_whitespace(config, equal.whitespace_after),
284 )
285 else:
286 raise Exception("Invalid parser state!")
288 return AnnAssignPartial(
289 annotation=Annotation(
290 whitespace_before_indicator=parse_simple_whitespace(
291 config, colon.whitespace_before
292 ),
293 whitespace_after_indicator=parse_simple_whitespace(
294 config, colon.whitespace_after
295 ),
296 annotation=annotation,
297 ),
298 equal=equal,
299 value=value,
300 )
303@with_production(
304 "augassign",
305 (
306 "('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | "
307 + "'>>=' | '**=' | '//=') (yield_expr | testlist)"
308 ),
309 version=">=3.5",
310)
311@with_production(
312 "augassign",
313 (
314 "('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | "
315 + "'>>=' | '**=' | '//=') (yield_expr | testlist)"
316 ),
317 version="<3.5",
318)
319def convert_augassign(config: ParserConfig, children: Sequence[Any]) -> Any:
320 op, expr = children
321 if op.string not in AUGOP_TOKEN_LUT:
322 raise Exception(f"Unexpected token '{op.string}'!")
323 return AugAssignPartial(
324 # pyre-ignore Pyre seems to think that the value of this LUT is CSTNode
325 operator=AUGOP_TOKEN_LUT[op.string](
326 whitespace_before=parse_simple_whitespace(config, op.whitespace_before),
327 whitespace_after=parse_simple_whitespace(config, op.whitespace_after),
328 ),
329 value=expr.value,
330 )
333@with_production("assign", "'=' (yield_expr|testlist_star_expr)")
334def convert_assign(config: ParserConfig, children: Sequence[Any]) -> Any:
335 equal, expr = children
336 return AssignPartial(
337 equal=AssignEqual(
338 whitespace_before=parse_simple_whitespace(config, equal.whitespace_before),
339 whitespace_after=parse_simple_whitespace(config, equal.whitespace_after),
340 ),
341 value=expr.value,
342 )
345@with_production("pass_stmt", "'pass'")
346def convert_pass_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
347 (name,) = children
348 return WithLeadingWhitespace(Pass(), name.whitespace_before)
351@with_production("del_stmt", "'del' exprlist")
352def convert_del_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
353 (del_name, exprlist) = children
354 return WithLeadingWhitespace(
355 Del(
356 target=exprlist.value,
357 whitespace_after_del=parse_simple_whitespace(
358 config, del_name.whitespace_after
359 ),
360 ),
361 del_name.whitespace_before,
362 )
365@with_production("continue_stmt", "'continue'")
366def convert_continue_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
367 (name,) = children
368 return WithLeadingWhitespace(Continue(), name.whitespace_before)
371@with_production("break_stmt", "'break'")
372def convert_break_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
373 (name,) = children
374 return WithLeadingWhitespace(Break(), name.whitespace_before)
377@with_production("return_stmt", "'return' [testlist]", version="<=3.7")
378@with_production("return_stmt", "'return' [testlist_star_expr]", version=">=3.8")
379def convert_return_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
380 if len(children) == 1:
381 (keyword,) = children
382 return WithLeadingWhitespace(
383 Return(whitespace_after_return=SimpleWhitespace("")),
384 keyword.whitespace_before,
385 )
386 else:
387 (keyword, testlist) = children
388 return WithLeadingWhitespace(
389 Return(
390 value=testlist.value,
391 whitespace_after_return=parse_simple_whitespace(
392 config, keyword.whitespace_after
393 ),
394 ),
395 keyword.whitespace_before,
396 )
399@with_production("import_stmt", "import_name | import_from")
400def convert_import_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
401 (child,) = children
402 return child
405@with_production("import_name", "'import' dotted_as_names")
406def convert_import_name(config: ParserConfig, children: Sequence[Any]) -> Any:
407 importtoken, names = children
408 return WithLeadingWhitespace(
409 Import(
410 names=names.names,
411 whitespace_after_import=parse_simple_whitespace(
412 config, importtoken.whitespace_after
413 ),
414 ),
415 importtoken.whitespace_before,
416 )
419@with_production("import_relative", "('.' | '...')* dotted_name | ('.' | '...')+")
420def convert_import_relative(config: ParserConfig, children: Sequence[Any]) -> Any:
421 dots = []
422 dotted_name = None
423 for child in children:
424 if isinstance(child, Token):
425 # Special case for "...", which is part of the grammar
426 if child.string == "...":
427 dots.extend(
428 [
429 Dot(),
430 Dot(),
431 Dot(
432 whitespace_after=parse_simple_whitespace(
433 config, child.whitespace_after
434 )
435 ),
436 ]
437 )
438 else:
439 dots.append(
440 Dot(
441 whitespace_after=parse_simple_whitespace(
442 config, child.whitespace_after
443 )
444 )
445 )
446 else:
447 # This should be the dotted name, and we can't get more than
448 # one, but lets be sure anyway
449 if dotted_name is not None:
450 raise Exception("Logic error!")
451 dotted_name = child
453 return ImportRelativePartial(relative=tuple(dots), module=dotted_name)
456@with_production(
457 "import_from",
458 "'from' import_relative 'import' ('*' | '(' import_as_names ')' | import_as_names)",
459)
460def convert_import_from(config: ParserConfig, children: Sequence[Any]) -> Any:
461 fromtoken, import_relative, importtoken, *importlist = children
463 if len(importlist) == 1:
464 (possible_star,) = importlist
465 if isinstance(possible_star, Token):
466 # Its a "*" import, so we must construct this node.
467 names = ImportStar()
468 else:
469 # Its an import as names partial, grab the names from that.
470 names = possible_star.names
471 lpar = None
472 rpar = None
473 else:
474 # Its an import as names partial with parens
475 lpartoken, namespartial, rpartoken = importlist
476 lpar = LeftParen(
477 whitespace_after=parse_parenthesizable_whitespace(
478 config, lpartoken.whitespace_after
479 )
480 )
481 names = namespartial.names
482 rpar = RightParen(
483 whitespace_before=parse_parenthesizable_whitespace(
484 config, rpartoken.whitespace_before
485 )
486 )
488 # If we have a relative-only import, then we need to relocate the space
489 # after the final dot to be owned by the import token.
490 if len(import_relative.relative) > 0 and import_relative.module is None:
491 whitespace_before_import = import_relative.relative[-1].whitespace_after
492 relative = (
493 *import_relative.relative[:-1],
494 import_relative.relative[-1].with_changes(
495 whitespace_after=SimpleWhitespace("")
496 ),
497 )
498 else:
499 whitespace_before_import = parse_simple_whitespace(
500 config, importtoken.whitespace_before
501 )
502 relative = import_relative.relative
504 return WithLeadingWhitespace(
505 ImportFrom(
506 whitespace_after_from=parse_simple_whitespace(
507 config, fromtoken.whitespace_after
508 ),
509 relative=relative,
510 module=import_relative.module,
511 whitespace_before_import=whitespace_before_import,
512 whitespace_after_import=parse_simple_whitespace(
513 config, importtoken.whitespace_after
514 ),
515 lpar=lpar,
516 names=names,
517 rpar=rpar,
518 ),
519 fromtoken.whitespace_before,
520 )
523@with_production("import_as_name", "NAME ['as' NAME]")
524def convert_import_as_name(config: ParserConfig, children: Sequence[Any]) -> Any:
525 if len(children) == 1:
526 (dotted_name,) = children
527 return ImportAlias(name=Name(dotted_name.string), asname=None)
528 else:
529 dotted_name, astoken, name = children
530 return ImportAlias(
531 name=Name(dotted_name.string),
532 asname=AsName(
533 whitespace_before_as=parse_simple_whitespace(
534 config, astoken.whitespace_before
535 ),
536 whitespace_after_as=parse_simple_whitespace(
537 config, astoken.whitespace_after
538 ),
539 name=Name(name.string),
540 ),
541 )
544@with_production("dotted_as_name", "dotted_name ['as' NAME]")
545def convert_dotted_as_name(config: ParserConfig, children: Sequence[Any]) -> Any:
546 if len(children) == 1:
547 (dotted_name,) = children
548 return ImportAlias(name=dotted_name, asname=None)
549 else:
550 dotted_name, astoken, name = children
551 return ImportAlias(
552 name=dotted_name,
553 asname=AsName(
554 whitespace_before_as=parse_parenthesizable_whitespace(
555 config, astoken.whitespace_before
556 ),
557 whitespace_after_as=parse_parenthesizable_whitespace(
558 config, astoken.whitespace_after
559 ),
560 name=Name(name.string),
561 ),
562 )
565@with_production("import_as_names", "import_as_name (',' import_as_name)* [',']")
566def convert_import_as_names(config: ParserConfig, children: Sequence[Any]) -> Any:
567 return _gather_import_names(config, children)
570@with_production("dotted_as_names", "dotted_as_name (',' dotted_as_name)*")
571def convert_dotted_as_names(config: ParserConfig, children: Sequence[Any]) -> Any:
572 return _gather_import_names(config, children)
575def _gather_import_names(
576 config: ParserConfig, children: Sequence[Any]
577) -> ImportPartial:
578 names = []
579 for name, comma in grouper(children, 2):
580 if comma is None:
581 names.append(name)
582 else:
583 names.append(
584 name.with_changes(
585 comma=Comma(
586 whitespace_before=parse_parenthesizable_whitespace(
587 config, comma.whitespace_before
588 ),
589 whitespace_after=parse_parenthesizable_whitespace(
590 config, comma.whitespace_after
591 ),
592 )
593 )
594 )
596 return ImportPartial(names=names)
599@with_production("dotted_name", "NAME ('.' NAME)*")
600def convert_dotted_name(config: ParserConfig, children: Sequence[Any]) -> Any:
601 left, *rest = children
602 node = Name(left.string)
604 for dot, right in grouper(rest, 2):
605 node = Attribute(
606 value=node,
607 dot=Dot(
608 whitespace_before=parse_parenthesizable_whitespace(
609 config, dot.whitespace_before
610 ),
611 whitespace_after=parse_parenthesizable_whitespace(
612 config, dot.whitespace_after
613 ),
614 ),
615 attr=Name(right.string),
616 )
618 return node
621@with_production("raise_stmt", "'raise' [test ['from' test]]")
622def convert_raise_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
623 if len(children) == 1:
624 (raise_token,) = children
625 whitespace_after_raise = MaybeSentinel.DEFAULT
626 exc = None
627 cause = None
628 elif len(children) == 2:
629 (raise_token, test) = children
630 whitespace_after_raise = parse_simple_whitespace(config, test.whitespace_before)
631 exc = test.value
632 cause = None
633 elif len(children) == 4:
634 (raise_token, test, from_token, source) = children
635 whitespace_after_raise = parse_simple_whitespace(config, test.whitespace_before)
636 exc = test.value
637 cause = From(
638 whitespace_before_from=parse_simple_whitespace(
639 config, from_token.whitespace_before
640 ),
641 whitespace_after_from=parse_simple_whitespace(
642 config, source.whitespace_before
643 ),
644 item=source.value,
645 )
646 else:
647 raise Exception("Logic error!")
649 return WithLeadingWhitespace(
650 Raise(whitespace_after_raise=whitespace_after_raise, exc=exc, cause=cause),
651 raise_token.whitespace_before,
652 )
655def _construct_nameitems(config: ParserConfig, names: Sequence[Any]) -> List[NameItem]:
656 nameitems: List[NameItem] = []
657 for name, maybe_comma in grouper(names, 2):
658 if maybe_comma is None:
659 nameitems.append(NameItem(Name(name.string)))
660 else:
661 nameitems.append(
662 NameItem(
663 Name(name.string),
664 comma=Comma(
665 whitespace_before=parse_simple_whitespace(
666 config, maybe_comma.whitespace_before
667 ),
668 whitespace_after=parse_simple_whitespace(
669 config, maybe_comma.whitespace_after
670 ),
671 ),
672 )
673 )
674 return nameitems
677@with_production("global_stmt", "'global' NAME (',' NAME)*")
678def convert_global_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
679 (global_token, *names) = children
680 return WithLeadingWhitespace(
681 Global(
682 names=tuple(_construct_nameitems(config, names)),
683 whitespace_after_global=parse_simple_whitespace(
684 config, names[0].whitespace_before
685 ),
686 ),
687 global_token.whitespace_before,
688 )
691@with_production("nonlocal_stmt", "'nonlocal' NAME (',' NAME)*")
692def convert_nonlocal_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
693 (nonlocal_token, *names) = children
694 return WithLeadingWhitespace(
695 Nonlocal(
696 names=tuple(_construct_nameitems(config, names)),
697 whitespace_after_nonlocal=parse_simple_whitespace(
698 config, names[0].whitespace_before
699 ),
700 ),
701 nonlocal_token.whitespace_before,
702 )
705@with_production("assert_stmt", "'assert' test [',' test]")
706def convert_assert_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
707 if len(children) == 2:
708 (assert_token, test) = children
709 assert_node = Assert(
710 whitespace_after_assert=parse_simple_whitespace(
711 config, test.whitespace_before
712 ),
713 test=test.value,
714 msg=None,
715 )
716 else:
717 (assert_token, test, comma_token, msg) = children
718 assert_node = Assert(
719 whitespace_after_assert=parse_simple_whitespace(
720 config, test.whitespace_before
721 ),
722 test=test.value,
723 comma=Comma(
724 whitespace_before=parse_simple_whitespace(
725 config, comma_token.whitespace_before
726 ),
727 whitespace_after=parse_simple_whitespace(config, msg.whitespace_before),
728 ),
729 msg=msg.value,
730 )
732 return WithLeadingWhitespace(assert_node, assert_token.whitespace_before)
735@with_production(
736 "compound_stmt",
737 ("if_stmt | while_stmt | asyncable_stmt | try_stmt | classdef | decorated"),
738)
739def convert_compound_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
740 (stmt,) = children
741 return stmt
744@with_production(
745 "if_stmt", "'if' test ':' suite [if_stmt_elif|if_stmt_else]", version="<=3.7"
746)
747@with_production(
748 "if_stmt",
749 "'if' namedexpr_test ':' suite [if_stmt_elif|if_stmt_else]",
750 version=">=3.8",
751)
752def convert_if_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
753 if_tok, test, colon_tok, suite, *tail = children
755 if len(tail) > 0:
756 (orelse,) = tail
757 else:
758 orelse = None
760 return If(
761 leading_lines=parse_empty_lines(config, if_tok.whitespace_before),
762 whitespace_before_test=parse_simple_whitespace(config, if_tok.whitespace_after),
763 test=test.value,
764 whitespace_after_test=parse_simple_whitespace(
765 config, colon_tok.whitespace_before
766 ),
767 body=suite,
768 orelse=orelse,
769 )
772@with_production(
773 "if_stmt_elif", "'elif' test ':' suite [if_stmt_elif|if_stmt_else]", version="<=3.7"
774)
775@with_production(
776 "if_stmt_elif",
777 "'elif' namedexpr_test ':' suite [if_stmt_elif|if_stmt_else]",
778 version=">=3.8",
779)
780def convert_if_stmt_elif(config: ParserConfig, children: Sequence[Any]) -> Any:
781 # this behaves exactly the same as `convert_if_stmt`, except that the leading token
782 # has a different string value.
783 return convert_if_stmt(config, children)
786@with_production("if_stmt_else", "'else' ':' suite")
787def convert_if_stmt_else(config: ParserConfig, children: Sequence[Any]) -> Any:
788 else_tok, colon_tok, suite = children
789 return Else(
790 leading_lines=parse_empty_lines(config, else_tok.whitespace_before),
791 whitespace_before_colon=parse_simple_whitespace(
792 config, colon_tok.whitespace_before
793 ),
794 body=suite,
795 )
798@with_production(
799 "while_stmt", "'while' test ':' suite ['else' ':' suite]", version="<=3.7"
800)
801@with_production(
802 "while_stmt", "'while' namedexpr_test ':' suite ['else' ':' suite]", version=">=3.8"
803)
804def convert_while_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
805 while_token, test, while_colon_token, while_suite, *else_block = children
807 if len(else_block) > 0:
808 (else_token, else_colon_token, else_suite) = else_block
809 orelse = Else(
810 leading_lines=parse_empty_lines(config, else_token.whitespace_before),
811 whitespace_before_colon=parse_simple_whitespace(
812 config, else_colon_token.whitespace_before
813 ),
814 body=else_suite,
815 )
816 else:
817 orelse = None
819 return While(
820 leading_lines=parse_empty_lines(config, while_token.whitespace_before),
821 whitespace_after_while=parse_simple_whitespace(
822 config, while_token.whitespace_after
823 ),
824 test=test.value,
825 whitespace_before_colon=parse_simple_whitespace(
826 config, while_colon_token.whitespace_before
827 ),
828 body=while_suite,
829 orelse=orelse,
830 )
833@with_production(
834 "for_stmt", "'for' exprlist 'in' testlist ':' suite ['else' ':' suite]"
835)
836def convert_for_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
837 (
838 for_token,
839 expr,
840 in_token,
841 test,
842 for_colon_token,
843 for_suite,
844 *else_block,
845 ) = children
847 if len(else_block) > 0:
848 (else_token, else_colon_token, else_suite) = else_block
849 orelse = Else(
850 leading_lines=parse_empty_lines(config, else_token.whitespace_before),
851 whitespace_before_colon=parse_simple_whitespace(
852 config, else_colon_token.whitespace_before
853 ),
854 body=else_suite,
855 )
856 else:
857 orelse = None
859 return WithLeadingWhitespace(
860 For(
861 whitespace_after_for=parse_simple_whitespace(
862 config, for_token.whitespace_after
863 ),
864 target=expr.value,
865 whitespace_before_in=parse_simple_whitespace(
866 config, in_token.whitespace_before
867 ),
868 whitespace_after_in=parse_simple_whitespace(
869 config, in_token.whitespace_after
870 ),
871 iter=test.value,
872 whitespace_before_colon=parse_simple_whitespace(
873 config, for_colon_token.whitespace_before
874 ),
875 body=for_suite,
876 orelse=orelse,
877 ),
878 for_token.whitespace_before,
879 )
882@with_production(
883 "try_stmt",
884 "('try' ':' suite ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] | 'finally' ':' suite))",
885)
886def convert_try_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
887 trytoken, try_colon_token, try_suite, *rest = children
888 handlers: List[ExceptHandler] = []
889 orelse: Optional[Else] = None
890 finalbody: Optional[Finally] = None
892 for clause, colon_token, suite in grouper(rest, 3):
893 if isinstance(clause, Token):
894 if clause.string == "else":
895 if orelse is not None:
896 raise Exception("Logic error!")
897 orelse = Else(
898 leading_lines=parse_empty_lines(config, clause.whitespace_before),
899 whitespace_before_colon=parse_simple_whitespace(
900 config, colon_token.whitespace_before
901 ),
902 body=suite,
903 )
904 elif clause.string == "finally":
905 if finalbody is not None:
906 raise Exception("Logic error!")
907 finalbody = Finally(
908 leading_lines=parse_empty_lines(config, clause.whitespace_before),
909 whitespace_before_colon=parse_simple_whitespace(
910 config, colon_token.whitespace_before
911 ),
912 body=suite,
913 )
914 else:
915 raise Exception("Logic error!")
916 elif isinstance(clause, ExceptClausePartial):
917 handlers.append(
918 ExceptHandler(
919 body=suite,
920 type=clause.type,
921 name=clause.name,
922 leading_lines=clause.leading_lines,
923 whitespace_after_except=clause.whitespace_after_except,
924 whitespace_before_colon=parse_simple_whitespace(
925 config, colon_token.whitespace_before
926 ),
927 )
928 )
929 else:
930 raise Exception("Logic error!")
932 return Try(
933 leading_lines=parse_empty_lines(config, trytoken.whitespace_before),
934 whitespace_before_colon=parse_simple_whitespace(
935 config, try_colon_token.whitespace_before
936 ),
937 body=try_suite,
938 handlers=tuple(handlers),
939 orelse=orelse,
940 finalbody=finalbody,
941 )
944@with_production("except_clause", "'except' [test ['as' NAME]]")
945def convert_except_clause(config: ParserConfig, children: Sequence[Any]) -> Any:
946 if len(children) == 1:
947 (except_token,) = children
948 whitespace_after_except = SimpleWhitespace("")
949 test = None
950 name = None
951 elif len(children) == 2:
952 (except_token, test_node) = children
953 whitespace_after_except = parse_simple_whitespace(
954 config, except_token.whitespace_after
955 )
956 test = test_node.value
957 name = None
958 else:
959 (except_token, test_node, as_token, name_token) = children
960 whitespace_after_except = parse_simple_whitespace(
961 config, except_token.whitespace_after
962 )
963 test = test_node.value
964 name = AsName(
965 whitespace_before_as=parse_simple_whitespace(
966 config, as_token.whitespace_before
967 ),
968 whitespace_after_as=parse_simple_whitespace(
969 config, as_token.whitespace_after
970 ),
971 name=Name(name_token.string),
972 )
974 return ExceptClausePartial(
975 leading_lines=parse_empty_lines(config, except_token.whitespace_before),
976 whitespace_after_except=whitespace_after_except,
977 type=test,
978 name=name,
979 )
982@with_production(
983 "with_stmt", "'with' with_item (',' with_item)* ':' suite", version=">=3.1"
984)
985@with_production("with_stmt", "'with' with_item ':' suite", version="<3.1")
986def convert_with_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
987 (with_token, *items, colon_token, suite) = children
988 item_nodes: List[WithItem] = []
990 for with_item, maybe_comma in grouper(items, 2):
991 if maybe_comma is not None:
992 item_nodes.append(
993 with_item.with_changes(
994 comma=Comma(
995 whitespace_before=parse_parenthesizable_whitespace(
996 config, maybe_comma.whitespace_before
997 ),
998 whitespace_after=parse_parenthesizable_whitespace(
999 config, maybe_comma.whitespace_after
1000 ),
1001 )
1002 )
1003 )
1004 else:
1005 item_nodes.append(with_item)
1007 return WithLeadingWhitespace(
1008 With(
1009 whitespace_after_with=parse_simple_whitespace(
1010 config, with_token.whitespace_after
1011 ),
1012 items=tuple(item_nodes),
1013 whitespace_before_colon=parse_simple_whitespace(
1014 config, colon_token.whitespace_before
1015 ),
1016 body=suite,
1017 ),
1018 with_token.whitespace_before,
1019 )
1022@with_production("with_item", "test ['as' expr]")
1023def convert_with_item(config: ParserConfig, children: Sequence[Any]) -> Any:
1024 if len(children) == 3:
1025 (test, as_token, expr_node) = children
1026 test_node = test.value
1027 asname = AsName(
1028 whitespace_before_as=parse_simple_whitespace(
1029 config, as_token.whitespace_before
1030 ),
1031 whitespace_after_as=parse_simple_whitespace(
1032 config, as_token.whitespace_after
1033 ),
1034 name=expr_node.value,
1035 )
1036 else:
1037 (test,) = children
1038 test_node = test.value
1039 asname = None
1041 return WithItem(item=test_node, asname=asname)
1044def _extract_async(
1045 config: ParserConfig, children: Sequence[Any]
1046) -> Tuple[List[EmptyLine], Optional[Asynchronous], Any]:
1047 if len(children) == 1:
1048 (stmt,) = children
1050 whitespace_before = stmt.whitespace_before
1051 asyncnode = None
1052 else:
1053 asynctoken, stmt = children
1055 whitespace_before = asynctoken.whitespace_before
1056 asyncnode = Asynchronous(
1057 whitespace_after=parse_simple_whitespace(
1058 config, asynctoken.whitespace_after
1059 )
1060 )
1062 return (parse_empty_lines(config, whitespace_before), asyncnode, stmt.value)
1065@with_production("asyncable_funcdef", "[ASYNC] funcdef", version=">=3.5")
1066@with_production("asyncable_funcdef", "funcdef", version="<3.5")
1067def convert_asyncable_funcdef(config: ParserConfig, children: Sequence[Any]) -> Any:
1068 leading_lines, asyncnode, funcdef = _extract_async(config, children)
1070 return funcdef.with_changes(
1071 asynchronous=asyncnode, leading_lines=leading_lines, lines_after_decorators=()
1072 )
1075@with_production("funcdef", "'def' NAME parameters [funcdef_annotation] ':' suite")
1076def convert_funcdef(config: ParserConfig, children: Sequence[Any]) -> Any:
1077 defnode, namenode, param_partial, *annotation, colon, suite = children
1079 # If the trailing paremeter doesn't have a comma, then it owns the trailing
1080 # whitespace before the rpar. Otherwise, the comma owns it (and will have
1081 # already parsed it). We don't check/update ParamStar because if it exists
1082 # then we are guaranteed have at least one kwonly_param.
1083 parameters = param_partial.params
1084 if parameters.star_kwarg is not None:
1085 if parameters.star_kwarg.comma == MaybeSentinel.DEFAULT:
1086 parameters = parameters.with_changes(
1087 star_kwarg=parameters.star_kwarg.with_changes(
1088 whitespace_after_param=param_partial.rpar.whitespace_before
1089 )
1090 )
1091 elif parameters.kwonly_params:
1092 if parameters.kwonly_params[-1].comma == MaybeSentinel.DEFAULT:
1093 parameters = parameters.with_changes(
1094 kwonly_params=(
1095 *parameters.kwonly_params[:-1],
1096 parameters.kwonly_params[-1].with_changes(
1097 whitespace_after_param=param_partial.rpar.whitespace_before
1098 ),
1099 )
1100 )
1101 elif isinstance(parameters.star_arg, Param):
1102 if parameters.star_arg.comma == MaybeSentinel.DEFAULT:
1103 parameters = parameters.with_changes(
1104 star_arg=parameters.star_arg.with_changes(
1105 whitespace_after_param=param_partial.rpar.whitespace_before
1106 )
1107 )
1108 elif parameters.params:
1109 if parameters.params[-1].comma == MaybeSentinel.DEFAULT:
1110 parameters = parameters.with_changes(
1111 params=(
1112 *parameters.params[:-1],
1113 parameters.params[-1].with_changes(
1114 whitespace_after_param=param_partial.rpar.whitespace_before
1115 ),
1116 )
1117 )
1119 return WithLeadingWhitespace(
1120 FunctionDef(
1121 whitespace_after_def=parse_simple_whitespace(
1122 config, defnode.whitespace_after
1123 ),
1124 name=Name(namenode.string),
1125 whitespace_after_name=parse_simple_whitespace(
1126 config, namenode.whitespace_after
1127 ),
1128 whitespace_before_params=param_partial.lpar.whitespace_after,
1129 params=parameters,
1130 returns=None if not annotation else annotation[0],
1131 whitespace_before_colon=parse_simple_whitespace(
1132 config, colon.whitespace_before
1133 ),
1134 body=suite,
1135 ),
1136 defnode.whitespace_before,
1137 )
1140@with_production("parameters", "'(' [typedargslist] ')'")
1141def convert_parameters(config: ParserConfig, children: Sequence[Any]) -> Any:
1142 lpar, *paramlist, rpar = children
1143 return FuncdefPartial(
1144 lpar=LeftParen(
1145 whitespace_after=parse_parenthesizable_whitespace(
1146 config, lpar.whitespace_after
1147 )
1148 ),
1149 params=Parameters() if not paramlist else paramlist[0],
1150 rpar=RightParen(
1151 whitespace_before=parse_parenthesizable_whitespace(
1152 config, rpar.whitespace_before
1153 )
1154 ),
1155 )
1158@with_production("funcdef_annotation", "'->' test")
1159def convert_funcdef_annotation(config: ParserConfig, children: Sequence[Any]) -> Any:
1160 arrow, typehint = children
1161 return Annotation(
1162 whitespace_before_indicator=parse_parenthesizable_whitespace(
1163 config, arrow.whitespace_before
1164 ),
1165 whitespace_after_indicator=parse_parenthesizable_whitespace(
1166 config, arrow.whitespace_after
1167 ),
1168 annotation=typehint.value,
1169 )
1172@with_production("classdef", "'class' NAME ['(' [arglist] ')'] ':' suite")
1173def convert_classdef(config: ParserConfig, children: Sequence[Any]) -> Any:
1174 classdef, name, *arglist, colon, suite = children
1176 # First, parse out the comments and empty lines before the statement.
1177 leading_lines = parse_empty_lines(config, classdef.whitespace_before)
1179 # Compute common whitespace and nodes
1180 whitespace_after_class = parse_simple_whitespace(config, classdef.whitespace_after)
1181 namenode = Name(name.string)
1182 whitespace_after_name = parse_simple_whitespace(config, name.whitespace_after)
1184 # Now, construct the classdef node itself
1185 if not arglist:
1186 # No arglist, so no arguments to this class
1187 return ClassDef(
1188 leading_lines=leading_lines,
1189 lines_after_decorators=(),
1190 whitespace_after_class=whitespace_after_class,
1191 name=namenode,
1192 whitespace_after_name=whitespace_after_name,
1193 body=suite,
1194 )
1195 else:
1196 # Unwrap arglist partial, because its valid to not have any
1197 lpar, *args, rpar = arglist
1198 args = args[0].args if args else []
1200 bases: List[Arg] = []
1201 keywords: List[Arg] = []
1203 current_arg = bases
1204 for arg in args:
1205 if arg.star == "**" or arg.keyword is not None:
1206 current_arg = keywords
1207 # Some quick validation
1208 if current_arg is keywords and (
1209 arg.star == "*" or (arg.star == "" and arg.keyword is None)
1210 ):
1211 raise PartialParserSyntaxError(
1212 "Positional argument follows keyword argument."
1213 )
1214 current_arg.append(arg)
1216 return ClassDef(
1217 leading_lines=leading_lines,
1218 lines_after_decorators=(),
1219 whitespace_after_class=whitespace_after_class,
1220 name=namenode,
1221 whitespace_after_name=whitespace_after_name,
1222 lpar=LeftParen(
1223 whitespace_after=parse_parenthesizable_whitespace(
1224 config, lpar.whitespace_after
1225 )
1226 ),
1227 bases=bases,
1228 keywords=keywords,
1229 rpar=RightParen(
1230 whitespace_before=parse_parenthesizable_whitespace(
1231 config, rpar.whitespace_before
1232 )
1233 ),
1234 whitespace_before_colon=parse_simple_whitespace(
1235 config, colon.whitespace_before
1236 ),
1237 body=suite,
1238 )
1241@with_production("decorator", "'@' dotted_name [ '(' [arglist] ')' ] NEWLINE")
1242def convert_decorator(config: ParserConfig, children: Sequence[Any]) -> Any:
1243 atsign, name, *arglist, newline = children
1244 if not arglist:
1245 # This is either a name or an attribute node, so just extract it.
1246 decoratornode = name
1247 else:
1248 # This needs to be converted into a call node, and we have the
1249 # arglist partial.
1250 lpar, *args, rpar = arglist
1251 args = args[0].args if args else []
1253 # If the trailing argument doesn't have a comma, then it owns the
1254 # trailing whitespace before the rpar. Otherwise, the comma owns
1255 # it.
1256 if len(args) > 0 and args[-1].comma == MaybeSentinel.DEFAULT:
1257 args[-1] = args[-1].with_changes(
1258 whitespace_after_arg=parse_parenthesizable_whitespace(
1259 config, rpar.whitespace_before
1260 )
1261 )
1263 decoratornode = Call(
1264 func=name,
1265 whitespace_after_func=parse_simple_whitespace(
1266 config, lpar.whitespace_before
1267 ),
1268 whitespace_before_args=parse_parenthesizable_whitespace(
1269 config, lpar.whitespace_after
1270 ),
1271 args=tuple(args),
1272 )
1274 return Decorator(
1275 leading_lines=parse_empty_lines(config, atsign.whitespace_before),
1276 whitespace_after_at=parse_simple_whitespace(config, atsign.whitespace_after),
1277 decorator=decoratornode,
1278 trailing_whitespace=newline,
1279 )
1282@with_production("decorators", "decorator+")
1283def convert_decorators(config: ParserConfig, children: Sequence[Any]) -> Any:
1284 return DecoratorPartial(decorators=children)
1287@with_production("decorated", "decorators (classdef | asyncable_funcdef)")
1288def convert_decorated(config: ParserConfig, children: Sequence[Any]) -> Any:
1289 partial, class_or_func = children
1291 # First, split up the spacing on the first decorator
1292 leading_lines = partial.decorators[0].leading_lines
1294 # Now, redistribute ownership of the whitespace
1295 decorators = (
1296 partial.decorators[0].with_changes(leading_lines=()),
1297 *partial.decorators[1:],
1298 )
1300 # Now, modify the original function or class to add the decorators.
1301 return class_or_func.with_changes(
1302 leading_lines=leading_lines,
1303 # pyre-fixme[60]: Concatenation not yet support for multiple variadic
1304 # tuples: `*class_or_func.leading_lines,
1305 # *class_or_func.lines_after_decorators`.
1306 # pyre-fixme[60]: Expected to unpack an iterable, but got `unknown`.
1307 lines_after_decorators=(
1308 *class_or_func.leading_lines,
1309 *class_or_func.lines_after_decorators,
1310 ),
1311 decorators=decorators,
1312 )
1315@with_production(
1316 "asyncable_stmt", "[ASYNC] (funcdef | with_stmt | for_stmt)", version=">=3.5"
1317)
1318@with_production("asyncable_stmt", "funcdef | with_stmt | for_stmt", version="<3.5")
1319def convert_asyncable_stmt(config: ParserConfig, children: Sequence[Any]) -> Any:
1320 leading_lines, asyncnode, stmtnode = _extract_async(config, children)
1321 if isinstance(stmtnode, FunctionDef):
1322 return stmtnode.with_changes(
1323 asynchronous=asyncnode,
1324 leading_lines=leading_lines,
1325 lines_after_decorators=(),
1326 )
1327 elif isinstance(stmtnode, With):
1328 return stmtnode.with_changes(
1329 asynchronous=asyncnode, leading_lines=leading_lines
1330 )
1331 elif isinstance(stmtnode, For):
1332 return stmtnode.with_changes(
1333 asynchronous=asyncnode, leading_lines=leading_lines
1334 )
1335 else:
1336 raise Exception("Logic error!")
1339@with_production("suite", "simple_stmt_suite | indented_suite")
1340def convert_suite(config: ParserConfig, children: Sequence[Any]) -> Any:
1341 (suite,) = children
1342 return suite
1345@with_production("indented_suite", "NEWLINE INDENT stmt+ DEDENT")
1346def convert_indented_suite(config: ParserConfig, children: Sequence[Any]) -> Any:
1347 newline, indent, *stmts, dedent = children
1348 return IndentedBlock(
1349 header=newline,
1350 indent=(
1351 None
1352 if indent.relative_indent == config.default_indent
1353 else indent.relative_indent
1354 ),
1355 body=stmts,
1356 # We want to be able to only keep comments in the footer that are actually for
1357 # this IndentedBlock. We do so by assuming that lines which are indented to the
1358 # same level as the block itself are comments that go at the footer of the
1359 # block. Comments that are indented to less than this indent are assumed to
1360 # belong to the next line of code. We override the indent here because the
1361 # dedent node's absolute indent is the resulting indentation after the dedent
1362 # is performed. Its this way because the whitespace state for both the dedent's
1363 # whitespace_after and the next BaseCompoundStatement's whitespace_before is
1364 # shared. This allows us to partially parse here and parse the rest of the
1365 # whitespace and comments on the next line, effectively making sure that
1366 # comments are attached to the correct node.
1367 footer=parse_empty_lines(
1368 config,
1369 dedent.whitespace_after,
1370 override_absolute_indent=indent.whitespace_before.absolute_indent,
1371 ),
1372 )