Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/IPython/core/inputtransformer2.py: 23%
380 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-20 06:09 +0000
1"""Input transformer machinery to support IPython special syntax.
3This includes the machinery to recognise and transform ``%magic`` commands,
4``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
6Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7deprecated in 7.0.
8"""
10# Copyright (c) IPython Development Team.
11# Distributed under the terms of the Modified BSD License.
13import ast
14from codeop import CommandCompiler, Compile
15import re
16import tokenize
17from typing import List, Tuple, Optional, Any
18import warnings
20_indent_re = re.compile(r'^[ \t]+')
22def leading_empty_lines(lines):
23 """Remove leading empty lines
25 If the leading lines are empty or contain only whitespace, they will be
26 removed.
27 """
28 if not lines:
29 return lines
30 for i, line in enumerate(lines):
31 if line and not line.isspace():
32 return lines[i:]
33 return lines
35def leading_indent(lines):
36 """Remove leading indentation.
38 If the first line starts with a spaces or tabs, the same whitespace will be
39 removed from each following line in the cell.
40 """
41 if not lines:
42 return lines
43 m = _indent_re.match(lines[0])
44 if not m:
45 return lines
46 space = m.group(0)
47 n = len(space)
48 return [l[n:] if l.startswith(space) else l
49 for l in lines]
51class PromptStripper:
52 """Remove matching input prompts from a block of input.
54 Parameters
55 ----------
56 prompt_re : regular expression
57 A regular expression matching any input prompt (including continuation,
58 e.g. ``...``)
59 initial_re : regular expression, optional
60 A regular expression matching only the initial prompt, but not continuation.
61 If no initial expression is given, prompt_re will be used everywhere.
62 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
63 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
65 Notes
66 -----
68 If initial_re and prompt_re differ,
69 only initial_re will be tested against the first line.
70 If any prompt is found on the first two lines,
71 prompts will be stripped from the rest of the block.
72 """
73 def __init__(self, prompt_re, initial_re=None):
74 self.prompt_re = prompt_re
75 self.initial_re = initial_re or prompt_re
77 def _strip(self, lines):
78 return [self.prompt_re.sub('', l, count=1) for l in lines]
80 def __call__(self, lines):
81 if not lines:
82 return lines
83 if self.initial_re.match(lines[0]) or \
84 (len(lines) > 1 and self.prompt_re.match(lines[1])):
85 return self._strip(lines)
86 return lines
88classic_prompt = PromptStripper(
89 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
90 initial_re=re.compile(r'^>>>( |$)')
91)
93ipython_prompt = PromptStripper(
94 re.compile(
95 r"""
96 ^( # Match from the beginning of a line, either:
98 # 1. First-line prompt:
99 ((\[nav\]|\[ins\])?\ )? # Vi editing mode prompt, if it's there
100 In\ # The 'In' of the prompt, with a space
101 \[\d+\]: # Command index, as displayed in the prompt
102 \ # With a mandatory trailing space
104 | # ... or ...
106 # 2. The three dots of the multiline prompt
107 \s* # All leading whitespace characters
108 \.{3,}: # The three (or more) dots
109 \ ? # With an optional trailing space
111 )
112 """,
113 re.VERBOSE,
114 )
115)
118def cell_magic(lines):
119 if not lines or not lines[0].startswith('%%'):
120 return lines
121 if re.match(r'%%\w+\?', lines[0]):
122 # This case will be handled by help_end
123 return lines
124 magic_name, _, first_line = lines[0][2:].rstrip().partition(' ')
125 body = ''.join(lines[1:])
126 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
127 % (magic_name, first_line, body)]
130def _find_assign_op(token_line) -> Optional[int]:
131 """Get the index of the first assignment in the line ('=' not inside brackets)
133 Note: We don't try to support multiple special assignment (a = b = %foo)
134 """
135 paren_level = 0
136 for i, ti in enumerate(token_line):
137 s = ti.string
138 if s == '=' and paren_level == 0:
139 return i
140 if s in {'(','[','{'}:
141 paren_level += 1
142 elif s in {')', ']', '}'}:
143 if paren_level > 0:
144 paren_level -= 1
145 return None
147def find_end_of_continued_line(lines, start_line: int):
148 """Find the last line of a line explicitly extended using backslashes.
150 Uses 0-indexed line numbers.
151 """
152 end_line = start_line
153 while lines[end_line].endswith('\\\n'):
154 end_line += 1
155 if end_line >= len(lines):
156 break
157 return end_line
159def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
160 r"""Assemble a single line from multiple continued line pieces
162 Continued lines are lines ending in ``\``, and the line following the last
163 ``\`` in the block.
165 For example, this code continues over multiple lines::
167 if (assign_ix is not None) \
168 and (len(line) >= assign_ix + 2) \
169 and (line[assign_ix+1].string == '%') \
170 and (line[assign_ix+2].type == tokenize.NAME):
172 This statement contains four continued line pieces.
173 Assembling these pieces into a single line would give::
175 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
177 This uses 0-indexed line numbers. *start* is (lineno, colno).
179 Used to allow ``%magic`` and ``!system`` commands to be continued over
180 multiple lines.
181 """
182 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
183 return ' '.join([p.rstrip()[:-1] for p in parts[:-1]] # Strip backslash+newline
184 + [parts[-1].rstrip()]) # Strip newline from last line
186class TokenTransformBase:
187 """Base class for transformations which examine tokens.
189 Special syntax should not be transformed when it occurs inside strings or
190 comments. This is hard to reliably avoid with regexes. The solution is to
191 tokenise the code as Python, and recognise the special syntax in the tokens.
193 IPython's special syntax is not valid Python syntax, so tokenising may go
194 wrong after the special syntax starts. These classes therefore find and
195 transform *one* instance of special syntax at a time into regular Python
196 syntax. After each transformation, tokens are regenerated to find the next
197 piece of special syntax.
199 Subclasses need to implement one class method (find)
200 and one regular method (transform).
202 The priority attribute can select which transformation to apply if multiple
203 transformers match in the same place. Lower numbers have higher priority.
204 This allows "%magic?" to be turned into a help call rather than a magic call.
205 """
206 # Lower numbers -> higher priority (for matches in the same location)
207 priority = 10
209 def sortby(self):
210 return self.start_line, self.start_col, self.priority
212 def __init__(self, start):
213 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
214 self.start_col = start[1]
216 @classmethod
217 def find(cls, tokens_by_line):
218 """Find one instance of special syntax in the provided tokens.
220 Tokens are grouped into logical lines for convenience,
221 so it is easy to e.g. look at the first token of each line.
222 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
224 This should return an instance of its class, pointing to the start
225 position it has found, or None if it found no match.
226 """
227 raise NotImplementedError
229 def transform(self, lines: List[str]):
230 """Transform one instance of special syntax found by ``find()``
232 Takes a list of strings representing physical lines,
233 returns a similar list of transformed lines.
234 """
235 raise NotImplementedError
237class MagicAssign(TokenTransformBase):
238 """Transformer for assignments from magics (a = %foo)"""
239 @classmethod
240 def find(cls, tokens_by_line):
241 """Find the first magic assignment (a = %foo) in the cell.
242 """
243 for line in tokens_by_line:
244 assign_ix = _find_assign_op(line)
245 if (assign_ix is not None) \
246 and (len(line) >= assign_ix + 2) \
247 and (line[assign_ix+1].string == '%') \
248 and (line[assign_ix+2].type == tokenize.NAME):
249 return cls(line[assign_ix+1].start)
251 def transform(self, lines: List[str]):
252 """Transform a magic assignment found by the ``find()`` classmethod.
253 """
254 start_line, start_col = self.start_line, self.start_col
255 lhs = lines[start_line][:start_col]
256 end_line = find_end_of_continued_line(lines, start_line)
257 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
258 assert rhs.startswith('%'), rhs
259 magic_name, _, args = rhs[1:].partition(' ')
261 lines_before = lines[:start_line]
262 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
263 new_line = lhs + call + '\n'
264 lines_after = lines[end_line+1:]
266 return lines_before + [new_line] + lines_after
269class SystemAssign(TokenTransformBase):
270 """Transformer for assignments from system commands (a = !foo)"""
271 @classmethod
272 def find(cls, tokens_by_line):
273 """Find the first system assignment (a = !foo) in the cell.
274 """
275 for line in tokens_by_line:
276 assign_ix = _find_assign_op(line)
277 if (assign_ix is not None) \
278 and not line[assign_ix].line.strip().startswith('=') \
279 and (len(line) >= assign_ix + 2) \
280 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
281 ix = assign_ix + 1
283 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
284 if line[ix].string == '!':
285 return cls(line[ix].start)
286 elif not line[ix].string.isspace():
287 break
288 ix += 1
290 def transform(self, lines: List[str]):
291 """Transform a system assignment found by the ``find()`` classmethod.
292 """
293 start_line, start_col = self.start_line, self.start_col
295 lhs = lines[start_line][:start_col]
296 end_line = find_end_of_continued_line(lines, start_line)
297 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
298 assert rhs.startswith('!'), rhs
299 cmd = rhs[1:]
301 lines_before = lines[:start_line]
302 call = "get_ipython().getoutput({!r})".format(cmd)
303 new_line = lhs + call + '\n'
304 lines_after = lines[end_line + 1:]
306 return lines_before + [new_line] + lines_after
308# The escape sequences that define the syntax transformations IPython will
309# apply to user input. These can NOT be just changed here: many regular
310# expressions and other parts of the code may use their hardcoded values, and
311# for all intents and purposes they constitute the 'IPython syntax', so they
312# should be considered fixed.
314ESC_SHELL = '!' # Send line to underlying system shell
315ESC_SH_CAP = '!!' # Send line to system shell and capture output
316ESC_HELP = '?' # Find information about object
317ESC_HELP2 = '??' # Find extra-detailed information about object
318ESC_MAGIC = '%' # Call magic function
319ESC_MAGIC2 = '%%' # Call cell-magic function
320ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
321ESC_QUOTE2 = ';' # Quote all args as a single string, call
322ESC_PAREN = '/' # Call first argument with rest of line as arguments
324ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
325ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
327def _make_help_call(target, esc):
328 """Prepares a pinfo(2)/psearch call from a target name and the escape
329 (i.e. ? or ??)"""
330 method = 'pinfo2' if esc == '??' \
331 else 'psearch' if '*' in target \
332 else 'pinfo'
333 arg = " ".join([method, target])
334 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
335 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
336 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
337 return "get_ipython().run_line_magic(%r, %r)" % (t_magic_name, t_magic_arg_s)
340def _tr_help(content):
341 """Translate lines escaped with: ?
343 A naked help line should fire the intro help screen (shell.show_usage())
344 """
345 if not content:
346 return 'get_ipython().show_usage()'
348 return _make_help_call(content, '?')
350def _tr_help2(content):
351 """Translate lines escaped with: ??
353 A naked help line should fire the intro help screen (shell.show_usage())
354 """
355 if not content:
356 return 'get_ipython().show_usage()'
358 return _make_help_call(content, '??')
360def _tr_magic(content):
361 "Translate lines escaped with a percent sign: %"
362 name, _, args = content.partition(' ')
363 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
365def _tr_quote(content):
366 "Translate lines escaped with a comma: ,"
367 name, _, args = content.partition(' ')
368 return '%s("%s")' % (name, '", "'.join(args.split()) )
370def _tr_quote2(content):
371 "Translate lines escaped with a semicolon: ;"
372 name, _, args = content.partition(' ')
373 return '%s("%s")' % (name, args)
375def _tr_paren(content):
376 "Translate lines escaped with a slash: /"
377 name, _, args = content.partition(' ')
378 return '%s(%s)' % (name, ", ".join(args.split()))
380tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
381 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
382 ESC_HELP : _tr_help,
383 ESC_HELP2 : _tr_help2,
384 ESC_MAGIC : _tr_magic,
385 ESC_QUOTE : _tr_quote,
386 ESC_QUOTE2 : _tr_quote2,
387 ESC_PAREN : _tr_paren }
389class EscapedCommand(TokenTransformBase):
390 """Transformer for escaped commands like %foo, !foo, or /foo"""
391 @classmethod
392 def find(cls, tokens_by_line):
393 """Find the first escaped command (%foo, !foo, etc.) in the cell.
394 """
395 for line in tokens_by_line:
396 if not line:
397 continue
398 ix = 0
399 ll = len(line)
400 while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
401 ix += 1
402 if ix >= ll:
403 continue
404 if line[ix].string in ESCAPE_SINGLES:
405 return cls(line[ix].start)
407 def transform(self, lines):
408 """Transform an escaped line found by the ``find()`` classmethod.
409 """
410 start_line, start_col = self.start_line, self.start_col
412 indent = lines[start_line][:start_col]
413 end_line = find_end_of_continued_line(lines, start_line)
414 line = assemble_continued_line(lines, (start_line, start_col), end_line)
416 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
417 escape, content = line[:2], line[2:]
418 else:
419 escape, content = line[:1], line[1:]
421 if escape in tr:
422 call = tr[escape](content)
423 else:
424 call = ''
426 lines_before = lines[:start_line]
427 new_line = indent + call + '\n'
428 lines_after = lines[end_line + 1:]
430 return lines_before + [new_line] + lines_after
433_help_end_re = re.compile(
434 r"""(%{0,2}
435 (?!\d)[\w*]+ # Variable name
436 (\.(?!\d)[\w*]+|\[-?[0-9]+\])* # .etc.etc or [0], we only support literal integers.
437 )
438 (\?\??)$ # ? or ??
439 """,
440 re.VERBOSE,
441)
444class HelpEnd(TokenTransformBase):
445 """Transformer for help syntax: obj? and obj??"""
446 # This needs to be higher priority (lower number) than EscapedCommand so
447 # that inspecting magics (%foo?) works.
448 priority = 5
450 def __init__(self, start, q_locn):
451 super().__init__(start)
452 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
453 self.q_col = q_locn[1]
455 @classmethod
456 def find(cls, tokens_by_line):
457 """Find the first help command (foo?) in the cell.
458 """
459 for line in tokens_by_line:
460 # Last token is NEWLINE; look at last but one
461 if len(line) > 2 and line[-2].string == '?':
462 # Find the first token that's not INDENT/DEDENT
463 ix = 0
464 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
465 ix += 1
466 return cls(line[ix].start, line[-2].start)
468 def transform(self, lines):
469 """Transform a help command found by the ``find()`` classmethod.
470 """
472 piece = "".join(lines[self.start_line : self.q_line + 1])
473 indent, content = piece[: self.start_col], piece[self.start_col :]
474 lines_before = lines[: self.start_line]
475 lines_after = lines[self.q_line + 1 :]
477 m = _help_end_re.search(content)
478 if not m:
479 raise SyntaxError(content)
480 assert m is not None, content
481 target = m.group(1)
482 esc = m.group(3)
485 call = _make_help_call(target, esc)
486 new_line = indent + call + '\n'
488 return lines_before + [new_line] + lines_after
490def make_tokens_by_line(lines:List[str]):
491 """Tokenize a series of lines and group tokens by line.
493 The tokens for a multiline Python string or expression are grouped as one
494 line. All lines except the last lines should keep their line ending ('\\n',
495 '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
496 for example when passing block of text to this function.
498 """
499 # NL tokens are used inside multiline expressions, but also after blank
500 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
501 # We want to group the former case together but split the latter, so we
502 # track parentheses level, similar to the internals of tokenize.
504 # reexported from token on 3.7+
505 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore
506 tokens_by_line: List[List[Any]] = [[]]
507 if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
508 warnings.warn(
509 "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
510 stacklevel=2,
511 )
512 parenlev = 0
513 try:
514 for token in tokenize.generate_tokens(iter(lines).__next__):
515 tokens_by_line[-1].append(token)
516 if (token.type == NEWLINE) \
517 or ((token.type == NL) and (parenlev <= 0)):
518 tokens_by_line.append([])
519 elif token.string in {'(', '[', '{'}:
520 parenlev += 1
521 elif token.string in {')', ']', '}'}:
522 if parenlev > 0:
523 parenlev -= 1
524 except tokenize.TokenError:
525 # Input ended in a multiline string or expression. That's OK for us.
526 pass
529 if not tokens_by_line[-1]:
530 tokens_by_line.pop()
533 return tokens_by_line
536def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):
537 """Check if the depth of brackets in the list of tokens drops below 0"""
538 parenlev = 0
539 for token in tokens:
540 if token.string in {"(", "[", "{"}:
541 parenlev += 1
542 elif token.string in {")", "]", "}"}:
543 parenlev -= 1
544 if parenlev < 0:
545 return True
546 return False
549def show_linewise_tokens(s: str):
550 """For investigation and debugging"""
551 warnings.warn(
552 "show_linewise_tokens is deprecated since IPython 8.6",
553 DeprecationWarning,
554 stacklevel=2,
555 )
556 if not s.endswith("\n"):
557 s += "\n"
558 lines = s.splitlines(keepends=True)
559 for line in make_tokens_by_line(lines):
560 print("Line -------")
561 for tokinfo in line:
562 print(" ", tokinfo)
564# Arbitrary limit to prevent getting stuck in infinite loops
565TRANSFORM_LOOP_LIMIT = 500
567class TransformerManager:
568 """Applies various transformations to a cell or code block.
570 The key methods for external use are ``transform_cell()``
571 and ``check_complete()``.
572 """
573 def __init__(self):
574 self.cleanup_transforms = [
575 leading_empty_lines,
576 leading_indent,
577 classic_prompt,
578 ipython_prompt,
579 ]
580 self.line_transforms = [
581 cell_magic,
582 ]
583 self.token_transformers = [
584 MagicAssign,
585 SystemAssign,
586 EscapedCommand,
587 HelpEnd,
588 ]
590 def do_one_token_transform(self, lines):
591 """Find and run the transform earliest in the code.
593 Returns (changed, lines).
595 This method is called repeatedly until changed is False, indicating
596 that all available transformations are complete.
598 The tokens following IPython special syntax might not be valid, so
599 the transformed code is retokenised every time to identify the next
600 piece of special syntax. Hopefully long code cells are mostly valid
601 Python, not using lots of IPython special syntax, so this shouldn't be
602 a performance issue.
603 """
604 tokens_by_line = make_tokens_by_line(lines)
605 candidates = []
606 for transformer_cls in self.token_transformers:
607 transformer = transformer_cls.find(tokens_by_line)
608 if transformer:
609 candidates.append(transformer)
611 if not candidates:
612 # Nothing to transform
613 return False, lines
614 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
615 for transformer in ordered_transformers:
616 try:
617 return True, transformer.transform(lines)
618 except SyntaxError:
619 pass
620 return False, lines
622 def do_token_transforms(self, lines):
623 for _ in range(TRANSFORM_LOOP_LIMIT):
624 changed, lines = self.do_one_token_transform(lines)
625 if not changed:
626 return lines
628 raise RuntimeError("Input transformation still changing after "
629 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
631 def transform_cell(self, cell: str) -> str:
632 """Transforms a cell of input code"""
633 if not cell.endswith('\n'):
634 cell += '\n' # Ensure the cell has a trailing newline
635 lines = cell.splitlines(keepends=True)
636 for transform in self.cleanup_transforms + self.line_transforms:
637 lines = transform(lines)
639 lines = self.do_token_transforms(lines)
640 return ''.join(lines)
642 def check_complete(self, cell: str):
643 """Return whether a block of code is ready to execute, or should be continued
645 Parameters
646 ----------
647 cell : string
648 Python input code, which can be multiline.
650 Returns
651 -------
652 status : str
653 One of 'complete', 'incomplete', or 'invalid' if source is not a
654 prefix of valid code.
655 indent_spaces : int or None
656 The number of spaces by which to indent the next line of code. If
657 status is not 'incomplete', this is None.
658 """
659 # Remember if the lines ends in a new line.
660 ends_with_newline = False
661 for character in reversed(cell):
662 if character == '\n':
663 ends_with_newline = True
664 break
665 elif character.strip():
666 break
667 else:
668 continue
670 if not ends_with_newline:
671 # Append an newline for consistent tokenization
672 # See https://bugs.python.org/issue33899
673 cell += '\n'
675 lines = cell.splitlines(keepends=True)
677 if not lines:
678 return 'complete', None
680 if lines[-1].endswith('\\'):
681 # Explicit backslash continuation
682 return 'incomplete', find_last_indent(lines)
684 try:
685 for transform in self.cleanup_transforms:
686 if not getattr(transform, 'has_side_effects', False):
687 lines = transform(lines)
688 except SyntaxError:
689 return 'invalid', None
691 if lines[0].startswith('%%'):
692 # Special case for cell magics - completion marked by blank line
693 if lines[-1].strip():
694 return 'incomplete', find_last_indent(lines)
695 else:
696 return 'complete', None
698 try:
699 for transform in self.line_transforms:
700 if not getattr(transform, 'has_side_effects', False):
701 lines = transform(lines)
702 lines = self.do_token_transforms(lines)
703 except SyntaxError:
704 return 'invalid', None
706 tokens_by_line = make_tokens_by_line(lines)
708 # Bail if we got one line and there are more closing parentheses than
709 # the opening ones
710 if (
711 len(lines) == 1
712 and tokens_by_line
713 and has_sunken_brackets(tokens_by_line[0])
714 ):
715 return "invalid", None
717 if not tokens_by_line:
718 return 'incomplete', find_last_indent(lines)
720 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
721 # We're in a multiline string or expression
722 return 'incomplete', find_last_indent(lines)
724 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore
726 # Pop the last line which only contains DEDENTs and ENDMARKER
727 last_token_line = None
728 if {t.type for t in tokens_by_line[-1]} in [
729 {tokenize.DEDENT, tokenize.ENDMARKER},
730 {tokenize.ENDMARKER}
731 ] and len(tokens_by_line) > 1:
732 last_token_line = tokens_by_line.pop()
734 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
735 tokens_by_line[-1].pop()
737 if not tokens_by_line[-1]:
738 return 'incomplete', find_last_indent(lines)
740 if tokens_by_line[-1][-1].string == ':':
741 # The last line starts a block (e.g. 'if foo:')
742 ix = 0
743 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
744 ix += 1
746 indent = tokens_by_line[-1][ix].start[1]
747 return 'incomplete', indent + 4
749 if tokens_by_line[-1][0].line.endswith('\\'):
750 return 'incomplete', None
752 # At this point, our checks think the code is complete (or invalid).
753 # We'll use codeop.compile_command to check this with the real parser
754 try:
755 with warnings.catch_warnings():
756 warnings.simplefilter('error', SyntaxWarning)
757 res = compile_command(''.join(lines), symbol='exec')
758 except (SyntaxError, OverflowError, ValueError, TypeError,
759 MemoryError, SyntaxWarning):
760 return 'invalid', None
761 else:
762 if res is None:
763 return 'incomplete', find_last_indent(lines)
765 if last_token_line and last_token_line[0].type == tokenize.DEDENT:
766 if ends_with_newline:
767 return 'complete', None
768 return 'incomplete', find_last_indent(lines)
770 # If there's a blank line at the end, assume we're ready to execute
771 if not lines[-1].strip():
772 return 'complete', None
774 return 'complete', None
777def find_last_indent(lines):
778 m = _indent_re.match(lines[-1])
779 if not m:
780 return 0
781 return len(m.group(0).replace('\t', ' '*4))
784class MaybeAsyncCompile(Compile):
785 def __init__(self, extra_flags=0):
786 super().__init__()
787 self.flags |= extra_flags
790class MaybeAsyncCommandCompiler(CommandCompiler):
791 def __init__(self, extra_flags=0):
792 self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)
795_extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
797compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)