Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Input transformer machinery to support IPython special syntax.
3This includes the machinery to recognise and transform ``%magic`` commands,
4``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
6Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7deprecated in 7.0 and removed in 9.0
8"""
10# Copyright (c) IPython Development Team.
11# Distributed under the terms of the Modified BSD License.
13import ast
14from codeop import CommandCompiler, Compile
15import re
16import sys
17import tokenize
18from typing import Any
19import warnings
20from textwrap import dedent
22from IPython.utils import tokenutil
24_indent_re = re.compile(r"^[ \t]+")
27def leading_empty_lines(lines):
28 """Remove leading empty lines
30 If the leading lines are empty or contain only whitespace, they will be
31 removed.
32 """
33 if not lines:
34 return lines
35 for i, line in enumerate(lines):
36 if line and not line.isspace():
37 return lines[i:]
38 return lines
41def leading_indent(lines):
42 """Remove leading indentation.
44 Removes the minimum common leading indentation from all lines.
45 """
46 if not lines:
47 return lines
48 return dedent("".join(lines)).splitlines(keepends=True)
51class PromptStripper:
52 """Remove matching input prompts from a block of input.
54 Parameters
55 ----------
56 prompt_re : regular expression
57 A regular expression matching any input prompt (including continuation,
58 e.g. ``...``)
59 initial_re : regular expression, optional
60 A regular expression matching only the initial prompt, but not continuation.
61 If no initial expression is given, prompt_re will be used everywhere.
62 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
63 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
65 Notes
66 -----
68 If initial_re and prompt_re differ,
69 only initial_re will be tested against the first line.
70 If any prompt is found on the first two lines,
71 prompts will be stripped from the rest of the block.
72 """
74 def __init__(self, prompt_re, initial_re=None, *, doctest=False):
75 self.prompt_re = prompt_re
76 self.initial_re = initial_re or prompt_re
77 self.doctest = doctest
78 if doctest:
79 # Doctest/xdoctest prompts may be indented (e.g. " >>>").
80 # We only treat "..." as a continuation prompt when the same pasted
81 # block contains at least one ">>>" line, to avoid ambiguity with the
82 # Python Ellipsis literal.
83 self._doctest_initial_re = re.compile(r"^\s*>>>")
84 self._doctest_ps1_re = re.compile(r"^\s*>>>[ \t]?")
85 self._doctest_ps2_re = re.compile(r"^\s*\.\.\.[ \t]?")
87 # Very small state machine to detect triple-quoted strings in the
88 # *same* input block (e.g. user typed """ then pasted doctest).
89 # We preserve literal >>> / ... inside triple-quoted strings.
90 self._triple_quote_re = re.compile(r"(?<!\\)(\"\"\"|''')")
92 def _triple_quote_mask(self, lines: list[str]) -> list[bool]:
93 """
94 Return a boolean mask: True if the corresponding line is considered
95 inside a triple-quoted string literal.
97 This is intentionally heuristic (fast + good enough for paste handling).
98 """
99 mask: list[bool] = []
100 in_triple: str | None = None # either ''' or """
101 preserve_prompt = False
102 seen_prompt = False
103 string_prefix_re = re.compile(r"(?i)^[rubf]*$")
105 for line in lines:
106 mask.append(in_triple is not None and preserve_prompt)
107 # Toggle state for each occurrence of """ or ''' in the line.
108 for m in self._triple_quote_re.finditer(line):
109 q = m.group(1)
110 if in_triple is None:
111 in_triple = q
112 before_quote = line[: m.start()]
113 stripped = self._doctest_ps1_re.sub("", before_quote, count=1)
114 stripped = self._doctest_ps2_re.sub("", stripped, count=1)
115 had_prompt = stripped != before_quote
116 prompted_code = had_prompt and seen_prompt
117 preserve_prompt = bool(
118 not prompted_code and string_prefix_re.match(stripped.strip())
119 )
120 mask[-1] = preserve_prompt
121 elif in_triple == q:
122 in_triple = None
123 preserve_prompt = False
124 # else: ignore mismatched triple quote while inside
125 seen_prompt = seen_prompt or bool(self._doctest_initial_re.match(line))
126 return mask
128 def _strip(self, lines):
129 return [self.prompt_re.sub("", l, count=1) for l in lines]
131 def __call__(self, lines):
132 if not lines:
133 return lines
135 if self.doctest:
136 triple_mask = self._triple_quote_mask(lines)
138 # Detect doctest prompts only outside triple-quoted strings.
139 has_doctest_outside = any(
140 (not in_triple) and self._doctest_initial_re.match(l)
141 for l, in_triple in zip(lines, triple_mask)
142 )
143 if not has_doctest_outside:
144 return lines
146 out_lines: list[str] = []
147 stripped_mask: list[bool] = []
149 for l, in_triple in zip(lines, triple_mask):
150 if in_triple:
151 out_lines.append(l)
152 stripped_mask.append(False)
153 continue
155 if self._doctest_ps1_re.match(l):
156 new_l = self._doctest_ps1_re.sub("", l, count=1)
157 elif self._doctest_ps2_re.match(l):
158 new_l = self._doctest_ps2_re.sub("", l, count=1)
159 else:
160 new_l = l
161 out_lines.append(new_l)
162 stripped_mask.append(new_l != l)
164 # Dedent only the non-triple-quoted segments where stripping occurred.
165 dedented: list[str] = []
166 i = 0
167 while i < len(out_lines):
168 j = i
169 in_triple = triple_mask[i]
170 while j < len(out_lines) and triple_mask[j] == in_triple:
171 j += 1
173 segment = out_lines[i:j]
174 seg_stripped = any(stripped_mask[i:j])
176 if (not in_triple) and seg_stripped:
177 dedented.extend(dedent("".join(segment)).splitlines(keepends=True))
178 else:
179 dedented.extend(segment)
181 i = j
183 return dedented
185 if self.initial_re.match(lines[0]) or (
186 len(lines) > 1 and self.prompt_re.match(lines[1])
187 ):
188 return self._strip(lines)
189 return lines
192classic_prompt = PromptStripper(
193 prompt_re=re.compile(r"^(>>>|\.\.\.)( |$)"),
194 initial_re=re.compile(r"^>>>( |$)"),
195 doctest=True,
196)
198ipython_prompt = PromptStripper(
199 re.compile(
200 r"""
201 ^( # Match from the beginning of a line, either:
203 # 1. First-line prompt:
204 ((\[nav\]|\[ins\])?\ )? # Vi editing mode prompt, if it's there
205 In\ # The 'In' of the prompt, with a space
206 \[\d+\]: # Command index, as displayed in the prompt
207 \ # With a mandatory trailing space
209 | # ... or ...
211 # 2. The three dots of the multiline prompt
212 \s* # All leading whitespace characters
213 \.{3,}: # The three (or more) dots
214 \ ? # With an optional trailing space
216 )
217 """,
218 re.VERBOSE,
219 )
220)
223def cell_magic(lines):
224 if not lines or not lines[0].startswith("%%"):
225 return lines
226 if re.match(r"%%\w+\?", lines[0]):
227 # This case will be handled by help_end
228 return lines
229 magic_name, _, first_line = lines[0][2:].rstrip().partition(" ")
230 body = "".join(lines[1:])
231 return [
232 "get_ipython().run_cell_magic(%r, %r, %r)\n" % (magic_name, first_line, body)
233 ]
236def _find_assign_op(token_line) -> int | None:
237 """Get the index of the first assignment in the line ('=' not inside brackets)
239 Note: We don't try to support multiple special assignment (a = b = %foo)
240 """
241 paren_level = 0
242 for i, ti in enumerate(token_line):
243 s = ti.string
244 if s == "=" and paren_level == 0:
245 return i
246 if s in {"(", "[", "{"}:
247 paren_level += 1
248 elif s in {")", "]", "}"}:
249 if paren_level > 0:
250 paren_level -= 1
251 return None
254def find_end_of_continued_line(lines, start_line: int):
255 """Find the last line of a line explicitly extended using backslashes.
257 Uses 0-indexed line numbers.
258 """
259 end_line = start_line
260 while lines[end_line].endswith("\\\n"):
261 end_line += 1
262 if end_line >= len(lines):
263 break
264 return end_line
267def assemble_continued_line(lines, start: tuple[int, int], end_line: int):
268 r"""Assemble a single line from multiple continued line pieces
270 Continued lines are lines ending in ``\``, and the line following the last
271 ``\`` in the block.
273 For example, this code continues over multiple lines::
275 if (assign_ix is not None) \
276 and (len(line) >= assign_ix + 2) \
277 and (line[assign_ix+1].string == '%') \
278 and (line[assign_ix+2].type == tokenize.NAME):
280 This statement contains four continued line pieces.
281 Assembling these pieces into a single line would give::
283 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
285 This uses 0-indexed line numbers. *start* is (lineno, colno).
287 Used to allow ``%magic`` and ``!system`` commands to be continued over
288 multiple lines.
289 """
290 parts = [lines[start[0]][start[1] :]] + lines[start[0] + 1 : end_line + 1]
291 return " ".join(
292 [p.rstrip()[:-1] for p in parts[:-1]] # Strip backslash+newline
293 + [parts[-1].rstrip()]
294 ) # Strip newline from last line
297class TokenTransformBase:
298 """Base class for transformations which examine tokens.
300 Special syntax should not be transformed when it occurs inside strings or
301 comments. This is hard to reliably avoid with regexes. The solution is to
302 tokenise the code as Python, and recognise the special syntax in the tokens.
304 IPython's special syntax is not valid Python syntax, so tokenising may go
305 wrong after the special syntax starts. These classes therefore find and
306 transform *one* instance of special syntax at a time into regular Python
307 syntax. After each transformation, tokens are regenerated to find the next
308 piece of special syntax.
310 Subclasses need to implement one class method (find)
311 and one regular method (transform).
313 The priority attribute can select which transformation to apply if multiple
314 transformers match in the same place. Lower numbers have higher priority.
315 This allows "%magic?" to be turned into a help call rather than a magic call.
316 """
318 # Lower numbers -> higher priority (for matches in the same location)
319 priority = 10
321 def sortby(self):
322 return self.start_line, self.start_col, self.priority
324 def __init__(self, start):
325 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
326 self.start_col = start[1]
328 @classmethod
329 def find(cls, tokens_by_line):
330 """Find one instance of special syntax in the provided tokens.
332 Tokens are grouped into logical lines for convenience,
333 so it is easy to e.g. look at the first token of each line.
334 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
336 This should return an instance of its class, pointing to the start
337 position it has found, or None if it found no match.
338 """
339 raise NotImplementedError
341 def transform(self, lines: list[str]):
342 """Transform one instance of special syntax found by ``find()``
344 Takes a list of strings representing physical lines,
345 returns a similar list of transformed lines.
346 """
347 raise NotImplementedError
350class MagicAssign(TokenTransformBase):
351 """Transformer for assignments from magics (a = %foo)"""
353 @classmethod
354 def find(cls, tokens_by_line):
355 """Find the first magic assignment (a = %foo) in the cell."""
356 for line in tokens_by_line:
357 assign_ix = _find_assign_op(line)
358 if (
359 (assign_ix is not None)
360 and (len(line) >= assign_ix + 2)
361 and (line[assign_ix + 1].string == "%")
362 and (line[assign_ix + 2].type == tokenize.NAME)
363 ):
364 return cls(line[assign_ix + 1].start)
366 def transform(self, lines: list[str]):
367 """Transform a magic assignment found by the ``find()`` classmethod."""
368 start_line, start_col = self.start_line, self.start_col
369 lhs = lines[start_line][:start_col]
370 end_line = find_end_of_continued_line(lines, start_line)
371 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
372 assert rhs.startswith("%"), rhs
373 magic_name, _, args = rhs[1:].partition(" ")
375 lines_before = lines[:start_line]
376 call = f"get_ipython().run_line_magic({magic_name!r}, {args!r})"
377 new_line = lhs + call + "\n"
378 lines_after = lines[end_line + 1 :]
380 return lines_before + [new_line] + lines_after
383class SystemAssign(TokenTransformBase):
384 """Transformer for assignments from system commands (a = !foo)"""
386 @classmethod
387 def find_pre_312(cls, tokens_by_line):
388 for line in tokens_by_line:
389 assign_ix = _find_assign_op(line)
390 if (
391 (assign_ix is not None)
392 and not line[assign_ix].line.strip().startswith("=")
393 and (len(line) >= assign_ix + 2)
394 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN)
395 ):
396 ix = assign_ix + 1
398 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
399 if line[ix].string == "!":
400 return cls(line[ix].start)
401 elif not line[ix].string.isspace():
402 break
403 ix += 1
405 @classmethod
406 def find_post_312(cls, tokens_by_line):
407 for line in tokens_by_line:
408 assign_ix = _find_assign_op(line)
409 if (
410 (assign_ix is not None)
411 and not line[assign_ix].line.strip().startswith("=")
412 and (len(line) >= assign_ix + 2)
413 and (line[assign_ix + 1].type == tokenize.OP)
414 and (line[assign_ix + 1].string == "!")
415 ):
416 return cls(line[assign_ix + 1].start)
418 @classmethod
419 def find(cls, tokens_by_line):
420 """Find the first system assignment (a = !foo) in the cell."""
421 if sys.version_info < (3, 12):
422 return cls.find_pre_312(tokens_by_line)
423 return cls.find_post_312(tokens_by_line)
425 def transform(self, lines: list[str]):
426 """Transform a system assignment found by the ``find()`` classmethod."""
427 start_line, start_col = self.start_line, self.start_col
429 lhs = lines[start_line][:start_col]
430 end_line = find_end_of_continued_line(lines, start_line)
431 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
432 assert rhs.startswith("!"), rhs
433 cmd = rhs[1:]
435 lines_before = lines[:start_line]
436 call = f"get_ipython().getoutput({cmd!r})"
437 new_line = lhs + call + "\n"
438 lines_after = lines[end_line + 1 :]
440 return lines_before + [new_line] + lines_after
443# The escape sequences that define the syntax transformations IPython will
444# apply to user input. These can NOT be just changed here: many regular
445# expressions and other parts of the code may use their hardcoded values, and
446# for all intents and purposes they constitute the 'IPython syntax', so they
447# should be considered fixed.
449ESC_SHELL = "!" # Send line to underlying system shell
450ESC_SH_CAP = "!!" # Send line to system shell and capture output
451ESC_HELP = "?" # Find information about object
452ESC_HELP2 = "??" # Find extra-detailed information about object
453ESC_MAGIC = "%" # Call magic function
454ESC_MAGIC2 = "%%" # Call cell-magic function
455ESC_QUOTE = "," # Split args on whitespace, quote each as string and call
456ESC_QUOTE2 = ";" # Quote all args as a single string, call
457ESC_PAREN = "/" # Call first argument with rest of line as arguments
459ESCAPE_SINGLES = {"!", "?", "%", ",", ";", "/"}
460ESCAPE_DOUBLES = {"!!", "??"} # %% (cell magic) is handled separately
463def _make_help_call(target, esc):
464 """Prepares a pinfo(2)/psearch call from a target name and the escape
465 (i.e. ? or ??)"""
466 method = "pinfo2" if esc == "??" else "psearch" if "*" in target else "pinfo"
467 arg = " ".join([method, target])
468 # Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
469 t_magic_name, _, t_magic_arg_s = arg.partition(" ")
470 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
471 return "get_ipython().run_line_magic({!r}, {!r})".format(
472 t_magic_name, t_magic_arg_s
473 )
476def _tr_help(content):
477 """Translate lines escaped with: ?
479 A naked help line should fire the intro help screen (shell.show_usage())
480 """
481 if not content:
482 return "get_ipython().show_usage()"
484 return _make_help_call(content, "?")
487def _tr_help2(content):
488 """Translate lines escaped with: ??
490 A naked help line should fire the intro help screen (shell.show_usage())
491 """
492 if not content:
493 return "get_ipython().show_usage()"
495 return _make_help_call(content, "??")
498def _tr_magic(content):
499 "Translate lines escaped with a percent sign: %"
500 name, _, args = content.partition(" ")
501 return "get_ipython().run_line_magic({!r}, {!r})".format(name, args)
504def _tr_quote(content):
505 "Translate lines escaped with a comma: ,"
506 name, _, args = content.partition(" ")
507 return '{}("{}")'.format(name, '", "'.join(args.split()))
510def _tr_quote2(content):
511 "Translate lines escaped with a semicolon: ;"
512 name, _, args = content.partition(" ")
513 return '{}("{}")'.format(name, args)
516def _tr_paren(content):
517 "Translate lines escaped with a slash: /"
518 name, _, args = content.partition(" ")
519 if name == "":
520 raise SyntaxError(f'"{ESC_SHELL}" must be followed by a callable name')
522 return "{}({})".format(name, ", ".join(args.split()))
525tr = {
526 ESC_SHELL: "get_ipython().system({!r})".format,
527 ESC_SH_CAP: "get_ipython().getoutput({!r})".format,
528 ESC_HELP: _tr_help,
529 ESC_HELP2: _tr_help2,
530 ESC_MAGIC: _tr_magic,
531 ESC_QUOTE: _tr_quote,
532 ESC_QUOTE2: _tr_quote2,
533 ESC_PAREN: _tr_paren,
534}
537class EscapedCommand(TokenTransformBase):
538 """Transformer for escaped commands like %foo, !foo, or /foo"""
540 @classmethod
541 def find(cls, tokens_by_line):
542 """Find the first escaped command (%foo, !foo, etc.) in the cell."""
543 for line in tokens_by_line:
544 if not line:
545 continue
546 ix = 0
547 ll = len(line)
548 while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
549 ix += 1
550 if ix >= ll:
551 continue
552 if line[ix].string in ESCAPE_SINGLES:
553 return cls(line[ix].start)
555 def transform(self, lines):
556 """Transform an escaped line found by the ``find()`` classmethod."""
557 start_line, start_col = self.start_line, self.start_col
559 indent = lines[start_line][:start_col]
560 end_line = find_end_of_continued_line(lines, start_line)
561 line = assemble_continued_line(lines, (start_line, start_col), end_line)
563 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
564 escape, content = line[:2], line[2:]
565 else:
566 escape, content = line[:1], line[1:]
568 if escape in tr:
569 call = tr[escape](content)
570 else:
571 call = ""
573 lines_before = lines[:start_line]
574 new_line = indent + call + "\n"
575 lines_after = lines[end_line + 1 :]
577 return lines_before + [new_line] + lines_after
580_help_end_re = re.compile(
581 r"""(%{0,2}
582 (?!\d)[\w*]+ # Variable name
583 (\.(?!\d)[\w*]+|\[-?[0-9]+\])* # .etc.etc or [0], we only support literal integers.
584 )
585 (\?\??)$ # ? or ??
586 """,
587 re.VERBOSE,
588)
591class HelpEnd(TokenTransformBase):
592 """Transformer for help syntax: obj? and obj??"""
594 # This needs to be higher priority (lower number) than EscapedCommand so
595 # that inspecting magics (%foo?) works.
596 priority = 5
598 def __init__(self, start, q_locn):
599 super().__init__(start)
600 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
601 self.q_col = q_locn[1]
603 @classmethod
604 def find(cls, tokens_by_line):
605 """Find the first help command (foo?) in the cell."""
606 for line in tokens_by_line:
607 # Last token is NEWLINE; look at last but one
608 if len(line) > 2 and line[-2].string == "?":
609 # Find the first token that's not INDENT/DEDENT
610 ix = 0
611 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
612 ix += 1
613 return cls(line[ix].start, line[-2].start)
615 def transform(self, lines):
616 """Transform a help command found by the ``find()`` classmethod."""
618 piece = "".join(lines[self.start_line : self.q_line + 1])
619 indent, content = piece[: self.start_col], piece[self.start_col :]
620 lines_before = lines[: self.start_line]
621 lines_after = lines[self.q_line + 1 :]
623 m = _help_end_re.search(content)
624 if not m:
625 raise SyntaxError(content)
626 assert m is not None, content
627 target = m.group(1)
628 esc = m.group(3)
630 call = _make_help_call(target, esc)
631 new_line = indent + call + "\n"
633 return lines_before + [new_line] + lines_after
636def make_tokens_by_line(lines: list[str]):
637 """Tokenize a series of lines and group tokens by line.
639 The tokens for a multiline Python string or expression are grouped as one
640 line. All lines except the last lines should keep their line ending ('\\n',
641 '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
642 for example when passing block of text to this function.
644 """
645 # NL tokens are used inside multiline expressions, but also after blank
646 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
647 # We want to group the former case together but split the latter, so we
648 # track parentheses level, similar to the internals of tokenize.
650 # reexported from token on 3.7+
651 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore
652 tokens_by_line: list[list[Any]] = [[]]
653 if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
654 warnings.warn(
655 "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
656 stacklevel=2,
657 )
658 parenlev = 0
659 try:
660 for token in tokenutil.generate_tokens_catch_errors(
661 iter(lines).__next__, extra_errors_to_catch=["expected EOF"]
662 ):
663 tokens_by_line[-1].append(token)
664 if (token.type == NEWLINE) or ((token.type == NL) and (parenlev <= 0)):
665 tokens_by_line.append([])
666 elif token.string in {"(", "[", "{"}:
667 parenlev += 1
668 elif token.string in {")", "]", "}"}:
669 if parenlev > 0:
670 parenlev -= 1
671 except tokenize.TokenError:
672 # Input ended in a multiline string or expression. That's OK for us.
673 pass
675 if not tokens_by_line[-1]:
676 tokens_by_line.pop()
678 return tokens_by_line
681def has_sunken_brackets(tokens: list[tokenize.TokenInfo]):
682 """Check if the depth of brackets in the list of tokens drops below 0"""
683 parenlev = 0
684 for token in tokens:
685 if token.string in {"(", "[", "{"}:
686 parenlev += 1
687 elif token.string in {")", "]", "}"}:
688 parenlev -= 1
689 if parenlev < 0:
690 return True
691 return False
694# Arbitrary limit to prevent getting stuck in infinite loops
695TRANSFORM_LOOP_LIMIT = 500
698class TransformerManager:
699 """Applies various transformations to a cell or code block.
701 The key methods for external use are ``transform_cell()``
702 and ``check_complete()``.
703 """
705 def __init__(self):
706 self.cleanup_transforms = [
707 leading_empty_lines,
708 leading_indent,
709 classic_prompt,
710 ipython_prompt,
711 ]
712 self.line_transforms = [
713 cell_magic,
714 ]
715 self.token_transformers = [
716 MagicAssign,
717 SystemAssign,
718 EscapedCommand,
719 HelpEnd,
720 ]
722 def do_one_token_transform(self, lines):
723 """Find and run the transform earliest in the code.
725 Returns (changed, lines).
727 This method is called repeatedly until changed is False, indicating
728 that all available transformations are complete.
730 The tokens following IPython special syntax might not be valid, so
731 the transformed code is retokenised every time to identify the next
732 piece of special syntax. Hopefully long code cells are mostly valid
733 Python, not using lots of IPython special syntax, so this shouldn't be
734 a performance issue.
735 """
736 tokens_by_line = make_tokens_by_line(lines)
737 candidates = []
738 for transformer_cls in self.token_transformers:
739 transformer = transformer_cls.find(tokens_by_line)
740 if transformer:
741 candidates.append(transformer)
743 if not candidates:
744 # Nothing to transform
745 return False, lines
746 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
747 for transformer in ordered_transformers:
748 try:
749 return True, transformer.transform(lines)
750 except SyntaxError:
751 pass
752 return False, lines
754 def do_token_transforms(self, lines):
755 for _ in range(TRANSFORM_LOOP_LIMIT):
756 changed, lines = self.do_one_token_transform(lines)
757 if not changed:
758 return lines
760 raise RuntimeError(
761 "Input transformation still changing after "
762 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT
763 )
765 def transform_cell(self, cell: str) -> str:
766 """Transforms a cell of input code"""
767 if not cell.endswith("\n"):
768 cell += "\n" # Ensure the cell has a trailing newline
769 lines = cell.splitlines(keepends=True)
770 for transform in self.cleanup_transforms + self.line_transforms:
771 lines = transform(lines)
773 lines = self.do_token_transforms(lines)
774 return "".join(lines)
776 def check_complete(self, cell: str):
777 """Return whether a block of code is ready to execute, or should be continued
779 Parameters
780 ----------
781 cell : string
782 Python input code, which can be multiline.
784 Returns
785 -------
786 status : str
787 One of 'complete', 'incomplete', or 'invalid' if source is not a
788 prefix of valid code.
789 indent_spaces : int or None
790 The number of spaces by which to indent the next line of code. If
791 status is not 'incomplete', this is None.
792 """
793 # Remember if the lines ends in a new line.
794 ends_with_newline = False
795 for character in reversed(cell):
796 if character == "\n":
797 ends_with_newline = True
798 break
799 elif character.strip():
800 break
801 else:
802 continue
804 if not ends_with_newline:
805 # Append an newline for consistent tokenization
806 # See https://bugs.python.org/issue33899
807 cell += "\n"
809 lines = cell.splitlines(keepends=True)
811 if not lines:
812 return "complete", None
814 for line in reversed(lines):
815 if not line.strip():
816 continue
817 elif line.strip("\n").endswith("\\"):
818 return "incomplete", find_last_indent(lines)
819 else:
820 break
822 try:
823 for transform in self.cleanup_transforms:
824 if not getattr(transform, "has_side_effects", False):
825 lines = transform(lines)
826 except SyntaxError:
827 return "invalid", None
829 if lines[0].startswith("%%"):
830 # Special case for cell magics - completion marked by blank line
831 if lines[-1].strip():
832 return "incomplete", find_last_indent(lines)
833 else:
834 return "complete", None
836 try:
837 for transform in self.line_transforms:
838 if not getattr(transform, "has_side_effects", False):
839 lines = transform(lines)
840 lines = self.do_token_transforms(lines)
841 except SyntaxError:
842 return "invalid", None
844 tokens_by_line = make_tokens_by_line(lines)
846 # Bail if we got one line and there are more closing parentheses than
847 # the opening ones
848 if (
849 len(lines) == 1
850 and tokens_by_line
851 and has_sunken_brackets(tokens_by_line[0])
852 ):
853 return "invalid", None
855 if not tokens_by_line:
856 return "incomplete", find_last_indent(lines)
858 if (
859 tokens_by_line[-1][-1].type != tokenize.ENDMARKER
860 and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN
861 ):
862 # We're in a multiline string or expression
863 return "incomplete", find_last_indent(lines)
865 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore
867 # Pop the last line which only contains DEDENTs and ENDMARKER
868 last_token_line = None
869 if {t.type for t in tokens_by_line[-1]} in [
870 {tokenize.DEDENT, tokenize.ENDMARKER},
871 {tokenize.ENDMARKER},
872 ] and len(tokens_by_line) > 1:
873 last_token_line = tokens_by_line.pop()
875 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
876 tokens_by_line[-1].pop()
878 if not tokens_by_line[-1]:
879 return "incomplete", find_last_indent(lines)
881 if tokens_by_line[-1][-1].string == ":":
882 # The last line starts a block (e.g. 'if foo:')
883 ix = 0
884 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
885 ix += 1
887 indent = tokens_by_line[-1][ix].start[1]
888 return "incomplete", indent + 4
890 if tokens_by_line[-1][0].line.endswith("\\"):
891 return "incomplete", None
893 # At this point, our checks think the code is complete (or invalid).
894 # We'll use codeop.compile_command to check this with the real parser
895 try:
896 with warnings.catch_warnings():
897 warnings.simplefilter("error", SyntaxWarning)
898 res = compile_command("".join(lines), symbol="exec")
899 except (
900 SyntaxError,
901 OverflowError,
902 ValueError,
903 TypeError,
904 MemoryError,
905 SyntaxWarning,
906 ):
907 return "invalid", None
908 else:
909 if res is None:
910 return "incomplete", find_last_indent(lines)
912 if last_token_line and last_token_line[0].type == tokenize.DEDENT:
913 if ends_with_newline:
914 return "complete", None
915 return "incomplete", find_last_indent(lines)
917 # If there's a blank line at the end, assume we're ready to execute
918 if not lines[-1].strip():
919 return "complete", None
921 return "complete", None
924def find_last_indent(lines):
925 m = _indent_re.match(lines[-1])
926 if not m:
927 return 0
928 return len(m.group(0).replace("\t", " " * 4))
931class MaybeAsyncCompile(Compile):
932 def __init__(self, extra_flags=0):
933 super().__init__()
934 self.flags |= extra_flags
937class MaybeAsyncCommandCompiler(CommandCompiler):
938 def __init__(self, extra_flags=0):
939 self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)
942_extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
944compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)