Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Input transformer machinery to support IPython special syntax.
3This includes the machinery to recognise and transform ``%magic`` commands,
4``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
6Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7deprecated in 7.0 and removed in 9.0
8"""
10# Copyright (c) IPython Development Team.
11# Distributed under the terms of the Modified BSD License.
13import ast
14from codeop import CommandCompiler, Compile
15import re
16import sys
17import tokenize
18from typing import Any
19import warnings
20from textwrap import dedent
22from IPython.utils import tokenutil
24_indent_re = re.compile(r"^[ \t]+")
27def leading_empty_lines(lines):
28 """Remove leading empty lines
30 If the leading lines are empty or contain only whitespace, they will be
31 removed.
32 """
33 if not lines:
34 return lines
35 for i, line in enumerate(lines):
36 if line and not line.isspace():
37 return lines[i:]
38 return lines
41def leading_indent(lines):
42 """Remove leading indentation.
44 Removes the minimum common leading indentation from all lines.
45 """
46 if not lines:
47 return lines
48 return dedent("".join(lines)).splitlines(keepends=True)
51class PromptStripper:
52 """Remove matching input prompts from a block of input.
54 Parameters
55 ----------
56 prompt_re : regular expression
57 A regular expression matching any input prompt (including continuation,
58 e.g. ``...``)
59 initial_re : regular expression, optional
60 A regular expression matching only the initial prompt, but not continuation.
61 If no initial expression is given, prompt_re will be used everywhere.
62 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
63 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
65 Notes
66 -----
68 If initial_re and prompt_re differ,
69 only initial_re will be tested against the first line.
70 If any prompt is found on the first two lines,
71 prompts will be stripped from the rest of the block.
72 """
74 def __init__(self, prompt_re, initial_re=None, *, doctest=False):
75 self.prompt_re = prompt_re
76 self.initial_re = initial_re or prompt_re
77 self.doctest = doctest
78 if doctest:
79 # Doctest/xdoctest prompts may be indented (e.g. " >>>").
80 # We only treat "..." as a continuation prompt when the same pasted
81 # block contains at least one ">>>" line, to avoid ambiguity with the
82 # Python Ellipsis literal.
83 self._doctest_initial_re = re.compile(r"^\s*>>>")
84 self._doctest_ps1_re = re.compile(r"^\s*>>>[ \t]?")
85 self._doctest_ps2_re = re.compile(r"^\s*\.\.\.[ \t]?")
87 # Very small state machine to detect triple-quoted strings in the
88 # *same* input block (e.g. user typed """ then pasted doctest).
89 # We preserve literal >>> / ... inside triple-quoted strings.
90 self._triple_quote_re = re.compile(r"(?<!\\)(\"\"\"|''')")
92 def _triple_quote_mask(self, lines: list[str]) -> list[bool]:
93 """
94 Return a boolean mask: True if the corresponding line is considered
95 inside a triple-quoted string literal.
97 This is intentionally heuristic (fast + good enough for paste handling).
98 """
99 mask: list[bool] = []
100 in_triple: str | None = None # either ''' or """
101 preserve_prompt = False
102 seen_prompt = False
103 string_prefix_re = re.compile(r"(?i)^[rubf]*$")
105 for line in lines:
106 mask.append(in_triple is not None and preserve_prompt)
107 # Toggle state for each occurrence of """ or ''' in the line.
108 for m in self._triple_quote_re.finditer(line):
109 q = m.group(1)
110 if in_triple is None:
111 in_triple = q
112 before_quote = line[: m.start()]
113 stripped = self._doctest_ps1_re.sub("", before_quote, count=1)
114 stripped = self._doctest_ps2_re.sub("", stripped, count=1)
115 had_prompt = stripped != before_quote
116 prompted_code = had_prompt and seen_prompt
117 preserve_prompt = bool(
118 not prompted_code and string_prefix_re.match(stripped.strip())
119 )
120 mask[-1] = preserve_prompt
121 elif in_triple == q:
122 in_triple = None
123 preserve_prompt = False
124 # else: ignore mismatched triple quote while inside
125 seen_prompt = seen_prompt or bool(self._doctest_initial_re.match(line))
126 return mask
128 def _strip(self, lines):
129 return [self.prompt_re.sub("", l, count=1) for l in lines]
131 def __call__(self, lines):
132 if not lines:
133 return lines
135 if self.doctest:
136 triple_mask = self._triple_quote_mask(lines)
138 # Detect doctest prompts only outside triple-quoted strings.
139 has_doctest_outside = any(
140 (not in_triple) and self._doctest_initial_re.match(l)
141 for l, in_triple in zip(lines, triple_mask)
142 )
143 if not has_doctest_outside:
144 return lines
146 out_lines: list[str] = []
147 stripped_mask: list[bool] = []
149 for l, in_triple in zip(lines, triple_mask):
150 if in_triple:
151 out_lines.append(l)
152 stripped_mask.append(False)
153 continue
155 if self._doctest_ps1_re.match(l):
156 new_l = self._doctest_ps1_re.sub("", l, count=1)
157 elif self._doctest_ps2_re.match(l):
158 new_l = self._doctest_ps2_re.sub("", l, count=1)
159 else:
160 new_l = l
161 out_lines.append(new_l)
162 stripped_mask.append(new_l != l)
164 # Dedent only the non-triple-quoted segments where stripping occurred.
165 dedented: list[str] = []
166 i = 0
167 while i < len(out_lines):
168 j = i
169 in_triple = triple_mask[i]
170 while j < len(out_lines) and triple_mask[j] == in_triple:
171 j += 1
173 segment = out_lines[i:j]
174 seg_stripped = any(stripped_mask[i:j])
176 if (not in_triple) and seg_stripped:
177 dedented.extend(dedent("".join(segment)).splitlines(keepends=True))
178 else:
179 dedented.extend(segment)
181 i = j
183 return dedented
185 if self.initial_re.match(lines[0]) or (
186 len(lines) > 1 and self.prompt_re.match(lines[1])
187 ):
188 return self._strip(lines)
189 return lines
192classic_prompt = PromptStripper(
193 prompt_re=re.compile(r"^(>>>|\.\.\.)( |$)"),
194 initial_re=re.compile(r"^>>>( |$)"),
195 doctest=True,
196)
198ipython_prompt = PromptStripper(
199 re.compile(
200 r"""
201 ^( # Match from the beginning of a line, either:
203 # 1. First-line prompt:
204 ((\[nav\]|\[ins\])?\ )? # Vi editing mode prompt, if it's there
205 In\ # The 'In' of the prompt, with a space
206 \[\d+\]: # Command index, as displayed in the prompt
207 \ # With a mandatory trailing space
209 | # ... or ...
211 # 2. The three dots of the multiline prompt
212 \s* # All leading whitespace characters
213 \.{3,}: # The three (or more) dots
214 \ ? # With an optional trailing space
216 )
217 """,
218 re.VERBOSE,
219 )
220)
223def cell_magic(lines):
224 if not lines or not lines[0].startswith("%%"):
225 return lines
226 if re.match(r"%%\w+\?", lines[0]):
227 # This case will be handled by help_end
228 return lines
229 magic_name, _, first_line = lines[0][2:].rstrip().partition(" ")
230 body = "".join(lines[1:])
231 return [
232 "get_ipython().run_cell_magic(%r, %r, %r)\n" % (magic_name, first_line, body)
233 ]
236def _find_assign_op(token_line) -> int | None:
237 """Get the index of the first assignment in the line ('=' not inside brackets)
239 Note: We don't try to support multiple special assignment (a = b = %foo)
240 """
241 paren_level = 0
242 for i, ti in enumerate(token_line):
243 s = ti.string
244 if s == "=" and paren_level == 0:
245 return i
246 if s in {"(", "[", "{"}:
247 paren_level += 1
248 elif s in {")", "]", "}"}:
249 if paren_level > 0:
250 paren_level -= 1
251 return None
254def find_end_of_continued_line(lines, start_line: int):
255 """Find the last line of a line explicitly extended using backslashes.
257 Uses 0-indexed line numbers.
258 """
259 end_line = start_line
260 while lines[end_line].endswith("\\\n"):
261 end_line += 1
262 if end_line >= len(lines):
263 break
264 return end_line
267def assemble_continued_line(lines, start: tuple[int, int], end_line: int):
268 r"""Assemble a single line from multiple continued line pieces
270 Continued lines are lines ending in ``\``, and the line following the last
271 ``\`` in the block.
273 For example, this code continues over multiple lines::
275 if (assign_ix is not None) \
276 and (len(line) >= assign_ix + 2) \
277 and (line[assign_ix+1].string == '%') \
278 and (line[assign_ix+2].type == tokenize.NAME):
280 This statement contains four continued line pieces.
281 Assembling these pieces into a single line would give::
283 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
285 This uses 0-indexed line numbers. *start* is (lineno, colno).
287 Used to allow ``%magic`` and ``!system`` commands to be continued over
288 multiple lines.
289 """
290 parts = [lines[start[0]][start[1] :]] + lines[start[0] + 1 : end_line + 1]
291 return " ".join(
292 [p.rstrip()[:-1] for p in parts[:-1]] # Strip backslash+newline
293 + [parts[-1].rstrip()]
294 ) # Strip newline from last line
297class TokenTransformBase:
298 """Base class for transformations which examine tokens.
300 Special syntax should not be transformed when it occurs inside strings or
301 comments. This is hard to reliably avoid with regexes. The solution is to
302 tokenise the code as Python, and recognise the special syntax in the tokens.
304 IPython's special syntax is not valid Python syntax, so tokenising may go
305 wrong after the special syntax starts. These classes therefore find and
306 transform *one* instance of special syntax at a time into regular Python
307 syntax. After each transformation, tokens are regenerated to find the next
308 piece of special syntax.
310 Subclasses need to implement one class method (find)
311 and one regular method (transform).
313 The priority attribute can select which transformation to apply if multiple
314 transformers match in the same place. Lower numbers have higher priority.
315 This allows "%magic?" to be turned into a help call rather than a magic call.
316 """
318 # Lower numbers -> higher priority (for matches in the same location)
319 priority = 10
321 def sortby(self):
322 return self.start_line, self.start_col, self.priority
324 def __init__(self, start):
325 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
326 self.start_col = start[1]
328 @classmethod
329 def find(cls, tokens_by_line):
330 """Find one instance of special syntax in the provided tokens.
332 Tokens are grouped into logical lines for convenience,
333 so it is easy to e.g. look at the first token of each line.
334 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
336 This should return an instance of its class, pointing to the start
337 position it has found, or None if it found no match.
338 """
339 raise NotImplementedError
341 def transform(self, lines: list[str]):
342 """Transform one instance of special syntax found by ``find()``
344 Takes a list of strings representing physical lines,
345 returns a similar list of transformed lines.
346 """
347 raise NotImplementedError
350class MagicAssign(TokenTransformBase):
351 """Transformer for assignments from magics (a = %foo)"""
353 @classmethod
354 def find(cls, tokens_by_line):
355 """Find the first magic assignment (a = %foo) in the cell."""
356 for line in tokens_by_line:
357 assign_ix = _find_assign_op(line)
358 if (
359 (assign_ix is not None)
360 and (len(line) >= assign_ix + 2)
361 and (line[assign_ix + 1].string == "%")
362 and (line[assign_ix + 2].type == tokenize.NAME)
363 ):
364 return cls(line[assign_ix + 1].start)
366 def transform(self, lines: list[str]):
367 """Transform a magic assignment found by the ``find()`` classmethod."""
368 start_line, start_col = self.start_line, self.start_col
369 lhs = lines[start_line][:start_col]
370 end_line = find_end_of_continued_line(lines, start_line)
371 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
372 assert rhs.startswith("%"), rhs
373 magic_name, _, args = rhs[1:].partition(" ")
375 lines_before = lines[:start_line]
376 call = f"get_ipython().run_line_magic({magic_name!r}, {args!r})"
377 new_line = lhs + call + "\n"
378 lines_after = lines[end_line + 1 :]
380 return lines_before + [new_line] + lines_after
383class SystemAssign(TokenTransformBase):
384 """Transformer for assignments from system commands (a = !foo)"""
386 @classmethod
387 def find_pre_312(cls, tokens_by_line):
388 for line in tokens_by_line:
389 assign_ix = _find_assign_op(line)
390 if (
391 (assign_ix is not None)
392 and not line[assign_ix].line.strip().startswith("=")
393 and (len(line) >= assign_ix + 2)
394 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN)
395 ):
396 ix = assign_ix + 1
398 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
399 if line[ix].string == "!":
400 return cls(line[ix].start)
401 elif not line[ix].string.isspace():
402 break
403 ix += 1
405 @classmethod
406 def find_post_312(cls, tokens_by_line):
407 for line in tokens_by_line:
408 assign_ix = _find_assign_op(line)
409 if (
410 (assign_ix is not None)
411 and not line[assign_ix].line.strip().startswith("=")
412 and (len(line) >= assign_ix + 2)
413 and (line[assign_ix + 1].type == tokenize.OP)
414 and (line[assign_ix + 1].string == "!")
415 ):
416 return cls(line[assign_ix + 1].start)
418 @classmethod
419 def find(cls, tokens_by_line):
420 """Find the first system assignment (a = !foo) in the cell."""
421 if sys.version_info < (3, 12):
422 return cls.find_pre_312(tokens_by_line)
423 return cls.find_post_312(tokens_by_line)
425 def transform(self, lines: list[str]):
426 """Transform a system assignment found by the ``find()`` classmethod."""
427 start_line, start_col = self.start_line, self.start_col
429 lhs = lines[start_line][:start_col]
430 end_line = find_end_of_continued_line(lines, start_line)
431 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
432 assert rhs.startswith("!"), rhs
433 cmd = rhs[1:]
435 lines_before = lines[:start_line]
436 call = f"get_ipython().getoutput({cmd!r})"
437 new_line = lhs + call + "\n"
438 lines_after = lines[end_line + 1 :]
440 return lines_before + [new_line] + lines_after
443# The escape sequences that define the syntax transformations IPython will
444# apply to user input. These can NOT be just changed here: many regular
445# expressions and other parts of the code may use their hardcoded values, and
446# for all intents and purposes they constitute the 'IPython syntax', so they
447# should be considered fixed.
449ESC_SHELL = "!" # Send line to underlying system shell
450ESC_SH_CAP = "!!" # Send line to system shell and capture output
451ESC_HELP = "?" # Find information about object
452ESC_HELP2 = "??" # Find extra-detailed information about object
453ESC_MAGIC = "%" # Call magic function
454ESC_MAGIC2 = "%%" # Call cell-magic function
455ESC_QUOTE = "," # Split args on whitespace, quote each as string and call
456ESC_QUOTE2 = ";" # Quote all args as a single string, call
457ESC_PAREN = "/" # Call first argument with rest of line as arguments
459ESCAPE_SINGLES = {"!", "?", "%", ",", ";", "/"}
460ESCAPE_DOUBLES = {"!!", "??"} # %% (cell magic) is handled separately
463def _make_help_call(target, esc):
464 """Prepares a pinfo(2)/psearch call from a target name and the escape
465 (i.e. ? or ??)"""
466 method = "pinfo2" if esc == "??" else "psearch" if "*" in target else "pinfo"
467 arg = " ".join([method, target])
468 # Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
469 t_magic_name, _, t_magic_arg_s = arg.partition(" ")
470 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
471 return "get_ipython().run_line_magic({!r}, {!r})".format(
472 t_magic_name, t_magic_arg_s
473 )
476def _tr_help(content):
477 """Translate lines escaped with: ?
479 A naked help line should fire the intro help screen (shell.show_usage())
480 """
481 if not content:
482 return "get_ipython().show_usage()"
484 return _make_help_call(content, "?")
487def _tr_help2(content):
488 """Translate lines escaped with: ??
490 A naked help line should fire the intro help screen (shell.show_usage())
491 """
492 if not content:
493 return "get_ipython().show_usage()"
495 return _make_help_call(content, "??")
498def _tr_magic(content):
499 "Translate lines escaped with a percent sign: %"
500 name, _, args = content.partition(" ")
501 return "get_ipython().run_line_magic({!r}, {!r})".format(name, args)
504def _tr_quote(content):
505 "Translate lines escaped with a comma: ,"
506 name, _, args = content.partition(" ")
507 return '{}("{}")'.format(name, '", "'.join(args.split()))
510def _tr_quote2(content):
511 "Translate lines escaped with a semicolon: ;"
512 name, _, args = content.partition(" ")
513 return '{}("{}")'.format(name, args)
516def _tr_paren(content):
517 "Translate lines escaped with a slash: /"
518 name, _, args = content.partition(" ")
519 if name == "":
520 raise SyntaxError(f'"{ESC_SHELL}" must be followed by a callable name')
522 return "{}({})".format(name, ", ".join(args.split()))
525tr = {
526 ESC_SHELL: "get_ipython().system({!r})".format,
527 ESC_SH_CAP: "get_ipython().getoutput({!r})".format,
528 ESC_HELP: _tr_help,
529 ESC_HELP2: _tr_help2,
530 ESC_MAGIC: _tr_magic,
531 ESC_QUOTE: _tr_quote,
532 ESC_QUOTE2: _tr_quote2,
533 ESC_PAREN: _tr_paren,
534}
537class EscapedCommand(TokenTransformBase):
538 """Transformer for escaped commands like %foo, !foo, or /foo"""
540 @classmethod
541 def find(cls, tokens_by_line):
542 """Find the first escaped command (%foo, !foo, etc.) in the cell."""
543 for line in tokens_by_line:
544 if not line:
545 continue
546 ix = 0
547 ll = len(line)
548 while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
549 ix += 1
550 if ix >= ll:
551 continue
552 if line[ix].string in ESCAPE_SINGLES:
553 return cls(line[ix].start)
555 def transform(self, lines):
556 """Transform an escaped line found by the ``find()`` classmethod."""
557 start_line, start_col = self.start_line, self.start_col
559 indent = lines[start_line][:start_col]
560 end_line = find_end_of_continued_line(lines, start_line)
561 line = assemble_continued_line(lines, (start_line, start_col), end_line)
563 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
564 escape, content = line[:2], line[2:]
565 else:
566 escape, content = line[:1], line[1:]
568 if escape in tr:
569 call = tr[escape](content)
570 else:
571 call = ""
573 lines_before = lines[:start_line]
574 new_line = indent + call + "\n"
575 lines_after = lines[end_line + 1 :]
577 return lines_before + [new_line] + lines_after
580_help_end_re = re.compile(
581 r"""(%{0,2}
582 (?!\d)[\w*]+ # Variable name
583 (\.(?!\d)[\w*]+|\[-?[0-9]+\])* # .etc.etc or [0], we only support literal integers.
584 )
585 (\?\??)$ # ? or ??
586 """,
587 re.VERBOSE,
588)
591class HelpEnd(TokenTransformBase):
592 """Transformer for help syntax: obj? and obj??"""
594 # This needs to be higher priority (lower number) than EscapedCommand so
595 # that inspecting magics (%foo?) works.
596 priority = 5
598 def __init__(self, start, q_locn):
599 super().__init__(start)
600 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
601 self.q_col = q_locn[1]
603 @classmethod
604 def find(cls, tokens_by_line):
605 """Find the first help command (foo?) in the cell."""
606 for line in tokens_by_line:
607 # Last token is NEWLINE; look at last but one
608 if len(line) > 2 and line[-2].string == "?":
609 # Find the first token that's not INDENT/DEDENT
610 ix = 0
611 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
612 ix += 1
613 return cls(line[ix].start, line[-2].start)
615 def transform(self, lines):
616 """Transform a help command found by the ``find()`` classmethod."""
618 piece = "".join(lines[self.start_line : self.q_line + 1])
619 indent, content = piece[: self.start_col], piece[self.start_col :]
620 lines_before = lines[: self.start_line]
621 lines_after = lines[self.q_line + 1 :]
623 m = _help_end_re.search(content)
624 if not m:
625 raise SyntaxError(content)
626 assert m is not None, content
627 target = m.group(1)
628 esc = m.group(3)
630 call = _make_help_call(target, esc)
631 new_line = indent + call + "\n"
633 return lines_before + [new_line] + lines_after
636def make_tokens_by_line(lines: list[str]):
637 """Tokenize a series of lines and group tokens by line.
639 The tokens for a multiline Python string or expression are grouped as one
640 line. All lines except the last lines should keep their line ending ('\\n',
641 '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
642 for example when passing block of text to this function.
644 """
645 # NL tokens are used inside multiline expressions, but also after blank
646 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
647 # We want to group the former case together but split the latter, so we
648 # track parentheses level, similar to the internals of tokenize.
650 # reexported from token on 3.7+
651 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore
652 tokens_by_line: list[list[Any]] = [[]]
653 if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
654 warnings.warn(
655 "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
656 stacklevel=2,
657 )
658 parenlev = 0
659 try:
660 for token in tokenutil.generate_tokens_catch_errors(
661 iter(lines).__next__, extra_errors_to_catch=["expected EOF"]
662 ):
663 tokens_by_line[-1].append(token)
664 if (token.type == NEWLINE) or ((token.type == NL) and (parenlev <= 0)):
665 tokens_by_line.append([])
666 elif token.type == tokenize.OP and token.string in {"(", "[", "{"}:
667 # Only OP tokens are real brackets; e.g. on Python 3.12+
668 # FSTRING_MIDDLE tokens may contain bracket characters as
669 # literal text (refs #15436).
670 parenlev += 1
671 elif token.type == tokenize.OP and token.string in {")", "]", "}"}:
672 if parenlev > 0:
673 parenlev -= 1
674 except tokenize.TokenError:
675 # Input ended in a multiline string or expression. That's OK for us.
676 pass
678 if not tokens_by_line[-1]:
679 tokens_by_line.pop()
681 return tokens_by_line
684def has_sunken_brackets(tokens: list[tokenize.TokenInfo]):
685 """Check if the depth of brackets in the list of tokens drops below 0"""
686 parenlev = 0
687 for token in tokens:
688 # Only OP tokens are real brackets; e.g. on Python 3.12+
689 # FSTRING_MIDDLE tokens may contain bracket characters as
690 # literal text (refs #15436).
691 if token.type == tokenize.OP and token.string in {"(", "[", "{"}:
692 parenlev += 1
693 elif token.type == tokenize.OP and token.string in {")", "]", "}"}:
694 parenlev -= 1
695 if parenlev < 0:
696 return True
697 return False
700# Arbitrary limit to prevent getting stuck in infinite loops
701TRANSFORM_LOOP_LIMIT = 500
704class TransformerManager:
705 """Applies various transformations to a cell or code block.
707 The key methods for external use are ``transform_cell()``
708 and ``check_complete()``.
709 """
711 def __init__(self):
712 self.cleanup_transforms = [
713 leading_empty_lines,
714 leading_indent,
715 classic_prompt,
716 ipython_prompt,
717 ]
718 self.line_transforms = [
719 cell_magic,
720 ]
721 self.token_transformers = [
722 MagicAssign,
723 SystemAssign,
724 EscapedCommand,
725 HelpEnd,
726 ]
728 def do_one_token_transform(self, lines):
729 """Find and run the transform earliest in the code.
731 Returns (changed, lines).
733 This method is called repeatedly until changed is False, indicating
734 that all available transformations are complete.
736 The tokens following IPython special syntax might not be valid, so
737 the transformed code is retokenised every time to identify the next
738 piece of special syntax. Hopefully long code cells are mostly valid
739 Python, not using lots of IPython special syntax, so this shouldn't be
740 a performance issue.
741 """
742 tokens_by_line = make_tokens_by_line(lines)
743 candidates = []
744 for transformer_cls in self.token_transformers:
745 transformer = transformer_cls.find(tokens_by_line)
746 if transformer:
747 candidates.append(transformer)
749 if not candidates:
750 # Nothing to transform
751 return False, lines
752 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
753 for transformer in ordered_transformers:
754 try:
755 return True, transformer.transform(lines)
756 except SyntaxError:
757 pass
758 return False, lines
760 def do_token_transforms(self, lines):
761 for _ in range(TRANSFORM_LOOP_LIMIT):
762 changed, lines = self.do_one_token_transform(lines)
763 if not changed:
764 return lines
766 raise RuntimeError(
767 "Input transformation still changing after "
768 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT
769 )
771 def transform_cell(self, cell: str) -> str:
772 """Transforms a cell of input code"""
773 if not cell.endswith("\n"):
774 cell += "\n" # Ensure the cell has a trailing newline
775 lines = cell.splitlines(keepends=True)
776 for transform in self.cleanup_transforms + self.line_transforms:
777 lines = transform(lines)
779 lines = self.do_token_transforms(lines)
780 return "".join(lines)
782 def check_complete(self, cell: str):
783 """Return whether a block of code is ready to execute, or should be continued
785 Parameters
786 ----------
787 cell : string
788 Python input code, which can be multiline.
790 Returns
791 -------
792 status : str
793 One of 'complete', 'incomplete', or 'invalid' if source is not a
794 prefix of valid code.
795 indent_spaces : int or None
796 The number of spaces by which to indent the next line of code. If
797 status is not 'incomplete', this is None.
798 """
799 # Remember if the lines ends in a new line.
800 ends_with_newline = False
801 for character in reversed(cell):
802 if character == "\n":
803 ends_with_newline = True
804 break
805 elif character.strip():
806 break
807 else:
808 continue
810 if not ends_with_newline:
811 # Append an newline for consistent tokenization
812 # See https://bugs.python.org/issue33899
813 cell += "\n"
815 lines = cell.splitlines(keepends=True)
817 if not lines:
818 return "complete", None
820 for line in reversed(lines):
821 if not line.strip():
822 continue
823 elif line.strip("\n").endswith("\\"):
824 return "incomplete", find_last_indent(lines)
825 else:
826 break
828 try:
829 for transform in self.cleanup_transforms:
830 if not getattr(transform, "has_side_effects", False):
831 lines = transform(lines)
832 except SyntaxError:
833 return "invalid", None
835 if lines[0].startswith("%%"):
836 # Special case for cell magics - completion marked by blank line
837 if lines[-1].strip():
838 return "incomplete", find_last_indent(lines)
839 else:
840 return "complete", None
842 try:
843 for transform in self.line_transforms:
844 if not getattr(transform, "has_side_effects", False):
845 lines = transform(lines)
846 lines = self.do_token_transforms(lines)
847 except SyntaxError:
848 return "invalid", None
850 tokens_by_line = make_tokens_by_line(lines)
852 # Bail if we got one line and there are more closing parentheses than
853 # the opening ones
854 if (
855 len(lines) == 1
856 and tokens_by_line
857 and has_sunken_brackets(tokens_by_line[0])
858 ):
859 return "invalid", None
861 if not tokens_by_line:
862 return "incomplete", find_last_indent(lines)
864 if (
865 tokens_by_line[-1][-1].type != tokenize.ENDMARKER
866 and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN
867 ):
868 # We're in a multiline string or expression
869 return "incomplete", find_last_indent(lines)
871 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore
873 # Pop the last line which only contains DEDENTs and ENDMARKER
874 last_token_line = None
875 if {t.type for t in tokens_by_line[-1]} in [
876 {tokenize.DEDENT, tokenize.ENDMARKER},
877 {tokenize.ENDMARKER},
878 ] and len(tokens_by_line) > 1:
879 last_token_line = tokens_by_line.pop()
881 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
882 tokens_by_line[-1].pop()
884 if not tokens_by_line[-1]:
885 return "incomplete", find_last_indent(lines)
887 if tokens_by_line[-1][-1].string == ":":
888 # The last line starts a block (e.g. 'if foo:')
889 ix = 0
890 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
891 ix += 1
893 indent = tokens_by_line[-1][ix].start[1]
894 return "incomplete", indent + 4
896 if tokens_by_line[-1][0].line.endswith("\\"):
897 return "incomplete", None
899 # At this point, our checks think the code is complete (or invalid).
900 # We'll use codeop.compile_command to check this with the real parser
901 try:
902 with warnings.catch_warnings():
903 warnings.simplefilter("error", SyntaxWarning)
904 res = compile_command("".join(lines), symbol="exec")
905 except (
906 SyntaxError,
907 OverflowError,
908 ValueError,
909 TypeError,
910 MemoryError,
911 SyntaxWarning,
912 ):
913 return "invalid", None
914 else:
915 if res is None:
916 return "incomplete", find_last_indent(lines)
918 if last_token_line and last_token_line[0].type == tokenize.DEDENT:
919 if ends_with_newline:
920 return "complete", None
921 return "incomplete", find_last_indent(lines)
923 # If there's a blank line at the end, assume we're ready to execute
924 if not lines[-1].strip():
925 return "complete", None
927 return "complete", None
930def find_last_indent(lines):
931 m = _indent_re.match(lines[-1])
932 if not m:
933 return 0
934 return len(m.group(0).replace("\t", " " * 4))
937class MaybeAsyncCompile(Compile):
938 def __init__(self, extra_flags=0):
939 super().__init__()
940 self.flags |= extra_flags
943class MaybeAsyncCommandCompiler(CommandCompiler):
944 def __init__(self, extra_flags=0):
945 self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)
948_extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
950compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)