Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.9/dist-packages/IPython/core/inputsplitter.py: 58%
317 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:05 +0000
1"""DEPRECATED: Input handling and transformation machinery.
3This module was deprecated in IPython 7.0, in favour of inputtransformer2.
5The first class in this module, :class:`InputSplitter`, is designed to tell when
6input from a line-oriented frontend is complete and should be executed, and when
7the user should be prompted for another line of code instead. The name 'input
8splitter' is largely for historical reasons.
10A companion, :class:`IPythonInputSplitter`, provides the same functionality but
11with full support for the extended IPython syntax (magics, system calls, etc).
12The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`.
13:class:`IPythonInputSplitter` feeds the raw code to the transformers in order
14and stores the results.
16For more details, see the class docstrings below.
17"""
19from warnings import warn
21warn('IPython.core.inputsplitter is deprecated since IPython 7 in favor of `IPython.core.inputtransformer2`',
22 DeprecationWarning)
24# Copyright (c) IPython Development Team.
25# Distributed under the terms of the Modified BSD License.
26import ast
27import codeop
28import io
29import re
30import sys
31import tokenize
32import warnings
34from typing import List
36from IPython.core.inputtransformer import (leading_indent,
37 classic_prompt,
38 ipy_prompt,
39 cellmagic,
40 assemble_logical_lines,
41 help_end,
42 escaped_commands,
43 assign_from_magic,
44 assign_from_system,
45 assemble_python_lines,
46 )
47from IPython.utils import tokenutil
49# These are available in this module for backwards compatibility.
50from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
51 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
52 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
54#-----------------------------------------------------------------------------
55# Utilities
56#-----------------------------------------------------------------------------
58# FIXME: These are general-purpose utilities that later can be moved to the
59# general ward. Kept here for now because we're being very strict about test
60# coverage with this code, and this lets us ensure that we keep 100% coverage
61# while developing.
63# compiled regexps for autoindent management
64dedent_re = re.compile('|'.join([
65 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
66 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
67 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
68 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
69 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
70 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
71 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
72]))
73ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
75# regexp to match pure comment lines so we don't accidentally insert 'if 1:'
76# before pure comments
77comment_line_re = re.compile(r'^\s*\#')
80def num_ini_spaces(s):
81 """Return the number of initial spaces in a string.
83 Note that tabs are counted as a single space. For now, we do *not* support
84 mixing of tabs and spaces in the user's input.
86 Parameters
87 ----------
88 s : string
90 Returns
91 -------
92 n : int
93 """
95 ini_spaces = ini_spaces_re.match(s)
96 if ini_spaces:
97 return ini_spaces.end()
98 else:
99 return 0
101# Fake token types for partial_tokenize:
102INCOMPLETE_STRING = tokenize.N_TOKENS
103IN_MULTILINE_STATEMENT = tokenize.N_TOKENS + 1
105# The 2 classes below have the same API as TokenInfo, but don't try to look up
106# a token type name that they won't find.
107class IncompleteString:
108 type = exact_type = INCOMPLETE_STRING
109 def __init__(self, s, start, end, line):
110 self.s = s
111 self.start = start
112 self.end = end
113 self.line = line
115class InMultilineStatement:
116 type = exact_type = IN_MULTILINE_STATEMENT
117 def __init__(self, pos, line):
118 self.s = ''
119 self.start = self.end = pos
120 self.line = line
122def partial_tokens(s):
123 """Iterate over tokens from a possibly-incomplete string of code.
125 This adds two special token types: INCOMPLETE_STRING and
126 IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and
127 represent the two main ways for code to be incomplete.
128 """
129 readline = io.StringIO(s).readline
130 token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '')
131 try:
132 for token in tokenutil.generate_tokens_catch_errors(readline):
133 yield token
134 except tokenize.TokenError as e:
135 # catch EOF error
136 lines = s.splitlines(keepends=True)
137 end = len(lines), len(lines[-1])
138 if 'multi-line string' in e.args[0]:
139 l, c = start = token.end
140 s = lines[l-1][c:] + ''.join(lines[l:])
141 yield IncompleteString(s, start, end, lines[-1])
142 elif 'multi-line statement' in e.args[0]:
143 yield InMultilineStatement(end, lines[-1])
144 else:
145 raise
147def find_next_indent(code):
148 """Find the number of spaces for the next line of indentation"""
149 tokens = list(partial_tokens(code))
150 if tokens[-1].type == tokenize.ENDMARKER:
151 tokens.pop()
152 if not tokens:
153 return 0
155 while tokens[-1].type in {
156 tokenize.DEDENT,
157 tokenize.NEWLINE,
158 tokenize.COMMENT,
159 tokenize.ERRORTOKEN,
160 }:
161 tokens.pop()
163 # Starting in Python 3.12, the tokenize module adds implicit newlines at the end
164 # of input. We need to remove those if we're in a multiline statement
165 if tokens[-1].type == IN_MULTILINE_STATEMENT:
166 while tokens[-2].type in {tokenize.NL}:
167 tokens.pop(-2)
170 if tokens[-1].type == INCOMPLETE_STRING:
171 # Inside a multiline string
172 return 0
174 # Find the indents used before
175 prev_indents = [0]
176 def _add_indent(n):
177 if n != prev_indents[-1]:
178 prev_indents.append(n)
180 tokiter = iter(tokens)
181 for tok in tokiter:
182 if tok.type in {tokenize.INDENT, tokenize.DEDENT}:
183 _add_indent(tok.end[1])
184 elif (tok.type == tokenize.NL):
185 try:
186 _add_indent(next(tokiter).start[1])
187 except StopIteration:
188 break
190 last_indent = prev_indents.pop()
192 # If we've just opened a multiline statement (e.g. 'a = ['), indent more
193 if tokens[-1].type == IN_MULTILINE_STATEMENT:
194 if tokens[-2].exact_type in {tokenize.LPAR, tokenize.LSQB, tokenize.LBRACE}:
195 return last_indent + 4
196 return last_indent
198 if tokens[-1].exact_type == tokenize.COLON:
199 # Line ends with colon - indent
200 return last_indent + 4
202 if last_indent:
203 # Examine the last line for dedent cues - statements like return or
204 # raise which normally end a block of code.
205 last_line_starts = 0
206 for i, tok in enumerate(tokens):
207 if tok.type == tokenize.NEWLINE:
208 last_line_starts = i + 1
210 last_line_tokens = tokens[last_line_starts:]
211 names = [t.string for t in last_line_tokens if t.type == tokenize.NAME]
212 if names and names[0] in {'raise', 'return', 'pass', 'break', 'continue'}:
213 # Find the most recent indentation less than the current level
214 for indent in reversed(prev_indents):
215 if indent < last_indent:
216 return indent
218 return last_indent
221def last_blank(src):
222 """Determine if the input source ends in a blank.
224 A blank is either a newline or a line consisting of whitespace.
226 Parameters
227 ----------
228 src : string
229 A single or multiline string.
230 """
231 if not src: return False
232 ll = src.splitlines()[-1]
233 return (ll == '') or ll.isspace()
236last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
237last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
239def last_two_blanks(src):
240 """Determine if the input source ends in two blanks.
242 A blank is either a newline or a line consisting of whitespace.
244 Parameters
245 ----------
246 src : string
247 A single or multiline string.
248 """
249 if not src: return False
250 # The logic here is tricky: I couldn't get a regexp to work and pass all
251 # the tests, so I took a different approach: split the source by lines,
252 # grab the last two and prepend '###\n' as a stand-in for whatever was in
253 # the body before the last two lines. Then, with that structure, it's
254 # possible to analyze with two regexps. Not the most elegant solution, but
255 # it works. If anyone tries to change this logic, make sure to validate
256 # the whole test suite first!
257 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
258 return (bool(last_two_blanks_re.match(new_src)) or
259 bool(last_two_blanks_re2.match(new_src)) )
262def remove_comments(src):
263 """Remove all comments from input source.
265 Note: comments are NOT recognized inside of strings!
267 Parameters
268 ----------
269 src : string
270 A single or multiline input string.
272 Returns
273 -------
274 String with all Python comments removed.
275 """
277 return re.sub('#.*', '', src)
280def get_input_encoding():
281 """Return the default standard input encoding.
283 If sys.stdin has no encoding, 'ascii' is returned."""
284 # There are strange environments for which sys.stdin.encoding is None. We
285 # ensure that a valid encoding is returned.
286 encoding = getattr(sys.stdin, 'encoding', None)
287 if encoding is None:
288 encoding = 'ascii'
289 return encoding
291#-----------------------------------------------------------------------------
292# Classes and functions for normal Python syntax handling
293#-----------------------------------------------------------------------------
295class InputSplitter(object):
296 r"""An object that can accumulate lines of Python source before execution.
298 This object is designed to be fed python source line-by-line, using
299 :meth:`push`. It will return on each push whether the currently pushed
300 code could be executed already. In addition, it provides a method called
301 :meth:`push_accepts_more` that can be used to query whether more input
302 can be pushed into a single interactive block.
304 This is a simple example of how an interactive terminal-based client can use
305 this tool::
307 isp = InputSplitter()
308 while isp.push_accepts_more():
309 indent = ' '*isp.indent_spaces
310 prompt = '>>> ' + indent
311 line = indent + raw_input(prompt)
312 isp.push(line)
313 print 'Input source was:\n', isp.source_reset(),
314 """
315 # A cache for storing the current indentation
316 # The first value stores the most recently processed source input
317 # The second value is the number of spaces for the current indentation
318 # If self.source matches the first value, the second value is a valid
319 # current indentation. Otherwise, the cache is invalid and the indentation
320 # must be recalculated.
321 _indent_spaces_cache = None, None
322 # String, indicating the default input encoding. It is computed by default
323 # at initialization time via get_input_encoding(), but it can be reset by a
324 # client with specific knowledge of the encoding.
325 encoding = ''
326 # String where the current full source input is stored, properly encoded.
327 # Reading this attribute is the normal way of querying the currently pushed
328 # source code, that has been properly encoded.
329 source = ''
330 # Code object corresponding to the current source. It is automatically
331 # synced to the source, so it can be queried at any time to obtain the code
332 # object; it will be None if the source doesn't compile to valid Python.
333 code = None
335 # Private attributes
337 # List with lines of input accumulated so far
338 _buffer: List[str]
339 # Command compiler
340 _compile: codeop.CommandCompiler
341 # Boolean indicating whether the current block is complete
342 _is_complete = None
343 # Boolean indicating whether the current block has an unrecoverable syntax error
344 _is_invalid = False
346 def __init__(self) -> None:
347 """Create a new InputSplitter instance."""
348 self._buffer = []
349 self._compile = codeop.CommandCompiler()
350 self.encoding = get_input_encoding()
352 def reset(self):
353 """Reset the input buffer and associated state."""
354 self._buffer[:] = []
355 self.source = ''
356 self.code = None
357 self._is_complete = False
358 self._is_invalid = False
360 def source_reset(self):
361 """Return the input source and perform a full reset.
362 """
363 out = self.source
364 self.reset()
365 return out
367 def check_complete(self, source):
368 """Return whether a block of code is ready to execute, or should be continued
370 This is a non-stateful API, and will reset the state of this InputSplitter.
372 Parameters
373 ----------
374 source : string
375 Python input code, which can be multiline.
377 Returns
378 -------
379 status : str
380 One of 'complete', 'incomplete', or 'invalid' if source is not a
381 prefix of valid code.
382 indent_spaces : int or None
383 The number of spaces by which to indent the next line of code. If
384 status is not 'incomplete', this is None.
385 """
386 self.reset()
387 try:
388 self.push(source)
389 except SyntaxError:
390 # Transformers in IPythonInputSplitter can raise SyntaxError,
391 # which push() will not catch.
392 return 'invalid', None
393 else:
394 if self._is_invalid:
395 return 'invalid', None
396 elif self.push_accepts_more():
397 return 'incomplete', self.get_indent_spaces()
398 else:
399 return 'complete', None
400 finally:
401 self.reset()
403 def push(self, lines:str) -> bool:
404 """Push one or more lines of input.
406 This stores the given lines and returns a status code indicating
407 whether the code forms a complete Python block or not.
409 Any exceptions generated in compilation are swallowed, but if an
410 exception was produced, the method returns True.
412 Parameters
413 ----------
414 lines : string
415 One or more lines of Python input.
417 Returns
418 -------
419 is_complete : boolean
420 True if the current input source (the result of the current input
421 plus prior inputs) forms a complete Python execution block. Note that
422 this value is also stored as a private attribute (``_is_complete``), so it
423 can be queried at any time.
424 """
425 assert isinstance(lines, str)
426 self._store(lines)
427 source = self.source
429 # Before calling _compile(), reset the code object to None so that if an
430 # exception is raised in compilation, we don't mislead by having
431 # inconsistent code/source attributes.
432 self.code, self._is_complete = None, None
433 self._is_invalid = False
435 # Honor termination lines properly
436 if source.endswith('\\\n'):
437 return False
439 try:
440 with warnings.catch_warnings():
441 warnings.simplefilter('error', SyntaxWarning)
442 self.code = self._compile(source, symbol="exec")
443 # Invalid syntax can produce any of a number of different errors from
444 # inside the compiler, so we have to catch them all. Syntax errors
445 # immediately produce a 'ready' block, so the invalid Python can be
446 # sent to the kernel for evaluation with possible ipython
447 # special-syntax conversion.
448 except (SyntaxError, OverflowError, ValueError, TypeError,
449 MemoryError, SyntaxWarning):
450 self._is_complete = True
451 self._is_invalid = True
452 else:
453 # Compilation didn't produce any exceptions (though it may not have
454 # given a complete code object)
455 self._is_complete = self.code is not None
457 return self._is_complete
459 def push_accepts_more(self):
460 """Return whether a block of interactive input can accept more input.
462 This method is meant to be used by line-oriented frontends, who need to
463 guess whether a block is complete or not based solely on prior and
464 current input lines. The InputSplitter considers it has a complete
465 interactive block and will not accept more input when either:
467 * A SyntaxError is raised
469 * The code is complete and consists of a single line or a single
470 non-compound statement
472 * The code is complete and has a blank line at the end
474 If the current input produces a syntax error, this method immediately
475 returns False but does *not* raise the syntax error exception, as
476 typically clients will want to send invalid syntax to an execution
477 backend which might convert the invalid syntax into valid Python via
478 one of the dynamic IPython mechanisms.
479 """
481 # With incomplete input, unconditionally accept more
482 # A syntax error also sets _is_complete to True - see push()
483 if not self._is_complete:
484 #print("Not complete") # debug
485 return True
487 # The user can make any (complete) input execute by leaving a blank line
488 last_line = self.source.splitlines()[-1]
489 if (not last_line) or last_line.isspace():
490 #print("Blank line") # debug
491 return False
493 # If there's just a single line or AST node, and we're flush left, as is
494 # the case after a simple statement such as 'a=1', we want to execute it
495 # straight away.
496 if self.get_indent_spaces() == 0:
497 if len(self.source.splitlines()) <= 1:
498 return False
500 try:
501 code_ast = ast.parse("".join(self._buffer))
502 except Exception:
503 #print("Can't parse AST") # debug
504 return False
505 else:
506 if len(code_ast.body) == 1 and \
507 not hasattr(code_ast.body[0], 'body'):
508 #print("Simple statement") # debug
509 return False
511 # General fallback - accept more code
512 return True
514 def get_indent_spaces(self):
515 sourcefor, n = self._indent_spaces_cache
516 if sourcefor == self.source:
517 return n
519 # self.source always has a trailing newline
520 n = find_next_indent(self.source[:-1])
521 self._indent_spaces_cache = (self.source, n)
522 return n
524 # Backwards compatibility. I think all code that used .indent_spaces was
525 # inside IPython, but we can leave this here until IPython 7 in case any
526 # other modules are using it. -TK, November 2017
527 indent_spaces = property(get_indent_spaces)
529 def _store(self, lines, buffer=None, store='source'):
530 """Store one or more lines of input.
532 If input lines are not newline-terminated, a newline is automatically
533 appended."""
535 if buffer is None:
536 buffer = self._buffer
538 if lines.endswith('\n'):
539 buffer.append(lines)
540 else:
541 buffer.append(lines+'\n')
542 setattr(self, store, self._set_source(buffer))
544 def _set_source(self, buffer):
545 return u''.join(buffer)
548class IPythonInputSplitter(InputSplitter):
549 """An input splitter that recognizes all of IPython's special syntax."""
551 # String with raw, untransformed input.
552 source_raw = ''
554 # Flag to track when a transformer has stored input that it hasn't given
555 # back yet.
556 transformer_accumulating = False
558 # Flag to track when assemble_python_lines has stored input that it hasn't
559 # given back yet.
560 within_python_line = False
562 # Private attributes
564 # List with lines of raw input accumulated so far.
565 _buffer_raw = None
567 def __init__(self, line_input_checker=True, physical_line_transforms=None,
568 logical_line_transforms=None, python_line_transforms=None):
569 super(IPythonInputSplitter, self).__init__()
570 self._buffer_raw = []
571 self._validate = True
573 if physical_line_transforms is not None:
574 self.physical_line_transforms = physical_line_transforms
575 else:
576 self.physical_line_transforms = [
577 leading_indent(),
578 classic_prompt(),
579 ipy_prompt(),
580 cellmagic(end_on_blank_line=line_input_checker),
581 ]
583 self.assemble_logical_lines = assemble_logical_lines()
584 if logical_line_transforms is not None:
585 self.logical_line_transforms = logical_line_transforms
586 else:
587 self.logical_line_transforms = [
588 help_end(),
589 escaped_commands(),
590 assign_from_magic(),
591 assign_from_system(),
592 ]
594 self.assemble_python_lines = assemble_python_lines()
595 if python_line_transforms is not None:
596 self.python_line_transforms = python_line_transforms
597 else:
598 # We don't use any of these at present
599 self.python_line_transforms = []
601 @property
602 def transforms(self):
603 "Quick access to all transformers."
604 return self.physical_line_transforms + \
605 [self.assemble_logical_lines] + self.logical_line_transforms + \
606 [self.assemble_python_lines] + self.python_line_transforms
608 @property
609 def transforms_in_use(self):
610 """Transformers, excluding logical line transformers if we're in a
611 Python line."""
612 t = self.physical_line_transforms[:]
613 if not self.within_python_line:
614 t += [self.assemble_logical_lines] + self.logical_line_transforms
615 return t + [self.assemble_python_lines] + self.python_line_transforms
617 def reset(self):
618 """Reset the input buffer and associated state."""
619 super(IPythonInputSplitter, self).reset()
620 self._buffer_raw[:] = []
621 self.source_raw = ''
622 self.transformer_accumulating = False
623 self.within_python_line = False
625 for t in self.transforms:
626 try:
627 t.reset()
628 except SyntaxError:
629 # Nothing that calls reset() expects to handle transformer
630 # errors
631 pass
633 def flush_transformers(self):
634 def _flush(transform, outs):
635 """yield transformed lines
637 always strings, never None
639 transform: the current transform
640 outs: an iterable of previously transformed inputs.
641 Each may be multiline, which will be passed
642 one line at a time to transform.
643 """
644 for out in outs:
645 for line in out.splitlines():
646 # push one line at a time
647 tmp = transform.push(line)
648 if tmp is not None:
649 yield tmp
651 # reset the transform
652 tmp = transform.reset()
653 if tmp is not None:
654 yield tmp
656 out = []
657 for t in self.transforms_in_use:
658 out = _flush(t, out)
660 out = list(out)
661 if out:
662 self._store('\n'.join(out))
664 def raw_reset(self):
665 """Return raw input only and perform a full reset.
666 """
667 out = self.source_raw
668 self.reset()
669 return out
671 def source_reset(self):
672 try:
673 self.flush_transformers()
674 return self.source
675 finally:
676 self.reset()
678 def push_accepts_more(self):
679 if self.transformer_accumulating:
680 return True
681 else:
682 return super(IPythonInputSplitter, self).push_accepts_more()
684 def transform_cell(self, cell):
685 """Process and translate a cell of input.
686 """
687 self.reset()
688 try:
689 self.push(cell)
690 self.flush_transformers()
691 return self.source
692 finally:
693 self.reset()
695 def push(self, lines:str) -> bool:
696 """Push one or more lines of IPython input.
698 This stores the given lines and returns a status code indicating
699 whether the code forms a complete Python block or not, after processing
700 all input lines for special IPython syntax.
702 Any exceptions generated in compilation are swallowed, but if an
703 exception was produced, the method returns True.
705 Parameters
706 ----------
707 lines : string
708 One or more lines of Python input.
710 Returns
711 -------
712 is_complete : boolean
713 True if the current input source (the result of the current input
714 plus prior inputs) forms a complete Python execution block. Note that
715 this value is also stored as a private attribute (_is_complete), so it
716 can be queried at any time.
717 """
718 assert isinstance(lines, str)
719 # We must ensure all input is pure unicode
720 # ''.splitlines() --> [], but we need to push the empty line to transformers
721 lines_list = lines.splitlines()
722 if not lines_list:
723 lines_list = ['']
725 # Store raw source before applying any transformations to it. Note
726 # that this must be done *after* the reset() call that would otherwise
727 # flush the buffer.
728 self._store(lines, self._buffer_raw, 'source_raw')
730 transformed_lines_list = []
731 for line in lines_list:
732 transformed = self._transform_line(line)
733 if transformed is not None:
734 transformed_lines_list.append(transformed)
736 if transformed_lines_list:
737 transformed_lines = '\n'.join(transformed_lines_list)
738 return super(IPythonInputSplitter, self).push(transformed_lines)
739 else:
740 # Got nothing back from transformers - they must be waiting for
741 # more input.
742 return False
744 def _transform_line(self, line):
745 """Push a line of input code through the various transformers.
747 Returns any output from the transformers, or None if a transformer
748 is accumulating lines.
750 Sets self.transformer_accumulating as a side effect.
751 """
752 def _accumulating(dbg):
753 #print(dbg)
754 self.transformer_accumulating = True
755 return None
757 for transformer in self.physical_line_transforms:
758 line = transformer.push(line)
759 if line is None:
760 return _accumulating(transformer)
762 if not self.within_python_line:
763 line = self.assemble_logical_lines.push(line)
764 if line is None:
765 return _accumulating('acc logical line')
767 for transformer in self.logical_line_transforms:
768 line = transformer.push(line)
769 if line is None:
770 return _accumulating(transformer)
772 line = self.assemble_python_lines.push(line)
773 if line is None:
774 self.within_python_line = True
775 return _accumulating('acc python line')
776 else:
777 self.within_python_line = False
779 for transformer in self.python_line_transforms:
780 line = transformer.push(line)
781 if line is None:
782 return _accumulating(transformer)
784 #print("transformers clear") #debug
785 self.transformer_accumulating = False
786 return line