Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Compiler tools with improved interactive support.
3Provides compilation machinery similar to codeop, but with caching support so
4we can provide interactive tracebacks.
6Authors
7-------
8* Robert Kern
9* Fernando Perez
10* Thomas Kluyver
11"""
13# Note: though it might be more natural to name this module 'compiler', that
14# name is in the stdlib and name collisions with the stdlib tend to produce
15# weird problems (often with third-party tools).
17#-----------------------------------------------------------------------------
18# Copyright (C) 2010-2011 The IPython Development Team.
19#
20# Distributed under the terms of the BSD License.
21#
22# The full license is in the file COPYING.txt, distributed with this software.
23#-----------------------------------------------------------------------------
25#-----------------------------------------------------------------------------
26# Imports
27#-----------------------------------------------------------------------------
29from __future__ import annotations
31import ast
33# Stdlib imports
34import __future__
35from ast import PyCF_ONLY_AST
36import codeop
37import functools
38import hashlib
39import linecache
40import operator
41from contextlib import contextmanager
42from collections.abc import Generator
44#-----------------------------------------------------------------------------
45# Constants
46#-----------------------------------------------------------------------------
48# Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h,
49# this is used as a bitmask to extract future-related code flags.
50PyCF_MASK = functools.reduce(operator.or_,
51 (getattr(__future__, fname).compiler_flag
52 for fname in __future__.all_feature_names))
54#-----------------------------------------------------------------------------
55# Local utilities
56#-----------------------------------------------------------------------------
58def code_name(code: str, number: int = 0) -> str:
59 """ Compute a (probably) unique name for code for caching.
61 This now expects code to be unicode.
62 """
63 hash_digest = hashlib.sha1(code.encode("utf-8"), usedforsecurity=False).hexdigest()
64 # Include the number and 12 characters of the hash in the name. It's
65 # pretty much impossible that in a single session we'll have collisions
66 # even with truncated hashes, and the full one makes tracebacks too long
67 return f'<ipython-input-{number}-{hash_digest[:12]}>'
69#-----------------------------------------------------------------------------
70# Classes and functions
71#-----------------------------------------------------------------------------
73class CachingCompiler(codeop.Compile):
74 """A compiler that caches code compiled from interactive statements.
75 """
77 def __init__(self):
78 codeop.Compile.__init__(self)
80 # Caching a dictionary { filename: execution_count } for nicely
81 # rendered tracebacks. The filename corresponds to the filename
82 # argument used for the builtins.compile function.
83 self._filename_map = {}
85 def ast_parse(self, source: str, filename: str = '<unknown>', symbol: str = 'exec') -> ast.AST:
86 """Parse code to an AST with the current compiler flags active.
88 Arguments are exactly the same as ast.parse (in the standard library),
89 and are passed to the built-in compile function."""
90 return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
92 def reset_compiler_flags(self) -> None:
93 """Reset compiler flags to default state."""
94 # This value is copied from codeop.Compile.__init__, so if that ever
95 # changes, it will need to be updated.
96 self.flags = codeop.PyCF_DONT_IMPLY_DEDENT
98 @property
99 def compiler_flags(self) -> int:
100 """Flags currently active in the compilation process.
101 """
102 return self.flags
104 def get_code_name(self, raw_code: str, transformed_code: str, number: int) -> str:
105 """Compute filename given the code, and the cell number.
107 Parameters
108 ----------
109 raw_code : str
110 The raw cell code.
111 transformed_code : str
112 The executable Python source code to cache and compile.
113 number : int
114 A number which forms part of the code's name. Used for the execution
115 counter.
117 Returns
118 -------
119 The computed filename.
120 """
121 return code_name(transformed_code, number)
123 def format_code_name(self, name: str) -> tuple[str, str] | None:
124 """Return a user-friendly label and name for a code block.
126 Parameters
127 ----------
128 name : str
129 The name for the code block returned from get_code_name
131 Returns
132 -------
133 A (label, name) pair that can be used in tracebacks, or None if the default formatting should be used.
134 """
135 if name in self._filename_map:
136 return "Cell", "In[%s]" % self._filename_map[name]
138 def cache(self, transformed_code: str, number: int = 0, raw_code: str | None = None) -> str:
139 """Make a name for a block of code, and cache the code.
141 Parameters
142 ----------
143 transformed_code : str
144 The executable Python source code to cache and compile.
145 number : int
146 A number which forms part of the code's name. Used for the execution
147 counter.
148 raw_code : str
149 The raw code before transformation, if None, set to `transformed_code`.
151 Returns
152 -------
153 The name of the cached code (as a string). Pass this as the filename
154 argument to compilation, so that tracebacks are correctly hooked up.
155 """
156 if raw_code is None:
157 raw_code = transformed_code
159 name = self.get_code_name(raw_code, transformed_code, number)
161 # Save the execution count
162 self._filename_map[name] = number
164 # Since Python 2.5, setting mtime to `None` means the lines will
165 # never be removed by `linecache.checkcache`. This means all the
166 # monkeypatching has *never* been necessary, since this code was
167 # only added in 2010, at which point IPython had already stopped
168 # supporting Python 2.4.
169 #
170 # Note that `linecache.clearcache` and `linecache.updatecache` may
171 # still remove our code from the cache, but those show explicit
172 # intent, and we should not try to interfere. Normally the former
173 # is never called except when out of memory, and the latter is only
174 # called for lines *not* in the cache.
175 entry = (
176 len(transformed_code),
177 None,
178 [line + "\n" for line in transformed_code.splitlines()],
179 name,
180 )
181 linecache.cache[name] = entry
182 return name
184 @contextmanager
185 def extra_flags(self, flags: int) -> Generator[None, None, None]:
186 ## bits that we'll set to 1
187 turn_on_bits = ~self.flags & flags
190 self.flags = self.flags | flags
191 try:
192 yield
193 finally:
194 # turn off only the bits we turned on so that something like
195 # __future__ that set flags stays.
196 self.flags &= ~turn_on_bits