Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Environments are a way to activate different Python versions or Virtualenvs for
3static analysis. The Python binary in that environment is going to be executed.
4"""
5import os
6import sys
7import hashlib
8import filecmp
9from collections import namedtuple
10from shutil import which
11from typing import TYPE_CHECKING, Any
13from jedi.cache import memoize_method, time_cache
14from jedi.inference.compiled.subprocess import CompiledSubprocess, \
15 InferenceStateSameProcess, InferenceStateSubprocess
17import parso
19if TYPE_CHECKING:
20 from jedi.inference import InferenceState
23_VersionInfo = namedtuple('VersionInfo', 'major minor micro') # type: ignore[name-match]
25_SUPPORTED_PYTHONS = ['3.14', '3.13', '3.12', '3.11', '3.10']
26_SAFE_PATHS = ['/usr/bin', '/usr/local/bin']
27_CONDA_VAR = 'CONDA_PREFIX'
28_CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
31class InvalidPythonEnvironment(Exception):
32 """
33 If you see this exception, the Python executable or Virtualenv you have
34 been trying to use is probably not a correct Python version.
35 """
38class _BaseEnvironment:
39 version_info: Any
40 executable: Any
42 @memoize_method
43 def get_grammar(self):
44 version_string = '%s.%s' % (self.version_info.major, self.version_info.minor)
45 return parso.load_grammar(version=version_string)
47 @property
48 def _sha256(self):
49 try:
50 return self._hash
51 except AttributeError:
52 self._hash = _calculate_sha256_for_file(self.executable)
53 return self._hash
56def _get_info():
57 return (
58 sys.executable,
59 sys.prefix,
60 sys.version_info[:3],
61 )
64class Environment(_BaseEnvironment):
65 """
66 This class is supposed to be created by internal Jedi architecture. You
67 should not create it directly. Please use create_environment or the other
68 functions instead. It is then returned by that function.
69 """
70 _subprocess = None
72 def __init__(self, executable, env_vars=None):
73 self._start_executable = executable
74 self._env_vars = env_vars
75 # Initialize the environment
76 self._get_subprocess()
78 def _get_subprocess(self):
79 if self._subprocess is not None and not self._subprocess.is_crashed:
80 return self._subprocess
82 try:
83 self._subprocess = CompiledSubprocess(self._start_executable,
84 env_vars=self._env_vars)
85 info = self._subprocess._send(None, _get_info)
86 except Exception as exc:
87 raise InvalidPythonEnvironment(
88 "Could not get version information for %r: %r" % (
89 self._start_executable,
90 exc))
92 # Since it could change and might not be the same(?) as the one given,
93 # set it here.
94 self.executable = info[0]
95 """
96 The Python executable, matches ``sys.executable``.
97 """
98 self.path = info[1]
99 """
100 The path to an environment, matches ``sys.prefix``.
101 """
102 self.version_info = _VersionInfo(*info[2])
103 """
104 Like :data:`sys.version_info`: a tuple to show the current
105 Environment's Python version.
106 """
107 return self._subprocess
109 def __repr__(self):
110 version = '.'.join(str(i) for i in self.version_info)
111 return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path)
113 def get_inference_state_subprocess(
114 self,
115 inference_state: 'InferenceState',
116 ) -> InferenceStateSubprocess:
117 return InferenceStateSubprocess(inference_state, self._get_subprocess())
119 @memoize_method
120 def get_sys_path(self):
121 """
122 The sys path for this environment. Does not include potential
123 modifications from e.g. appending to :data:`sys.path`.
125 :returns: list of str
126 """
127 # It's pretty much impossible to generate the sys path without actually
128 # executing Python. The sys path (when starting with -S) itself depends
129 # on how the Python version was compiled (ENV variables).
130 # If you omit -S when starting Python (normal case), additionally
131 # site.py gets executed.
132 return self._get_subprocess().get_sys_path()
135class _SameEnvironmentMixin:
136 def __init__(self):
137 self._start_executable = self.executable = sys.executable
138 self.path = sys.prefix
139 self.version_info = _VersionInfo(*sys.version_info[:3])
140 self._env_vars = None
143class SameEnvironment(_SameEnvironmentMixin, Environment):
144 pass
147class InterpreterEnvironment(_SameEnvironmentMixin, _BaseEnvironment):
148 def get_inference_state_subprocess(
149 self,
150 inference_state: 'InferenceState',
151 ) -> InferenceStateSameProcess:
152 return InferenceStateSameProcess(inference_state)
154 def get_sys_path(self):
155 return sys.path
158def _get_virtual_env_from_var(env_var='VIRTUAL_ENV'):
159 """Get virtualenv environment from VIRTUAL_ENV environment variable.
161 It uses `safe=False` with ``create_environment``, because the environment
162 variable is considered to be safe / controlled by the user solely.
163 """
164 var = os.environ.get(env_var)
165 if var:
166 # Under macOS in some cases - notably when using Pipenv - the
167 # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of
168 # /path/to/env so we need to fully resolve the paths in order to
169 # compare them.
170 if os.path.realpath(var) == os.path.realpath(sys.prefix):
171 return _try_get_same_env()
173 try:
174 return create_environment(var, safe=False)
175 except InvalidPythonEnvironment:
176 pass
179def _calculate_sha256_for_file(path):
180 sha256 = hashlib.sha256()
181 with open(path, 'rb') as f:
182 for block in iter(lambda: f.read(filecmp.BUFSIZE), b''):
183 sha256.update(block)
184 return sha256.hexdigest()
187def get_default_environment():
188 """
189 Tries to return an active Virtualenv or conda environment.
190 If there is no VIRTUAL_ENV variable or no CONDA_PREFIX variable set
191 set it will return the latest Python version installed on the system. This
192 makes it possible to use as many new Python features as possible when using
193 autocompletion and other functionality.
195 :returns: :class:`.Environment`
196 """
197 virtual_env = _get_virtual_env_from_var()
198 if virtual_env is not None:
199 return virtual_env
201 conda_env = _get_virtual_env_from_var(_CONDA_VAR)
202 if conda_env is not None:
203 return conda_env
205 return _try_get_same_env()
208def _try_get_same_env():
209 env = SameEnvironment()
210 if not os.path.basename(env.executable).lower().startswith('python'):
211 # This tries to counter issues with embedding. In some cases (e.g.
212 # VIM's Python Mac/Windows, sys.executable is /foo/bar/vim. This
213 # happens, because for Mac a function called `_NSGetExecutablePath` is
214 # used and for Windows `GetModuleFileNameW`. These are both platform
215 # specific functions. For all other systems sys.executable should be
216 # alright. However here we try to generalize:
217 #
218 # 1. Check if the executable looks like python (heuristic)
219 # 2. In case it's not try to find the executable
220 # 3. In case we don't find it use an interpreter environment.
221 #
222 # The last option will always work, but leads to potential crashes of
223 # Jedi - which is ok, because it happens very rarely and even less,
224 # because the code below should work for most cases.
225 if os.name == 'nt':
226 # The first case would be a virtualenv and the second a normal
227 # Python installation.
228 checks = (r'Scripts\python.exe', 'python.exe')
229 else:
230 # For unix it looks like Python is always in a bin folder.
231 checks = (
232 'bin/python%s.%s' % (sys.version_info[0], sys.version[1]),
233 'bin/python%s' % (sys.version_info[0]),
234 'bin/python',
235 )
236 for check in checks:
237 guess = os.path.join(sys.exec_prefix, check)
238 if os.path.isfile(guess):
239 # Bingo - We think we have our Python.
240 return Environment(guess)
241 # It looks like there is no reasonable Python to be found.
242 return InterpreterEnvironment()
243 # If no virtualenv is found, use the environment we're already
244 # using.
245 return env
248def get_cached_default_environment():
249 var = os.environ.get('VIRTUAL_ENV') or os.environ.get(_CONDA_VAR)
250 environment = _get_cached_default_environment()
252 # Under macOS in some cases - notably when using Pipenv - the
253 # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of
254 # /path/to/env so we need to fully resolve the paths in order to
255 # compare them.
256 if var and os.path.realpath(var) != os.path.realpath(environment.path):
257 _get_cached_default_environment.clear_cache() # type: ignore[attr-defined]
258 return _get_cached_default_environment()
259 return environment
262@time_cache(seconds=10 * 60) # 10 Minutes
263def _get_cached_default_environment():
264 try:
265 return get_default_environment()
266 except InvalidPythonEnvironment:
267 # It's possible that `sys.executable` is wrong. Typically happens
268 # when Jedi is used in an executable that embeds Python. For further
269 # information, have a look at:
270 # https://github.com/davidhalter/jedi/issues/1531
271 return InterpreterEnvironment()
274def find_virtualenvs(paths=None, *, safe=True, use_environment_vars=True):
275 """
276 :param paths: A list of paths in your file system to be scanned for
277 Virtualenvs. It will search in these paths and potentially execute the
278 Python binaries.
279 :param safe: Default True. In case this is False, it will allow this
280 function to execute potential `python` environments. An attacker might
281 be able to drop an executable in a path this function is searching by
282 default. If the executable has not been installed by root, it will not
283 be executed.
284 :param use_environment_vars: Default True. If True, the VIRTUAL_ENV
285 variable will be checked if it contains a valid VirtualEnv.
286 CONDA_PREFIX will be checked to see if it contains a valid conda
287 environment.
289 :yields: :class:`.Environment`
290 """
291 if paths is None:
292 paths = []
294 _used_paths = set()
296 if use_environment_vars:
297 # Using this variable should be safe, because attackers might be
298 # able to drop files (via git) but not environment variables.
299 virtual_env = _get_virtual_env_from_var()
300 if virtual_env is not None:
301 yield virtual_env
302 _used_paths.add(virtual_env.path)
304 conda_env = _get_virtual_env_from_var(_CONDA_VAR)
305 if conda_env is not None:
306 yield conda_env
307 _used_paths.add(conda_env.path)
309 for directory in paths:
310 if not os.path.isdir(directory):
311 continue
313 directory = os.path.abspath(directory)
314 for path in os.listdir(directory):
315 path = os.path.join(directory, path)
316 if path in _used_paths:
317 # A path shouldn't be inferred twice.
318 continue
319 _used_paths.add(path)
321 try:
322 executable = _get_executable_path(path, safe=safe)
323 yield Environment(executable)
324 except InvalidPythonEnvironment:
325 pass
328def find_system_environments(*, env_vars=None):
329 """
330 Ignores virtualenvs and returns the Python versions that were installed on
331 your system. This might return nothing, if you're running Python e.g. from
332 a portable version.
334 The environments are sorted from latest to oldest Python version.
336 :yields: :class:`.Environment`
337 """
338 for version_string in _SUPPORTED_PYTHONS:
339 try:
340 yield get_system_environment(version_string, env_vars=env_vars)
341 except InvalidPythonEnvironment:
342 pass
345# TODO: this function should probably return a list of environments since
346# multiple Python installations can be found on a system for the same version.
347def get_system_environment(version, *, env_vars=None):
348 """
349 Return the first Python environment found for a string of the form 'X.Y'
350 where X and Y are the major and minor versions of Python.
352 :raises: :exc:`.InvalidPythonEnvironment`
353 :returns: :class:`.Environment`
354 """
355 exe = which('python' + version)
356 if exe:
357 if exe == sys.executable:
358 return SameEnvironment()
359 return Environment(exe)
361 if sys.platform == "win32":
362 for exe in _get_executables_from_windows_registry(version):
363 try:
364 return Environment(exe, env_vars=env_vars)
365 except InvalidPythonEnvironment:
366 pass
367 raise InvalidPythonEnvironment("Cannot find executable python%s." % version)
370def create_environment(path, *, safe=True, env_vars=None):
371 """
372 Make it possible to manually create an Environment object by specifying a
373 Virtualenv path or an executable path and optional environment variables.
375 :raises: :exc:`.InvalidPythonEnvironment`
376 :returns: :class:`.Environment`
377 """
378 if os.path.isfile(path):
379 _assert_safe(path, safe)
380 return Environment(path, env_vars=env_vars)
381 return Environment(_get_executable_path(path, safe=safe), env_vars=env_vars)
384def _get_executable_path(path, safe=True):
385 """
386 Returns None if it's not actually a virtual env.
387 """
389 if sys.platform == "win32":
390 pythons = [os.path.join(path, 'Scripts', 'python.exe'), os.path.join(path, 'python.exe')]
391 else:
392 pythons = [os.path.join(path, 'bin', 'python')]
393 for python in pythons:
394 if os.path.exists(python):
395 break
396 else:
397 raise InvalidPythonEnvironment("%s seems to be missing." % python)
399 _assert_safe(python, safe)
400 return python
403if sys.platform == "win32":
404 def _get_executables_from_windows_registry(version):
405 import winreg
407 # TODO: support Python Anaconda.
408 sub_keys = [
409 r'SOFTWARE\Python\PythonCore\{version}\InstallPath',
410 r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath',
411 r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath',
412 r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath'
413 ]
414 for root_key in [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]:
415 for sub_key in sub_keys:
416 sub_key = sub_key.format(version=version)
417 try:
418 with winreg.OpenKey(root_key, sub_key) as key:
419 prefix = winreg.QueryValueEx(key, '')[0]
420 exe = os.path.join(prefix, 'python.exe')
421 if os.path.isfile(exe):
422 yield exe
423 except WindowsError:
424 pass
427def _assert_safe(executable_path, safe):
428 if safe and not _is_safe(executable_path):
429 raise InvalidPythonEnvironment(
430 "The python binary is potentially unsafe.")
433def _is_safe(executable_path):
434 # Resolve sym links. A venv typically is a symlink to a known Python
435 # binary. Only virtualenvs copy symlinks around.
436 real_path = os.path.realpath(executable_path)
438 if _is_unix_safe_simple(real_path):
439 return True
441 # Just check the list of known Python versions. If it's not in there,
442 # it's likely an attacker or some Python that was not properly
443 # installed in the system.
444 for environment in find_system_environments():
445 if environment.executable == real_path:
446 return True
448 # If the versions don't match, just compare the binary files. If we
449 # don't do that, only venvs will be working and not virtualenvs.
450 # venvs are symlinks while virtualenvs are actual copies of the
451 # Python files.
452 # This still means that if the system Python is updated and the
453 # virtualenv's Python is not (which is probably never going to get
454 # upgraded), it will not work with Jedi. IMO that's fine, because
455 # people should just be using venv. ~ dave
456 if environment._sha256 == _calculate_sha256_for_file(real_path):
457 return True
458 return False
461def _is_unix_safe_simple(real_path):
462 if _is_unix_admin():
463 # In case we are root, just be conservative and
464 # only execute known paths.
465 return any(real_path.startswith(p) for p in _SAFE_PATHS)
467 uid = os.stat(real_path).st_uid
468 # The interpreter needs to be owned by root. This means that it wasn't
469 # written by a user and therefore attacking Jedi is not as simple.
470 # The attack could look like the following:
471 # 1. A user clones a repository.
472 # 2. The repository has an innocent looking folder called foobar. jedi
473 # searches for the folder and executes foobar/bin/python --version if
474 # there's also a foobar/bin/activate.
475 # 3. The attacker has gained code execution, since he controls
476 # foobar/bin/python.
477 return uid == 0
480def _is_unix_admin():
481 try:
482 return os.getuid() == 0
483 except AttributeError:
484 return False # Windows