Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/jedi/api/environment.py: 25%
212 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-20 06:09 +0000
1"""
2Environments are a way to activate different Python versions or Virtualenvs for
3static analysis. The Python binary in that environment is going to be executed.
4"""
5import os
6import sys
7import hashlib
8import filecmp
9from collections import namedtuple
10from shutil import which
12from jedi.cache import memoize_method, time_cache
13from jedi.inference.compiled.subprocess import CompiledSubprocess, \
14 InferenceStateSameProcess, InferenceStateSubprocess
16import parso
18_VersionInfo = namedtuple('VersionInfo', 'major minor micro') # type: ignore[name-match]
20_SUPPORTED_PYTHONS = ['3.12', '3.11', '3.10', '3.9', '3.8', '3.7', '3.6']
21_SAFE_PATHS = ['/usr/bin', '/usr/local/bin']
22_CONDA_VAR = 'CONDA_PREFIX'
23_CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
26class InvalidPythonEnvironment(Exception):
27 """
28 If you see this exception, the Python executable or Virtualenv you have
29 been trying to use is probably not a correct Python version.
30 """
33class _BaseEnvironment:
34 @memoize_method
35 def get_grammar(self):
36 version_string = '%s.%s' % (self.version_info.major, self.version_info.minor)
37 return parso.load_grammar(version=version_string)
39 @property
40 def _sha256(self):
41 try:
42 return self._hash
43 except AttributeError:
44 self._hash = _calculate_sha256_for_file(self.executable)
45 return self._hash
48def _get_info():
49 return (
50 sys.executable,
51 sys.prefix,
52 sys.version_info[:3],
53 )
56class Environment(_BaseEnvironment):
57 """
58 This class is supposed to be created by internal Jedi architecture. You
59 should not create it directly. Please use create_environment or the other
60 functions instead. It is then returned by that function.
61 """
62 _subprocess = None
64 def __init__(self, executable, env_vars=None):
65 self._start_executable = executable
66 self._env_vars = env_vars
67 # Initialize the environment
68 self._get_subprocess()
70 def _get_subprocess(self):
71 if self._subprocess is not None and not self._subprocess.is_crashed:
72 return self._subprocess
74 try:
75 self._subprocess = CompiledSubprocess(self._start_executable,
76 env_vars=self._env_vars)
77 info = self._subprocess._send(None, _get_info)
78 except Exception as exc:
79 raise InvalidPythonEnvironment(
80 "Could not get version information for %r: %r" % (
81 self._start_executable,
82 exc))
84 # Since it could change and might not be the same(?) as the one given,
85 # set it here.
86 self.executable = info[0]
87 """
88 The Python executable, matches ``sys.executable``.
89 """
90 self.path = info[1]
91 """
92 The path to an environment, matches ``sys.prefix``.
93 """
94 self.version_info = _VersionInfo(*info[2])
95 """
96 Like :data:`sys.version_info`: a tuple to show the current
97 Environment's Python version.
98 """
99 return self._subprocess
101 def __repr__(self):
102 version = '.'.join(str(i) for i in self.version_info)
103 return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path)
105 def get_inference_state_subprocess(self, inference_state):
106 return InferenceStateSubprocess(inference_state, self._get_subprocess())
108 @memoize_method
109 def get_sys_path(self):
110 """
111 The sys path for this environment. Does not include potential
112 modifications from e.g. appending to :data:`sys.path`.
114 :returns: list of str
115 """
116 # It's pretty much impossible to generate the sys path without actually
117 # executing Python. The sys path (when starting with -S) itself depends
118 # on how the Python version was compiled (ENV variables).
119 # If you omit -S when starting Python (normal case), additionally
120 # site.py gets executed.
121 return self._get_subprocess().get_sys_path()
124class _SameEnvironmentMixin:
125 def __init__(self):
126 self._start_executable = self.executable = sys.executable
127 self.path = sys.prefix
128 self.version_info = _VersionInfo(*sys.version_info[:3])
129 self._env_vars = None
132class SameEnvironment(_SameEnvironmentMixin, Environment):
133 pass
136class InterpreterEnvironment(_SameEnvironmentMixin, _BaseEnvironment):
137 def get_inference_state_subprocess(self, inference_state):
138 return InferenceStateSameProcess(inference_state)
140 def get_sys_path(self):
141 return sys.path
144def _get_virtual_env_from_var(env_var='VIRTUAL_ENV'):
145 """Get virtualenv environment from VIRTUAL_ENV environment variable.
147 It uses `safe=False` with ``create_environment``, because the environment
148 variable is considered to be safe / controlled by the user solely.
149 """
150 var = os.environ.get(env_var)
151 if var:
152 # Under macOS in some cases - notably when using Pipenv - the
153 # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of
154 # /path/to/env so we need to fully resolve the paths in order to
155 # compare them.
156 if os.path.realpath(var) == os.path.realpath(sys.prefix):
157 return _try_get_same_env()
159 try:
160 return create_environment(var, safe=False)
161 except InvalidPythonEnvironment:
162 pass
165def _calculate_sha256_for_file(path):
166 sha256 = hashlib.sha256()
167 with open(path, 'rb') as f:
168 for block in iter(lambda: f.read(filecmp.BUFSIZE), b''):
169 sha256.update(block)
170 return sha256.hexdigest()
173def get_default_environment():
174 """
175 Tries to return an active Virtualenv or conda environment.
176 If there is no VIRTUAL_ENV variable or no CONDA_PREFIX variable set
177 set it will return the latest Python version installed on the system. This
178 makes it possible to use as many new Python features as possible when using
179 autocompletion and other functionality.
181 :returns: :class:`.Environment`
182 """
183 virtual_env = _get_virtual_env_from_var()
184 if virtual_env is not None:
185 return virtual_env
187 conda_env = _get_virtual_env_from_var(_CONDA_VAR)
188 if conda_env is not None:
189 return conda_env
191 return _try_get_same_env()
194def _try_get_same_env():
195 env = SameEnvironment()
196 if not os.path.basename(env.executable).lower().startswith('python'):
197 # This tries to counter issues with embedding. In some cases (e.g.
198 # VIM's Python Mac/Windows, sys.executable is /foo/bar/vim. This
199 # happens, because for Mac a function called `_NSGetExecutablePath` is
200 # used and for Windows `GetModuleFileNameW`. These are both platform
201 # specific functions. For all other systems sys.executable should be
202 # alright. However here we try to generalize:
203 #
204 # 1. Check if the executable looks like python (heuristic)
205 # 2. In case it's not try to find the executable
206 # 3. In case we don't find it use an interpreter environment.
207 #
208 # The last option will always work, but leads to potential crashes of
209 # Jedi - which is ok, because it happens very rarely and even less,
210 # because the code below should work for most cases.
211 if os.name == 'nt':
212 # The first case would be a virtualenv and the second a normal
213 # Python installation.
214 checks = (r'Scripts\python.exe', 'python.exe')
215 else:
216 # For unix it looks like Python is always in a bin folder.
217 checks = (
218 'bin/python%s.%s' % (sys.version_info[0], sys.version[1]),
219 'bin/python%s' % (sys.version_info[0]),
220 'bin/python',
221 )
222 for check in checks:
223 guess = os.path.join(sys.exec_prefix, check)
224 if os.path.isfile(guess):
225 # Bingo - We think we have our Python.
226 return Environment(guess)
227 # It looks like there is no reasonable Python to be found.
228 return InterpreterEnvironment()
229 # If no virtualenv is found, use the environment we're already
230 # using.
231 return env
234def get_cached_default_environment():
235 var = os.environ.get('VIRTUAL_ENV') or os.environ.get(_CONDA_VAR)
236 environment = _get_cached_default_environment()
238 # Under macOS in some cases - notably when using Pipenv - the
239 # sys.prefix of the virtualenv is /path/to/env/bin/.. instead of
240 # /path/to/env so we need to fully resolve the paths in order to
241 # compare them.
242 if var and os.path.realpath(var) != os.path.realpath(environment.path):
243 _get_cached_default_environment.clear_cache()
244 return _get_cached_default_environment()
245 return environment
248@time_cache(seconds=10 * 60) # 10 Minutes
249def _get_cached_default_environment():
250 try:
251 return get_default_environment()
252 except InvalidPythonEnvironment:
253 # It's possible that `sys.executable` is wrong. Typically happens
254 # when Jedi is used in an executable that embeds Python. For further
255 # information, have a look at:
256 # https://github.com/davidhalter/jedi/issues/1531
257 return InterpreterEnvironment()
260def find_virtualenvs(paths=None, *, safe=True, use_environment_vars=True):
261 """
262 :param paths: A list of paths in your file system to be scanned for
263 Virtualenvs. It will search in these paths and potentially execute the
264 Python binaries.
265 :param safe: Default True. In case this is False, it will allow this
266 function to execute potential `python` environments. An attacker might
267 be able to drop an executable in a path this function is searching by
268 default. If the executable has not been installed by root, it will not
269 be executed.
270 :param use_environment_vars: Default True. If True, the VIRTUAL_ENV
271 variable will be checked if it contains a valid VirtualEnv.
272 CONDA_PREFIX will be checked to see if it contains a valid conda
273 environment.
275 :yields: :class:`.Environment`
276 """
277 if paths is None:
278 paths = []
280 _used_paths = set()
282 if use_environment_vars:
283 # Using this variable should be safe, because attackers might be
284 # able to drop files (via git) but not environment variables.
285 virtual_env = _get_virtual_env_from_var()
286 if virtual_env is not None:
287 yield virtual_env
288 _used_paths.add(virtual_env.path)
290 conda_env = _get_virtual_env_from_var(_CONDA_VAR)
291 if conda_env is not None:
292 yield conda_env
293 _used_paths.add(conda_env.path)
295 for directory in paths:
296 if not os.path.isdir(directory):
297 continue
299 directory = os.path.abspath(directory)
300 for path in os.listdir(directory):
301 path = os.path.join(directory, path)
302 if path in _used_paths:
303 # A path shouldn't be inferred twice.
304 continue
305 _used_paths.add(path)
307 try:
308 executable = _get_executable_path(path, safe=safe)
309 yield Environment(executable)
310 except InvalidPythonEnvironment:
311 pass
314def find_system_environments(*, env_vars=None):
315 """
316 Ignores virtualenvs and returns the Python versions that were installed on
317 your system. This might return nothing, if you're running Python e.g. from
318 a portable version.
320 The environments are sorted from latest to oldest Python version.
322 :yields: :class:`.Environment`
323 """
324 for version_string in _SUPPORTED_PYTHONS:
325 try:
326 yield get_system_environment(version_string, env_vars=env_vars)
327 except InvalidPythonEnvironment:
328 pass
331# TODO: this function should probably return a list of environments since
332# multiple Python installations can be found on a system for the same version.
333def get_system_environment(version, *, env_vars=None):
334 """
335 Return the first Python environment found for a string of the form 'X.Y'
336 where X and Y are the major and minor versions of Python.
338 :raises: :exc:`.InvalidPythonEnvironment`
339 :returns: :class:`.Environment`
340 """
341 exe = which('python' + version)
342 if exe:
343 if exe == sys.executable:
344 return SameEnvironment()
345 return Environment(exe)
347 if os.name == 'nt':
348 for exe in _get_executables_from_windows_registry(version):
349 try:
350 return Environment(exe, env_vars=env_vars)
351 except InvalidPythonEnvironment:
352 pass
353 raise InvalidPythonEnvironment("Cannot find executable python%s." % version)
356def create_environment(path, *, safe=True, env_vars=None):
357 """
358 Make it possible to manually create an Environment object by specifying a
359 Virtualenv path or an executable path and optional environment variables.
361 :raises: :exc:`.InvalidPythonEnvironment`
362 :returns: :class:`.Environment`
363 """
364 if os.path.isfile(path):
365 _assert_safe(path, safe)
366 return Environment(path, env_vars=env_vars)
367 return Environment(_get_executable_path(path, safe=safe), env_vars=env_vars)
370def _get_executable_path(path, safe=True):
371 """
372 Returns None if it's not actually a virtual env.
373 """
375 if os.name == 'nt':
376 python = os.path.join(path, 'Scripts', 'python.exe')
377 else:
378 python = os.path.join(path, 'bin', 'python')
379 if not os.path.exists(python):
380 raise InvalidPythonEnvironment("%s seems to be missing." % python)
382 _assert_safe(python, safe)
383 return python
386def _get_executables_from_windows_registry(version):
387 import winreg
389 # TODO: support Python Anaconda.
390 sub_keys = [
391 r'SOFTWARE\Python\PythonCore\{version}\InstallPath',
392 r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}\InstallPath',
393 r'SOFTWARE\Python\PythonCore\{version}-32\InstallPath',
394 r'SOFTWARE\Wow6432Node\Python\PythonCore\{version}-32\InstallPath'
395 ]
396 for root_key in [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]:
397 for sub_key in sub_keys:
398 sub_key = sub_key.format(version=version)
399 try:
400 with winreg.OpenKey(root_key, sub_key) as key:
401 prefix = winreg.QueryValueEx(key, '')[0]
402 exe = os.path.join(prefix, 'python.exe')
403 if os.path.isfile(exe):
404 yield exe
405 except WindowsError:
406 pass
409def _assert_safe(executable_path, safe):
410 if safe and not _is_safe(executable_path):
411 raise InvalidPythonEnvironment(
412 "The python binary is potentially unsafe.")
415def _is_safe(executable_path):
416 # Resolve sym links. A venv typically is a symlink to a known Python
417 # binary. Only virtualenvs copy symlinks around.
418 real_path = os.path.realpath(executable_path)
420 if _is_unix_safe_simple(real_path):
421 return True
423 # Just check the list of known Python versions. If it's not in there,
424 # it's likely an attacker or some Python that was not properly
425 # installed in the system.
426 for environment in find_system_environments():
427 if environment.executable == real_path:
428 return True
430 # If the versions don't match, just compare the binary files. If we
431 # don't do that, only venvs will be working and not virtualenvs.
432 # venvs are symlinks while virtualenvs are actual copies of the
433 # Python files.
434 # This still means that if the system Python is updated and the
435 # virtualenv's Python is not (which is probably never going to get
436 # upgraded), it will not work with Jedi. IMO that's fine, because
437 # people should just be using venv. ~ dave
438 if environment._sha256 == _calculate_sha256_for_file(real_path):
439 return True
440 return False
443def _is_unix_safe_simple(real_path):
444 if _is_unix_admin():
445 # In case we are root, just be conservative and
446 # only execute known paths.
447 return any(real_path.startswith(p) for p in _SAFE_PATHS)
449 uid = os.stat(real_path).st_uid
450 # The interpreter needs to be owned by root. This means that it wasn't
451 # written by a user and therefore attacking Jedi is not as simple.
452 # The attack could look like the following:
453 # 1. A user clones a repository.
454 # 2. The repository has an innocent looking folder called foobar. jedi
455 # searches for the folder and executes foobar/bin/python --version if
456 # there's also a foobar/bin/activate.
457 # 3. The attacker has gained code execution, since he controls
458 # foobar/bin/python.
459 return uid == 0
462def _is_unix_admin():
463 try:
464 return os.getuid() == 0
465 except AttributeError:
466 return False # Windows