Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/modutils.py: 26%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Python modules manipulation utility functions.
7:type PY_SOURCE_EXTS: tuple(str)
8:var PY_SOURCE_EXTS: list of possible python source file extension
10:type STD_LIB_DIRS: set of str
11:var STD_LIB_DIRS: directories where standard modules are located
13:type BUILTIN_MODULES: dict
14:var BUILTIN_MODULES: dictionary with builtin module names has key
15"""
17from __future__ import annotations
19import importlib
20import importlib.machinery
21import importlib.util
22import io
23import itertools
24import os
25import sys
26import sysconfig
27import types
28from collections.abc import Callable, Iterable, Sequence
29from contextlib import redirect_stderr, redirect_stdout
30from functools import lru_cache
31from sys import stdlib_module_names
33from astroid.const import IS_JYTHON
34from astroid.interpreter._import import spec, util
36if sys.platform.startswith("win"):
37 PY_SOURCE_EXTS = ("py", "pyw", "pyi")
38 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "pyw", "py")
39 PY_COMPILED_EXTS = ("dll", "pyd")
40else:
41 PY_SOURCE_EXTS = ("py", "pyi")
42 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "py")
43 PY_COMPILED_EXTS = ("so",)
45# Bytecode extensions are platform-independent, unlike PY_COMPILED_EXTS above.
46PY_BYTECODE_EXTS = ("pyc", "pyo")
49# TODO: Adding `platstdlib` is a fix for a workaround in virtualenv. At some point we should
50# revisit whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1323.
51STD_LIB_DIRS = {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
53if os.name == "nt":
54 STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls"))
55 try:
56 # real_prefix is defined when running inside virtual environments,
57 # created with the **virtualenv** library.
58 # Deprecated in virtualenv==16.7.9
59 # See: https://github.com/pypa/virtualenv/issues/1622
60 STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) # type: ignore[attr-defined]
61 except AttributeError:
62 # sys.base_exec_prefix is always defined, but in a virtual environment
63 # created with the stdlib **venv** module, it points to the original
64 # installation, if the virtual env is activated.
65 try:
66 STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls"))
67 except AttributeError:
68 pass
70if os.name == "posix":
71 # Need the real prefix if we're in a virtualenv, otherwise
72 # the usual one will do.
73 # Deprecated in virtualenv==16.7.9
74 # See: https://github.com/pypa/virtualenv/issues/1622
75 try:
76 prefix: str = sys.real_prefix # type: ignore[attr-defined]
77 except AttributeError:
78 prefix = sys.prefix
80 def _posix_path(path: str) -> str:
81 base_python = f"python{sys.version_info.major}.{sys.version_info.minor}"
82 return os.path.join(prefix, path, base_python)
84 STD_LIB_DIRS.add(_posix_path("lib"))
85 if sys.maxsize > 2**32:
86 # This tries to fix a problem with /usr/lib64 builds,
87 # where systems are running both 32-bit and 64-bit code
88 # on the same machine, which reflects into the places where
89 # standard library could be found. More details can be found
90 # here http://bugs.python.org/issue1294959.
91 # An easy reproducing case would be
92 # https://github.com/pylint-dev/pylint/issues/712#issuecomment-163178753
93 STD_LIB_DIRS.add(_posix_path("lib64"))
95EXT_LIB_DIRS = {sysconfig.get_path("purelib"), sysconfig.get_path("platlib")}
96BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
99class NoSourceFile(Exception):
100 """Exception raised when we are not able to get a python
101 source file for a precompiled file.
102 """
105def _normalize_path(path: str) -> str:
106 """Resolve symlinks in path and convert to absolute path.
108 Note that environment variables and ~ in the path need to be expanded in
109 advance.
111 This can be cached by using _cache_normalize_path.
112 """
113 return os.path.normcase(os.path.realpath(path))
116def _path_from_filename(filename: str, is_jython: bool = IS_JYTHON) -> str:
117 if not is_jython:
118 return filename
119 head, has_pyclass, _ = filename.partition("$py.class")
120 if has_pyclass:
121 return head + ".py"
122 return filename
125def _handle_blacklist(
126 blacklist: Sequence[str], dirnames: list[str], filenames: list[str]
127) -> None:
128 """Remove files/directories in the black list.
130 dirnames/filenames are usually from os.walk
131 """
132 for norecurs in blacklist:
133 if norecurs in dirnames:
134 dirnames.remove(norecurs)
135 elif norecurs in filenames:
136 filenames.remove(norecurs)
139@lru_cache
140def _cache_normalize_path_(path: str) -> str:
141 return _normalize_path(path)
144def _cache_normalize_path(path: str) -> str:
145 """Normalize path with caching."""
146 # _module_file calls abspath on every path in sys.path every time it's
147 # called; on a larger codebase this easily adds up to half a second just
148 # assembling path components. This cache alleviates that.
149 if not path: # don't cache result for ''
150 return _normalize_path(path)
151 return _cache_normalize_path_(path)
154def load_module_from_name(dotted_name: str) -> types.ModuleType:
155 """Load a Python module from its name.
157 :type dotted_name: str
158 :param dotted_name: python name of a module or package
160 :raise ImportError: if the module or package is not found
162 :rtype: module
163 :return: the loaded module
164 """
165 try:
166 return sys.modules[dotted_name]
167 except KeyError:
168 pass
170 # Capture and log anything emitted during import to avoid
171 # contaminating JSON reports in pylint
172 with (
173 redirect_stderr(io.StringIO()) as stderr,
174 redirect_stdout(io.StringIO()) as stdout,
175 ):
176 module = importlib.import_module(dotted_name)
178 stderr_value = stderr.getvalue()
179 stdout_value = stdout.getvalue()
180 if stderr_value or stdout_value:
181 import logging # pylint: disable=import-outside-toplevel
183 logger = logging.getLogger(__name__)
184 if stderr_value:
185 logger.error(
186 "Captured stderr while importing %s:\n%s", dotted_name, stderr_value
187 )
188 if stdout_value:
189 logger.info(
190 "Captured stdout while importing %s:\n%s", dotted_name, stdout_value
191 )
193 return module
196def load_module_from_modpath(parts: Sequence[str]) -> types.ModuleType:
197 """Load a python module from its split name.
199 :param parts:
200 python name of a module or package split on '.'
202 :raise ImportError: if the module or package is not found
204 :return: the loaded module
205 """
206 return load_module_from_name(".".join(parts))
209def load_module_from_file(filepath: str) -> types.ModuleType:
210 """Load a Python module from it's path.
212 :type filepath: str
213 :param filepath: path to the python module or package
215 :raise ImportError: if the module or package is not found
217 :rtype: module
218 :return: the loaded module
219 """
220 modpath = modpath_from_file(filepath)
221 return load_module_from_modpath(modpath)
224def check_modpath_has_init(path: str, mod_path: list[str]) -> bool:
225 """Check there are some __init__.py all along the way."""
226 modpath: list[str] = []
227 for part in mod_path:
228 modpath.append(part)
229 path = os.path.join(path, part)
230 if not _has_init(path):
231 old_namespace = util.is_namespace(".".join(modpath))
232 if not old_namespace:
233 return False
234 return True
237def _is_subpath(path: str, base: str) -> bool:
238 path = os.path.normcase(os.path.normpath(path))
239 base = os.path.normcase(os.path.normpath(base))
240 if not path.startswith(base):
241 return False
242 return (
243 (len(path) == len(base))
244 or (path[len(base)] == os.path.sep)
245 or (base.endswith(os.path.sep) and path[len(base) - 1] == os.path.sep)
246 )
249def _get_relative_base_path(filename: str, path_to_check: str) -> list[str] | None:
250 """Extracts the relative mod path of the file to import from.
252 Check if a file is within the passed in path and if so, returns the
253 relative mod path from the one passed in.
255 If the filename is no in path_to_check, returns None
257 Note this function will look for both abs and realpath of the file,
258 this allows to find the relative base path even if the file is a
259 symlink of a file in the passed in path
261 Examples:
262 _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"]
263 _get_relative_base_path("/a/b/c/d.py", "/dev") -> None
264 """
265 path_to_check = os.path.normcase(os.path.normpath(path_to_check))
267 abs_filename = os.path.abspath(filename)
268 if _is_subpath(abs_filename, path_to_check):
269 base_path = os.path.splitext(abs_filename)[0]
270 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
271 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
273 real_filename = os.path.realpath(filename)
274 if _is_subpath(real_filename, path_to_check):
275 base_path = os.path.splitext(real_filename)[0]
276 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
277 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
279 return None
282def modpath_from_file_with_callback(
283 filename: str,
284 path: list[str] | None = None,
285 is_package_cb: Callable[[str, list[str]], bool] | None = None,
286) -> list[str]:
287 filename = os.path.expanduser(_path_from_filename(filename))
288 paths_to_check = sys.path.copy()
289 if path:
290 paths_to_check = path + paths_to_check
291 for pathname in itertools.chain(
292 paths_to_check, map(_cache_normalize_path, paths_to_check)
293 ):
294 if not pathname:
295 continue
296 modpath = _get_relative_base_path(filename, pathname)
297 if not modpath:
298 continue
299 assert is_package_cb is not None
300 if is_package_cb(pathname, modpath[:-1]):
301 return modpath
303 raise ImportError(
304 "Unable to find module for {} in {}".format(
305 filename, ", \n".join(paths_to_check)
306 )
307 )
310def modpath_from_file(filename: str, path: list[str] | None = None) -> list[str]:
311 """Get the corresponding split module's name from a filename.
313 This function will return the name of a module or package split on `.`.
315 :type filename: str
316 :param filename: file's path for which we want the module's name
318 :type Optional[List[str]] path:
319 Optional list of paths where the module or package should be
320 searched, additionally to sys.path
322 :raise ImportError:
323 if the corresponding module's name has not been found
325 :rtype: list(str)
326 :return: the corresponding split module's name
327 """
328 return modpath_from_file_with_callback(filename, path, check_modpath_has_init)
331def file_from_modpath(
332 modpath: list[str],
333 path: Sequence[str] | None = None,
334 context_file: str | None = None,
335) -> str | None:
336 return file_info_from_modpath(modpath, path, context_file).location
339def file_info_from_modpath(
340 modpath: list[str],
341 path: Sequence[str] | None = None,
342 context_file: str | None = None,
343) -> spec.ModuleSpec:
344 """Given a mod path (i.e. split module / package name), return the
345 corresponding file.
347 Giving priority to source file over precompiled file if it exists.
349 :param modpath:
350 split module's name (i.e name of a module or package split
351 on '.')
352 (this means explicit relative imports that start with dots have
353 empty strings in this list!)
355 :param path:
356 optional list of path where the module or package should be
357 searched (use sys.path if nothing or None is given)
359 :param context_file:
360 context file to consider, necessary if the identifier has been
361 introduced using a relative import unresolvable in the actual
362 context (i.e. modutils)
364 :raise ImportError: if there is no such module in the directory
366 :return:
367 the path to the module's file or None if it's an integrated
368 builtin module such as 'sys'
369 """
370 if context_file is not None:
371 context: str | None = os.path.dirname(context_file)
372 else:
373 context = context_file
374 if modpath[0] == "xml":
375 # handle _xmlplus
376 try:
377 return _spec_from_modpath(["_xmlplus", *modpath[1:]], path, context)
378 except ImportError:
379 return _spec_from_modpath(modpath, path, context)
380 elif modpath == ["os", "path"]:
381 # FIXME: currently ignoring search_path...
382 return spec.ModuleSpec(
383 name="os.path",
384 location=os.path.__file__,
385 type=spec.ModuleType.PY_SOURCE,
386 )
387 return _spec_from_modpath(modpath, path, context)
390def get_module_part(dotted_name: str, context_file: str | None = None) -> str:
391 """Given a dotted name return the module part of the name :
393 >>> get_module_part('astroid.as_string.dump')
394 'astroid.as_string'
396 :param dotted_name: full name of the identifier we are interested in
398 :param context_file:
399 context file to consider, necessary if the identifier has been
400 introduced using a relative import unresolvable in the actual
401 context (i.e. modutils)
403 :raise ImportError: if there is no such module in the directory
405 :return:
406 the module part of the name or None if we have not been able at
407 all to import the given name
409 XXX: deprecated, since it doesn't handle package precedence over module
410 (see #10066)
411 """
412 # os.path trick
413 if dotted_name.startswith("os.path"):
414 return "os.path"
415 parts = dotted_name.split(".")
416 if context_file is not None:
417 # first check for builtin module which won't be considered latter
418 # in that case (path != None)
419 if parts[0] in BUILTIN_MODULES:
420 if len(parts) > 2:
421 raise ImportError(dotted_name)
422 return parts[0]
423 # don't use += or insert, we want a new list to be created !
424 path: list[str] | None = None
425 starti = 0
426 if parts[0] == "":
427 assert (
428 context_file is not None
429 ), "explicit relative import, but no context_file?"
430 path = [] # prevent resolving the import non-relatively
431 starti = 1
432 # for all further dots: change context
433 while starti < len(parts) and parts[starti] == "":
434 starti += 1
435 assert (
436 context_file is not None
437 ), "explicit relative import, but no context_file?"
438 context_file = os.path.dirname(context_file)
439 for i in range(starti, len(parts)):
440 try:
441 file_from_modpath(
442 parts[starti : i + 1], path=path, context_file=context_file
443 )
444 except ImportError:
445 if i < max(1, len(parts) - 2):
446 raise
447 return ".".join(parts[:i])
448 return dotted_name
451def get_module_files(
452 src_directory: str, blacklist: Sequence[str], list_all: bool = False
453) -> list[str]:
454 """Given a package directory return a list of all available python
455 module's files in the package and its subpackages.
457 :param src_directory:
458 path of the directory corresponding to the package
460 :param blacklist: iterable
461 list of files or directories to ignore.
463 :param list_all:
464 get files from all paths, including ones without __init__.py
466 :return:
467 the list of all available python module's files in the package and
468 its subpackages
469 """
470 files: list[str] = []
471 for directory, dirnames, filenames in os.walk(src_directory):
472 if directory in blacklist:
473 continue
474 _handle_blacklist(blacklist, dirnames, filenames)
475 # check for __init__.py
476 if not list_all and {"__init__.py", "__init__.pyi"}.isdisjoint(filenames):
477 dirnames[:] = ()
478 continue
479 for filename in filenames:
480 if _is_python_file(filename):
481 src = os.path.join(directory, filename)
482 files.append(src)
483 return files
486def _is_compiled_ext(ext: str) -> bool:
487 """Return whether ``ext`` (without leading dot) names a compiled module:
488 bytecode (.pyc/.pyo) or a binary extension module (.so/.pyd/.dll). Such
489 files are never source.
490 """
491 return ext in PY_BYTECODE_EXTS or ext in PY_COMPILED_EXTS
494def get_source_file(
495 filename: str, include_no_ext: bool = False, prefer_stubs: bool = False
496) -> str:
497 """Given a python module's file name return the matching source file
498 name (the filename will be returned identically if it's already an
499 absolute path to a python source file).
501 :param filename: python module's file name
503 :raise NoSourceFile: if no source file exists on the file system
505 :return: the absolute path of the source file if it exists
506 """
507 filename = os.path.abspath(_path_from_filename(filename))
508 base, orig_ext = os.path.splitext(filename)
509 orig_ext = orig_ext.lstrip(".")
510 # A non-standard extension (e.g. a custom suffix) is returned as-is, but a
511 # compiled file falls through to the lookup below so its .py/.pyi stub is
512 # found instead.
513 if (
514 orig_ext not in PY_SOURCE_EXTS
515 and not _is_compiled_ext(orig_ext)
516 and os.path.exists(f"{base}.{orig_ext}")
517 ):
518 return f"{base}.{orig_ext}"
519 for ext in PY_SOURCE_EXTS_STUBS_FIRST if prefer_stubs else PY_SOURCE_EXTS:
520 source_path = f"{base}.{ext}"
521 if os.path.exists(source_path):
522 return source_path
523 if include_no_ext and not orig_ext and os.path.exists(base):
524 return base
525 raise NoSourceFile(filename)
528def is_python_source(filename: str | None) -> bool:
529 """Return: True if the filename is a python source file."""
530 if not filename:
531 return False
532 return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
535def is_stdlib_module(modname: str) -> bool:
536 """Return: True if the modname is in the standard library"""
537 return modname.split(".")[0] in stdlib_module_names
540def module_in_path(modname: str, path: str | Iterable[str]) -> bool:
541 """Try to determine if a module is imported from one of the specified paths
543 :param modname: name of the module
545 :param path: paths to consider
547 :return:
548 true if the module:
549 - is located on the path listed in one of the directory in `paths`
550 """
552 modname = modname.split(".")[0]
553 try:
554 filename = file_from_modpath([modname])
555 except ImportError:
556 # Import failed, we can't check path if we don't know it
557 return False
559 if filename is None:
560 # No filename likely means it's compiled in, or potentially a namespace
561 return False
562 filename = _normalize_path(filename)
564 if isinstance(path, str):
565 return _is_subpath(filename, _cache_normalize_path(path))
567 return any(_is_subpath(filename, _cache_normalize_path(entry)) for entry in path)
570def is_relative(modname: str, from_file: str) -> bool:
571 """Return true if the given module name is relative to the given
572 file name.
574 :param modname: name of the module we are interested in
576 :param from_file:
577 path of the module from which modname has been imported
579 :return:
580 true if the module has been imported relatively to `from_file`
581 """
582 if not os.path.isdir(from_file):
583 from_file = os.path.dirname(from_file)
584 if from_file in sys.path:
585 return False
586 return bool(
587 importlib.machinery.PathFinder.find_spec(
588 modname.split(".", maxsplit=1)[0], [from_file]
589 )
590 )
593@lru_cache(maxsize=1024)
594def cached_os_path_isfile(path: str | os.PathLike[str]) -> bool:
595 """A cached version of os.path.isfile that helps avoid repetitive I/O"""
596 return os.path.isfile(path)
599# internal only functions #####################################################
602def _spec_from_modpath(
603 modpath: list[str],
604 path: Sequence[str] | None = None,
605 context: str | None = None,
606) -> spec.ModuleSpec:
607 """Given a mod path (i.e. split module / package name), return the
608 corresponding spec.
610 this function is used internally, see `file_from_modpath`'s
611 documentation for more information
612 """
613 assert modpath
614 location = None
615 if context is not None:
616 try:
617 found_spec = spec.find_spec(modpath, [context])
618 location = found_spec.location
619 except ImportError:
620 found_spec = spec.find_spec(modpath, path)
621 location = found_spec.location
622 else:
623 found_spec = spec.find_spec(modpath, path)
624 if found_spec.type == spec.ModuleType.PY_COMPILED:
625 try:
626 assert found_spec.location is not None
627 location = get_source_file(found_spec.location)
628 return found_spec._replace(
629 location=location, type=spec.ModuleType.PY_SOURCE
630 )
631 except NoSourceFile:
632 return found_spec._replace(location=location)
633 elif found_spec.type == spec.ModuleType.C_BUILTIN:
634 # integrated builtin module
635 return found_spec._replace(location=None)
636 elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:
637 assert found_spec.location is not None
638 location = _has_init(found_spec.location)
639 return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
640 return found_spec
643def _is_python_file(filename: str) -> bool:
644 """Return true if the given filename should be considered as a python file.
646 .pyc and .pyo are ignored
647 """
648 return filename.endswith((".py", ".pyi", ".so", ".pyd", ".pyw"))
651@lru_cache(maxsize=1024)
652def _has_init(directory: str) -> str | None:
653 """If the given directory has a valid __init__ file, return its path,
654 else return None.
655 """
656 mod_or_pack = os.path.join(directory, "__init__")
657 for ext in (*PY_SOURCE_EXTS, *PY_BYTECODE_EXTS):
658 if os.path.exists(mod_or_pack + "." + ext):
659 return mod_or_pack + "." + ext
660 return None
663def is_namespace(specobj: spec.ModuleSpec) -> bool:
664 return specobj.type == spec.ModuleType.PY_NAMESPACE
667def is_directory(specobj: spec.ModuleSpec) -> bool:
668 return specobj.type == spec.ModuleType.PKG_DIRECTORY
671def is_module_name_part_of_extension_package_whitelist(
672 module_name: str, package_whitelist: set[str]
673) -> bool:
674 """
675 Returns True if one part of the module name is in the package whitelist.
677 >>> is_module_name_part_of_extension_package_whitelist('numpy.core.umath', {'numpy'})
678 True
679 """
680 parts = module_name.split(".")
681 return any(
682 ".".join(parts[:x]) in package_whitelist for x in range(1, len(parts) + 1)
683 )