Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/modutils.py: 25%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Python modules manipulation utility functions.
7:var PY_SOURCE_EXTS: list of possible python source file extension
9:var STD_LIB_DIRS: directories where standard modules are located
11:var BUILTIN_MODULES: dictionary with builtin module names has key
12"""
14from __future__ import annotations
16import importlib
17import importlib.machinery
18import importlib.util
19import io
20import itertools
21import os
22import sys
23import sysconfig
24import types
25from collections.abc import Callable, Iterable, Sequence
26from contextlib import redirect_stderr, redirect_stdout
27from functools import lru_cache
28from sys import stdlib_module_names
30from astroid.const import IS_JYTHON
31from astroid.interpreter._import import spec, util
33if sys.platform.startswith("win"):
34 PY_SOURCE_EXTS: tuple[str, ...] = ("py", "pyw", "pyi")
35 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "pyw", "py")
36 PY_COMPILED_EXTS = ("dll", "pyd")
37else:
38 PY_SOURCE_EXTS = ("py", "pyi")
39 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "py")
40 PY_COMPILED_EXTS = ("so",)
42# Bytecode extensions are platform-independent, unlike PY_COMPILED_EXTS above.
43PY_BYTECODE_EXTS = ("pyc", "pyo")
46# TODO: Adding `platstdlib` is a fix for a workaround in virtualenv. At some point we should
47# revisit whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1323.
48STD_LIB_DIRS: set[str] = {
49 sysconfig.get_path("stdlib"),
50 sysconfig.get_path("platstdlib"),
51}
53if os.name == "nt":
54 STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls"))
55 try:
56 # real_prefix is defined when running inside virtual environments,
57 # created with the **virtualenv** library.
58 # Deprecated in virtualenv==16.7.9
59 # See: https://github.com/pypa/virtualenv/issues/1622
60 STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) # type: ignore[attr-defined]
61 except AttributeError:
62 # sys.base_exec_prefix is always defined, but in a virtual environment
63 # created with the stdlib **venv** module, it points to the original
64 # installation, if the virtual env is activated.
65 try:
66 STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls"))
67 except AttributeError:
68 pass
70if os.name == "posix":
71 # Need the real prefix if we're in a virtualenv, otherwise
72 # the usual one will do.
73 # Deprecated in virtualenv==16.7.9
74 # See: https://github.com/pypa/virtualenv/issues/1622
75 try:
76 prefix: str = sys.real_prefix # type: ignore[attr-defined]
77 except AttributeError:
78 prefix = sys.prefix
80 def _posix_path(path: str) -> str:
81 base_python = f"python{sys.version_info.major}.{sys.version_info.minor}"
82 return os.path.join(prefix, path, base_python)
84 STD_LIB_DIRS.add(_posix_path("lib"))
85 if sys.maxsize > 2**32:
86 # This tries to fix a problem with /usr/lib64 builds,
87 # where systems are running both 32-bit and 64-bit code
88 # on the same machine, which reflects into the places where
89 # standard library could be found. More details can be found
90 # here http://bugs.python.org/issue1294959.
91 # An easy reproducing case would be
92 # https://github.com/pylint-dev/pylint/issues/712#issuecomment-163178753
93 STD_LIB_DIRS.add(_posix_path("lib64"))
95EXT_LIB_DIRS = {sysconfig.get_path("purelib"), sysconfig.get_path("platlib")}
96BUILTIN_MODULES: dict[str, bool] = dict.fromkeys(sys.builtin_module_names, True)
99class NoSourceFile(Exception):
100 """Exception raised when we are not able to get a python
101 source file for a precompiled file.
102 """
105def _normalize_path(path: str) -> str:
106 """Resolve symlinks in path and convert to absolute path.
108 Note that environment variables and ~ in the path need to be expanded in
109 advance.
111 This can be cached by using _cache_normalize_path.
112 """
113 return os.path.normcase(os.path.realpath(path))
116def _path_from_filename(filename: str, is_jython: bool = IS_JYTHON) -> str:
117 if not is_jython:
118 return filename
119 head, has_pyclass, _ = filename.partition("$py.class")
120 if has_pyclass:
121 return head + ".py"
122 return filename
125def _handle_blacklist(
126 blacklist: Sequence[str], dirnames: list[str], filenames: list[str]
127) -> None:
128 """Remove files/directories in the black list.
130 dirnames/filenames are usually from os.walk
131 """
132 for norecurs in blacklist:
133 if norecurs in dirnames:
134 dirnames.remove(norecurs)
135 elif norecurs in filenames:
136 filenames.remove(norecurs)
139@lru_cache
140def _cache_normalize_path_(path: str) -> str:
141 return _normalize_path(path)
144def _cache_normalize_path(path: str) -> str:
145 """Normalize path with caching."""
146 # _module_file calls abspath on every path in sys.path every time it's
147 # called; on a larger codebase this easily adds up to half a second just
148 # assembling path components. This cache alleviates that.
149 if not path: # don't cache result for ''
150 return _normalize_path(path)
151 return _cache_normalize_path_(path)
154def load_module_from_name(dotted_name: str) -> types.ModuleType:
155 """Load a Python module from its name.
157 :param dotted_name: python name of a module or package
159 :raise ImportError: if the module or package is not found
161 :return: the loaded module
162 """
163 try:
164 return sys.modules[dotted_name]
165 except KeyError:
166 pass
168 # Capture and log anything emitted during import to avoid
169 # contaminating JSON reports in pylint
170 with (
171 redirect_stderr(io.StringIO()) as stderr,
172 redirect_stdout(io.StringIO()) as stdout,
173 ):
174 module = importlib.import_module(dotted_name)
176 stderr_value = stderr.getvalue()
177 stdout_value = stdout.getvalue()
178 if stderr_value or stdout_value:
179 import logging # pylint: disable=import-outside-toplevel
181 logger = logging.getLogger(__name__)
182 if stderr_value:
183 logger.error(
184 "Captured stderr while importing %s:\n%s", dotted_name, stderr_value
185 )
186 if stdout_value:
187 logger.info(
188 "Captured stdout while importing %s:\n%s", dotted_name, stdout_value
189 )
191 return module
194def load_module_from_modpath(parts: Sequence[str]) -> types.ModuleType:
195 """Load a python module from its split name.
197 :param parts:
198 python name of a module or package split on '.'
200 :raise ImportError: if the module or package is not found
202 :return: the loaded module
203 """
204 return load_module_from_name(".".join(parts))
207def load_module_from_file(filepath: str) -> types.ModuleType:
208 """Load a Python module from it's path.
210 :param filepath: path to the python module or package
212 :raise ImportError: if the module or package is not found
214 :return: the loaded module
215 """
216 modpath = modpath_from_file(filepath)
217 return load_module_from_modpath(modpath)
220def check_modpath_has_init(path: str, mod_path: list[str]) -> bool:
221 """Check there are some __init__.py all along the way."""
222 modpath: list[str] = []
223 for part in mod_path:
224 modpath.append(part)
225 path = os.path.join(path, part)
226 if not _has_init(path):
227 old_namespace = util.is_namespace(".".join(modpath))
228 if not old_namespace:
229 return False
230 return True
233def _is_subpath(path: str, base: str) -> bool:
234 path = os.path.normcase(os.path.normpath(path))
235 base = os.path.normcase(os.path.normpath(base))
236 if not path.startswith(base):
237 return False
238 return (
239 (len(path) == len(base))
240 or (path[len(base)] == os.path.sep)
241 or (base.endswith(os.path.sep) and path[len(base) - 1] == os.path.sep)
242 )
245def _get_relative_base_path(filename: str, path_to_check: str) -> list[str] | None:
246 """Extracts the relative mod path of the file to import from.
248 Check if a file is within the passed in path and if so, returns the
249 relative mod path from the one passed in.
251 If the filename is no in path_to_check, returns None
253 Note this function will look for both abs and realpath of the file,
254 this allows to find the relative base path even if the file is a
255 symlink of a file in the passed in path
257 Examples:
258 _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"]
259 _get_relative_base_path("/a/b/c/d.py", "/dev") -> None
260 """
261 path_to_check = os.path.normcase(os.path.normpath(path_to_check))
263 abs_filename = os.path.abspath(filename)
264 if _is_subpath(abs_filename, path_to_check):
265 base_path = os.path.splitext(abs_filename)[0]
266 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
267 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
269 real_filename = os.path.realpath(filename)
270 if _is_subpath(real_filename, path_to_check):
271 base_path = os.path.splitext(real_filename)[0]
272 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
273 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
275 return None
278def modpath_from_file_with_callback(
279 filename: str,
280 path: list[str] | None = None,
281 is_package_cb: Callable[[str, list[str]], bool] | None = None,
282) -> list[str]:
283 filename = os.path.expanduser(_path_from_filename(filename))
284 paths_to_check = sys.path.copy()
285 if path:
286 paths_to_check = path + paths_to_check
287 for pathname in itertools.chain(
288 paths_to_check, map(_cache_normalize_path, paths_to_check)
289 ):
290 if not pathname:
291 continue
292 modpath = _get_relative_base_path(filename, pathname)
293 if not modpath:
294 continue
295 assert is_package_cb is not None
296 if is_package_cb(pathname, modpath[:-1]):
297 return modpath
299 raise ImportError(
300 "Unable to find module for {} in {}".format(
301 filename, ", \n".join(paths_to_check)
302 )
303 )
306def modpath_from_file(filename: str, path: list[str] | None = None) -> list[str]:
307 """Get the corresponding split module's name from a filename.
309 This function will return the name of a module or package split on `.`.
311 :param filename: file's path for which we want the module's name
313 :param path:
314 Optional list of paths where the module or package should be
315 searched, additionally to sys.path
317 :raise ImportError:
318 if the corresponding module's name has not been found
320 :return: the corresponding split module's name
321 """
322 return modpath_from_file_with_callback(filename, path, check_modpath_has_init)
325def file_from_modpath(
326 modpath: list[str],
327 path: Sequence[str] | None = None,
328 context_file: str | None = None,
329) -> str | None:
330 return file_info_from_modpath(modpath, path, context_file).location
333def file_info_from_modpath(
334 modpath: list[str],
335 path: Sequence[str] | None = None,
336 context_file: str | None = None,
337) -> spec.ModuleSpec:
338 """Given a mod path (i.e. split module / package name), return the
339 corresponding file.
341 Giving priority to source file over precompiled file if it exists.
343 :param modpath:
344 split module's name (i.e name of a module or package split
345 on '.')
346 (this means explicit relative imports that start with dots have
347 empty strings in this list!)
349 :param path:
350 optional list of path where the module or package should be
351 searched (use sys.path if nothing or None is given)
353 :param context_file:
354 context file to consider, necessary if the identifier has been
355 introduced using a relative import unresolvable in the actual
356 context (i.e. modutils)
358 :raise ImportError: if there is no such module in the directory
360 :return:
361 the path to the module's file or None if it's an integrated
362 builtin module such as 'sys'
363 """
364 if context_file is not None:
365 context: str | None = os.path.dirname(context_file)
366 else:
367 context = context_file
368 if modpath[0] == "xml":
369 # handle _xmlplus
370 try:
371 return _spec_from_modpath(["_xmlplus", *modpath[1:]], path, context)
372 except ImportError:
373 return _spec_from_modpath(modpath, path, context)
374 elif modpath == ["os", "path"]:
375 # FIXME: currently ignoring search_path...
376 return spec.ModuleSpec(
377 name="os.path",
378 location=os.path.__file__,
379 type=spec.ModuleType.PY_SOURCE,
380 )
381 return _spec_from_modpath(modpath, path, context)
384def get_module_part(dotted_name: str, context_file: str | None = None) -> str:
385 """Given a dotted name return the module part of the name :
387 >>> get_module_part('astroid.as_string.dump')
388 'astroid.as_string'
390 :param dotted_name: full name of the identifier we are interested in
392 :param context_file:
393 context file to consider, necessary if the identifier has been
394 introduced using a relative import unresolvable in the actual
395 context (i.e. modutils)
397 :raise ImportError: if there is no such module in the directory
399 :return:
400 the module part of the name or None if we have not been able at
401 all to import the given name
403 XXX: deprecated, since it doesn't handle package precedence over module
404 (see #10066)
405 """
406 # os.path trick
407 if dotted_name.startswith("os.path"):
408 return "os.path"
409 parts = dotted_name.split(".")
410 if context_file is not None:
411 # first check for builtin module which won't be considered latter
412 # in that case (path != None)
413 if parts[0] in BUILTIN_MODULES:
414 if len(parts) > 2:
415 raise ImportError(dotted_name)
416 return parts[0]
417 # don't use += or insert, we want a new list to be created !
418 path: list[str] | None = None
419 starti = 0
420 if parts[0] == "":
421 assert (
422 context_file is not None
423 ), "explicit relative import, but no context_file?"
424 path = [] # prevent resolving the import non-relatively
425 starti = 1
426 # for all further dots: change context
427 while starti < len(parts) and parts[starti] == "":
428 starti += 1
429 assert (
430 context_file is not None
431 ), "explicit relative import, but no context_file?"
432 context_file = os.path.dirname(context_file)
433 for i in range(starti, len(parts)):
434 try:
435 file_from_modpath(
436 parts[starti : i + 1], path=path, context_file=context_file
437 )
438 except ImportError:
439 if i < max(1, len(parts) - 2):
440 raise
441 return ".".join(parts[:i])
442 return dotted_name
445def get_module_files(
446 src_directory: str, blacklist: Sequence[str], list_all: bool = False
447) -> list[str]:
448 """Given a package directory return a list of all available python
449 module's files in the package and its subpackages.
451 :param src_directory:
452 path of the directory corresponding to the package
454 :param blacklist: iterable
455 list of files or directories to ignore.
457 :param list_all:
458 get files from all paths, including ones without __init__.py
460 :return:
461 the list of all available python module's files in the package and
462 its subpackages
463 """
464 files: list[str] = []
465 for directory, dirnames, filenames in os.walk(src_directory):
466 if directory in blacklist:
467 continue
468 _handle_blacklist(blacklist, dirnames, filenames)
469 # check for __init__.py
470 if not list_all and {"__init__.py", "__init__.pyi"}.isdisjoint(filenames):
471 dirnames[:] = ()
472 continue
473 for filename in filenames:
474 if _is_python_file(filename):
475 src = os.path.join(directory, filename)
476 files.append(src)
477 return files
480def _is_compiled_ext(ext: str) -> bool:
481 """Return whether ``ext`` (without leading dot) names a compiled module:
482 bytecode (.pyc/.pyo) or a binary extension module (.so/.pyd/.dll). Such
483 files are never source.
484 """
485 return ext in PY_BYTECODE_EXTS or ext in PY_COMPILED_EXTS
488def get_source_file(
489 filename: str, include_no_ext: bool = False, prefer_stubs: bool = False
490) -> str:
491 """Given a python module's file name return the matching source file
492 name (the filename will be returned identically if it's already an
493 absolute path to a python source file).
495 :param filename: python module's file name
497 :raise NoSourceFile: if no source file exists on the file system
499 :return: the absolute path of the source file if it exists
500 """
501 filename = os.path.abspath(_path_from_filename(filename))
502 base, orig_ext = os.path.splitext(filename)
503 orig_ext = orig_ext.lstrip(".")
504 extension_suffix = max(
505 (
506 suffix
507 for suffix in importlib.machinery.EXTENSION_SUFFIXES
508 if filename.endswith(suffix)
509 ),
510 key=len,
511 default=None,
512 )
513 if extension_suffix is not None:
514 base = filename[: -len(extension_suffix)]
515 # A non-standard extension (e.g. a custom suffix) is returned as-is, but a
516 # compiled file falls through to the lookup below so its .py/.pyi stub is
517 # found instead.
518 if (
519 orig_ext not in PY_SOURCE_EXTS
520 and not _is_compiled_ext(orig_ext)
521 and os.path.exists(f"{base}.{orig_ext}")
522 ):
523 return f"{base}.{orig_ext}"
524 for ext in PY_SOURCE_EXTS_STUBS_FIRST if prefer_stubs else PY_SOURCE_EXTS:
525 source_path = f"{base}.{ext}"
526 if os.path.exists(source_path):
527 return source_path
528 if include_no_ext and not orig_ext and os.path.exists(base):
529 return base
530 raise NoSourceFile(filename)
533def is_python_source(filename: str | None) -> bool:
534 """Return: True if the filename is a python source file."""
535 if not filename:
536 return False
537 return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
540def is_stdlib_module(modname: str) -> bool:
541 """Return: True if the modname is in the standard library"""
542 return modname.split(".")[0] in stdlib_module_names
545def is_stdlib_path(filename: str) -> bool:
546 """Return: True if the file is located in the standard library.
548 site-packages sits below one of the standard library directories in a
549 virtualenv, so external locations are rejected first.
550 """
551 filename = _normalize_path(filename)
552 if any(
553 filename.startswith(_cache_normalize_path(entry) + os.sep)
554 for entry in EXT_LIB_DIRS
555 ):
556 return False
557 return any(
558 filename.startswith(_cache_normalize_path(entry) + os.sep)
559 for entry in STD_LIB_DIRS
560 )
563def module_in_path(modname: str, path: str | Iterable[str]) -> bool:
564 """Try to determine if a module is imported from one of the specified paths
566 :param modname: name of the module
568 :param path: paths to consider
570 :return:
571 true if the module:
572 - is located on the path listed in one of the directory in `paths`
573 """
575 modname = modname.split(".")[0]
576 try:
577 filename = file_from_modpath([modname])
578 except ImportError:
579 # Import failed, we can't check path if we don't know it
580 return False
582 if filename is None:
583 # No filename likely means it's compiled in, or potentially a namespace
584 return False
585 filename = _normalize_path(filename)
587 if isinstance(path, str):
588 return _is_subpath(filename, _cache_normalize_path(path))
590 return any(_is_subpath(filename, _cache_normalize_path(entry)) for entry in path)
593def is_relative(modname: str, from_file: str) -> bool:
594 """Return true if the given module name is relative to the given
595 file name.
597 :param modname: name of the module we are interested in
599 :param from_file:
600 path of the module from which modname has been imported
602 :return:
603 true if the module has been imported relatively to `from_file`
604 """
605 if not os.path.isdir(from_file):
606 from_file = os.path.dirname(from_file)
607 if from_file in sys.path:
608 return False
609 return bool(
610 importlib.machinery.PathFinder.find_spec(
611 modname.split(".", maxsplit=1)[0], [from_file]
612 )
613 )
616@lru_cache(maxsize=1024)
617def cached_os_path_isfile(path: str | os.PathLike[str]) -> bool:
618 """A cached version of os.path.isfile that helps avoid repetitive I/O"""
619 return os.path.isfile(path)
622# internal only functions #####################################################
625def _spec_from_modpath(
626 modpath: list[str],
627 path: Sequence[str] | None = None,
628 context: str | None = None,
629) -> spec.ModuleSpec:
630 """Given a mod path (i.e. split module / package name), return the
631 corresponding spec.
633 this function is used internally, see `file_from_modpath`'s
634 documentation for more information
635 """
636 assert modpath
637 location = None
638 if context is not None:
639 try:
640 found_spec = spec.find_spec(modpath, [context])
641 location = found_spec.location
642 except ImportError:
643 found_spec = spec.find_spec(modpath, path)
644 location = found_spec.location
645 else:
646 found_spec = spec.find_spec(modpath, path)
647 if found_spec.type == spec.ModuleType.PY_COMPILED:
648 try:
649 assert found_spec.location is not None
650 location = get_source_file(found_spec.location)
651 return found_spec._replace(
652 location=location, type=spec.ModuleType.PY_SOURCE
653 )
654 except NoSourceFile:
655 return found_spec._replace(location=location)
656 elif found_spec.type == spec.ModuleType.C_BUILTIN:
657 # integrated builtin module
658 return found_spec._replace(location=None)
659 elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:
660 assert found_spec.location is not None
661 location = _has_init(found_spec.location)
662 return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
663 return found_spec
666def _is_python_file(filename: str) -> bool:
667 """Return true if the given filename should be considered as a python file.
669 .pyc and .pyo are ignored
670 """
671 return filename.endswith((".py", ".pyi", ".so", ".pyd", ".pyw"))
674@lru_cache(maxsize=1024)
675def _has_init(directory: str) -> str | None:
676 """If the given directory has a valid __init__ file, return its path,
677 else return None.
678 """
679 mod_or_pack = os.path.join(directory, "__init__")
680 for ext in (*PY_SOURCE_EXTS, *PY_BYTECODE_EXTS):
681 if os.path.exists(mod_or_pack + "." + ext):
682 return mod_or_pack + "." + ext
683 return None
686def is_namespace(specobj: spec.ModuleSpec) -> bool:
687 return specobj.type == spec.ModuleType.PY_NAMESPACE
690def is_directory(specobj: spec.ModuleSpec) -> bool:
691 return specobj.type == spec.ModuleType.PKG_DIRECTORY
694def is_module_name_part_of_extension_package_whitelist(
695 module_name: str, package_whitelist: set[str]
696) -> bool:
697 """
698 Returns True if one part of the module name is in the package whitelist.
700 >>> is_module_name_part_of_extension_package_whitelist('numpy.core.umath', {'numpy'})
701 True
702 """
703 parts = module_name.split(".")
704 return any(
705 ".".join(parts[:x]) in package_whitelist for x in range(1, len(parts) + 1)
706 )