Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/modutils.py: 25%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Python modules manipulation utility functions.
7:type PY_SOURCE_EXTS: tuple(str)
8:var PY_SOURCE_EXTS: list of possible python source file extension
10:type STD_LIB_DIRS: set of str
11:var STD_LIB_DIRS: directories where standard modules are located
13:type BUILTIN_MODULES: dict
14:var BUILTIN_MODULES: dictionary with builtin module names has key
15"""
17from __future__ import annotations
19import importlib
20import importlib.machinery
21import importlib.util
22import io
23import itertools
24import logging
25import os
26import sys
27import sysconfig
28import types
29import warnings
30from collections.abc import Callable, Iterable, Sequence
31from contextlib import redirect_stderr, redirect_stdout
32from functools import lru_cache
33from sys import stdlib_module_names
35from astroid.const import IS_JYTHON
36from astroid.interpreter._import import spec, util
38logger = logging.getLogger(__name__)
41if sys.platform.startswith("win"):
42 PY_SOURCE_EXTS = ("py", "pyw", "pyi")
43 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "pyw", "py")
44 PY_COMPILED_EXTS = ("dll", "pyd")
45else:
46 PY_SOURCE_EXTS = ("py", "pyi")
47 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "py")
48 PY_COMPILED_EXTS = ("so",)
51# TODO: Adding `platstdlib` is a fix for a workaround in virtualenv. At some point we should
52# revisit whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1323.
53STD_LIB_DIRS = {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
55if os.name == "nt":
56 STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls"))
57 try:
58 # real_prefix is defined when running inside virtual environments,
59 # created with the **virtualenv** library.
60 # Deprecated in virtualenv==16.7.9
61 # See: https://github.com/pypa/virtualenv/issues/1622
62 STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) # type: ignore[attr-defined]
63 except AttributeError:
64 # sys.base_exec_prefix is always defined, but in a virtual environment
65 # created with the stdlib **venv** module, it points to the original
66 # installation, if the virtual env is activated.
67 try:
68 STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls"))
69 except AttributeError:
70 pass
72if os.name == "posix":
73 # Need the real prefix if we're in a virtualenv, otherwise
74 # the usual one will do.
75 # Deprecated in virtualenv==16.7.9
76 # See: https://github.com/pypa/virtualenv/issues/1622
77 try:
78 prefix: str = sys.real_prefix # type: ignore[attr-defined]
79 except AttributeError:
80 prefix = sys.prefix
82 def _posix_path(path: str) -> str:
83 base_python = "python%d.%d" % sys.version_info[:2]
84 return os.path.join(prefix, path, base_python)
86 STD_LIB_DIRS.add(_posix_path("lib"))
87 if sys.maxsize > 2**32:
88 # This tries to fix a problem with /usr/lib64 builds,
89 # where systems are running both 32-bit and 64-bit code
90 # on the same machine, which reflects into the places where
91 # standard library could be found. More details can be found
92 # here http://bugs.python.org/issue1294959.
93 # An easy reproducing case would be
94 # https://github.com/pylint-dev/pylint/issues/712#issuecomment-163178753
95 STD_LIB_DIRS.add(_posix_path("lib64"))
97EXT_LIB_DIRS = {sysconfig.get_path("purelib"), sysconfig.get_path("platlib")}
98BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
101class NoSourceFile(Exception):
102 """Exception raised when we are not able to get a python
103 source file for a precompiled file.
104 """
107def _normalize_path(path: str) -> str:
108 """Resolve symlinks in path and convert to absolute path.
110 Note that environment variables and ~ in the path need to be expanded in
111 advance.
113 This can be cached by using _cache_normalize_path.
114 """
115 return os.path.normcase(os.path.realpath(path))
118def _path_from_filename(filename: str, is_jython: bool = IS_JYTHON) -> str:
119 if not is_jython:
120 return filename
121 head, has_pyclass, _ = filename.partition("$py.class")
122 if has_pyclass:
123 return head + ".py"
124 return filename
127def _handle_blacklist(
128 blacklist: Sequence[str], dirnames: list[str], filenames: list[str]
129) -> None:
130 """Remove files/directories in the black list.
132 dirnames/filenames are usually from os.walk
133 """
134 for norecurs in blacklist:
135 if norecurs in dirnames:
136 dirnames.remove(norecurs)
137 elif norecurs in filenames:
138 filenames.remove(norecurs)
141@lru_cache
142def _cache_normalize_path_(path: str) -> str:
143 return _normalize_path(path)
146def _cache_normalize_path(path: str) -> str:
147 """Normalize path with caching."""
148 # _module_file calls abspath on every path in sys.path every time it's
149 # called; on a larger codebase this easily adds up to half a second just
150 # assembling path components. This cache alleviates that.
151 if not path: # don't cache result for ''
152 return _normalize_path(path)
153 return _cache_normalize_path_(path)
156def load_module_from_name(dotted_name: str) -> types.ModuleType:
157 """Load a Python module from its name.
159 :type dotted_name: str
160 :param dotted_name: python name of a module or package
162 :raise ImportError: if the module or package is not found
164 :rtype: module
165 :return: the loaded module
166 """
167 try:
168 return sys.modules[dotted_name]
169 except KeyError:
170 pass
172 # Capture and log anything emitted during import to avoid
173 # contaminating JSON reports in pylint
174 with (
175 redirect_stderr(io.StringIO()) as stderr,
176 redirect_stdout(io.StringIO()) as stdout,
177 ):
178 module = importlib.import_module(dotted_name)
180 stderr_value = stderr.getvalue()
181 if stderr_value:
182 logger.error(
183 "Captured stderr while importing %s:\n%s", dotted_name, stderr_value
184 )
185 stdout_value = stdout.getvalue()
186 if stdout_value:
187 logger.info(
188 "Captured stdout while importing %s:\n%s", dotted_name, stdout_value
189 )
191 return module
194def load_module_from_modpath(parts: Sequence[str]) -> types.ModuleType:
195 """Load a python module from its split name.
197 :param parts:
198 python name of a module or package split on '.'
200 :raise ImportError: if the module or package is not found
202 :return: the loaded module
203 """
204 return load_module_from_name(".".join(parts))
207def load_module_from_file(filepath: str) -> types.ModuleType:
208 """Load a Python module from it's path.
210 :type filepath: str
211 :param filepath: path to the python module or package
213 :raise ImportError: if the module or package is not found
215 :rtype: module
216 :return: the loaded module
217 """
218 modpath = modpath_from_file(filepath)
219 return load_module_from_modpath(modpath)
222def check_modpath_has_init(path: str, mod_path: list[str]) -> bool:
223 """Check there are some __init__.py all along the way."""
224 modpath: list[str] = []
225 for part in mod_path:
226 modpath.append(part)
227 path = os.path.join(path, part)
228 if not _has_init(path):
229 old_namespace = util.is_namespace(".".join(modpath))
230 if not old_namespace:
231 return False
232 return True
235def _is_subpath(path: str, base: str) -> bool:
236 path = os.path.normcase(os.path.normpath(path))
237 base = os.path.normcase(os.path.normpath(base))
238 if not path.startswith(base):
239 return False
240 return (len(path) == len(base)) or (path[len(base)] == os.path.sep)
243def _get_relative_base_path(filename: str, path_to_check: str) -> list[str] | None:
244 """Extracts the relative mod path of the file to import from.
246 Check if a file is within the passed in path and if so, returns the
247 relative mod path from the one passed in.
249 If the filename is no in path_to_check, returns None
251 Note this function will look for both abs and realpath of the file,
252 this allows to find the relative base path even if the file is a
253 symlink of a file in the passed in path
255 Examples:
256 _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"]
257 _get_relative_base_path("/a/b/c/d.py", "/dev") -> None
258 """
259 path_to_check = os.path.normcase(os.path.normpath(path_to_check))
261 abs_filename = os.path.abspath(filename)
262 if _is_subpath(abs_filename, path_to_check):
263 base_path = os.path.splitext(abs_filename)[0]
264 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
265 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
267 real_filename = os.path.realpath(filename)
268 if _is_subpath(real_filename, path_to_check):
269 base_path = os.path.splitext(real_filename)[0]
270 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
271 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
273 return None
276def modpath_from_file_with_callback(
277 filename: str,
278 path: list[str] | None = None,
279 is_package_cb: Callable[[str, list[str]], bool] | None = None,
280) -> list[str]:
281 filename = os.path.expanduser(_path_from_filename(filename))
282 paths_to_check = sys.path.copy()
283 if path:
284 paths_to_check = path + paths_to_check
285 for pathname in itertools.chain(
286 paths_to_check, map(_cache_normalize_path, paths_to_check)
287 ):
288 if not pathname:
289 continue
290 modpath = _get_relative_base_path(filename, pathname)
291 if not modpath:
292 continue
293 assert is_package_cb is not None
294 if is_package_cb(pathname, modpath[:-1]):
295 return modpath
297 raise ImportError(
298 "Unable to find module for {} in {}".format(
299 filename, ", \n".join(paths_to_check)
300 )
301 )
304def modpath_from_file(filename: str, path: list[str] | None = None) -> list[str]:
305 """Get the corresponding split module's name from a filename.
307 This function will return the name of a module or package split on `.`.
309 :type filename: str
310 :param filename: file's path for which we want the module's name
312 :type Optional[List[str]] path:
313 Optional list of paths where the module or package should be
314 searched, additionally to sys.path
316 :raise ImportError:
317 if the corresponding module's name has not been found
319 :rtype: list(str)
320 :return: the corresponding split module's name
321 """
322 return modpath_from_file_with_callback(filename, path, check_modpath_has_init)
325def file_from_modpath(
326 modpath: list[str],
327 path: Sequence[str] | None = None,
328 context_file: str | None = None,
329) -> str | None:
330 return file_info_from_modpath(modpath, path, context_file).location
333def file_info_from_modpath(
334 modpath: list[str],
335 path: Sequence[str] | None = None,
336 context_file: str | None = None,
337) -> spec.ModuleSpec:
338 """Given a mod path (i.e. split module / package name), return the
339 corresponding file.
341 Giving priority to source file over precompiled file if it exists.
343 :param modpath:
344 split module's name (i.e name of a module or package split
345 on '.')
346 (this means explicit relative imports that start with dots have
347 empty strings in this list!)
349 :param path:
350 optional list of path where the module or package should be
351 searched (use sys.path if nothing or None is given)
353 :param context_file:
354 context file to consider, necessary if the identifier has been
355 introduced using a relative import unresolvable in the actual
356 context (i.e. modutils)
358 :raise ImportError: if there is no such module in the directory
360 :return:
361 the path to the module's file or None if it's an integrated
362 builtin module such as 'sys'
363 """
364 if context_file is not None:
365 context: str | None = os.path.dirname(context_file)
366 else:
367 context = context_file
368 if modpath[0] == "xml":
369 # handle _xmlplus
370 try:
371 return _spec_from_modpath(["_xmlplus", *modpath[1:]], path, context)
372 except ImportError:
373 return _spec_from_modpath(modpath, path, context)
374 elif modpath == ["os", "path"]:
375 # FIXME: currently ignoring search_path...
376 return spec.ModuleSpec(
377 name="os.path",
378 location=os.path.__file__,
379 type=spec.ModuleType.PY_SOURCE,
380 )
381 return _spec_from_modpath(modpath, path, context)
384def get_module_part(dotted_name: str, context_file: str | None = None) -> str:
385 """Given a dotted name return the module part of the name :
387 >>> get_module_part('astroid.as_string.dump')
388 'astroid.as_string'
390 :param dotted_name: full name of the identifier we are interested in
392 :param context_file:
393 context file to consider, necessary if the identifier has been
394 introduced using a relative import unresolvable in the actual
395 context (i.e. modutils)
397 :raise ImportError: if there is no such module in the directory
399 :return:
400 the module part of the name or None if we have not been able at
401 all to import the given name
403 XXX: deprecated, since it doesn't handle package precedence over module
404 (see #10066)
405 """
406 # os.path trick
407 if dotted_name.startswith("os.path"):
408 return "os.path"
409 parts = dotted_name.split(".")
410 if context_file is not None:
411 # first check for builtin module which won't be considered latter
412 # in that case (path != None)
413 if parts[0] in BUILTIN_MODULES:
414 if len(parts) > 2:
415 raise ImportError(dotted_name)
416 return parts[0]
417 # don't use += or insert, we want a new list to be created !
418 path: list[str] | None = None
419 starti = 0
420 if parts[0] == "":
421 assert (
422 context_file is not None
423 ), "explicit relative import, but no context_file?"
424 path = [] # prevent resolving the import non-relatively
425 starti = 1
426 # for all further dots: change context
427 while starti < len(parts) and parts[starti] == "":
428 starti += 1
429 assert (
430 context_file is not None
431 ), "explicit relative import, but no context_file?"
432 context_file = os.path.dirname(context_file)
433 for i in range(starti, len(parts)):
434 try:
435 file_from_modpath(
436 parts[starti : i + 1], path=path, context_file=context_file
437 )
438 except ImportError:
439 if i < max(1, len(parts) - 2):
440 raise
441 return ".".join(parts[:i])
442 return dotted_name
445def get_module_files(
446 src_directory: str, blacklist: Sequence[str], list_all: bool = False
447) -> list[str]:
448 """Given a package directory return a list of all available python
449 module's files in the package and its subpackages.
451 :param src_directory:
452 path of the directory corresponding to the package
454 :param blacklist: iterable
455 list of files or directories to ignore.
457 :param list_all:
458 get files from all paths, including ones without __init__.py
460 :return:
461 the list of all available python module's files in the package and
462 its subpackages
463 """
464 files: list[str] = []
465 for directory, dirnames, filenames in os.walk(src_directory):
466 if directory in blacklist:
467 continue
468 _handle_blacklist(blacklist, dirnames, filenames)
469 # check for __init__.py
470 if not list_all and {"__init__.py", "__init__.pyi"}.isdisjoint(filenames):
471 dirnames[:] = ()
472 continue
473 for filename in filenames:
474 if _is_python_file(filename):
475 src = os.path.join(directory, filename)
476 files.append(src)
477 return files
480def get_source_file(
481 filename: str, include_no_ext: bool = False, prefer_stubs: bool = False
482) -> str:
483 """Given a python module's file name return the matching source file
484 name (the filename will be returned identically if it's already an
485 absolute path to a python source file).
487 :param filename: python module's file name
489 :raise NoSourceFile: if no source file exists on the file system
491 :return: the absolute path of the source file if it exists
492 """
493 filename = os.path.abspath(_path_from_filename(filename))
494 base, orig_ext = os.path.splitext(filename)
495 orig_ext = orig_ext.lstrip(".")
496 if orig_ext not in PY_SOURCE_EXTS and os.path.exists(f"{base}.{orig_ext}"):
497 return f"{base}.{orig_ext}"
498 for ext in PY_SOURCE_EXTS_STUBS_FIRST if prefer_stubs else PY_SOURCE_EXTS:
499 source_path = f"{base}.{ext}"
500 if os.path.exists(source_path):
501 return source_path
502 if include_no_ext and not orig_ext and os.path.exists(base):
503 return base
504 raise NoSourceFile(filename)
507def is_python_source(filename: str | None) -> bool:
508 """Return: True if the filename is a python source file."""
509 if not filename:
510 return False
511 return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
514def is_stdlib_module(modname: str) -> bool:
515 """Return: True if the modname is in the standard library"""
516 return modname.split(".")[0] in stdlib_module_names
519def module_in_path(modname: str, path: str | Iterable[str]) -> bool:
520 """Try to determine if a module is imported from one of the specified paths
522 :param modname: name of the module
524 :param path: paths to consider
526 :return:
527 true if the module:
528 - is located on the path listed in one of the directory in `paths`
529 """
531 modname = modname.split(".")[0]
532 try:
533 filename = file_from_modpath([modname])
534 except ImportError:
535 # Import failed, we can't check path if we don't know it
536 return False
538 if filename is None:
539 # No filename likely means it's compiled in, or potentially a namespace
540 return False
541 filename = _normalize_path(filename)
543 if isinstance(path, str):
544 return filename.startswith(_cache_normalize_path(path))
546 return any(filename.startswith(_cache_normalize_path(entry)) for entry in path)
549def is_standard_module(modname: str, std_path: Iterable[str] | None = None) -> bool:
550 """Try to guess if a module is a standard python module (by default,
551 see `std_path` parameter's description).
553 :param modname: name of the module we are interested in
555 :param std_path: list of path considered has standard
557 :return:
558 true if the module:
559 - is located on the path listed in one of the directory in `std_path`
560 - is a built-in module
561 """
562 warnings.warn(
563 "is_standard_module() is deprecated. Use, is_stdlib_module() or module_in_path() instead",
564 DeprecationWarning,
565 stacklevel=2,
566 )
568 modname = modname.split(".")[0]
569 try:
570 filename = file_from_modpath([modname])
571 except ImportError:
572 # import failed, i'm probably not so wrong by supposing it's
573 # not standard...
574 return False
575 # modules which are not living in a file are considered standard
576 # (sys and __builtin__ for instance)
577 if filename is None:
578 # we assume there are no namespaces in stdlib
579 return not util.is_namespace(modname)
580 filename = _normalize_path(filename)
581 for path in EXT_LIB_DIRS:
582 if filename.startswith(_cache_normalize_path(path)):
583 return False
584 if std_path is None:
585 std_path = STD_LIB_DIRS
587 return any(filename.startswith(_cache_normalize_path(path)) for path in std_path)
590def is_relative(modname: str, from_file: str) -> bool:
591 """Return true if the given module name is relative to the given
592 file name.
594 :param modname: name of the module we are interested in
596 :param from_file:
597 path of the module from which modname has been imported
599 :return:
600 true if the module has been imported relatively to `from_file`
601 """
602 if not os.path.isdir(from_file):
603 from_file = os.path.dirname(from_file)
604 if from_file in sys.path:
605 return False
606 return bool(
607 importlib.machinery.PathFinder.find_spec(
608 modname.split(".", maxsplit=1)[0], [from_file]
609 )
610 )
613@lru_cache(maxsize=1024)
614def cached_os_path_isfile(path: str | os.PathLike[str]) -> bool:
615 """A cached version of os.path.isfile that helps avoid repetitive I/O"""
616 return os.path.isfile(path)
619# internal only functions #####################################################
622def _spec_from_modpath(
623 modpath: list[str],
624 path: Sequence[str] | None = None,
625 context: str | None = None,
626) -> spec.ModuleSpec:
627 """Given a mod path (i.e. split module / package name), return the
628 corresponding spec.
630 this function is used internally, see `file_from_modpath`'s
631 documentation for more information
632 """
633 assert modpath
634 location = None
635 if context is not None:
636 try:
637 found_spec = spec.find_spec(modpath, [context])
638 location = found_spec.location
639 except ImportError:
640 found_spec = spec.find_spec(modpath, path)
641 location = found_spec.location
642 else:
643 found_spec = spec.find_spec(modpath, path)
644 if found_spec.type == spec.ModuleType.PY_COMPILED:
645 try:
646 assert found_spec.location is not None
647 location = get_source_file(found_spec.location)
648 return found_spec._replace(
649 location=location, type=spec.ModuleType.PY_SOURCE
650 )
651 except NoSourceFile:
652 return found_spec._replace(location=location)
653 elif found_spec.type == spec.ModuleType.C_BUILTIN:
654 # integrated builtin module
655 return found_spec._replace(location=None)
656 elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:
657 assert found_spec.location is not None
658 location = _has_init(found_spec.location)
659 return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
660 return found_spec
663def _is_python_file(filename: str) -> bool:
664 """Return true if the given filename should be considered as a python file.
666 .pyc and .pyo are ignored
667 """
668 return filename.endswith((".py", ".pyi", ".so", ".pyd", ".pyw"))
671@lru_cache(maxsize=1024)
672def _has_init(directory: str) -> str | None:
673 """If the given directory has a valid __init__ file, return its path,
674 else return None.
675 """
676 mod_or_pack = os.path.join(directory, "__init__")
677 for ext in (*PY_SOURCE_EXTS, "pyc", "pyo"):
678 if os.path.exists(mod_or_pack + "." + ext):
679 return mod_or_pack + "." + ext
680 return None
683def is_namespace(specobj: spec.ModuleSpec) -> bool:
684 return specobj.type == spec.ModuleType.PY_NAMESPACE
687def is_directory(specobj: spec.ModuleSpec) -> bool:
688 return specobj.type == spec.ModuleType.PKG_DIRECTORY
691def is_module_name_part_of_extension_package_whitelist(
692 module_name: str, package_whitelist: set[str]
693) -> bool:
694 """
695 Returns True if one part of the module name is in the package whitelist.
697 >>> is_module_name_part_of_extension_package_whitelist('numpy.core.umath', {'numpy'})
698 True
699 """
700 parts = module_name.split(".")
701 return any(
702 ".".join(parts[:x]) in package_whitelist for x in range(1, len(parts) + 1)
703 )