Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/astroid/modutils.py: 34%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Python modules manipulation utility functions.
7:type PY_SOURCE_EXTS: tuple(str)
8:var PY_SOURCE_EXTS: list of possible python source file extension
10:type STD_LIB_DIRS: set of str
11:var STD_LIB_DIRS: directories where standard modules are located
13:type BUILTIN_MODULES: dict
14:var BUILTIN_MODULES: dictionary with builtin module names has key
15"""
17from __future__ import annotations
19import importlib
20import importlib.machinery
21import importlib.util
22import io
23import itertools
24import logging
25import os
26import sys
27import sysconfig
28import types
29import warnings
30from collections.abc import Callable, Iterable, Sequence
31from contextlib import redirect_stderr, redirect_stdout
32from functools import lru_cache
33from pathlib import Path
35from astroid.const import IS_JYTHON, IS_PYPY, PY310_PLUS
36from astroid.interpreter._import import spec, util
38if PY310_PLUS:
39 from sys import stdlib_module_names
40else:
41 from astroid._backport_stdlib_names import stdlib_module_names
43logger = logging.getLogger(__name__)
46if sys.platform.startswith("win"):
47 PY_SOURCE_EXTS = ("py", "pyw", "pyi")
48 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "pyw", "py")
49 PY_COMPILED_EXTS = ("dll", "pyd")
50else:
51 PY_SOURCE_EXTS = ("py", "pyi")
52 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "py")
53 PY_COMPILED_EXTS = ("so",)
56# TODO: Adding `platstdlib` is a fix for a workaround in virtualenv. At some point we should
57# revisit whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1323.
58STD_LIB_DIRS = {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
60if os.name == "nt":
61 STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls"))
62 try:
63 # real_prefix is defined when running inside virtual environments,
64 # created with the **virtualenv** library.
65 # Deprecated in virtualenv==16.7.9
66 # See: https://github.com/pypa/virtualenv/issues/1622
67 STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) # type: ignore[attr-defined]
68 except AttributeError:
69 # sys.base_exec_prefix is always defined, but in a virtual environment
70 # created with the stdlib **venv** module, it points to the original
71 # installation, if the virtual env is activated.
72 try:
73 STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls"))
74 except AttributeError:
75 pass
77if IS_PYPY and sys.version_info < (3, 8):
78 # PyPy stores the stdlib in two places: sys.prefix/lib_pypy and sys.prefix/lib-python/3
79 # sysconfig.get_path on PyPy returns the first, but without an underscore so we patch this manually.
80 # Beginning with 3.8 the stdlib is only stored in: sys.prefix/pypy{py_version_short}
81 STD_LIB_DIRS.add(str(Path(sysconfig.get_path("stdlib")).parent / "lib_pypy"))
82 STD_LIB_DIRS.add(str(Path(sysconfig.get_path("stdlib")).parent / "lib-python/3"))
84 # TODO: This is a fix for a workaround in virtualenv. At some point we should revisit
85 # whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1324.
86 STD_LIB_DIRS.add(str(Path(sysconfig.get_path("platstdlib")).parent / "lib_pypy"))
87 STD_LIB_DIRS.add(
88 str(Path(sysconfig.get_path("platstdlib")).parent / "lib-python/3")
89 )
91if os.name == "posix":
92 # Need the real prefix if we're in a virtualenv, otherwise
93 # the usual one will do.
94 # Deprecated in virtualenv==16.7.9
95 # See: https://github.com/pypa/virtualenv/issues/1622
96 try:
97 prefix: str = sys.real_prefix # type: ignore[attr-defined]
98 except AttributeError:
99 prefix = sys.prefix
101 def _posix_path(path: str) -> str:
102 base_python = "python%d.%d" % sys.version_info[:2]
103 return os.path.join(prefix, path, base_python)
105 STD_LIB_DIRS.add(_posix_path("lib"))
106 if sys.maxsize > 2**32:
107 # This tries to fix a problem with /usr/lib64 builds,
108 # where systems are running both 32-bit and 64-bit code
109 # on the same machine, which reflects into the places where
110 # standard library could be found. More details can be found
111 # here http://bugs.python.org/issue1294959.
112 # An easy reproducing case would be
113 # https://github.com/pylint-dev/pylint/issues/712#issuecomment-163178753
114 STD_LIB_DIRS.add(_posix_path("lib64"))
116EXT_LIB_DIRS = {sysconfig.get_path("purelib"), sysconfig.get_path("platlib")}
117BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
120class NoSourceFile(Exception):
121 """Exception raised when we are not able to get a python
122 source file for a precompiled file.
123 """
126def _normalize_path(path: str) -> str:
127 """Resolve symlinks in path and convert to absolute path.
129 Note that environment variables and ~ in the path need to be expanded in
130 advance.
132 This can be cached by using _cache_normalize_path.
133 """
134 return os.path.normcase(os.path.realpath(path))
137def _path_from_filename(filename: str, is_jython: bool = IS_JYTHON) -> str:
138 if not is_jython:
139 return filename
140 head, has_pyclass, _ = filename.partition("$py.class")
141 if has_pyclass:
142 return head + ".py"
143 return filename
146def _handle_blacklist(
147 blacklist: Sequence[str], dirnames: list[str], filenames: list[str]
148) -> None:
149 """Remove files/directories in the black list.
151 dirnames/filenames are usually from os.walk
152 """
153 for norecurs in blacklist:
154 if norecurs in dirnames:
155 dirnames.remove(norecurs)
156 elif norecurs in filenames:
157 filenames.remove(norecurs)
160@lru_cache
161def _cache_normalize_path_(path: str) -> str:
162 return _normalize_path(path)
165def _cache_normalize_path(path: str) -> str:
166 """Normalize path with caching."""
167 # _module_file calls abspath on every path in sys.path every time it's
168 # called; on a larger codebase this easily adds up to half a second just
169 # assembling path components. This cache alleviates that.
170 if not path: # don't cache result for ''
171 return _normalize_path(path)
172 return _cache_normalize_path_(path)
175def load_module_from_name(dotted_name: str) -> types.ModuleType:
176 """Load a Python module from its name.
178 :type dotted_name: str
179 :param dotted_name: python name of a module or package
181 :raise ImportError: if the module or package is not found
183 :rtype: module
184 :return: the loaded module
185 """
186 try:
187 return sys.modules[dotted_name]
188 except KeyError:
189 pass
191 # Capture and log anything emitted during import to avoid
192 # contaminating JSON reports in pylint
193 with redirect_stderr(io.StringIO()) as stderr, redirect_stdout(
194 io.StringIO()
195 ) as stdout:
196 module = importlib.import_module(dotted_name)
198 stderr_value = stderr.getvalue()
199 if stderr_value:
200 logger.error(
201 "Captured stderr while importing %s:\n%s", dotted_name, stderr_value
202 )
203 stdout_value = stdout.getvalue()
204 if stdout_value:
205 logger.info(
206 "Captured stdout while importing %s:\n%s", dotted_name, stdout_value
207 )
209 return module
212def load_module_from_modpath(parts: Sequence[str]) -> types.ModuleType:
213 """Load a python module from its split name.
215 :param parts:
216 python name of a module or package split on '.'
218 :raise ImportError: if the module or package is not found
220 :return: the loaded module
221 """
222 return load_module_from_name(".".join(parts))
225def load_module_from_file(filepath: str) -> types.ModuleType:
226 """Load a Python module from it's path.
228 :type filepath: str
229 :param filepath: path to the python module or package
231 :raise ImportError: if the module or package is not found
233 :rtype: module
234 :return: the loaded module
235 """
236 modpath = modpath_from_file(filepath)
237 return load_module_from_modpath(modpath)
240def check_modpath_has_init(path: str, mod_path: list[str]) -> bool:
241 """Check there are some __init__.py all along the way."""
242 modpath: list[str] = []
243 for part in mod_path:
244 modpath.append(part)
245 path = os.path.join(path, part)
246 if not _has_init(path):
247 old_namespace = util.is_namespace(".".join(modpath))
248 if not old_namespace:
249 return False
250 return True
253def _get_relative_base_path(filename: str, path_to_check: str) -> list[str] | None:
254 """Extracts the relative mod path of the file to import from.
256 Check if a file is within the passed in path and if so, returns the
257 relative mod path from the one passed in.
259 If the filename is no in path_to_check, returns None
261 Note this function will look for both abs and realpath of the file,
262 this allows to find the relative base path even if the file is a
263 symlink of a file in the passed in path
265 Examples:
266 _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"]
267 _get_relative_base_path("/a/b/c/d.py", "/dev") -> None
268 """
269 importable_path = None
270 path_to_check = os.path.normcase(path_to_check)
271 abs_filename = os.path.abspath(filename)
272 if os.path.normcase(abs_filename).startswith(path_to_check):
273 importable_path = abs_filename
275 real_filename = os.path.realpath(filename)
276 if os.path.normcase(real_filename).startswith(path_to_check):
277 importable_path = real_filename
279 if importable_path:
280 base_path = os.path.splitext(importable_path)[0]
281 relative_base_path = base_path[len(path_to_check) :]
282 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
284 return None
287def modpath_from_file_with_callback(
288 filename: str,
289 path: Sequence[str] | None = None,
290 is_package_cb: Callable[[str, list[str]], bool] | None = None,
291) -> list[str]:
292 filename = os.path.expanduser(_path_from_filename(filename))
293 paths_to_check = sys.path.copy()
294 if path:
295 paths_to_check += path
296 for pathname in itertools.chain(
297 paths_to_check, map(_cache_normalize_path, paths_to_check)
298 ):
299 if not pathname:
300 continue
301 modpath = _get_relative_base_path(filename, pathname)
302 if not modpath:
303 continue
304 assert is_package_cb is not None
305 if is_package_cb(pathname, modpath[:-1]):
306 return modpath
308 raise ImportError(
309 "Unable to find module for {} in {}".format(filename, ", \n".join(sys.path))
310 )
313def modpath_from_file(filename: str, path: Sequence[str] | None = None) -> list[str]:
314 """Get the corresponding split module's name from a filename.
316 This function will return the name of a module or package split on `.`.
318 :type filename: str
319 :param filename: file's path for which we want the module's name
321 :type Optional[List[str]] path:
322 Optional list of path where the module or package should be
323 searched (use sys.path if nothing or None is given)
325 :raise ImportError:
326 if the corresponding module's name has not been found
328 :rtype: list(str)
329 :return: the corresponding split module's name
330 """
331 return modpath_from_file_with_callback(filename, path, check_modpath_has_init)
334def file_from_modpath(
335 modpath: list[str],
336 path: Sequence[str] | None = None,
337 context_file: str | None = None,
338) -> str | None:
339 return file_info_from_modpath(modpath, path, context_file).location
342def file_info_from_modpath(
343 modpath: list[str],
344 path: Sequence[str] | None = None,
345 context_file: str | None = None,
346) -> spec.ModuleSpec:
347 """Given a mod path (i.e. split module / package name), return the
348 corresponding file.
350 Giving priority to source file over precompiled file if it exists.
352 :param modpath:
353 split module's name (i.e name of a module or package split
354 on '.')
355 (this means explicit relative imports that start with dots have
356 empty strings in this list!)
358 :param path:
359 optional list of path where the module or package should be
360 searched (use sys.path if nothing or None is given)
362 :param context_file:
363 context file to consider, necessary if the identifier has been
364 introduced using a relative import unresolvable in the actual
365 context (i.e. modutils)
367 :raise ImportError: if there is no such module in the directory
369 :return:
370 the path to the module's file or None if it's an integrated
371 builtin module such as 'sys'
372 """
373 if context_file is not None:
374 context: str | None = os.path.dirname(context_file)
375 else:
376 context = context_file
377 if modpath[0] == "xml":
378 # handle _xmlplus
379 try:
380 return _spec_from_modpath(["_xmlplus"] + modpath[1:], path, context)
381 except ImportError:
382 return _spec_from_modpath(modpath, path, context)
383 elif modpath == ["os", "path"]:
384 # FIXME: currently ignoring search_path...
385 return spec.ModuleSpec(
386 name="os.path",
387 location=os.path.__file__,
388 type=spec.ModuleType.PY_SOURCE,
389 )
390 return _spec_from_modpath(modpath, path, context)
393def get_module_part(dotted_name: str, context_file: str | None = None) -> str:
394 """Given a dotted name return the module part of the name :
396 >>> get_module_part('astroid.as_string.dump')
397 'astroid.as_string'
399 :param dotted_name: full name of the identifier we are interested in
401 :param context_file:
402 context file to consider, necessary if the identifier has been
403 introduced using a relative import unresolvable in the actual
404 context (i.e. modutils)
406 :raise ImportError: if there is no such module in the directory
408 :return:
409 the module part of the name or None if we have not been able at
410 all to import the given name
412 XXX: deprecated, since it doesn't handle package precedence over module
413 (see #10066)
414 """
415 # os.path trick
416 if dotted_name.startswith("os.path"):
417 return "os.path"
418 parts = dotted_name.split(".")
419 if context_file is not None:
420 # first check for builtin module which won't be considered latter
421 # in that case (path != None)
422 if parts[0] in BUILTIN_MODULES:
423 if len(parts) > 2:
424 raise ImportError(dotted_name)
425 return parts[0]
426 # don't use += or insert, we want a new list to be created !
427 path: list[str] | None = None
428 starti = 0
429 if parts[0] == "":
430 assert (
431 context_file is not None
432 ), "explicit relative import, but no context_file?"
433 path = [] # prevent resolving the import non-relatively
434 starti = 1
435 # for all further dots: change context
436 while starti < len(parts) and parts[starti] == "":
437 starti += 1
438 assert (
439 context_file is not None
440 ), "explicit relative import, but no context_file?"
441 context_file = os.path.dirname(context_file)
442 for i in range(starti, len(parts)):
443 try:
444 file_from_modpath(
445 parts[starti : i + 1], path=path, context_file=context_file
446 )
447 except ImportError:
448 if i < max(1, len(parts) - 2):
449 raise
450 return ".".join(parts[:i])
451 return dotted_name
454def get_module_files(
455 src_directory: str, blacklist: Sequence[str], list_all: bool = False
456) -> list[str]:
457 """Given a package directory return a list of all available python
458 module's files in the package and its subpackages.
460 :param src_directory:
461 path of the directory corresponding to the package
463 :param blacklist: iterable
464 list of files or directories to ignore.
466 :param list_all:
467 get files from all paths, including ones without __init__.py
469 :return:
470 the list of all available python module's files in the package and
471 its subpackages
472 """
473 files: list[str] = []
474 for directory, dirnames, filenames in os.walk(src_directory):
475 if directory in blacklist:
476 continue
477 _handle_blacklist(blacklist, dirnames, filenames)
478 # check for __init__.py
479 if not list_all and {"__init__.py", "__init__.pyi"}.isdisjoint(filenames):
480 dirnames[:] = ()
481 continue
482 for filename in filenames:
483 if _is_python_file(filename):
484 src = os.path.join(directory, filename)
485 files.append(src)
486 return files
489def get_source_file(
490 filename: str, include_no_ext: bool = False, prefer_stubs: bool = False
491) -> str:
492 """Given a python module's file name return the matching source file
493 name (the filename will be returned identically if it's already an
494 absolute path to a python source file).
496 :param filename: python module's file name
498 :raise NoSourceFile: if no source file exists on the file system
500 :return: the absolute path of the source file if it exists
501 """
502 filename = os.path.abspath(_path_from_filename(filename))
503 base, orig_ext = os.path.splitext(filename)
504 if orig_ext == ".pyi" and os.path.exists(f"{base}{orig_ext}"):
505 return f"{base}{orig_ext}"
506 for ext in PY_SOURCE_EXTS_STUBS_FIRST if prefer_stubs else PY_SOURCE_EXTS:
507 source_path = f"{base}.{ext}"
508 if os.path.exists(source_path):
509 return source_path
510 if include_no_ext and not orig_ext and os.path.exists(base):
511 return base
512 raise NoSourceFile(filename)
515def is_python_source(filename: str | None) -> bool:
516 """Return: True if the filename is a python source file."""
517 if not filename:
518 return False
519 return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
522def is_stdlib_module(modname: str) -> bool:
523 """Return: True if the modname is in the standard library"""
524 return modname.split(".")[0] in stdlib_module_names
527def module_in_path(modname: str, path: str | Iterable[str]) -> bool:
528 """Try to determine if a module is imported from one of the specified paths
530 :param modname: name of the module
532 :param path: paths to consider
534 :return:
535 true if the module:
536 - is located on the path listed in one of the directory in `paths`
537 """
539 modname = modname.split(".")[0]
540 try:
541 filename = file_from_modpath([modname])
542 except ImportError:
543 # Import failed, we can't check path if we don't know it
544 return False
546 if filename is None:
547 # No filename likely means it's compiled in, or potentially a namespace
548 return False
549 filename = _normalize_path(filename)
551 if isinstance(path, str):
552 return filename.startswith(_cache_normalize_path(path))
554 return any(filename.startswith(_cache_normalize_path(entry)) for entry in path)
557def is_standard_module(modname: str, std_path: Iterable[str] | None = None) -> bool:
558 """Try to guess if a module is a standard python module (by default,
559 see `std_path` parameter's description).
561 :param modname: name of the module we are interested in
563 :param std_path: list of path considered has standard
565 :return:
566 true if the module:
567 - is located on the path listed in one of the directory in `std_path`
568 - is a built-in module
569 """
570 warnings.warn(
571 "is_standard_module() is deprecated. Use, is_stdlib_module() or module_in_path() instead",
572 DeprecationWarning,
573 stacklevel=2,
574 )
576 modname = modname.split(".")[0]
577 try:
578 filename = file_from_modpath([modname])
579 except ImportError:
580 # import failed, i'm probably not so wrong by supposing it's
581 # not standard...
582 return False
583 # modules which are not living in a file are considered standard
584 # (sys and __builtin__ for instance)
585 if filename is None:
586 # we assume there are no namespaces in stdlib
587 return not util.is_namespace(modname)
588 filename = _normalize_path(filename)
589 for path in EXT_LIB_DIRS:
590 if filename.startswith(_cache_normalize_path(path)):
591 return False
592 if std_path is None:
593 std_path = STD_LIB_DIRS
595 return any(filename.startswith(_cache_normalize_path(path)) for path in std_path)
598def is_relative(modname: str, from_file: str) -> bool:
599 """Return true if the given module name is relative to the given
600 file name.
602 :param modname: name of the module we are interested in
604 :param from_file:
605 path of the module from which modname has been imported
607 :return:
608 true if the module has been imported relatively to `from_file`
609 """
610 if not os.path.isdir(from_file):
611 from_file = os.path.dirname(from_file)
612 if from_file in sys.path:
613 return False
614 return bool(
615 importlib.machinery.PathFinder.find_spec(
616 modname.split(".", maxsplit=1)[0], [from_file]
617 )
618 )
621# internal only functions #####################################################
624def _spec_from_modpath(
625 modpath: list[str],
626 path: Sequence[str] | None = None,
627 context: str | None = None,
628) -> spec.ModuleSpec:
629 """Given a mod path (i.e. split module / package name), return the
630 corresponding spec.
632 this function is used internally, see `file_from_modpath`'s
633 documentation for more information
634 """
635 assert modpath
636 location = None
637 if context is not None:
638 try:
639 found_spec = spec.find_spec(modpath, [context])
640 location = found_spec.location
641 except ImportError:
642 found_spec = spec.find_spec(modpath, path)
643 location = found_spec.location
644 else:
645 found_spec = spec.find_spec(modpath, path)
646 if found_spec.type == spec.ModuleType.PY_COMPILED:
647 try:
648 assert found_spec.location is not None
649 location = get_source_file(found_spec.location)
650 return found_spec._replace(
651 location=location, type=spec.ModuleType.PY_SOURCE
652 )
653 except NoSourceFile:
654 return found_spec._replace(location=location)
655 elif found_spec.type == spec.ModuleType.C_BUILTIN:
656 # integrated builtin module
657 return found_spec._replace(location=None)
658 elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:
659 assert found_spec.location is not None
660 location = _has_init(found_spec.location)
661 return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
662 return found_spec
665def _is_python_file(filename: str) -> bool:
666 """Return true if the given filename should be considered as a python file.
668 .pyc and .pyo are ignored
669 """
670 return filename.endswith((".py", ".pyi", ".so", ".pyd", ".pyw"))
673def _has_init(directory: str) -> str | None:
674 """If the given directory has a valid __init__ file, return its path,
675 else return None.
676 """
677 mod_or_pack = os.path.join(directory, "__init__")
678 for ext in (*PY_SOURCE_EXTS, "pyc", "pyo"):
679 if os.path.exists(mod_or_pack + "." + ext):
680 return mod_or_pack + "." + ext
681 return None
684def is_namespace(specobj: spec.ModuleSpec) -> bool:
685 return specobj.type == spec.ModuleType.PY_NAMESPACE
688def is_directory(specobj: spec.ModuleSpec) -> bool:
689 return specobj.type == spec.ModuleType.PKG_DIRECTORY
692def is_module_name_part_of_extension_package_whitelist(
693 module_name: str, package_whitelist: set[str]
694) -> bool:
695 """
696 Returns True if one part of the module name is in the package whitelist.
698 >>> is_module_name_part_of_extension_package_whitelist('numpy.core.umath', {'numpy'})
699 True
700 """
701 parts = module_name.split(".")
702 return any(
703 ".".join(parts[:x]) in package_whitelist for x in range(1, len(parts) + 1)
704 )