Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/modutils.py: 25%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Python modules manipulation utility functions.
7:type PY_SOURCE_EXTS: tuple(str)
8:var PY_SOURCE_EXTS: list of possible python source file extension
10:type STD_LIB_DIRS: set of str
11:var STD_LIB_DIRS: directories where standard modules are located
13:type BUILTIN_MODULES: dict
14:var BUILTIN_MODULES: dictionary with builtin module names has key
15"""
17from __future__ import annotations
19import importlib
20import importlib.machinery
21import importlib.util
22import io
23import itertools
24import os
25import sys
26import sysconfig
27import types
28from collections.abc import Callable, Iterable, Sequence
29from contextlib import redirect_stderr, redirect_stdout
30from functools import lru_cache
31from sys import stdlib_module_names
33from astroid.const import IS_JYTHON
34from astroid.interpreter._import import spec, util
36if sys.platform.startswith("win"):
37 PY_SOURCE_EXTS = ("py", "pyw", "pyi")
38 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "pyw", "py")
39 PY_COMPILED_EXTS = ("dll", "pyd")
40else:
41 PY_SOURCE_EXTS = ("py", "pyi")
42 PY_SOURCE_EXTS_STUBS_FIRST = ("pyi", "py")
43 PY_COMPILED_EXTS = ("so",)
46# TODO: Adding `platstdlib` is a fix for a workaround in virtualenv. At some point we should
47# revisit whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1323.
48STD_LIB_DIRS = {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
50if os.name == "nt":
51 STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls"))
52 try:
53 # real_prefix is defined when running inside virtual environments,
54 # created with the **virtualenv** library.
55 # Deprecated in virtualenv==16.7.9
56 # See: https://github.com/pypa/virtualenv/issues/1622
57 STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) # type: ignore[attr-defined]
58 except AttributeError:
59 # sys.base_exec_prefix is always defined, but in a virtual environment
60 # created with the stdlib **venv** module, it points to the original
61 # installation, if the virtual env is activated.
62 try:
63 STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls"))
64 except AttributeError:
65 pass
67if os.name == "posix":
68 # Need the real prefix if we're in a virtualenv, otherwise
69 # the usual one will do.
70 # Deprecated in virtualenv==16.7.9
71 # See: https://github.com/pypa/virtualenv/issues/1622
72 try:
73 prefix: str = sys.real_prefix # type: ignore[attr-defined]
74 except AttributeError:
75 prefix = sys.prefix
77 def _posix_path(path: str) -> str:
78 base_python = f"python{sys.version_info.major}.{sys.version_info.minor}"
79 return os.path.join(prefix, path, base_python)
81 STD_LIB_DIRS.add(_posix_path("lib"))
82 if sys.maxsize > 2**32:
83 # This tries to fix a problem with /usr/lib64 builds,
84 # where systems are running both 32-bit and 64-bit code
85 # on the same machine, which reflects into the places where
86 # standard library could be found. More details can be found
87 # here http://bugs.python.org/issue1294959.
88 # An easy reproducing case would be
89 # https://github.com/pylint-dev/pylint/issues/712#issuecomment-163178753
90 STD_LIB_DIRS.add(_posix_path("lib64"))
92EXT_LIB_DIRS = {sysconfig.get_path("purelib"), sysconfig.get_path("platlib")}
93BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)
96class NoSourceFile(Exception):
97 """Exception raised when we are not able to get a python
98 source file for a precompiled file.
99 """
102def _normalize_path(path: str) -> str:
103 """Resolve symlinks in path and convert to absolute path.
105 Note that environment variables and ~ in the path need to be expanded in
106 advance.
108 This can be cached by using _cache_normalize_path.
109 """
110 return os.path.normcase(os.path.realpath(path))
113def _path_from_filename(filename: str, is_jython: bool = IS_JYTHON) -> str:
114 if not is_jython:
115 return filename
116 head, has_pyclass, _ = filename.partition("$py.class")
117 if has_pyclass:
118 return head + ".py"
119 return filename
122def _handle_blacklist(
123 blacklist: Sequence[str], dirnames: list[str], filenames: list[str]
124) -> None:
125 """Remove files/directories in the black list.
127 dirnames/filenames are usually from os.walk
128 """
129 for norecurs in blacklist:
130 if norecurs in dirnames:
131 dirnames.remove(norecurs)
132 elif norecurs in filenames:
133 filenames.remove(norecurs)
136@lru_cache
137def _cache_normalize_path_(path: str) -> str:
138 return _normalize_path(path)
141def _cache_normalize_path(path: str) -> str:
142 """Normalize path with caching."""
143 # _module_file calls abspath on every path in sys.path every time it's
144 # called; on a larger codebase this easily adds up to half a second just
145 # assembling path components. This cache alleviates that.
146 if not path: # don't cache result for ''
147 return _normalize_path(path)
148 return _cache_normalize_path_(path)
151def load_module_from_name(dotted_name: str) -> types.ModuleType:
152 """Load a Python module from its name.
154 :type dotted_name: str
155 :param dotted_name: python name of a module or package
157 :raise ImportError: if the module or package is not found
159 :rtype: module
160 :return: the loaded module
161 """
162 try:
163 return sys.modules[dotted_name]
164 except KeyError:
165 pass
167 # Capture and log anything emitted during import to avoid
168 # contaminating JSON reports in pylint
169 with (
170 redirect_stderr(io.StringIO()) as stderr,
171 redirect_stdout(io.StringIO()) as stdout,
172 ):
173 module = importlib.import_module(dotted_name)
175 stderr_value = stderr.getvalue()
176 stdout_value = stdout.getvalue()
177 if stderr_value or stdout_value:
178 import logging # pylint: disable=import-outside-toplevel
180 logger = logging.getLogger(__name__)
181 if stderr_value:
182 logger.error(
183 "Captured stderr while importing %s:\n%s", dotted_name, stderr_value
184 )
185 if stdout_value:
186 logger.info(
187 "Captured stdout while importing %s:\n%s", dotted_name, stdout_value
188 )
190 return module
193def load_module_from_modpath(parts: Sequence[str]) -> types.ModuleType:
194 """Load a python module from its split name.
196 :param parts:
197 python name of a module or package split on '.'
199 :raise ImportError: if the module or package is not found
201 :return: the loaded module
202 """
203 return load_module_from_name(".".join(parts))
206def load_module_from_file(filepath: str) -> types.ModuleType:
207 """Load a Python module from it's path.
209 :type filepath: str
210 :param filepath: path to the python module or package
212 :raise ImportError: if the module or package is not found
214 :rtype: module
215 :return: the loaded module
216 """
217 modpath = modpath_from_file(filepath)
218 return load_module_from_modpath(modpath)
221def check_modpath_has_init(path: str, mod_path: list[str]) -> bool:
222 """Check there are some __init__.py all along the way."""
223 modpath: list[str] = []
224 for part in mod_path:
225 modpath.append(part)
226 path = os.path.join(path, part)
227 if not _has_init(path):
228 old_namespace = util.is_namespace(".".join(modpath))
229 if not old_namespace:
230 return False
231 return True
234def _is_subpath(path: str, base: str) -> bool:
235 path = os.path.normcase(os.path.normpath(path))
236 base = os.path.normcase(os.path.normpath(base))
237 if not path.startswith(base):
238 return False
239 return (
240 (len(path) == len(base))
241 or (path[len(base)] == os.path.sep)
242 or (base.endswith(os.path.sep) and path[len(base) - 1] == os.path.sep)
243 )
246def _get_relative_base_path(filename: str, path_to_check: str) -> list[str] | None:
247 """Extracts the relative mod path of the file to import from.
249 Check if a file is within the passed in path and if so, returns the
250 relative mod path from the one passed in.
252 If the filename is no in path_to_check, returns None
254 Note this function will look for both abs and realpath of the file,
255 this allows to find the relative base path even if the file is a
256 symlink of a file in the passed in path
258 Examples:
259 _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"]
260 _get_relative_base_path("/a/b/c/d.py", "/dev") -> None
261 """
262 path_to_check = os.path.normcase(os.path.normpath(path_to_check))
264 abs_filename = os.path.abspath(filename)
265 if _is_subpath(abs_filename, path_to_check):
266 base_path = os.path.splitext(abs_filename)[0]
267 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
268 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
270 real_filename = os.path.realpath(filename)
271 if _is_subpath(real_filename, path_to_check):
272 base_path = os.path.splitext(real_filename)[0]
273 relative_base_path = base_path[len(path_to_check) :].lstrip(os.path.sep)
274 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]
276 return None
279def modpath_from_file_with_callback(
280 filename: str,
281 path: list[str] | None = None,
282 is_package_cb: Callable[[str, list[str]], bool] | None = None,
283) -> list[str]:
284 filename = os.path.expanduser(_path_from_filename(filename))
285 paths_to_check = sys.path.copy()
286 if path:
287 paths_to_check = path + paths_to_check
288 for pathname in itertools.chain(
289 paths_to_check, map(_cache_normalize_path, paths_to_check)
290 ):
291 if not pathname:
292 continue
293 modpath = _get_relative_base_path(filename, pathname)
294 if not modpath:
295 continue
296 assert is_package_cb is not None
297 if is_package_cb(pathname, modpath[:-1]):
298 return modpath
300 raise ImportError(
301 "Unable to find module for {} in {}".format(
302 filename, ", \n".join(paths_to_check)
303 )
304 )
307def modpath_from_file(filename: str, path: list[str] | None = None) -> list[str]:
308 """Get the corresponding split module's name from a filename.
310 This function will return the name of a module or package split on `.`.
312 :type filename: str
313 :param filename: file's path for which we want the module's name
315 :type Optional[List[str]] path:
316 Optional list of paths where the module or package should be
317 searched, additionally to sys.path
319 :raise ImportError:
320 if the corresponding module's name has not been found
322 :rtype: list(str)
323 :return: the corresponding split module's name
324 """
325 return modpath_from_file_with_callback(filename, path, check_modpath_has_init)
328def file_from_modpath(
329 modpath: list[str],
330 path: Sequence[str] | None = None,
331 context_file: str | None = None,
332) -> str | None:
333 return file_info_from_modpath(modpath, path, context_file).location
336def file_info_from_modpath(
337 modpath: list[str],
338 path: Sequence[str] | None = None,
339 context_file: str | None = None,
340) -> spec.ModuleSpec:
341 """Given a mod path (i.e. split module / package name), return the
342 corresponding file.
344 Giving priority to source file over precompiled file if it exists.
346 :param modpath:
347 split module's name (i.e name of a module or package split
348 on '.')
349 (this means explicit relative imports that start with dots have
350 empty strings in this list!)
352 :param path:
353 optional list of path where the module or package should be
354 searched (use sys.path if nothing or None is given)
356 :param context_file:
357 context file to consider, necessary if the identifier has been
358 introduced using a relative import unresolvable in the actual
359 context (i.e. modutils)
361 :raise ImportError: if there is no such module in the directory
363 :return:
364 the path to the module's file or None if it's an integrated
365 builtin module such as 'sys'
366 """
367 if context_file is not None:
368 context: str | None = os.path.dirname(context_file)
369 else:
370 context = context_file
371 if modpath[0] == "xml":
372 # handle _xmlplus
373 try:
374 return _spec_from_modpath(["_xmlplus", *modpath[1:]], path, context)
375 except ImportError:
376 return _spec_from_modpath(modpath, path, context)
377 elif modpath == ["os", "path"]:
378 # FIXME: currently ignoring search_path...
379 return spec.ModuleSpec(
380 name="os.path",
381 location=os.path.__file__,
382 type=spec.ModuleType.PY_SOURCE,
383 )
384 return _spec_from_modpath(modpath, path, context)
387def get_module_part(dotted_name: str, context_file: str | None = None) -> str:
388 """Given a dotted name return the module part of the name :
390 >>> get_module_part('astroid.as_string.dump')
391 'astroid.as_string'
393 :param dotted_name: full name of the identifier we are interested in
395 :param context_file:
396 context file to consider, necessary if the identifier has been
397 introduced using a relative import unresolvable in the actual
398 context (i.e. modutils)
400 :raise ImportError: if there is no such module in the directory
402 :return:
403 the module part of the name or None if we have not been able at
404 all to import the given name
406 XXX: deprecated, since it doesn't handle package precedence over module
407 (see #10066)
408 """
409 # os.path trick
410 if dotted_name.startswith("os.path"):
411 return "os.path"
412 parts = dotted_name.split(".")
413 if context_file is not None:
414 # first check for builtin module which won't be considered latter
415 # in that case (path != None)
416 if parts[0] in BUILTIN_MODULES:
417 if len(parts) > 2:
418 raise ImportError(dotted_name)
419 return parts[0]
420 # don't use += or insert, we want a new list to be created !
421 path: list[str] | None = None
422 starti = 0
423 if parts[0] == "":
424 assert (
425 context_file is not None
426 ), "explicit relative import, but no context_file?"
427 path = [] # prevent resolving the import non-relatively
428 starti = 1
429 # for all further dots: change context
430 while starti < len(parts) and parts[starti] == "":
431 starti += 1
432 assert (
433 context_file is not None
434 ), "explicit relative import, but no context_file?"
435 context_file = os.path.dirname(context_file)
436 for i in range(starti, len(parts)):
437 try:
438 file_from_modpath(
439 parts[starti : i + 1], path=path, context_file=context_file
440 )
441 except ImportError:
442 if i < max(1, len(parts) - 2):
443 raise
444 return ".".join(parts[:i])
445 return dotted_name
448def get_module_files(
449 src_directory: str, blacklist: Sequence[str], list_all: bool = False
450) -> list[str]:
451 """Given a package directory return a list of all available python
452 module's files in the package and its subpackages.
454 :param src_directory:
455 path of the directory corresponding to the package
457 :param blacklist: iterable
458 list of files or directories to ignore.
460 :param list_all:
461 get files from all paths, including ones without __init__.py
463 :return:
464 the list of all available python module's files in the package and
465 its subpackages
466 """
467 files: list[str] = []
468 for directory, dirnames, filenames in os.walk(src_directory):
469 if directory in blacklist:
470 continue
471 _handle_blacklist(blacklist, dirnames, filenames)
472 # check for __init__.py
473 if not list_all and {"__init__.py", "__init__.pyi"}.isdisjoint(filenames):
474 dirnames[:] = ()
475 continue
476 for filename in filenames:
477 if _is_python_file(filename):
478 src = os.path.join(directory, filename)
479 files.append(src)
480 return files
483def get_source_file(
484 filename: str, include_no_ext: bool = False, prefer_stubs: bool = False
485) -> str:
486 """Given a python module's file name return the matching source file
487 name (the filename will be returned identically if it's already an
488 absolute path to a python source file).
490 :param filename: python module's file name
492 :raise NoSourceFile: if no source file exists on the file system
494 :return: the absolute path of the source file if it exists
495 """
496 filename = os.path.abspath(_path_from_filename(filename))
497 base, orig_ext = os.path.splitext(filename)
498 orig_ext = orig_ext.lstrip(".")
499 if orig_ext not in PY_SOURCE_EXTS and os.path.exists(f"{base}.{orig_ext}"):
500 return f"{base}.{orig_ext}"
501 for ext in PY_SOURCE_EXTS_STUBS_FIRST if prefer_stubs else PY_SOURCE_EXTS:
502 source_path = f"{base}.{ext}"
503 if os.path.exists(source_path):
504 return source_path
505 if include_no_ext and not orig_ext and os.path.exists(base):
506 return base
507 raise NoSourceFile(filename)
510def is_python_source(filename: str | None) -> bool:
511 """Return: True if the filename is a python source file."""
512 if not filename:
513 return False
514 return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS
517def is_stdlib_module(modname: str) -> bool:
518 """Return: True if the modname is in the standard library"""
519 return modname.split(".")[0] in stdlib_module_names
522def module_in_path(modname: str, path: str | Iterable[str]) -> bool:
523 """Try to determine if a module is imported from one of the specified paths
525 :param modname: name of the module
527 :param path: paths to consider
529 :return:
530 true if the module:
531 - is located on the path listed in one of the directory in `paths`
532 """
534 modname = modname.split(".")[0]
535 try:
536 filename = file_from_modpath([modname])
537 except ImportError:
538 # Import failed, we can't check path if we don't know it
539 return False
541 if filename is None:
542 # No filename likely means it's compiled in, or potentially a namespace
543 return False
544 filename = _normalize_path(filename)
546 if isinstance(path, str):
547 return filename.startswith(_cache_normalize_path(path))
549 return any(filename.startswith(_cache_normalize_path(entry)) for entry in path)
552def is_relative(modname: str, from_file: str) -> bool:
553 """Return true if the given module name is relative to the given
554 file name.
556 :param modname: name of the module we are interested in
558 :param from_file:
559 path of the module from which modname has been imported
561 :return:
562 true if the module has been imported relatively to `from_file`
563 """
564 if not os.path.isdir(from_file):
565 from_file = os.path.dirname(from_file)
566 if from_file in sys.path:
567 return False
568 return bool(
569 importlib.machinery.PathFinder.find_spec(
570 modname.split(".", maxsplit=1)[0], [from_file]
571 )
572 )
575@lru_cache(maxsize=1024)
576def cached_os_path_isfile(path: str | os.PathLike[str]) -> bool:
577 """A cached version of os.path.isfile that helps avoid repetitive I/O"""
578 return os.path.isfile(path)
581# internal only functions #####################################################
584def _spec_from_modpath(
585 modpath: list[str],
586 path: Sequence[str] | None = None,
587 context: str | None = None,
588) -> spec.ModuleSpec:
589 """Given a mod path (i.e. split module / package name), return the
590 corresponding spec.
592 this function is used internally, see `file_from_modpath`'s
593 documentation for more information
594 """
595 assert modpath
596 location = None
597 if context is not None:
598 try:
599 found_spec = spec.find_spec(modpath, [context])
600 location = found_spec.location
601 except ImportError:
602 found_spec = spec.find_spec(modpath, path)
603 location = found_spec.location
604 else:
605 found_spec = spec.find_spec(modpath, path)
606 if found_spec.type == spec.ModuleType.PY_COMPILED:
607 try:
608 assert found_spec.location is not None
609 location = get_source_file(found_spec.location)
610 return found_spec._replace(
611 location=location, type=spec.ModuleType.PY_SOURCE
612 )
613 except NoSourceFile:
614 return found_spec._replace(location=location)
615 elif found_spec.type == spec.ModuleType.C_BUILTIN:
616 # integrated builtin module
617 return found_spec._replace(location=None)
618 elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:
619 assert found_spec.location is not None
620 location = _has_init(found_spec.location)
621 return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)
622 return found_spec
625def _is_python_file(filename: str) -> bool:
626 """Return true if the given filename should be considered as a python file.
628 .pyc and .pyo are ignored
629 """
630 return filename.endswith((".py", ".pyi", ".so", ".pyd", ".pyw"))
633@lru_cache(maxsize=1024)
634def _has_init(directory: str) -> str | None:
635 """If the given directory has a valid __init__ file, return its path,
636 else return None.
637 """
638 mod_or_pack = os.path.join(directory, "__init__")
639 for ext in (*PY_SOURCE_EXTS, "pyc", "pyo"):
640 if os.path.exists(mod_or_pack + "." + ext):
641 return mod_or_pack + "." + ext
642 return None
645def is_namespace(specobj: spec.ModuleSpec) -> bool:
646 return specobj.type == spec.ModuleType.PY_NAMESPACE
649def is_directory(specobj: spec.ModuleSpec) -> bool:
650 return specobj.type == spec.ModuleType.PKG_DIRECTORY
653def is_module_name_part_of_extension_package_whitelist(
654 module_name: str, package_whitelist: set[str]
655) -> bool:
656 """
657 Returns True if one part of the module name is in the package whitelist.
659 >>> is_module_name_part_of_extension_package_whitelist('numpy.core.umath', {'numpy'})
660 True
661 """
662 parts = module_name.split(".")
663 return any(
664 ".".join(parts[:x]) in package_whitelist for x in range(1, len(parts) + 1)
665 )