Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5from __future__ import annotations
7import abc
8import enum
9import importlib
10import importlib.machinery
11import importlib.util
12import os
13import pathlib
14import sys
15import types
16import warnings
17import zipimport
18from collections.abc import Iterable, Iterator, Sequence
19from functools import lru_cache
20from pathlib import Path
21from typing import Literal, NamedTuple, Protocol
23from astroid.const import PY310_PLUS
24from astroid.modutils import EXT_LIB_DIRS, cached_os_path_isfile
26from . import util
29# The MetaPathFinder protocol comes from typeshed, which says:
30# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
31class _MetaPathFinder(Protocol):
32 def find_spec(
33 self,
34 fullname: str,
35 path: Sequence[str] | None,
36 target: types.ModuleType | None = ...,
37 ) -> importlib.machinery.ModuleSpec | None: ... # pragma: no cover
40class ModuleType(enum.Enum):
41 """Python module types used for ModuleSpec."""
43 C_BUILTIN = enum.auto()
44 C_EXTENSION = enum.auto()
45 PKG_DIRECTORY = enum.auto()
46 PY_CODERESOURCE = enum.auto()
47 PY_COMPILED = enum.auto()
48 PY_FROZEN = enum.auto()
49 PY_RESOURCE = enum.auto()
50 PY_SOURCE = enum.auto()
51 PY_ZIPMODULE = enum.auto()
52 PY_NAMESPACE = enum.auto()
55_MetaPathFinderModuleTypes: dict[str, ModuleType] = {
56 # Finders created by setuptools editable installs
57 "_EditableFinder": ModuleType.PY_SOURCE,
58 "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE,
59 # Finders create by six
60 "_SixMetaPathImporter": ModuleType.PY_SOURCE,
61}
63_EditableFinderClasses: set[str] = {
64 "_EditableFinder",
65 "_EditableNamespaceFinder",
66}
69class ModuleSpec(NamedTuple):
70 """Defines a class similar to PEP 420's ModuleSpec.
72 A module spec defines a name of a module, its type, location
73 and where submodules can be found, if the module is a package.
74 """
76 name: str
77 type: ModuleType | None
78 location: str | None = None
79 origin: str | None = None
80 submodule_search_locations: Sequence[str] | None = None
83class Finder:
84 """A finder is a class which knows how to find a particular module."""
86 def __init__(self, path: Sequence[str] | None = None) -> None:
87 self._path = path or sys.path
89 @staticmethod
90 @abc.abstractmethod
91 def find_module(
92 modname: str,
93 module_parts: tuple[str, ...],
94 processed: tuple[str, ...],
95 submodule_path: tuple[str, ...] | None,
96 ) -> ModuleSpec | None:
97 """Find the given module.
99 Each finder is responsible for each protocol of finding, as long as
100 they all return a ModuleSpec.
102 :param modname: The module which needs to be searched.
103 :param module_parts: It should be a tuple of strings,
104 where each part contributes to the module's
105 namespace.
106 :param processed: What parts from the module parts were processed
107 so far.
108 :param submodule_path: A tuple of paths where the module
109 can be looked into.
110 :returns: A ModuleSpec, describing how and where the module was found,
111 None, otherwise.
112 """
114 def contribute_to_path(
115 self, spec: ModuleSpec, processed: list[str]
116 ) -> Sequence[str] | None:
117 """Get a list of extra paths where this finder can search."""
120class ImportlibFinder(Finder):
121 """A finder based on the importlib module."""
123 _SUFFIXES: Sequence[tuple[str, ModuleType]] = (
124 [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES]
125 + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES]
126 + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES]
127 )
129 @staticmethod
130 @lru_cache(maxsize=1024)
131 def find_module(
132 modname: str,
133 module_parts: tuple[str, ...],
134 processed: tuple[str, ...],
135 submodule_path: tuple[str, ...] | None,
136 ) -> ModuleSpec | None:
137 # Although we should be able to use `find_spec` this doesn't work on PyPy for builtins.
138 # Therefore, we use the `builtin_module_nams` heuristic for these.
139 if submodule_path is None and modname in sys.builtin_module_names:
140 return ModuleSpec(
141 name=modname,
142 location=None,
143 type=ModuleType.C_BUILTIN,
144 )
146 if submodule_path is not None:
147 search_paths = list(submodule_path)
148 else:
149 search_paths = sys.path
151 suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0])
152 for entry in search_paths:
153 package_directory = os.path.join(entry, modname)
154 for suffix in suffixes:
155 package_file_name = "__init__" + suffix
156 file_path = os.path.join(package_directory, package_file_name)
157 if cached_os_path_isfile(file_path):
158 return ModuleSpec(
159 name=modname,
160 location=package_directory,
161 type=ModuleType.PKG_DIRECTORY,
162 )
163 for suffix, type_ in ImportlibFinder._SUFFIXES:
164 file_name = modname + suffix
165 file_path = os.path.join(entry, file_name)
166 if cached_os_path_isfile(file_path):
167 return ModuleSpec(name=modname, location=file_path, type=type_)
169 # sys.stdlib_module_names was added in Python 3.10
170 if PY310_PLUS:
171 # If the module name matches a stdlib module name, check whether this is a frozen
172 # module. Note that `find_spec` actually imports parent modules, so we want to make
173 # sure we only run this code for stuff that can be expected to be frozen. For now
174 # this is only stdlib.
175 if modname in sys.stdlib_module_names or (
176 processed and processed[0] in sys.stdlib_module_names
177 ):
178 try:
179 with warnings.catch_warnings():
180 warnings.filterwarnings("ignore", category=Warning)
181 spec = importlib.util.find_spec(".".join((*processed, modname)))
182 except ValueError:
183 spec = None
185 if (
186 spec
187 and spec.loader # type: ignore[comparison-overlap] # noqa: E501
188 is importlib.machinery.FrozenImporter
189 ):
190 return ModuleSpec(
191 name=modname,
192 location=getattr(spec.loader_state, "filename", None),
193 type=ModuleType.PY_FROZEN,
194 )
195 else:
196 # NOTE: This is broken code. It doesn't work on Python 3.13+ where submodules can also
197 # be frozen. However, we don't want to worry about this and we don't want to break
198 # support for older versions of Python. This is just copy-pasted from the old non
199 # working version to at least have no functional behaviour change on <=3.10.
200 # It can be removed after 3.10 is no longer supported in favour of the logic above.
201 if submodule_path is None: # pylint: disable=else-if-used
202 try:
203 with warnings.catch_warnings():
204 warnings.filterwarnings("ignore", category=UserWarning)
205 spec = importlib.util.find_spec(modname)
206 if (
207 spec
208 and spec.loader # type: ignore[comparison-overlap] # noqa: E501
209 is importlib.machinery.FrozenImporter
210 ):
211 # No need for BuiltinImporter; builtins handled above
212 return ModuleSpec(
213 name=modname,
214 location=getattr(spec.loader_state, "filename", None),
215 type=ModuleType.PY_FROZEN,
216 )
217 except ValueError:
218 pass
220 return None
222 def contribute_to_path(
223 self, spec: ModuleSpec, processed: list[str]
224 ) -> Sequence[str] | None:
225 if spec.location is None:
226 # Builtin.
227 return None
229 if _is_setuptools_namespace(Path(spec.location)):
230 # extend_path is called, search sys.path for module/packages
231 # of this name see pkgutil.extend_path documentation
232 path = [
233 os.path.join(p, *processed)
234 for p in sys.path
235 if os.path.isdir(os.path.join(p, *processed))
236 ]
237 elif spec.name == "distutils" and not any(
238 spec.location.lower().startswith(ext_lib_dir.lower())
239 for ext_lib_dir in EXT_LIB_DIRS
240 ):
241 # virtualenv below 20.0 patches distutils in an unexpected way
242 # so we just find the location of distutils that will be
243 # imported to avoid spurious import-error messages
244 # https://github.com/pylint-dev/pylint/issues/5645
245 # A regression test to create this scenario exists in release-tests.yml
246 # and can be triggered manually from GitHub Actions
247 distutils_spec = importlib.util.find_spec("distutils")
248 if distutils_spec and distutils_spec.origin:
249 origin_path = Path(
250 distutils_spec.origin
251 ) # e.g. .../distutils/__init__.py
252 path = [str(origin_path.parent)] # e.g. .../distutils
253 else:
254 path = [spec.location]
255 else:
256 path = [spec.location]
257 return path
260class ExplicitNamespacePackageFinder(ImportlibFinder):
261 """A finder for the explicit namespace packages."""
263 @staticmethod
264 @lru_cache(maxsize=1024)
265 def find_module(
266 modname: str,
267 module_parts: tuple[str, ...],
268 processed: tuple[str, ...],
269 submodule_path: tuple[str, ...] | None,
270 ) -> ModuleSpec | None:
271 if processed:
272 modname = ".".join([*processed, modname])
273 if util.is_namespace(modname) and modname in sys.modules:
274 return ModuleSpec(
275 name=modname,
276 location="",
277 origin="namespace",
278 type=ModuleType.PY_NAMESPACE,
279 submodule_search_locations=sys.modules[modname].__path__,
280 )
281 return None
283 def contribute_to_path(
284 self, spec: ModuleSpec, processed: list[str]
285 ) -> Sequence[str] | None:
286 return spec.submodule_search_locations
289class ZipFinder(Finder):
290 """Finder that knows how to find a module inside zip files."""
292 def __init__(self, path: Sequence[str]) -> None:
293 super().__init__(path)
294 for entry_path in path:
295 if entry_path not in sys.path_importer_cache:
296 try:
297 sys.path_importer_cache[entry_path] = zipimport.zipimporter( # type: ignore[assignment]
298 entry_path
299 )
300 except zipimport.ZipImportError:
301 continue
303 @staticmethod
304 @lru_cache(maxsize=1024)
305 def find_module(
306 modname: str,
307 module_parts: tuple[str, ...],
308 processed: tuple[str, ...],
309 submodule_path: tuple[str, ...] | None,
310 ) -> ModuleSpec | None:
311 try:
312 file_type, filename, path = _search_zip(module_parts)
313 except ImportError:
314 return None
316 return ModuleSpec(
317 name=modname,
318 location=filename,
319 origin="egg",
320 type=file_type,
321 submodule_search_locations=path,
322 )
325class PathSpecFinder(Finder):
326 """Finder based on importlib.machinery.PathFinder."""
328 @staticmethod
329 @lru_cache(maxsize=1024)
330 def find_module(
331 modname: str,
332 module_parts: tuple[str, ...],
333 processed: tuple[str, ...],
334 submodule_path: tuple[str, ...] | None,
335 ) -> ModuleSpec | None:
336 spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path)
337 if spec is not None:
338 is_namespace_pkg = spec.origin is None
339 location = spec.origin if not is_namespace_pkg else None
340 module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None
341 return ModuleSpec(
342 name=spec.name,
343 location=location,
344 origin=spec.origin,
345 type=module_type,
346 submodule_search_locations=list(spec.submodule_search_locations or []),
347 )
348 return spec
350 def contribute_to_path(
351 self, spec: ModuleSpec, processed: list[str]
352 ) -> Sequence[str] | None:
353 if spec.type == ModuleType.PY_NAMESPACE:
354 return spec.submodule_search_locations
355 return None
358_SPEC_FINDERS = (
359 ImportlibFinder,
360 ZipFinder,
361 PathSpecFinder,
362 ExplicitNamespacePackageFinder,
363)
366def _is_setuptools_namespace(location: pathlib.Path) -> bool:
367 try:
368 with open(location / "__init__.py", "rb") as stream:
369 data = stream.read(4096)
370 except OSError:
371 return False
372 extend_path = b"pkgutil" in data and b"extend_path" in data
373 declare_namespace = (
374 b"pkg_resources" in data and b"declare_namespace(__name__)" in data
375 )
376 return extend_path or declare_namespace
379def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]:
380 for filepath, importer in sys.path_importer_cache.items():
381 if importer is not None and isinstance(importer, zipimport.zipimporter):
382 yield filepath, importer
385def _search_zip(
386 modpath: tuple[str, ...],
387) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]:
388 for filepath, importer in _get_zipimporters():
389 if PY310_PLUS:
390 found = importer.find_spec(modpath[0])
391 else:
392 found = importer.find_module(modpath[0])
393 if found:
394 if PY310_PLUS:
395 if not importer.find_spec(os.path.sep.join(modpath)):
396 raise ImportError(
397 "No module named {} in {}/{}".format(
398 ".".join(modpath[1:]), filepath, modpath
399 )
400 )
401 elif not importer.find_module(os.path.sep.join(modpath)):
402 raise ImportError(
403 "No module named {} in {}/{}".format(
404 ".".join(modpath[1:]), filepath, modpath
405 )
406 )
407 return (
408 ModuleType.PY_ZIPMODULE,
409 os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath),
410 filepath,
411 )
412 raise ImportError(f"No module named {'.'.join(modpath)}")
415def _find_spec_with_path(
416 search_path: Sequence[str],
417 modname: str,
418 module_parts: tuple[str, ...],
419 processed: tuple[str, ...],
420 submodule_path: tuple[str, ...] | None,
421) -> tuple[Finder | _MetaPathFinder, ModuleSpec]:
422 for finder in _SPEC_FINDERS:
423 finder_instance = finder(search_path)
424 mod_spec = finder.find_module(modname, module_parts, processed, submodule_path)
425 if mod_spec is None:
426 continue
427 return finder_instance, mod_spec
429 # Support for custom finders
430 for meta_finder in sys.meta_path:
431 # See if we support the customer import hook of the meta_finder
432 meta_finder_name = meta_finder.__class__.__name__
433 if meta_finder_name not in _MetaPathFinderModuleTypes:
434 # Setuptools>62 creates its EditableFinders dynamically and have
435 # "type" as their __class__.__name__. We check __name__ as well
436 # to see if we can support the finder.
437 try:
438 meta_finder_name = meta_finder.__name__ # type: ignore[attr-defined]
439 except AttributeError:
440 continue
441 if meta_finder_name not in _MetaPathFinderModuleTypes:
442 continue
444 module_type = _MetaPathFinderModuleTypes[meta_finder_name]
446 # Meta path finders are supposed to have a find_spec method since
447 # Python 3.4. However, some third-party finders do not implement it.
448 # PEP302 does not refer to find_spec as well.
449 # See: https://github.com/pylint-dev/astroid/pull/1752/
450 if not hasattr(meta_finder, "find_spec"):
451 continue
453 spec = meta_finder.find_spec(modname, submodule_path)
454 if spec:
455 return (
456 meta_finder,
457 ModuleSpec(
458 spec.name,
459 module_type,
460 spec.origin,
461 spec.origin,
462 spec.submodule_search_locations,
463 ),
464 )
466 raise ImportError(f"No module named {'.'.join(module_parts)}")
469def find_spec(modpath: Iterable[str], path: Iterable[str] | None = None) -> ModuleSpec:
470 """Find a spec for the given module.
472 :type modpath: list or tuple
473 :param modpath:
474 split module's name (i.e name of a module or package split
475 on '.'), with leading empty strings for explicit relative import
477 :type path: list or None
478 :param path:
479 optional list of path where the module or package should be
480 searched (use sys.path if nothing or None is given)
482 :rtype: ModuleSpec
483 :return: A module spec, which describes how the module was
484 found and where.
485 """
486 return _find_spec(tuple(modpath), tuple(path) if path else None)
489@lru_cache(maxsize=1024)
490def _find_spec(
491 module_path: tuple[str, ...], path: tuple[str, ...] | None
492) -> ModuleSpec:
493 _path = path or sys.path
495 # Need a copy for not mutating the argument.
496 modpath = list(module_path)
498 search_paths = None
499 processed: list[str] = []
501 while modpath:
502 modname = modpath.pop(0)
504 submodule_path = search_paths or path
505 if submodule_path is not None:
506 submodule_path = tuple(submodule_path)
508 finder, spec = _find_spec_with_path(
509 _path, modname, module_path, tuple(processed), submodule_path
510 )
511 processed.append(modname)
512 if modpath:
513 if isinstance(finder, Finder):
514 search_paths = finder.contribute_to_path(spec, processed)
515 # If modname is a package from an editable install, update search_paths
516 # so that the next module in the path will be found inside of it using importlib.
517 # Existence of __name__ is guaranteed by _find_spec_with_path.
518 elif finder.__name__ in _EditableFinderClasses: # type: ignore[attr-defined]
519 search_paths = spec.submodule_search_locations
521 if spec.type == ModuleType.PKG_DIRECTORY:
522 spec = spec._replace(submodule_search_locations=search_paths)
524 return spec