Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5from __future__ import annotations
7import abc
8import enum
9import importlib
10import importlib.machinery
11import importlib.util
12import os
13import pathlib
14import sys
15import types
16import warnings
17import zipimport
18from collections.abc import Iterable, Iterator, Sequence
19from functools import lru_cache
20from pathlib import Path
21from typing import Any, Literal, NamedTuple, Protocol
23from astroid.const import PY310_PLUS
24from astroid.modutils import EXT_LIB_DIRS
26from . import util
29# The MetaPathFinder protocol comes from typeshed, which says:
30# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
31class _MetaPathFinder(Protocol):
32 def find_spec(
33 self,
34 fullname: str,
35 path: Sequence[str] | None,
36 target: types.ModuleType | None = ...,
37 ) -> importlib.machinery.ModuleSpec | None: ... # pragma: no cover
40class ModuleType(enum.Enum):
41 """Python module types used for ModuleSpec."""
43 C_BUILTIN = enum.auto()
44 C_EXTENSION = enum.auto()
45 PKG_DIRECTORY = enum.auto()
46 PY_CODERESOURCE = enum.auto()
47 PY_COMPILED = enum.auto()
48 PY_FROZEN = enum.auto()
49 PY_RESOURCE = enum.auto()
50 PY_SOURCE = enum.auto()
51 PY_ZIPMODULE = enum.auto()
52 PY_NAMESPACE = enum.auto()
55_MetaPathFinderModuleTypes: dict[str, ModuleType] = {
56 # Finders created by setuptools editable installs
57 "_EditableFinder": ModuleType.PY_SOURCE,
58 "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE,
59 # Finders create by six
60 "_SixMetaPathImporter": ModuleType.PY_SOURCE,
61}
63_EditableFinderClasses: set[str] = {
64 "_EditableFinder",
65 "_EditableNamespaceFinder",
66}
69class ModuleSpec(NamedTuple):
70 """Defines a class similar to PEP 420's ModuleSpec.
72 A module spec defines a name of a module, its type, location
73 and where submodules can be found, if the module is a package.
74 """
76 name: str
77 type: ModuleType | None
78 location: str | None = None
79 origin: str | None = None
80 submodule_search_locations: Sequence[str] | None = None
83class Finder:
84 """A finder is a class which knows how to find a particular module."""
86 def __init__(self, path: Sequence[str] | None = None) -> None:
87 self._path = path or sys.path
89 @abc.abstractmethod
90 def find_module(
91 self,
92 modname: str,
93 module_parts: Sequence[str],
94 processed: list[str],
95 submodule_path: Sequence[str] | None,
96 ) -> ModuleSpec | None:
97 """Find the given module.
99 Each finder is responsible for each protocol of finding, as long as
100 they all return a ModuleSpec.
102 :param modname: The module which needs to be searched.
103 :param module_parts: It should be a list of strings,
104 where each part contributes to the module's
105 namespace.
106 :param processed: What parts from the module parts were processed
107 so far.
108 :param submodule_path: A list of paths where the module
109 can be looked into.
110 :returns: A ModuleSpec, describing how and where the module was found,
111 None, otherwise.
112 """
114 def contribute_to_path(
115 self, spec: ModuleSpec, processed: list[str]
116 ) -> Sequence[str] | None:
117 """Get a list of extra paths where this finder can search."""
120class ImportlibFinder(Finder):
121 """A finder based on the importlib module."""
123 _SUFFIXES: Sequence[tuple[str, ModuleType]] = (
124 [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES]
125 + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES]
126 + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES]
127 )
129 def find_module(
130 self,
131 modname: str,
132 module_parts: Sequence[str],
133 processed: list[str],
134 submodule_path: Sequence[str] | None,
135 ) -> ModuleSpec | None:
136 if submodule_path is not None:
137 submodule_path = list(submodule_path)
138 elif modname in sys.builtin_module_names:
139 return ModuleSpec(
140 name=modname,
141 location=None,
142 type=ModuleType.C_BUILTIN,
143 )
144 else:
145 try:
146 with warnings.catch_warnings():
147 warnings.filterwarnings("ignore", category=UserWarning)
148 spec = importlib.util.find_spec(modname)
149 if (
150 spec
151 and spec.loader # type: ignore[comparison-overlap] # noqa: E501
152 is importlib.machinery.FrozenImporter
153 ):
154 # No need for BuiltinImporter; builtins handled above
155 return ModuleSpec(
156 name=modname,
157 location=getattr(spec.loader_state, "filename", None),
158 type=ModuleType.PY_FROZEN,
159 )
160 except ValueError:
161 pass
162 submodule_path = sys.path
164 suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0])
165 for entry in submodule_path:
166 package_directory = os.path.join(entry, modname)
167 for suffix in suffixes:
168 package_file_name = "__init__" + suffix
169 file_path = os.path.join(package_directory, package_file_name)
170 if os.path.isfile(file_path):
171 return ModuleSpec(
172 name=modname,
173 location=package_directory,
174 type=ModuleType.PKG_DIRECTORY,
175 )
176 for suffix, type_ in ImportlibFinder._SUFFIXES:
177 file_name = modname + suffix
178 file_path = os.path.join(entry, file_name)
179 if os.path.isfile(file_path):
180 return ModuleSpec(name=modname, location=file_path, type=type_)
181 return None
183 def contribute_to_path(
184 self, spec: ModuleSpec, processed: list[str]
185 ) -> Sequence[str] | None:
186 if spec.location is None:
187 # Builtin.
188 return None
190 if _is_setuptools_namespace(Path(spec.location)):
191 # extend_path is called, search sys.path for module/packages
192 # of this name see pkgutil.extend_path documentation
193 path = [
194 os.path.join(p, *processed)
195 for p in sys.path
196 if os.path.isdir(os.path.join(p, *processed))
197 ]
198 elif spec.name == "distutils" and not any(
199 spec.location.lower().startswith(ext_lib_dir.lower())
200 for ext_lib_dir in EXT_LIB_DIRS
201 ):
202 # virtualenv below 20.0 patches distutils in an unexpected way
203 # so we just find the location of distutils that will be
204 # imported to avoid spurious import-error messages
205 # https://github.com/pylint-dev/pylint/issues/5645
206 # A regression test to create this scenario exists in release-tests.yml
207 # and can be triggered manually from GitHub Actions
208 distutils_spec = importlib.util.find_spec("distutils")
209 if distutils_spec and distutils_spec.origin:
210 origin_path = Path(
211 distutils_spec.origin
212 ) # e.g. .../distutils/__init__.py
213 path = [str(origin_path.parent)] # e.g. .../distutils
214 else:
215 path = [spec.location]
216 else:
217 path = [spec.location]
218 return path
221class ExplicitNamespacePackageFinder(ImportlibFinder):
222 """A finder for the explicit namespace packages."""
224 def find_module(
225 self,
226 modname: str,
227 module_parts: Sequence[str],
228 processed: list[str],
229 submodule_path: Sequence[str] | None,
230 ) -> ModuleSpec | None:
231 if processed:
232 modname = ".".join([*processed, modname])
233 if util.is_namespace(modname) and modname in sys.modules:
234 submodule_path = sys.modules[modname].__path__
235 return ModuleSpec(
236 name=modname,
237 location="",
238 origin="namespace",
239 type=ModuleType.PY_NAMESPACE,
240 submodule_search_locations=submodule_path,
241 )
242 return None
244 def contribute_to_path(
245 self, spec: ModuleSpec, processed: list[str]
246 ) -> Sequence[str] | None:
247 return spec.submodule_search_locations
250class ZipFinder(Finder):
251 """Finder that knows how to find a module inside zip files."""
253 def __init__(self, path: Sequence[str]) -> None:
254 super().__init__(path)
255 for entry_path in path:
256 if entry_path not in sys.path_importer_cache:
257 try:
258 sys.path_importer_cache[entry_path] = zipimport.zipimporter( # type: ignore[assignment]
259 entry_path
260 )
261 except zipimport.ZipImportError:
262 continue
264 def find_module(
265 self,
266 modname: str,
267 module_parts: Sequence[str],
268 processed: list[str],
269 submodule_path: Sequence[str] | None,
270 ) -> ModuleSpec | None:
271 try:
272 file_type, filename, path = _search_zip(module_parts)
273 except ImportError:
274 return None
276 return ModuleSpec(
277 name=modname,
278 location=filename,
279 origin="egg",
280 type=file_type,
281 submodule_search_locations=path,
282 )
285class PathSpecFinder(Finder):
286 """Finder based on importlib.machinery.PathFinder."""
288 def find_module(
289 self,
290 modname: str,
291 module_parts: Sequence[str],
292 processed: list[str],
293 submodule_path: Sequence[str] | None,
294 ) -> ModuleSpec | None:
295 spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path)
296 if spec is not None:
297 is_namespace_pkg = spec.origin is None
298 location = spec.origin if not is_namespace_pkg else None
299 module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None
300 return ModuleSpec(
301 name=spec.name,
302 location=location,
303 origin=spec.origin,
304 type=module_type,
305 submodule_search_locations=list(spec.submodule_search_locations or []),
306 )
307 return spec
309 def contribute_to_path(
310 self, spec: ModuleSpec, processed: list[str]
311 ) -> Sequence[str] | None:
312 if spec.type == ModuleType.PY_NAMESPACE:
313 return spec.submodule_search_locations
314 return None
317_SPEC_FINDERS = (
318 ImportlibFinder,
319 ZipFinder,
320 PathSpecFinder,
321 ExplicitNamespacePackageFinder,
322)
325def _is_setuptools_namespace(location: pathlib.Path) -> bool:
326 try:
327 with open(location / "__init__.py", "rb") as stream:
328 data = stream.read(4096)
329 except OSError:
330 return False
331 extend_path = b"pkgutil" in data and b"extend_path" in data
332 declare_namespace = (
333 b"pkg_resources" in data and b"declare_namespace(__name__)" in data
334 )
335 return extend_path or declare_namespace
338def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]:
339 for filepath, importer in sys.path_importer_cache.items():
340 if isinstance(importer, zipimport.zipimporter):
341 yield filepath, importer
344def _search_zip(
345 modpath: Sequence[str],
346) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]:
347 for filepath, importer in _get_zipimporters():
348 if PY310_PLUS:
349 found: Any = importer.find_spec(modpath[0])
350 else:
351 found = importer.find_module(modpath[0])
352 if found:
353 if PY310_PLUS:
354 if not importer.find_spec(os.path.sep.join(modpath)):
355 raise ImportError(
356 "No module named %s in %s/%s"
357 % (".".join(modpath[1:]), filepath, modpath)
358 )
359 elif not importer.find_module(os.path.sep.join(modpath)):
360 raise ImportError(
361 "No module named %s in %s/%s"
362 % (".".join(modpath[1:]), filepath, modpath)
363 )
364 return (
365 ModuleType.PY_ZIPMODULE,
366 os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath),
367 filepath,
368 )
369 raise ImportError(f"No module named {'.'.join(modpath)}")
372def _find_spec_with_path(
373 search_path: Sequence[str],
374 modname: str,
375 module_parts: list[str],
376 processed: list[str],
377 submodule_path: Sequence[str] | None,
378) -> tuple[Finder | _MetaPathFinder, ModuleSpec]:
379 for finder in _SPEC_FINDERS:
380 finder_instance = finder(search_path)
381 spec = finder_instance.find_module(
382 modname, module_parts, processed, submodule_path
383 )
384 if spec is None:
385 continue
386 return finder_instance, spec
388 # Support for custom finders
389 for meta_finder in sys.meta_path:
390 # See if we support the customer import hook of the meta_finder
391 meta_finder_name = meta_finder.__class__.__name__
392 if meta_finder_name not in _MetaPathFinderModuleTypes:
393 # Setuptools>62 creates its EditableFinders dynamically and have
394 # "type" as their __class__.__name__. We check __name__ as well
395 # to see if we can support the finder.
396 try:
397 meta_finder_name = meta_finder.__name__ # type: ignore[attr-defined]
398 except AttributeError:
399 continue
400 if meta_finder_name not in _MetaPathFinderModuleTypes:
401 continue
403 module_type = _MetaPathFinderModuleTypes[meta_finder_name]
405 # Meta path finders are supposed to have a find_spec method since
406 # Python 3.4. However, some third-party finders do not implement it.
407 # PEP302 does not refer to find_spec as well.
408 # See: https://github.com/pylint-dev/astroid/pull/1752/
409 if not hasattr(meta_finder, "find_spec"):
410 continue
412 spec = meta_finder.find_spec(modname, submodule_path)
413 if spec:
414 return (
415 meta_finder,
416 ModuleSpec(
417 spec.name,
418 module_type,
419 spec.origin,
420 spec.origin,
421 spec.submodule_search_locations,
422 ),
423 )
425 raise ImportError(f"No module named {'.'.join(module_parts)}")
428def find_spec(modpath: Iterable[str], path: Iterable[str] | None = None) -> ModuleSpec:
429 """Find a spec for the given module.
431 :type modpath: list or tuple
432 :param modpath:
433 split module's name (i.e name of a module or package split
434 on '.'), with leading empty strings for explicit relative import
436 :type path: list or None
437 :param path:
438 optional list of path where the module or package should be
439 searched (use sys.path if nothing or None is given)
441 :rtype: ModuleSpec
442 :return: A module spec, which describes how the module was
443 found and where.
444 """
445 return _find_spec(tuple(modpath), tuple(path) if path else None)
448@lru_cache(maxsize=1024)
449def _find_spec(module_path: tuple, path: tuple) -> ModuleSpec:
450 _path = path or sys.path
452 # Need a copy for not mutating the argument.
453 modpath = list(module_path)
455 submodule_path = None
456 module_parts = modpath[:]
457 processed: list[str] = []
459 while modpath:
460 modname = modpath.pop(0)
461 finder, spec = _find_spec_with_path(
462 _path, modname, module_parts, processed, submodule_path or path
463 )
464 processed.append(modname)
465 if modpath:
466 if isinstance(finder, Finder):
467 submodule_path = finder.contribute_to_path(spec, processed)
468 # If modname is a package from an editable install, update submodule_path
469 # so that the next module in the path will be found inside of it using importlib.
470 # Existence of __name__ is guaranteed by _find_spec_with_path.
471 elif finder.__name__ in _EditableFinderClasses: # type: ignore[attr-defined]
472 submodule_path = spec.submodule_search_locations
474 if spec.type == ModuleType.PKG_DIRECTORY:
475 spec = spec._replace(submodule_search_locations=submodule_path)
477 return spec