1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
4
5from __future__ import annotations
6
7import abc
8import enum
9import importlib
10import importlib.machinery
11import importlib.util
12import os
13import pathlib
14import sys
15import types
16import warnings
17import zipimport
18from collections.abc import Iterable, Iterator, Sequence
19from functools import lru_cache
20from pathlib import Path
21from typing import Literal, NamedTuple, Protocol
22
23from . import util
24
25
26# The MetaPathFinder protocol comes from typeshed, which says:
27# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
28class _MetaPathFinder(Protocol):
29 def find_spec(
30 self,
31 fullname: str,
32 path: Sequence[str] | None,
33 target: types.ModuleType | None = ...,
34 ) -> importlib.machinery.ModuleSpec | None: ... # pragma: no cover
35
36
37class ModuleType(enum.Enum):
38 """Python module types used for ModuleSpec."""
39
40 C_BUILTIN = enum.auto()
41 C_EXTENSION = enum.auto()
42 PKG_DIRECTORY = enum.auto()
43 PY_CODERESOURCE = enum.auto()
44 PY_COMPILED = enum.auto()
45 PY_FROZEN = enum.auto()
46 PY_RESOURCE = enum.auto()
47 PY_SOURCE = enum.auto()
48 PY_ZIPMODULE = enum.auto()
49 PY_NAMESPACE = enum.auto()
50
51
52_MetaPathFinderModuleTypes: dict[str, ModuleType] = {
53 # Finders created by setuptools editable installs
54 "_EditableFinder": ModuleType.PY_SOURCE,
55 "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE,
56 # Finders create by six
57 "_SixMetaPathImporter": ModuleType.PY_SOURCE,
58}
59
60_EditableFinderClasses: set[str] = {
61 "_EditableFinder",
62 "_EditableNamespaceFinder",
63}
64
65
66class ModuleSpec(NamedTuple):
67 """Defines a class similar to PEP 420's ModuleSpec.
68
69 A module spec defines a name of a module, its type, location
70 and where submodules can be found, if the module is a package.
71 """
72
73 name: str
74 type: ModuleType | None
75 location: str | None = None
76 origin: str | None = None
77 submodule_search_locations: Sequence[str] | None = None
78
79
80class Finder:
81 """A finder is a class which knows how to find a particular module."""
82
83 def __init__(self, path: Sequence[str] | None = None) -> None:
84 self._path = path or sys.path
85
86 @staticmethod
87 @abc.abstractmethod
88 def find_module(
89 modname: str,
90 module_parts: tuple[str, ...],
91 processed: tuple[str, ...],
92 submodule_path: tuple[str, ...] | None,
93 ) -> ModuleSpec | None:
94 """Find the given module.
95
96 Each finder is responsible for each protocol of finding, as long as
97 they all return a ModuleSpec.
98
99 :param modname: The module which needs to be searched.
100 :param module_parts: It should be a tuple of strings,
101 where each part contributes to the module's
102 namespace.
103 :param processed: What parts from the module parts were processed
104 so far.
105 :param submodule_path: A tuple of paths where the module
106 can be looked into.
107 :returns: A ModuleSpec, describing how and where the module was found,
108 None, otherwise.
109 """
110
111 def contribute_to_path(
112 self, spec: ModuleSpec, processed: list[str]
113 ) -> Sequence[str] | None:
114 """Get a list of extra paths where this finder can search."""
115
116
117class ImportlibFinder(Finder):
118 """A finder based on the importlib module."""
119
120 _SUFFIXES: Sequence[tuple[str, ModuleType]] = (
121 [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES]
122 + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES]
123 + [(".pyi", ModuleType.PY_SOURCE)]
124 + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES]
125 )
126
127 @staticmethod
128 @lru_cache(maxsize=1024)
129 def find_module(
130 modname: str,
131 module_parts: tuple[str, ...],
132 processed: tuple[str, ...],
133 submodule_path: tuple[str, ...] | None,
134 ) -> ModuleSpec | None:
135 # pylint: disable-next=import-outside-toplevel
136 from astroid.modutils import cached_os_path_isfile
137
138 # Although we should be able to use `find_spec` this doesn't work on PyPy for builtins.
139 # Therefore, we use the `builtin_module_nams` heuristic for these.
140 if submodule_path is None and modname in sys.builtin_module_names:
141 return ModuleSpec(
142 name=modname,
143 location=None,
144 type=ModuleType.C_BUILTIN,
145 )
146
147 if submodule_path is not None:
148 search_paths = list(submodule_path)
149 else:
150 search_paths = sys.path
151
152 suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0])
153 for entry in search_paths:
154 package_directory = os.path.join(entry, modname)
155 for suffix in suffixes:
156 package_file_name = "__init__" + suffix
157 file_path = os.path.join(package_directory, package_file_name)
158 if cached_os_path_isfile(file_path):
159 return ModuleSpec(
160 name=modname,
161 location=package_directory,
162 type=ModuleType.PKG_DIRECTORY,
163 )
164 for suffix, type_ in ImportlibFinder._SUFFIXES:
165 file_name = modname + suffix
166 file_path = os.path.join(entry, file_name)
167 if cached_os_path_isfile(file_path):
168 return ModuleSpec(name=modname, location=file_path, type=type_)
169
170 # If the module name matches a stdlib module name, check whether this is a frozen
171 # module. Note that `find_spec` actually imports parent modules, so we want to make
172 # sure we only run this code for stuff that can be expected to be frozen. For now
173 # this is only stdlib.
174 if (modname in sys.stdlib_module_names and not processed) or (
175 processed and processed[0] in sys.stdlib_module_names
176 ):
177 try:
178 with warnings.catch_warnings():
179 warnings.filterwarnings("ignore", category=Warning)
180 spec = importlib.util.find_spec(".".join((*processed, modname)))
181 except ValueError:
182 spec = None
183
184 if (
185 spec
186 and spec.loader # type: ignore[comparison-overlap] # noqa: E501
187 is importlib.machinery.FrozenImporter
188 ):
189 return ModuleSpec( # type: ignore[unreachable]
190 name=modname,
191 location=getattr(spec.loader_state, "filename", None),
192 type=ModuleType.PY_FROZEN,
193 )
194
195 return None
196
197 def contribute_to_path(
198 self, spec: ModuleSpec, processed: list[str]
199 ) -> Sequence[str] | None:
200 if spec.location is None:
201 # Builtin.
202 return None
203 # pylint: disable-next=import-outside-toplevel
204 from astroid.modutils import EXT_LIB_DIRS
205
206 if _is_setuptools_namespace(Path(spec.location)):
207 # extend_path is called, search sys.path for module/packages
208 # of this name see pkgutil.extend_path documentation
209 path = [
210 os.path.join(p, *processed)
211 for p in sys.path
212 if os.path.isdir(os.path.join(p, *processed))
213 ]
214 elif spec.name == "distutils" and not any(
215 spec.location.lower().startswith(ext_lib_dir.lower())
216 for ext_lib_dir in EXT_LIB_DIRS
217 ):
218 # virtualenv below 20.0 patches distutils in an unexpected way
219 # so we just find the location of distutils that will be
220 # imported to avoid spurious import-error messages
221 # https://github.com/pylint-dev/pylint/issues/5645
222 # A regression test to create this scenario exists in release-tests.yml
223 # and can be triggered manually from GitHub Actions
224 distutils_spec = importlib.util.find_spec("distutils")
225 if distutils_spec and distutils_spec.origin:
226 origin_path = Path(
227 distutils_spec.origin
228 ) # e.g. .../distutils/__init__.py
229 path = [str(origin_path.parent)] # e.g. .../distutils
230 else:
231 path = [spec.location]
232 else:
233 path = [spec.location]
234 return path
235
236
237class ExplicitNamespacePackageFinder(ImportlibFinder):
238 """A finder for the explicit namespace packages."""
239
240 @staticmethod
241 @lru_cache(maxsize=1024)
242 def find_module(
243 modname: str,
244 module_parts: tuple[str, ...],
245 processed: tuple[str, ...],
246 submodule_path: tuple[str, ...] | None,
247 ) -> ModuleSpec | None:
248 if processed:
249 modname = ".".join([*processed, modname])
250 if util.is_namespace(modname) and modname in sys.modules:
251 return ModuleSpec(
252 name=modname,
253 location="",
254 origin="namespace",
255 type=ModuleType.PY_NAMESPACE,
256 submodule_search_locations=sys.modules[modname].__path__,
257 )
258 return None
259
260 def contribute_to_path(
261 self, spec: ModuleSpec, processed: list[str]
262 ) -> Sequence[str] | None:
263 return spec.submodule_search_locations
264
265
266class ZipFinder(Finder):
267 """Finder that knows how to find a module inside zip files."""
268
269 def __init__(self, path: Sequence[str]) -> None:
270 super().__init__(path)
271 for entry_path in path:
272 if entry_path not in sys.path_importer_cache:
273 try:
274 sys.path_importer_cache[entry_path] = zipimport.zipimporter(
275 entry_path
276 )
277 except zipimport.ZipImportError:
278 continue
279
280 @staticmethod
281 @lru_cache(maxsize=1024)
282 def find_module(
283 modname: str,
284 module_parts: tuple[str, ...],
285 processed: tuple[str, ...],
286 submodule_path: tuple[str, ...] | None,
287 ) -> ModuleSpec | None:
288 try:
289 file_type, filename, path = _search_zip(module_parts)
290 except ImportError:
291 return None
292
293 return ModuleSpec(
294 name=modname,
295 location=filename,
296 origin="egg",
297 type=file_type,
298 submodule_search_locations=path,
299 )
300
301 def contribute_to_path(
302 self, spec: ModuleSpec, processed: list[str]
303 ) -> Sequence[str] | None:
304 return spec.submodule_search_locations
305
306
307class PathSpecFinder(Finder):
308 """Finder based on importlib.machinery.PathFinder."""
309
310 @staticmethod
311 @lru_cache(maxsize=1024)
312 def find_module(
313 modname: str,
314 module_parts: tuple[str, ...],
315 processed: tuple[str, ...],
316 submodule_path: tuple[str, ...] | None,
317 ) -> ModuleSpec | None:
318 spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path)
319 if spec is not None:
320 is_namespace_pkg = spec.origin is None
321 location = spec.origin if not is_namespace_pkg else None
322 module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None
323 return ModuleSpec(
324 name=spec.name,
325 location=location,
326 origin=spec.origin,
327 type=module_type,
328 submodule_search_locations=list(spec.submodule_search_locations or []),
329 )
330 return spec
331
332 def contribute_to_path(
333 self, spec: ModuleSpec, processed: list[str]
334 ) -> Sequence[str] | None:
335 if spec.type == ModuleType.PY_NAMESPACE:
336 return spec.submodule_search_locations
337 return None
338
339
340_SPEC_FINDERS = (
341 ImportlibFinder,
342 ZipFinder,
343 PathSpecFinder,
344 ExplicitNamespacePackageFinder,
345)
346
347
348@lru_cache(maxsize=1024)
349def _is_setuptools_namespace(location: pathlib.Path) -> bool:
350 try:
351 with open(location / "__init__.py", "rb") as stream:
352 data = stream.read(4096)
353 except OSError:
354 return False
355 extend_path = b"pkgutil" in data and b"extend_path" in data
356 declare_namespace = (
357 b"pkg_resources" in data and b"declare_namespace(__name__)" in data
358 )
359 return extend_path or declare_namespace
360
361
362def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]:
363 for filepath, importer in sys.path_importer_cache.items():
364 if importer is not None and isinstance(importer, zipimport.zipimporter):
365 yield filepath, importer
366
367
368def _search_zip(
369 modpath: tuple[str, ...],
370) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]:
371 for filepath, importer in _get_zipimporters():
372 found = importer.find_spec(modpath[0])
373 if found:
374 if not importer.find_spec(os.path.sep.join(modpath)):
375 raise ImportError(
376 "No module named {} in {}/{}".format(
377 ".".join(modpath[1:]), filepath, modpath
378 )
379 )
380 return (
381 ModuleType.PY_ZIPMODULE,
382 os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath),
383 filepath,
384 )
385 raise ImportError(f"No module named {'.'.join(modpath)}")
386
387
388def _find_spec_with_path(
389 search_path: Sequence[str],
390 modname: str,
391 module_parts: tuple[str, ...],
392 processed: tuple[str, ...],
393 submodule_path: tuple[str, ...] | None,
394) -> tuple[Finder | _MetaPathFinder, ModuleSpec]:
395 for finder in _SPEC_FINDERS:
396 finder_instance = finder(search_path)
397 mod_spec = finder.find_module(modname, module_parts, processed, submodule_path)
398 if mod_spec is None:
399 continue
400 return finder_instance, mod_spec
401
402 # Support for custom finders
403 for meta_finder in sys.meta_path:
404 # See if we support the customer import hook of the meta_finder
405 meta_finder_name = meta_finder.__class__.__name__
406 if meta_finder_name not in _MetaPathFinderModuleTypes:
407 # Setuptools>62 creates its EditableFinders dynamically and have
408 # "type" as their __class__.__name__. We check __name__ as well
409 # to see if we can support the finder.
410 try:
411 meta_finder_name = meta_finder.__name__ # type: ignore[attr-defined]
412 except AttributeError:
413 continue
414 if meta_finder_name not in _MetaPathFinderModuleTypes:
415 continue
416
417 module_type = _MetaPathFinderModuleTypes[meta_finder_name]
418
419 # Meta path finders are supposed to have a find_spec method since
420 # Python 3.4. However, some third-party finders do not implement it.
421 # PEP302 does not refer to find_spec as well.
422 # See: https://github.com/pylint-dev/astroid/pull/1752/
423 if not hasattr(meta_finder, "find_spec"):
424 continue
425
426 spec = meta_finder.find_spec(modname, submodule_path)
427 if spec:
428 return (
429 meta_finder,
430 ModuleSpec(
431 spec.name,
432 module_type,
433 spec.origin,
434 spec.origin,
435 spec.submodule_search_locations,
436 ),
437 )
438
439 raise ImportError(f"No module named {'.'.join(module_parts)}")
440
441
442def find_spec(modpath: Iterable[str], path: Iterable[str] | None = None) -> ModuleSpec:
443 """Find a spec for the given module.
444
445 :param modpath:
446 split module's name (i.e name of a module or package split
447 on '.'), with leading empty strings for explicit relative import
448
449 :param path:
450 optional list of path where the module or package should be
451 searched (use sys.path if nothing or None is given)
452
453 :return: A module spec, which describes how the module was
454 found and where.
455 """
456 return _find_spec(tuple(modpath), tuple(path) if path else None)
457
458
459@lru_cache(maxsize=1024)
460def _find_spec(
461 module_path: tuple[str, ...], path: tuple[str, ...] | None
462) -> ModuleSpec:
463 _path = path or sys.path
464
465 # Need a copy for not mutating the argument.
466 modpath = list(module_path)
467
468 search_paths = None
469 processed: list[str] = []
470
471 while modpath:
472 modname = modpath.pop(0)
473
474 submodule_path = search_paths or path
475 if submodule_path is not None:
476 submodule_path = tuple(submodule_path)
477
478 finder, spec = _find_spec_with_path(
479 _path, modname, module_path, tuple(processed), submodule_path
480 )
481 processed.append(modname)
482 if modpath:
483 if isinstance(finder, Finder):
484 search_paths = finder.contribute_to_path(spec, processed)
485 # If modname is a package from an editable install, update search_paths
486 # so that the next module in the path will be found inside of it using importlib.
487 # Existence of __name__ is guaranteed by _find_spec_with_path.
488 elif finder.__name__ in _EditableFinderClasses: # type: ignore[attr-defined]
489 search_paths = spec.submodule_search_locations
490
491 if spec.type == ModuleType.PKG_DIRECTORY:
492 spec = spec._replace(submodule_search_locations=search_paths)
493
494 return spec