Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/astroid/interpreter/_import/spec.py: 33%
193 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 06:53 +0000
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5from __future__ import annotations
7import abc
8import enum
9import importlib
10import importlib.machinery
11import importlib.util
12import os
13import pathlib
14import sys
15import types
16import warnings
17import zipimport
18from collections.abc import Iterator, Sequence
19from pathlib import Path
20from typing import Any, Literal, NamedTuple, Protocol
22from astroid.const import PY310_PLUS
23from astroid.modutils import EXT_LIB_DIRS
25from . import util
28# The MetaPathFinder protocol comes from typeshed, which says:
29# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
30class _MetaPathFinder(Protocol):
31 def find_spec(
32 self,
33 fullname: str,
34 path: Sequence[str] | None,
35 target: types.ModuleType | None = ...,
36 ) -> importlib.machinery.ModuleSpec | None:
37 ... # pragma: no cover
40class ModuleType(enum.Enum):
41 """Python module types used for ModuleSpec."""
43 C_BUILTIN = enum.auto()
44 C_EXTENSION = enum.auto()
45 PKG_DIRECTORY = enum.auto()
46 PY_CODERESOURCE = enum.auto()
47 PY_COMPILED = enum.auto()
48 PY_FROZEN = enum.auto()
49 PY_RESOURCE = enum.auto()
50 PY_SOURCE = enum.auto()
51 PY_ZIPMODULE = enum.auto()
52 PY_NAMESPACE = enum.auto()
55_MetaPathFinderModuleTypes: dict[str, ModuleType] = {
56 # Finders created by setuptools editable installs
57 "_EditableFinder": ModuleType.PY_SOURCE,
58 "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE,
59 # Finders create by six
60 "_SixMetaPathImporter": ModuleType.PY_SOURCE,
61}
63_EditableFinderClasses: set[str] = {
64 "_EditableFinder",
65 "_EditableNamespaceFinder",
66}
69class ModuleSpec(NamedTuple):
70 """Defines a class similar to PEP 420's ModuleSpec.
72 A module spec defines a name of a module, its type, location
73 and where submodules can be found, if the module is a package.
74 """
76 name: str
77 type: ModuleType | None
78 location: str | None = None
79 origin: str | None = None
80 submodule_search_locations: Sequence[str] | None = None
83class Finder:
84 """A finder is a class which knows how to find a particular module."""
86 def __init__(self, path: Sequence[str] | None = None) -> None:
87 self._path = path or sys.path
89 @abc.abstractmethod
90 def find_module(
91 self,
92 modname: str,
93 module_parts: Sequence[str],
94 processed: list[str],
95 submodule_path: Sequence[str] | None,
96 ) -> ModuleSpec | None:
97 """Find the given module.
99 Each finder is responsible for each protocol of finding, as long as
100 they all return a ModuleSpec.
102 :param modname: The module which needs to be searched.
103 :param module_parts: It should be a list of strings,
104 where each part contributes to the module's
105 namespace.
106 :param processed: What parts from the module parts were processed
107 so far.
108 :param submodule_path: A list of paths where the module
109 can be looked into.
110 :returns: A ModuleSpec, describing how and where the module was found,
111 None, otherwise.
112 """
114 def contribute_to_path(
115 self, spec: ModuleSpec, processed: list[str]
116 ) -> Sequence[str] | None:
117 """Get a list of extra paths where this finder can search."""
120class ImportlibFinder(Finder):
121 """A finder based on the importlib module."""
123 _SUFFIXES: Sequence[tuple[str, ModuleType]] = (
124 [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES]
125 + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES]
126 + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES]
127 )
129 def find_module(
130 self,
131 modname: str,
132 module_parts: Sequence[str],
133 processed: list[str],
134 submodule_path: Sequence[str] | None,
135 ) -> ModuleSpec | None:
136 if submodule_path is not None:
137 submodule_path = list(submodule_path)
138 elif modname in sys.builtin_module_names:
139 return ModuleSpec(
140 name=modname,
141 location=None,
142 type=ModuleType.C_BUILTIN,
143 )
144 else:
145 try:
146 with warnings.catch_warnings():
147 warnings.filterwarnings("ignore", category=UserWarning)
148 spec = importlib.util.find_spec(modname)
149 if (
150 spec
151 and spec.loader # type: ignore[comparison-overlap] # noqa: E501
152 is importlib.machinery.FrozenImporter
153 ):
154 # No need for BuiltinImporter; builtins handled above
155 return ModuleSpec(
156 name=modname,
157 location=getattr(spec.loader_state, "filename", None),
158 type=ModuleType.PY_FROZEN,
159 )
160 except ValueError:
161 pass
162 submodule_path = sys.path
164 for entry in submodule_path:
165 package_directory = os.path.join(entry, modname)
166 for suffix in (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0]):
167 package_file_name = "__init__" + suffix
168 file_path = os.path.join(package_directory, package_file_name)
169 if os.path.isfile(file_path):
170 return ModuleSpec(
171 name=modname,
172 location=package_directory,
173 type=ModuleType.PKG_DIRECTORY,
174 )
175 for suffix, type_ in ImportlibFinder._SUFFIXES:
176 file_name = modname + suffix
177 file_path = os.path.join(entry, file_name)
178 if os.path.isfile(file_path):
179 return ModuleSpec(name=modname, location=file_path, type=type_)
180 return None
182 def contribute_to_path(
183 self, spec: ModuleSpec, processed: list[str]
184 ) -> Sequence[str] | None:
185 if spec.location is None:
186 # Builtin.
187 return None
189 if _is_setuptools_namespace(Path(spec.location)):
190 # extend_path is called, search sys.path for module/packages
191 # of this name see pkgutil.extend_path documentation
192 path = [
193 os.path.join(p, *processed)
194 for p in sys.path
195 if os.path.isdir(os.path.join(p, *processed))
196 ]
197 elif spec.name == "distutils" and not any(
198 spec.location.lower().startswith(ext_lib_dir.lower())
199 for ext_lib_dir in EXT_LIB_DIRS
200 ):
201 # virtualenv below 20.0 patches distutils in an unexpected way
202 # so we just find the location of distutils that will be
203 # imported to avoid spurious import-error messages
204 # https://github.com/pylint-dev/pylint/issues/5645
205 # A regression test to create this scenario exists in release-tests.yml
206 # and can be triggered manually from GitHub Actions
207 distutils_spec = importlib.util.find_spec("distutils")
208 if distutils_spec and distutils_spec.origin:
209 origin_path = Path(
210 distutils_spec.origin
211 ) # e.g. .../distutils/__init__.py
212 path = [str(origin_path.parent)] # e.g. .../distutils
213 else:
214 path = [spec.location]
215 else:
216 path = [spec.location]
217 return path
220class ExplicitNamespacePackageFinder(ImportlibFinder):
221 """A finder for the explicit namespace packages."""
223 def find_module(
224 self,
225 modname: str,
226 module_parts: Sequence[str],
227 processed: list[str],
228 submodule_path: Sequence[str] | None,
229 ) -> ModuleSpec | None:
230 if processed:
231 modname = ".".join([*processed, modname])
232 if util.is_namespace(modname) and modname in sys.modules:
233 submodule_path = sys.modules[modname].__path__
234 return ModuleSpec(
235 name=modname,
236 location="",
237 origin="namespace",
238 type=ModuleType.PY_NAMESPACE,
239 submodule_search_locations=submodule_path,
240 )
241 return None
243 def contribute_to_path(
244 self, spec: ModuleSpec, processed: list[str]
245 ) -> Sequence[str] | None:
246 return spec.submodule_search_locations
249class ZipFinder(Finder):
250 """Finder that knows how to find a module inside zip files."""
252 def __init__(self, path: Sequence[str]) -> None:
253 super().__init__(path)
254 for entry_path in path:
255 if entry_path not in sys.path_importer_cache:
256 try:
257 sys.path_importer_cache[entry_path] = zipimport.zipimporter( # type: ignore[assignment]
258 entry_path
259 )
260 except zipimport.ZipImportError:
261 continue
263 def find_module(
264 self,
265 modname: str,
266 module_parts: Sequence[str],
267 processed: list[str],
268 submodule_path: Sequence[str] | None,
269 ) -> ModuleSpec | None:
270 try:
271 file_type, filename, path = _search_zip(module_parts)
272 except ImportError:
273 return None
275 return ModuleSpec(
276 name=modname,
277 location=filename,
278 origin="egg",
279 type=file_type,
280 submodule_search_locations=path,
281 )
284class PathSpecFinder(Finder):
285 """Finder based on importlib.machinery.PathFinder."""
287 def find_module(
288 self,
289 modname: str,
290 module_parts: Sequence[str],
291 processed: list[str],
292 submodule_path: Sequence[str] | None,
293 ) -> ModuleSpec | None:
294 spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path)
295 if spec is not None:
296 is_namespace_pkg = spec.origin is None
297 location = spec.origin if not is_namespace_pkg else None
298 module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None
299 return ModuleSpec(
300 name=spec.name,
301 location=location,
302 origin=spec.origin,
303 type=module_type,
304 submodule_search_locations=list(spec.submodule_search_locations or []),
305 )
306 return spec
308 def contribute_to_path(
309 self, spec: ModuleSpec, processed: list[str]
310 ) -> Sequence[str] | None:
311 if spec.type == ModuleType.PY_NAMESPACE:
312 return spec.submodule_search_locations
313 return None
316_SPEC_FINDERS = (
317 ImportlibFinder,
318 ZipFinder,
319 PathSpecFinder,
320 ExplicitNamespacePackageFinder,
321)
324def _is_setuptools_namespace(location: pathlib.Path) -> bool:
325 try:
326 with open(location / "__init__.py", "rb") as stream:
327 data = stream.read(4096)
328 except OSError:
329 return False
330 extend_path = b"pkgutil" in data and b"extend_path" in data
331 declare_namespace = (
332 b"pkg_resources" in data and b"declare_namespace(__name__)" in data
333 )
334 return extend_path or declare_namespace
337def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]:
338 for filepath, importer in sys.path_importer_cache.items():
339 if isinstance(importer, zipimport.zipimporter):
340 yield filepath, importer
343def _search_zip(
344 modpath: Sequence[str],
345) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]:
346 for filepath, importer in _get_zipimporters():
347 if PY310_PLUS:
348 found: Any = importer.find_spec(modpath[0])
349 else:
350 found = importer.find_module(modpath[0])
351 if found:
352 if PY310_PLUS:
353 if not importer.find_spec(os.path.sep.join(modpath)):
354 raise ImportError(
355 "No module named %s in %s/%s"
356 % (".".join(modpath[1:]), filepath, modpath)
357 )
358 elif not importer.find_module(os.path.sep.join(modpath)):
359 raise ImportError(
360 "No module named %s in %s/%s"
361 % (".".join(modpath[1:]), filepath, modpath)
362 )
363 return (
364 ModuleType.PY_ZIPMODULE,
365 os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath),
366 filepath,
367 )
368 raise ImportError(f"No module named {'.'.join(modpath)}")
371def _find_spec_with_path(
372 search_path: Sequence[str],
373 modname: str,
374 module_parts: list[str],
375 processed: list[str],
376 submodule_path: Sequence[str] | None,
377) -> tuple[Finder | _MetaPathFinder, ModuleSpec]:
378 for finder in _SPEC_FINDERS:
379 finder_instance = finder(search_path)
380 spec = finder_instance.find_module(
381 modname, module_parts, processed, submodule_path
382 )
383 if spec is None:
384 continue
385 return finder_instance, spec
387 # Support for custom finders
388 for meta_finder in sys.meta_path:
389 # See if we support the customer import hook of the meta_finder
390 meta_finder_name = meta_finder.__class__.__name__
391 if meta_finder_name not in _MetaPathFinderModuleTypes:
392 # Setuptools>62 creates its EditableFinders dynamically and have
393 # "type" as their __class__.__name__. We check __name__ as well
394 # to see if we can support the finder.
395 try:
396 meta_finder_name = meta_finder.__name__ # type: ignore[attr-defined]
397 except AttributeError:
398 continue
399 if meta_finder_name not in _MetaPathFinderModuleTypes:
400 continue
402 module_type = _MetaPathFinderModuleTypes[meta_finder_name]
404 # Meta path finders are supposed to have a find_spec method since
405 # Python 3.4. However, some third-party finders do not implement it.
406 # PEP302 does not refer to find_spec as well.
407 # See: https://github.com/pylint-dev/astroid/pull/1752/
408 if not hasattr(meta_finder, "find_spec"):
409 continue
411 spec = meta_finder.find_spec(modname, submodule_path)
412 if spec:
413 return (
414 meta_finder,
415 ModuleSpec(
416 spec.name,
417 module_type,
418 spec.origin,
419 spec.origin,
420 spec.submodule_search_locations,
421 ),
422 )
424 raise ImportError(f"No module named {'.'.join(module_parts)}")
427def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSpec:
428 """Find a spec for the given module.
430 :type modpath: list or tuple
431 :param modpath:
432 split module's name (i.e name of a module or package split
433 on '.'), with leading empty strings for explicit relative import
435 :type path: list or None
436 :param path:
437 optional list of path where the module or package should be
438 searched (use sys.path if nothing or None is given)
440 :rtype: ModuleSpec
441 :return: A module spec, which describes how the module was
442 found and where.
443 """
444 _path = path or sys.path
446 # Need a copy for not mutating the argument.
447 modpath = modpath[:]
449 submodule_path = None
450 module_parts = modpath[:]
451 processed: list[str] = []
453 while modpath:
454 modname = modpath.pop(0)
455 finder, spec = _find_spec_with_path(
456 _path, modname, module_parts, processed, submodule_path or path
457 )
458 processed.append(modname)
459 if modpath:
460 if isinstance(finder, Finder):
461 submodule_path = finder.contribute_to_path(spec, processed)
462 # If modname is a package from an editable install, update submodule_path
463 # so that the next module in the path will be found inside of it using importlib.
464 # Existence of __name__ is guaranteed by _find_spec_with_path.
465 elif finder.__name__ in _EditableFinderClasses: # type: ignore[attr-defined]
466 submodule_path = spec.submodule_search_locations
468 if spec.type == ModuleType.PKG_DIRECTORY:
469 spec = spec._replace(submodule_search_locations=submodule_path)
471 return spec