Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/interpreter/_import/spec.py: 36%

1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html 

2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE 

3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt 

4 

5from __future__ import annotations 

6 

7import abc 

8import enum 

9import importlib 

10import importlib.machinery 

11import importlib.util 

12import os 

13import pathlib 

14import sys 

15import types 

16import warnings 

17import zipimport 

18from collections.abc import Iterable, Iterator, Sequence 

19from functools import lru_cache 

20from pathlib import Path 

21from typing import Literal, NamedTuple, Protocol 

22 

23from . import util 

24 

25 

26# The MetaPathFinder protocol comes from typeshed, which says: 

27# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder` 

28class _MetaPathFinder(Protocol): 

29 def find_spec( 

30 self, 

31 fullname: str, 

32 path: Sequence[str] | None, 

33 target: types.ModuleType | None = ..., 

34 ) -> importlib.machinery.ModuleSpec | None: ... # pragma: no cover 

35 

36 

37class ModuleType(enum.Enum): 

38 """Python module types used for ModuleSpec.""" 

39 

40 C_BUILTIN = enum.auto() 

41 C_EXTENSION = enum.auto() 

42 PKG_DIRECTORY = enum.auto() 

43 PY_CODERESOURCE = enum.auto() 

44 PY_COMPILED = enum.auto() 

45 PY_FROZEN = enum.auto() 

46 PY_RESOURCE = enum.auto() 

47 PY_SOURCE = enum.auto() 

48 PY_ZIPMODULE = enum.auto() 

49 PY_NAMESPACE = enum.auto() 

50 

51 

52_MetaPathFinderModuleTypes: dict[str, ModuleType] = { 

53 # Finders created by setuptools editable installs 

54 "_EditableFinder": ModuleType.PY_SOURCE, 

55 "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE, 

56 # Finders create by six 

57 "_SixMetaPathImporter": ModuleType.PY_SOURCE, 

58} 

59 

60_EditableFinderClasses: set[str] = { 

61 "_EditableFinder", 

62 "_EditableNamespaceFinder", 

63} 

64 

65 

66class ModuleSpec(NamedTuple): 

67 """Defines a class similar to PEP 420's ModuleSpec. 

68 

69 A module spec defines a name of a module, its type, location 

70 and where submodules can be found, if the module is a package. 

71 """ 

72 

73 name: str 

74 type: ModuleType | None 

75 location: str | None = None 

76 origin: str | None = None 

77 submodule_search_locations: Sequence[str] | None = None 

78 

79 

80class Finder: 

81 """A finder is a class which knows how to find a particular module.""" 

82 

83 def __init__(self, path: Sequence[str] | None = None) -> None: 

84 self._path = path or sys.path 

85 

86 @staticmethod 

87 @abc.abstractmethod 

88 def find_module( 

89 modname: str, 

90 module_parts: tuple[str, ...], 

91 processed: tuple[str, ...], 

92 submodule_path: tuple[str, ...] | None, 

93 ) -> ModuleSpec | None: 

94 """Find the given module. 

95 

96 Each finder is responsible for each protocol of finding, as long as 

97 they all return a ModuleSpec. 

98 

99 :param modname: The module which needs to be searched. 

100 :param module_parts: It should be a tuple of strings, 

101 where each part contributes to the module's 

102 namespace. 

103 :param processed: What parts from the module parts were processed 

104 so far. 

105 :param submodule_path: A tuple of paths where the module 

106 can be looked into. 

107 :returns: A ModuleSpec, describing how and where the module was found, 

108 None, otherwise. 

109 """ 

110 

111 def contribute_to_path( 

112 self, spec: ModuleSpec, processed: list[str] 

113 ) -> Sequence[str] | None: 

114 """Get a list of extra paths where this finder can search.""" 

115 

116 

117class ImportlibFinder(Finder): 

118 """A finder based on the importlib module.""" 

119 

120 _SUFFIXES: Sequence[tuple[str, ModuleType]] = ( 

121 [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES] 

122 + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES] 

123 + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES] 

124 ) 

125 

126 @staticmethod 

127 @lru_cache(maxsize=1024) 

128 def find_module( 

129 modname: str, 

130 module_parts: tuple[str, ...], 

131 processed: tuple[str, ...], 

132 submodule_path: tuple[str, ...] | None, 

133 ) -> ModuleSpec | None: 

134 # pylint: disable-next=import-outside-toplevel 

135 from astroid.modutils import cached_os_path_isfile 

136 

137 # Although we should be able to use `find_spec` this doesn't work on PyPy for builtins. 

138 # Therefore, we use the `builtin_module_nams` heuristic for these. 

139 if submodule_path is None and modname in sys.builtin_module_names: 

140 return ModuleSpec( 

141 name=modname, 

142 location=None, 

143 type=ModuleType.C_BUILTIN, 

144 ) 

145 

146 if submodule_path is not None: 

147 search_paths = list(submodule_path) 

148 else: 

149 search_paths = sys.path 

150 

151 suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0]) 

152 for entry in search_paths: 

153 package_directory = os.path.join(entry, modname) 

154 for suffix in suffixes: 

155 package_file_name = "__init__" + suffix 

156 file_path = os.path.join(package_directory, package_file_name) 

157 if cached_os_path_isfile(file_path): 

158 return ModuleSpec( 

159 name=modname, 

160 location=package_directory, 

161 type=ModuleType.PKG_DIRECTORY, 

162 ) 

163 for suffix, type_ in ImportlibFinder._SUFFIXES: 

164 file_name = modname + suffix 

165 file_path = os.path.join(entry, file_name) 

166 if cached_os_path_isfile(file_path): 

167 return ModuleSpec(name=modname, location=file_path, type=type_) 

168 

169 # If the module name matches a stdlib module name, check whether this is a frozen 

170 # module. Note that `find_spec` actually imports parent modules, so we want to make 

171 # sure we only run this code for stuff that can be expected to be frozen. For now 

172 # this is only stdlib. 

173 if (modname in sys.stdlib_module_names and not processed) or ( 

174 processed and processed[0] in sys.stdlib_module_names 

175 ): 

176 try: 

177 with warnings.catch_warnings(): 

178 warnings.filterwarnings("ignore", category=Warning) 

179 spec = importlib.util.find_spec(".".join((*processed, modname))) 

180 except ValueError: 

181 spec = None 

182 

183 if ( 

184 spec 

185 and spec.loader # type: ignore[comparison-overlap] # noqa: E501 

186 is importlib.machinery.FrozenImporter 

187 ): 

188 return ModuleSpec( 

189 name=modname, 

190 location=getattr(spec.loader_state, "filename", None), 

191 type=ModuleType.PY_FROZEN, 

192 ) 

193 

194 return None 

195 

196 def contribute_to_path( 

197 self, spec: ModuleSpec, processed: list[str] 

198 ) -> Sequence[str] | None: 

199 if spec.location is None: 

200 # Builtin. 

201 return None 

202 # pylint: disable-next=import-outside-toplevel 

203 from astroid.modutils import EXT_LIB_DIRS 

204 

205 if _is_setuptools_namespace(Path(spec.location)): 

206 # extend_path is called, search sys.path for module/packages 

207 # of this name see pkgutil.extend_path documentation 

208 path = [ 

209 os.path.join(p, *processed) 

210 for p in sys.path 

211 if os.path.isdir(os.path.join(p, *processed)) 

212 ] 

213 elif spec.name == "distutils" and not any( 

214 spec.location.lower().startswith(ext_lib_dir.lower()) 

215 for ext_lib_dir in EXT_LIB_DIRS 

216 ): 

217 # virtualenv below 20.0 patches distutils in an unexpected way 

218 # so we just find the location of distutils that will be 

219 # imported to avoid spurious import-error messages 

220 # https://github.com/pylint-dev/pylint/issues/5645 

221 # A regression test to create this scenario exists in release-tests.yml 

222 # and can be triggered manually from GitHub Actions 

223 distutils_spec = importlib.util.find_spec("distutils") 

224 if distutils_spec and distutils_spec.origin: 

225 origin_path = Path( 

226 distutils_spec.origin 

227 ) # e.g. .../distutils/__init__.py 

228 path = [str(origin_path.parent)] # e.g. .../distutils 

229 else: 

230 path = [spec.location] 

231 else: 

232 path = [spec.location] 

233 return path 

234 

235 

236class ExplicitNamespacePackageFinder(ImportlibFinder): 

237 """A finder for the explicit namespace packages.""" 

238 

239 @staticmethod 

240 @lru_cache(maxsize=1024) 

241 def find_module( 

242 modname: str, 

243 module_parts: tuple[str, ...], 

244 processed: tuple[str, ...], 

245 submodule_path: tuple[str, ...] | None, 

246 ) -> ModuleSpec | None: 

247 if processed: 

248 modname = ".".join([*processed, modname]) 

249 if util.is_namespace(modname) and modname in sys.modules: 

250 return ModuleSpec( 

251 name=modname, 

252 location="", 

253 origin="namespace", 

254 type=ModuleType.PY_NAMESPACE, 

255 submodule_search_locations=sys.modules[modname].__path__, 

256 ) 

257 return None 

258 

259 def contribute_to_path( 

260 self, spec: ModuleSpec, processed: list[str] 

261 ) -> Sequence[str] | None: 

262 return spec.submodule_search_locations 

263 

264 

265class ZipFinder(Finder): 

266 """Finder that knows how to find a module inside zip files.""" 

267 

268 def __init__(self, path: Sequence[str]) -> None: 

269 super().__init__(path) 

270 for entry_path in path: 

271 if entry_path not in sys.path_importer_cache: 

272 try: 

273 sys.path_importer_cache[entry_path] = zipimport.zipimporter( 

274 entry_path 

275 ) 

276 except zipimport.ZipImportError: 

277 continue 

278 

279 @staticmethod 

280 @lru_cache(maxsize=1024) 

281 def find_module( 

282 modname: str, 

283 module_parts: tuple[str, ...], 

284 processed: tuple[str, ...], 

285 submodule_path: tuple[str, ...] | None, 

286 ) -> ModuleSpec | None: 

287 try: 

288 file_type, filename, path = _search_zip(module_parts) 

289 except ImportError: 

290 return None 

291 

292 return ModuleSpec( 

293 name=modname, 

294 location=filename, 

295 origin="egg", 

296 type=file_type, 

297 submodule_search_locations=path, 

298 ) 

299 

300 def contribute_to_path( 

301 self, spec: ModuleSpec, processed: list[str] 

302 ) -> Sequence[str] | None: 

303 return spec.submodule_search_locations 

304 

305 

306class PathSpecFinder(Finder): 

307 """Finder based on importlib.machinery.PathFinder.""" 

308 

309 @staticmethod 

310 @lru_cache(maxsize=1024) 

311 def find_module( 

312 modname: str, 

313 module_parts: tuple[str, ...], 

314 processed: tuple[str, ...], 

315 submodule_path: tuple[str, ...] | None, 

316 ) -> ModuleSpec | None: 

317 spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path) 

318 if spec is not None: 

319 is_namespace_pkg = spec.origin is None 

320 location = spec.origin if not is_namespace_pkg else None 

321 module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None 

322 return ModuleSpec( 

323 name=spec.name, 

324 location=location, 

325 origin=spec.origin, 

326 type=module_type, 

327 submodule_search_locations=list(spec.submodule_search_locations or []), 

328 ) 

329 return spec 

330 

331 def contribute_to_path( 

332 self, spec: ModuleSpec, processed: list[str] 

333 ) -> Sequence[str] | None: 

334 if spec.type == ModuleType.PY_NAMESPACE: 

335 return spec.submodule_search_locations 

336 return None 

337 

338 

339_SPEC_FINDERS = ( 

340 ImportlibFinder, 

341 ZipFinder, 

342 PathSpecFinder, 

343 ExplicitNamespacePackageFinder, 

344) 

345 

346 

347@lru_cache(maxsize=1024) 

348def _is_setuptools_namespace(location: pathlib.Path) -> bool: 

349 try: 

350 with open(location / "__init__.py", "rb") as stream: 

351 data = stream.read(4096) 

352 except OSError: 

353 return False 

354 extend_path = b"pkgutil" in data and b"extend_path" in data 

355 declare_namespace = ( 

356 b"pkg_resources" in data and b"declare_namespace(__name__)" in data 

357 ) 

358 return extend_path or declare_namespace 

359 

360 

361def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]: 

362 for filepath, importer in sys.path_importer_cache.items(): 

363 if importer is not None and isinstance(importer, zipimport.zipimporter): 

364 yield filepath, importer 

365 

366 

367def _search_zip( 

368 modpath: tuple[str, ...], 

369) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]: 

370 for filepath, importer in _get_zipimporters(): 

371 found = importer.find_spec(modpath[0]) 

372 if found: 

373 if not importer.find_spec(os.path.sep.join(modpath)): 

374 raise ImportError( 

375 "No module named {} in {}/{}".format( 

376 ".".join(modpath[1:]), filepath, modpath 

377 ) 

378 ) 

379 return ( 

380 ModuleType.PY_ZIPMODULE, 

381 os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath), 

382 filepath, 

383 ) 

384 raise ImportError(f"No module named {'.'.join(modpath)}") 

385 

386 

387def _find_spec_with_path( 

388 search_path: Sequence[str], 

389 modname: str, 

390 module_parts: tuple[str, ...], 

391 processed: tuple[str, ...], 

392 submodule_path: tuple[str, ...] | None, 

393) -> tuple[Finder | _MetaPathFinder, ModuleSpec]: 

394 for finder in _SPEC_FINDERS: 

395 finder_instance = finder(search_path) 

396 mod_spec = finder.find_module(modname, module_parts, processed, submodule_path) 

397 if mod_spec is None: 

398 continue 

399 return finder_instance, mod_spec 

400 

401 # Support for custom finders 

402 for meta_finder in sys.meta_path: 

403 # See if we support the customer import hook of the meta_finder 

404 meta_finder_name = meta_finder.__class__.__name__ 

405 if meta_finder_name not in _MetaPathFinderModuleTypes: 

406 # Setuptools>62 creates its EditableFinders dynamically and have 

407 # "type" as their __class__.__name__. We check __name__ as well 

408 # to see if we can support the finder. 

409 try: 

410 meta_finder_name = meta_finder.__name__ # type: ignore[attr-defined] 

411 except AttributeError: 

412 continue 

413 if meta_finder_name not in _MetaPathFinderModuleTypes: 

414 continue 

415 

416 module_type = _MetaPathFinderModuleTypes[meta_finder_name] 

417 

418 # Meta path finders are supposed to have a find_spec method since 

419 # Python 3.4. However, some third-party finders do not implement it. 

420 # PEP302 does not refer to find_spec as well. 

421 # See: https://github.com/pylint-dev/astroid/pull/1752/ 

422 if not hasattr(meta_finder, "find_spec"): 

423 continue 

424 

425 spec = meta_finder.find_spec(modname, submodule_path) 

426 if spec: 

427 return ( 

428 meta_finder, 

429 ModuleSpec( 

430 spec.name, 

431 module_type, 

432 spec.origin, 

433 spec.origin, 

434 spec.submodule_search_locations, 

435 ), 

436 ) 

437 

438 raise ImportError(f"No module named {'.'.join(module_parts)}") 

439 

440 

441def find_spec(modpath: Iterable[str], path: Iterable[str] | None = None) -> ModuleSpec: 

442 """Find a spec for the given module. 

443 

444 :type modpath: list or tuple 

445 :param modpath: 

446 split module's name (i.e name of a module or package split 

447 on '.'), with leading empty strings for explicit relative import 

448 

449 :type path: list or None 

450 :param path: 

451 optional list of path where the module or package should be 

452 searched (use sys.path if nothing or None is given) 

453 

454 :rtype: ModuleSpec 

455 :return: A module spec, which describes how the module was 

456 found and where. 

457 """ 

458 return _find_spec(tuple(modpath), tuple(path) if path else None) 

459 

460 

461@lru_cache(maxsize=1024) 

462def _find_spec( 

463 module_path: tuple[str, ...], path: tuple[str, ...] | None 

464) -> ModuleSpec: 

465 _path = path or sys.path 

466 

467 # Need a copy for not mutating the argument. 

468 modpath = list(module_path) 

469 

470 search_paths = None 

471 processed: list[str] = [] 

472 

473 while modpath: 

474 modname = modpath.pop(0) 

475 

476 submodule_path = search_paths or path 

477 if submodule_path is not None: 

478 submodule_path = tuple(submodule_path) 

479 

480 finder, spec = _find_spec_with_path( 

481 _path, modname, module_path, tuple(processed), submodule_path 

482 ) 

483 processed.append(modname) 

484 if modpath: 

485 if isinstance(finder, Finder): 

486 search_paths = finder.contribute_to_path(spec, processed) 

487 # If modname is a package from an editable install, update search_paths 

488 # so that the next module in the path will be found inside of it using importlib. 

489 # Existence of __name__ is guaranteed by _find_spec_with_path. 

490 elif finder.__name__ in _EditableFinderClasses: # type: ignore[attr-defined] 

491 search_paths = spec.submodule_search_locations 

492 

493 if spec.type == ModuleType.PKG_DIRECTORY: 

494 spec = spec._replace(submodule_search_locations=search_paths) 

495 

496 return spec