Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/interpreter/_import/spec.py: 36%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

207 statements  

1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html 

2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE 

3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt 

4 

5from __future__ import annotations 

6 

7import abc 

8import enum 

9import importlib 

10import importlib.machinery 

11import importlib.util 

12import os 

13import pathlib 

14import sys 

15import types 

16import warnings 

17import zipimport 

18from collections.abc import Iterable, Iterator, Sequence 

19from functools import lru_cache 

20from pathlib import Path 

21from typing import Literal, NamedTuple, Protocol 

22 

23from . import util 

24 

25 

26# The MetaPathFinder protocol comes from typeshed, which says: 

27# Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder` 

28class _MetaPathFinder(Protocol): 

29 def find_spec( 

30 self, 

31 fullname: str, 

32 path: Sequence[str] | None, 

33 target: types.ModuleType | None = ..., 

34 ) -> importlib.machinery.ModuleSpec | None: ... # pragma: no cover 

35 

36 

37class ModuleType(enum.Enum): 

38 """Python module types used for ModuleSpec.""" 

39 

40 C_BUILTIN = enum.auto() 

41 C_EXTENSION = enum.auto() 

42 PKG_DIRECTORY = enum.auto() 

43 PY_CODERESOURCE = enum.auto() 

44 PY_COMPILED = enum.auto() 

45 PY_FROZEN = enum.auto() 

46 PY_RESOURCE = enum.auto() 

47 PY_SOURCE = enum.auto() 

48 PY_ZIPMODULE = enum.auto() 

49 PY_NAMESPACE = enum.auto() 

50 

51 

52_MetaPathFinderModuleTypes: dict[str, ModuleType] = { 

53 # Finders created by setuptools editable installs 

54 "_EditableFinder": ModuleType.PY_SOURCE, 

55 "_EditableNamespaceFinder": ModuleType.PY_NAMESPACE, 

56 # Finders create by six 

57 "_SixMetaPathImporter": ModuleType.PY_SOURCE, 

58} 

59 

60_EditableFinderClasses: set[str] = { 

61 "_EditableFinder", 

62 "_EditableNamespaceFinder", 

63} 

64 

65 

66class ModuleSpec(NamedTuple): 

67 """Defines a class similar to PEP 420's ModuleSpec. 

68 

69 A module spec defines a name of a module, its type, location 

70 and where submodules can be found, if the module is a package. 

71 """ 

72 

73 name: str 

74 type: ModuleType | None 

75 location: str | None = None 

76 origin: str | None = None 

77 submodule_search_locations: Sequence[str] | None = None 

78 

79 

80class Finder: 

81 """A finder is a class which knows how to find a particular module.""" 

82 

83 def __init__(self, path: Sequence[str] | None = None) -> None: 

84 self._path = path or sys.path 

85 

86 @staticmethod 

87 @abc.abstractmethod 

88 def find_module( 

89 modname: str, 

90 module_parts: tuple[str, ...], 

91 processed: tuple[str, ...], 

92 submodule_path: tuple[str, ...] | None, 

93 ) -> ModuleSpec | None: 

94 """Find the given module. 

95 

96 Each finder is responsible for each protocol of finding, as long as 

97 they all return a ModuleSpec. 

98 

99 :param modname: The module which needs to be searched. 

100 :param module_parts: It should be a tuple of strings, 

101 where each part contributes to the module's 

102 namespace. 

103 :param processed: What parts from the module parts were processed 

104 so far. 

105 :param submodule_path: A tuple of paths where the module 

106 can be looked into. 

107 :returns: A ModuleSpec, describing how and where the module was found, 

108 None, otherwise. 

109 """ 

110 

111 def contribute_to_path( 

112 self, spec: ModuleSpec, processed: list[str] 

113 ) -> Sequence[str] | None: 

114 """Get a list of extra paths where this finder can search.""" 

115 

116 

117class ImportlibFinder(Finder): 

118 """A finder based on the importlib module.""" 

119 

120 _SUFFIXES: Sequence[tuple[str, ModuleType]] = ( 

121 [(s, ModuleType.C_EXTENSION) for s in importlib.machinery.EXTENSION_SUFFIXES] 

122 + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES] 

123 + [(".pyi", ModuleType.PY_SOURCE)] 

124 + [(s, ModuleType.PY_COMPILED) for s in importlib.machinery.BYTECODE_SUFFIXES] 

125 ) 

126 

127 @staticmethod 

128 @lru_cache(maxsize=1024) 

129 def find_module( 

130 modname: str, 

131 module_parts: tuple[str, ...], 

132 processed: tuple[str, ...], 

133 submodule_path: tuple[str, ...] | None, 

134 ) -> ModuleSpec | None: 

135 # pylint: disable-next=import-outside-toplevel 

136 from astroid.modutils import cached_os_path_isfile 

137 

138 # Although we should be able to use `find_spec` this doesn't work on PyPy for builtins. 

139 # Therefore, we use the `builtin_module_nams` heuristic for these. 

140 if submodule_path is None and modname in sys.builtin_module_names: 

141 return ModuleSpec( 

142 name=modname, 

143 location=None, 

144 type=ModuleType.C_BUILTIN, 

145 ) 

146 

147 if submodule_path is not None: 

148 search_paths = list(submodule_path) 

149 else: 

150 search_paths = sys.path 

151 

152 suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0]) 

153 for entry in search_paths: 

154 package_directory = os.path.join(entry, modname) 

155 for suffix in suffixes: 

156 package_file_name = "__init__" + suffix 

157 file_path = os.path.join(package_directory, package_file_name) 

158 if cached_os_path_isfile(file_path): 

159 return ModuleSpec( 

160 name=modname, 

161 location=package_directory, 

162 type=ModuleType.PKG_DIRECTORY, 

163 ) 

164 for suffix, type_ in ImportlibFinder._SUFFIXES: 

165 file_name = modname + suffix 

166 file_path = os.path.join(entry, file_name) 

167 if cached_os_path_isfile(file_path): 

168 return ModuleSpec(name=modname, location=file_path, type=type_) 

169 

170 # If the module name matches a stdlib module name, check whether this is a frozen 

171 # module. Note that `find_spec` actually imports parent modules, so we want to make 

172 # sure we only run this code for stuff that can be expected to be frozen. For now 

173 # this is only stdlib. 

174 if (modname in sys.stdlib_module_names and not processed) or ( 

175 processed and processed[0] in sys.stdlib_module_names 

176 ): 

177 try: 

178 with warnings.catch_warnings(): 

179 warnings.filterwarnings("ignore", category=Warning) 

180 spec = importlib.util.find_spec(".".join((*processed, modname))) 

181 except ValueError: 

182 spec = None 

183 

184 if ( 

185 spec 

186 and spec.loader # type: ignore[comparison-overlap] # noqa: E501 

187 is importlib.machinery.FrozenImporter 

188 ): 

189 return ModuleSpec( # type: ignore[unreachable] 

190 name=modname, 

191 location=getattr(spec.loader_state, "filename", None), 

192 type=ModuleType.PY_FROZEN, 

193 ) 

194 

195 return None 

196 

197 def contribute_to_path( 

198 self, spec: ModuleSpec, processed: list[str] 

199 ) -> Sequence[str] | None: 

200 if spec.location is None: 

201 # Builtin. 

202 return None 

203 # pylint: disable-next=import-outside-toplevel 

204 from astroid.modutils import EXT_LIB_DIRS 

205 

206 if _is_setuptools_namespace(Path(spec.location)): 

207 # extend_path is called, search sys.path for module/packages 

208 # of this name see pkgutil.extend_path documentation 

209 path = [ 

210 os.path.join(p, *processed) 

211 for p in sys.path 

212 if os.path.isdir(os.path.join(p, *processed)) 

213 ] 

214 elif spec.name == "distutils" and not any( 

215 spec.location.lower().startswith(ext_lib_dir.lower()) 

216 for ext_lib_dir in EXT_LIB_DIRS 

217 ): 

218 # virtualenv below 20.0 patches distutils in an unexpected way 

219 # so we just find the location of distutils that will be 

220 # imported to avoid spurious import-error messages 

221 # https://github.com/pylint-dev/pylint/issues/5645 

222 # A regression test to create this scenario exists in release-tests.yml 

223 # and can be triggered manually from GitHub Actions 

224 distutils_spec = importlib.util.find_spec("distutils") 

225 if distutils_spec and distutils_spec.origin: 

226 origin_path = Path( 

227 distutils_spec.origin 

228 ) # e.g. .../distutils/__init__.py 

229 path = [str(origin_path.parent)] # e.g. .../distutils 

230 else: 

231 path = [spec.location] 

232 else: 

233 path = [spec.location] 

234 return path 

235 

236 

237class ExplicitNamespacePackageFinder(ImportlibFinder): 

238 """A finder for the explicit namespace packages.""" 

239 

240 @staticmethod 

241 @lru_cache(maxsize=1024) 

242 def find_module( 

243 modname: str, 

244 module_parts: tuple[str, ...], 

245 processed: tuple[str, ...], 

246 submodule_path: tuple[str, ...] | None, 

247 ) -> ModuleSpec | None: 

248 if processed: 

249 modname = ".".join([*processed, modname]) 

250 if util.is_namespace(modname) and modname in sys.modules: 

251 return ModuleSpec( 

252 name=modname, 

253 location="", 

254 origin="namespace", 

255 type=ModuleType.PY_NAMESPACE, 

256 submodule_search_locations=sys.modules[modname].__path__, 

257 ) 

258 return None 

259 

260 def contribute_to_path( 

261 self, spec: ModuleSpec, processed: list[str] 

262 ) -> Sequence[str] | None: 

263 return spec.submodule_search_locations 

264 

265 

266class ZipFinder(Finder): 

267 """Finder that knows how to find a module inside zip files.""" 

268 

269 def __init__(self, path: Sequence[str]) -> None: 

270 super().__init__(path) 

271 for entry_path in path: 

272 if entry_path not in sys.path_importer_cache: 

273 try: 

274 sys.path_importer_cache[entry_path] = zipimport.zipimporter( 

275 entry_path 

276 ) 

277 except zipimport.ZipImportError: 

278 continue 

279 

280 @staticmethod 

281 @lru_cache(maxsize=1024) 

282 def find_module( 

283 modname: str, 

284 module_parts: tuple[str, ...], 

285 processed: tuple[str, ...], 

286 submodule_path: tuple[str, ...] | None, 

287 ) -> ModuleSpec | None: 

288 try: 

289 file_type, filename, path = _search_zip(module_parts) 

290 except ImportError: 

291 return None 

292 

293 return ModuleSpec( 

294 name=modname, 

295 location=filename, 

296 origin="egg", 

297 type=file_type, 

298 submodule_search_locations=path, 

299 ) 

300 

301 def contribute_to_path( 

302 self, spec: ModuleSpec, processed: list[str] 

303 ) -> Sequence[str] | None: 

304 return spec.submodule_search_locations 

305 

306 

307class PathSpecFinder(Finder): 

308 """Finder based on importlib.machinery.PathFinder.""" 

309 

310 @staticmethod 

311 @lru_cache(maxsize=1024) 

312 def find_module( 

313 modname: str, 

314 module_parts: tuple[str, ...], 

315 processed: tuple[str, ...], 

316 submodule_path: tuple[str, ...] | None, 

317 ) -> ModuleSpec | None: 

318 spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path) 

319 if spec is not None: 

320 is_namespace_pkg = spec.origin is None 

321 location = spec.origin if not is_namespace_pkg else None 

322 module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None 

323 return ModuleSpec( 

324 name=spec.name, 

325 location=location, 

326 origin=spec.origin, 

327 type=module_type, 

328 submodule_search_locations=list(spec.submodule_search_locations or []), 

329 ) 

330 return spec 

331 

332 def contribute_to_path( 

333 self, spec: ModuleSpec, processed: list[str] 

334 ) -> Sequence[str] | None: 

335 if spec.type == ModuleType.PY_NAMESPACE: 

336 return spec.submodule_search_locations 

337 return None 

338 

339 

340_SPEC_FINDERS = ( 

341 ImportlibFinder, 

342 ZipFinder, 

343 PathSpecFinder, 

344 ExplicitNamespacePackageFinder, 

345) 

346 

347 

348@lru_cache(maxsize=1024) 

349def _is_setuptools_namespace(location: pathlib.Path) -> bool: 

350 try: 

351 with open(location / "__init__.py", "rb") as stream: 

352 data = stream.read(4096) 

353 except OSError: 

354 return False 

355 extend_path = b"pkgutil" in data and b"extend_path" in data 

356 declare_namespace = ( 

357 b"pkg_resources" in data and b"declare_namespace(__name__)" in data 

358 ) 

359 return extend_path or declare_namespace 

360 

361 

362def _get_zipimporters() -> Iterator[tuple[str, zipimport.zipimporter]]: 

363 for filepath, importer in sys.path_importer_cache.items(): 

364 if importer is not None and isinstance(importer, zipimport.zipimporter): 

365 yield filepath, importer 

366 

367 

368def _search_zip( 

369 modpath: tuple[str, ...], 

370) -> tuple[Literal[ModuleType.PY_ZIPMODULE], str, str]: 

371 for filepath, importer in _get_zipimporters(): 

372 found = importer.find_spec(modpath[0]) 

373 if found: 

374 if not importer.find_spec(os.path.sep.join(modpath)): 

375 raise ImportError( 

376 "No module named {} in {}/{}".format( 

377 ".".join(modpath[1:]), filepath, modpath 

378 ) 

379 ) 

380 return ( 

381 ModuleType.PY_ZIPMODULE, 

382 os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath), 

383 filepath, 

384 ) 

385 raise ImportError(f"No module named {'.'.join(modpath)}") 

386 

387 

388def _find_spec_with_path( 

389 search_path: Sequence[str], 

390 modname: str, 

391 module_parts: tuple[str, ...], 

392 processed: tuple[str, ...], 

393 submodule_path: tuple[str, ...] | None, 

394) -> tuple[Finder | _MetaPathFinder, ModuleSpec]: 

395 for finder in _SPEC_FINDERS: 

396 finder_instance = finder(search_path) 

397 mod_spec = finder.find_module(modname, module_parts, processed, submodule_path) 

398 if mod_spec is None: 

399 continue 

400 return finder_instance, mod_spec 

401 

402 # Support for custom finders 

403 for meta_finder in sys.meta_path: 

404 # See if we support the customer import hook of the meta_finder 

405 meta_finder_name = meta_finder.__class__.__name__ 

406 if meta_finder_name not in _MetaPathFinderModuleTypes: 

407 # Setuptools>62 creates its EditableFinders dynamically and have 

408 # "type" as their __class__.__name__. We check __name__ as well 

409 # to see if we can support the finder. 

410 try: 

411 meta_finder_name = meta_finder.__name__ # type: ignore[attr-defined] 

412 except AttributeError: 

413 continue 

414 if meta_finder_name not in _MetaPathFinderModuleTypes: 

415 continue 

416 

417 module_type = _MetaPathFinderModuleTypes[meta_finder_name] 

418 

419 # Meta path finders are supposed to have a find_spec method since 

420 # Python 3.4. However, some third-party finders do not implement it. 

421 # PEP302 does not refer to find_spec as well. 

422 # See: https://github.com/pylint-dev/astroid/pull/1752/ 

423 if not hasattr(meta_finder, "find_spec"): 

424 continue 

425 

426 spec = meta_finder.find_spec(modname, submodule_path) 

427 if spec: 

428 return ( 

429 meta_finder, 

430 ModuleSpec( 

431 spec.name, 

432 module_type, 

433 spec.origin, 

434 spec.origin, 

435 spec.submodule_search_locations, 

436 ), 

437 ) 

438 

439 raise ImportError(f"No module named {'.'.join(module_parts)}") 

440 

441 

442def find_spec(modpath: Iterable[str], path: Iterable[str] | None = None) -> ModuleSpec: 

443 """Find a spec for the given module. 

444 

445 :param modpath: 

446 split module's name (i.e name of a module or package split 

447 on '.'), with leading empty strings for explicit relative import 

448 

449 :param path: 

450 optional list of path where the module or package should be 

451 searched (use sys.path if nothing or None is given) 

452 

453 :return: A module spec, which describes how the module was 

454 found and where. 

455 """ 

456 return _find_spec(tuple(modpath), tuple(path) if path else None) 

457 

458 

459@lru_cache(maxsize=1024) 

460def _find_spec( 

461 module_path: tuple[str, ...], path: tuple[str, ...] | None 

462) -> ModuleSpec: 

463 _path = path or sys.path 

464 

465 # Need a copy for not mutating the argument. 

466 modpath = list(module_path) 

467 

468 search_paths = None 

469 processed: list[str] = [] 

470 

471 while modpath: 

472 modname = modpath.pop(0) 

473 

474 submodule_path = search_paths or path 

475 if submodule_path is not None: 

476 submodule_path = tuple(submodule_path) 

477 

478 finder, spec = _find_spec_with_path( 

479 _path, modname, module_path, tuple(processed), submodule_path 

480 ) 

481 processed.append(modname) 

482 if modpath: 

483 if isinstance(finder, Finder): 

484 search_paths = finder.contribute_to_path(spec, processed) 

485 # If modname is a package from an editable install, update search_paths 

486 # so that the next module in the path will be found inside of it using importlib. 

487 # Existence of __name__ is guaranteed by _find_spec_with_path. 

488 elif finder.__name__ in _EditableFinderClasses: # type: ignore[attr-defined] 

489 search_paths = spec.submodule_search_locations 

490 

491 if spec.type == ModuleType.PKG_DIRECTORY: 

492 spec = spec._replace(submodule_search_locations=search_paths) 

493 

494 return spec