Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/manager.py: 31%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

264 statements  

1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html 

2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE 

3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt 

4 

5"""astroid manager: avoid multiple astroid build of a same module when 

6possible by providing a class responsible to get astroid representation 

7from various source and using a cache of built modules) 

8""" 

9 

10from __future__ import annotations 

11 

12import collections 

13import os 

14import types 

15import zipimport 

16from collections.abc import Callable, Iterator, Sequence 

17from typing import Any, ClassVar 

18 

19from astroid import nodes 

20from astroid.builder import AstroidBuilder, build_namespace_package_module 

21from astroid.context import InferenceContext, _invalidate_cache 

22from astroid.exceptions import AstroidBuildingError, AstroidImportError 

23from astroid.interpreter._import import spec, util 

24from astroid.modutils import ( 

25 NoSourceFile, 

26 _cache_normalize_path_, 

27 _has_init, 

28 cached_os_path_isfile, 

29 file_info_from_modpath, 

30 get_source_file, 

31 is_module_name_part_of_extension_package_whitelist, 

32 is_python_source, 

33 is_stdlib_module, 

34 is_stdlib_path, 

35 load_module_from_name, 

36 modpath_from_file, 

37) 

38from astroid.transforms import TransformVisitor 

39from astroid.typing import AstroidManagerBrain, InferenceResult 

40 

41ZIP_IMPORT_EXTS = (".zip", ".egg", ".whl", ".pyz", ".pyzw") 

42 

43 

44def safe_repr(obj: Any) -> str: 

45 try: 

46 return repr(obj) 

47 except Exception: # pylint: disable=broad-except 

48 return "???" 

49 

50 

51class AstroidManager: 

52 """Responsible to build astroid from files or modules. 

53 

54 Use the Borg (singleton) pattern. 

55 """ 

56 

57 name = "astroid loader" 

58 brain: ClassVar[AstroidManagerBrain] = { 

59 "astroid_cache": {}, 

60 "_mod_file_cache": {}, 

61 "_failed_import_hooks": [], 

62 "always_load_extensions": False, 

63 "optimize_ast": False, 

64 "max_inferable_values": 100, 

65 "extension_package_whitelist": set(), 

66 "module_denylist": set(), 

67 "_transform": TransformVisitor(), 

68 "prefer_stubs": False, 

69 } 

70 

71 def __init__(self) -> None: 

72 # NOTE: cache entries are added by the [re]builder 

73 self.astroid_cache = AstroidManager.brain["astroid_cache"] 

74 self._mod_file_cache = AstroidManager.brain["_mod_file_cache"] 

75 self._failed_import_hooks = AstroidManager.brain["_failed_import_hooks"] 

76 self.extension_package_whitelist = AstroidManager.brain[ 

77 "extension_package_whitelist" 

78 ] 

79 self.module_denylist = AstroidManager.brain["module_denylist"] 

80 self._transform = AstroidManager.brain["_transform"] 

81 self.prefer_stubs = AstroidManager.brain["prefer_stubs"] 

82 

83 @property 

84 def always_load_extensions(self) -> bool: 

85 return AstroidManager.brain["always_load_extensions"] 

86 

87 @always_load_extensions.setter 

88 def always_load_extensions(self, value: bool) -> None: 

89 AstroidManager.brain["always_load_extensions"] = value 

90 

91 @property 

92 def optimize_ast(self) -> bool: 

93 return AstroidManager.brain["optimize_ast"] 

94 

95 @optimize_ast.setter 

96 def optimize_ast(self, value: bool) -> None: 

97 AstroidManager.brain["optimize_ast"] = value 

98 

99 @property 

100 def max_inferable_values(self) -> int: 

101 return AstroidManager.brain["max_inferable_values"] 

102 

103 @max_inferable_values.setter 

104 def max_inferable_values(self, value: int) -> None: 

105 AstroidManager.brain["max_inferable_values"] = value 

106 

107 @property 

108 def register_transform(self): 

109 # This and unregister_transform below are exported for convenience 

110 return self._transform.register_transform 

111 

112 @property 

113 def unregister_transform(self): 

114 return self._transform.unregister_transform 

115 

116 @property 

117 def builtins_module(self) -> nodes.Module: 

118 return self.astroid_cache["builtins"] 

119 

120 @property 

121 def prefer_stubs(self) -> bool: 

122 return AstroidManager.brain["prefer_stubs"] 

123 

124 @prefer_stubs.setter 

125 def prefer_stubs(self, value: bool) -> None: 

126 AstroidManager.brain["prefer_stubs"] = value 

127 

128 def visit_transforms(self, node: nodes.NodeNG) -> InferenceResult: 

129 """Visit the transforms and apply them to the given *node*.""" 

130 return self._transform.visit(node) 

131 

132 def ast_from_file( 

133 self, 

134 filepath: str, 

135 modname: str | None = None, 

136 fallback: bool = True, 

137 source: bool = False, 

138 ) -> nodes.Module: 

139 """Given a module name, return the astroid object.""" 

140 if modname is None: 

141 try: 

142 modname = ".".join(modpath_from_file(filepath)) 

143 except ImportError: 

144 modname = filepath 

145 if ( 

146 modname in self.astroid_cache 

147 and self.astroid_cache[modname].file == filepath 

148 ): 

149 return self.astroid_cache[modname] 

150 # Call get_source_file() only after a cache miss, 

151 # since it calls os.path.exists(). 

152 try: 

153 filepath = get_source_file( 

154 filepath, include_no_ext=True, prefer_stubs=self.prefer_stubs 

155 ) 

156 source = True 

157 except NoSourceFile: 

158 pass 

159 # Second attempt on the cache after get_source_file(). 

160 if ( 

161 modname in self.astroid_cache 

162 and self.astroid_cache[modname].file == filepath 

163 ): 

164 return self.astroid_cache[modname] 

165 if source: 

166 try: 

167 return AstroidBuilder(self).file_build(filepath, modname) 

168 except RecursionError as e: 

169 e.add_note( 

170 f"Astroid was unable to parse {filepath}.\n" 

171 "From pylint, try: --init-hook='import sys; sys.setrecursionlimit(2000)' or higher.", 

172 ) 

173 raise e 

174 if fallback and modname: 

175 return self.ast_from_module_name(modname) 

176 raise AstroidBuildingError("Unable to build an AST for {path}.", path=filepath) 

177 

178 def ast_from_string( 

179 self, data: str, modname: str = "", filepath: str | None = None 

180 ) -> nodes.Module: 

181 """Given some source code as a string, return its corresponding astroid 

182 object. 

183 """ 

184 return AstroidBuilder(self).string_build(data, modname, filepath) 

185 

186 def _build_stub_module(self, modname: str) -> nodes.Module: 

187 return AstroidBuilder(self).string_build("", modname) 

188 

189 def _build_namespace_module( 

190 self, modname: str, path: Sequence[str] 

191 ) -> nodes.Module: 

192 return build_namespace_package_module(modname, path) 

193 

194 def _can_load_extension(self, modname: str, location: str | None) -> bool: 

195 if self.always_load_extensions: 

196 return True 

197 # Only trust a stdlib name that resolves to the stdlib itself. 

198 if ( 

199 is_stdlib_module(modname) 

200 and location is not None 

201 and is_stdlib_path(location) 

202 ): 

203 return True 

204 return is_module_name_part_of_extension_package_whitelist( 

205 modname, self.extension_package_whitelist 

206 ) 

207 

208 def _ast_from_c_module( 

209 self, modname: str, found_spec: spec.ModuleSpec 

210 ) -> nodes.Module: 

211 if ( 

212 found_spec.type == spec.ModuleType.C_EXTENSION 

213 and not self._can_load_extension(modname, found_spec.location) 

214 ): 

215 if not self.prefer_stubs or found_spec.location is None: 

216 return self._build_stub_module(modname) 

217 try: 

218 source = get_source_file( 

219 found_spec.location, prefer_stubs=self.prefer_stubs 

220 ) 

221 except NoSourceFile: 

222 return self._build_stub_module(modname) 

223 return AstroidBuilder(self).file_build(source, modname) 

224 try: 

225 named_module = load_module_from_name(modname) 

226 except Exception as e: 

227 raise AstroidImportError( 

228 "Loading {modname} failed with:\n{error}", 

229 modname=modname, 

230 path=found_spec.location, 

231 ) from e 

232 return self.ast_from_module(named_module, modname) 

233 

234 def ast_from_module_name( # noqa: C901 

235 self, 

236 modname: str | None, 

237 context_file: str | None = None, 

238 use_cache: bool = True, 

239 ) -> nodes.Module: 

240 """Given a module name, return the astroid object.""" 

241 if modname is None: 

242 raise AstroidBuildingError("No module name given.") 

243 # Sometimes we don't want to use the cache. For example, when we're 

244 # importing a module with the same name as the file that is importing 

245 # we want to fallback on the import system to make sure we get the correct 

246 # module. 

247 if modname in self.module_denylist: 

248 raise AstroidImportError(f"Skipping ignored module {modname!r}") 

249 if modname in self.astroid_cache and use_cache: 

250 return self.astroid_cache[modname] 

251 if modname == "__main__": 

252 return self._build_stub_module(modname) 

253 if context_file: 

254 old_cwd = os.getcwd() 

255 os.chdir(os.path.dirname(context_file)) 

256 try: 

257 found_spec = self.file_from_module_name(modname, context_file) 

258 if found_spec.type == spec.ModuleType.PY_ZIPMODULE: 

259 module = self.zip_import_data(found_spec.location) 

260 if module is not None: 

261 return module 

262 

263 elif found_spec.type in ( 

264 spec.ModuleType.C_BUILTIN, 

265 spec.ModuleType.C_EXTENSION, 

266 ): 

267 return self._ast_from_c_module(modname, found_spec) 

268 

269 elif found_spec.type == spec.ModuleType.PY_COMPILED: 

270 raise AstroidImportError( 

271 "Unable to load compiled module {modname}.", 

272 modname=modname, 

273 path=found_spec.location, 

274 ) 

275 

276 elif found_spec.type == spec.ModuleType.PY_NAMESPACE: 

277 return self._build_namespace_module( 

278 modname, found_spec.submodule_search_locations or [] 

279 ) 

280 elif found_spec.type == spec.ModuleType.PY_FROZEN: 

281 if found_spec.location is None: 

282 return self._build_stub_module(modname) 

283 # For stdlib frozen modules we can determine the location and 

284 # can therefore create a module from the source file 

285 return self.ast_from_file(found_spec.location, modname, fallback=False) 

286 

287 if found_spec.location is None: 

288 raise AstroidImportError( 

289 "Can't find a file for module {modname}.", modname=modname 

290 ) 

291 

292 return self.ast_from_file(found_spec.location, modname, fallback=False) 

293 except AstroidBuildingError as e: 

294 for hook in self._failed_import_hooks: 

295 try: 

296 return hook(modname) 

297 except AstroidBuildingError: 

298 pass 

299 raise e 

300 finally: 

301 if context_file: 

302 os.chdir(old_cwd) 

303 

304 def zip_import_data(self, filepath: str) -> nodes.Module | None: 

305 if zipimport is None: 

306 return None 

307 

308 builder = AstroidBuilder(self) 

309 for ext in ZIP_IMPORT_EXTS: 

310 try: 

311 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1) 

312 except ValueError: 

313 continue 

314 try: 

315 importer = zipimport.zipimporter(eggpath + ext) 

316 zmodname = resource.replace(os.path.sep, ".") 

317 if importer.is_package(resource): 

318 zmodname = zmodname + ".__init__" 

319 module = builder.string_build( 

320 importer.get_source(resource), zmodname, filepath 

321 ) 

322 return module 

323 except Exception: # pylint: disable=broad-except 

324 continue 

325 return None 

326 

327 def file_from_module_name( 

328 self, modname: str, contextfile: str | None 

329 ) -> spec.ModuleSpec: 

330 cache_key = (modname, contextfile) 

331 try: 

332 cached = self._mod_file_cache[cache_key] 

333 except KeyError: 

334 try: 

335 module_spec = file_info_from_modpath( 

336 modname.split("."), context_file=contextfile 

337 ) 

338 except ImportError as e: 

339 error = AstroidImportError( 

340 "Failed to import module {modname} with error:\n{error}.", 

341 modname=modname, 

342 # we remove the traceback here to save on memory usage (since these exceptions are cached) 

343 error=e.with_traceback(None), 

344 ) 

345 self._mod_file_cache[cache_key] = error 

346 raise error from None 

347 self._mod_file_cache[cache_key] = module_spec 

348 return module_spec 

349 if isinstance(cached, AstroidImportError): 

350 # we remove the traceback here to save on memory usage (since these exceptions are cached) 

351 raise cached.with_traceback(None) 

352 return cached 

353 

354 def ast_from_module( 

355 self, module: types.ModuleType, modname: str | None = None 

356 ) -> nodes.Module: 

357 """Given an imported module, return the astroid object.""" 

358 modname = modname or module.__name__ 

359 if modname in self.astroid_cache: 

360 return self.astroid_cache[modname] 

361 try: 

362 # some builtin modules don't have __file__ attribute 

363 filepath = module.__file__ 

364 if is_python_source(filepath): 

365 # Type is checked in is_python_source 

366 return self.ast_from_file(filepath, modname) # type: ignore[arg-type] 

367 except AttributeError: 

368 pass 

369 

370 return AstroidBuilder(self).module_build(module, modname) 

371 

372 def ast_from_class(self, klass: type, modname: str | None = None) -> nodes.ClassDef: 

373 """Get astroid for the given class.""" 

374 if modname is None: 

375 try: 

376 modname = klass.__module__ 

377 except AttributeError as exc: 

378 raise AstroidBuildingError( 

379 "Unable to get module for class {class_name}.", 

380 cls=klass, 

381 class_repr=safe_repr(klass), 

382 modname=modname, 

383 ) from exc 

384 modastroid = self.ast_from_module_name(modname) 

385 ret = modastroid.getattr(klass.__name__)[0] 

386 assert isinstance(ret, nodes.ClassDef) 

387 return ret 

388 

389 def infer_ast_from_something( 

390 self, obj: object, context: InferenceContext | None = None 

391 ) -> Iterator[InferenceResult]: 

392 """Infer astroid for the given class.""" 

393 if hasattr(obj, "__class__") and not isinstance(obj, type): 

394 klass = obj.__class__ 

395 elif isinstance(obj, type): 

396 klass = obj 

397 else: 

398 raise AstroidBuildingError( # pragma: no cover 

399 "Unable to get type for {class_repr}.", 

400 cls=None, 

401 class_repr=safe_repr(obj), 

402 ) 

403 try: 

404 modname = klass.__module__ 

405 except AttributeError as exc: 

406 raise AstroidBuildingError( 

407 "Unable to get module for {class_repr}.", 

408 cls=klass, 

409 class_repr=safe_repr(klass), 

410 ) from exc 

411 except Exception as exc: 

412 raise AstroidImportError( 

413 "Unexpected error while retrieving module for {class_repr}:\n" 

414 "{error}", 

415 cls=klass, 

416 class_repr=safe_repr(klass), 

417 ) from exc 

418 try: 

419 name = klass.__name__ 

420 except AttributeError as exc: 

421 raise AstroidBuildingError( 

422 "Unable to get name for {class_repr}:\n", 

423 cls=klass, 

424 class_repr=safe_repr(klass), 

425 ) from exc 

426 except Exception as exc: 

427 raise AstroidImportError( 

428 "Unexpected error while retrieving name for {class_repr}:\n{error}", 

429 cls=klass, 

430 class_repr=safe_repr(klass), 

431 ) from exc 

432 # take care, on living object __module__ is regularly wrong :( 

433 modastroid = self.ast_from_module_name(modname) 

434 if klass is obj: 

435 yield from modastroid.igetattr(name, context) 

436 else: 

437 for inferred in modastroid.igetattr(name, context): 

438 yield inferred.instantiate_class() 

439 

440 def register_failed_import_hook(self, hook: Callable[[str], nodes.Module]) -> None: 

441 """Registers a hook to resolve imports that cannot be found otherwise. 

442 

443 `hook` must be a function that accepts a single argument `modname` which 

444 contains the name of the module or package that could not be imported. 

445 If `hook` can resolve the import, must return a node of type `nodes.Module`, 

446 otherwise, it must raise `AstroidBuildingError`. 

447 """ 

448 self._failed_import_hooks.append(hook) 

449 

450 def cache_module(self, module: nodes.Module) -> None: 

451 """Cache a module if no module with the same name is known yet.""" 

452 self.astroid_cache.setdefault(module.name, module) 

453 

454 def bootstrap(self) -> None: 

455 """Bootstrap the required AST modules needed for the manager to work. 

456 

457 The bootstrap usually involves building the AST for the builtins 

458 module, which is required by the rest of astroid to work correctly. 

459 """ 

460 from astroid import raw_building # pylint: disable=import-outside-toplevel 

461 

462 raw_building._astroid_bootstrapping() 

463 

464 def clear_cache(self) -> None: 

465 """Clear the underlying caches, bootstrap the builtins module and 

466 re-register transforms. 

467 """ 

468 # import here because of cyclic imports 

469 # pylint: disable=import-outside-toplevel 

470 from astroid.brain.helpers import register_all_brains 

471 from astroid.constraint import clear_preceding_guards_cache 

472 from astroid.inference_tip import clear_inference_tip_cache 

473 from astroid.interpreter._import.spec import ( 

474 _find_spec, 

475 _is_setuptools_namespace, 

476 ) 

477 from astroid.interpreter.objectmodel import ObjectModel 

478 from astroid.nodes._base_nodes import LookupMixIn 

479 from astroid.nodes.scoped_nodes import ClassDef 

480 

481 clear_inference_tip_cache() 

482 clear_preceding_guards_cache() 

483 _invalidate_cache() # inference context cache 

484 

485 self.astroid_cache.clear() 

486 self._mod_file_cache.clear() 

487 

488 # NB: not a new TransformVisitor() 

489 AstroidManager.brain["_transform"].transforms = collections.defaultdict(list) 

490 

491 for lru_cache in ( 

492 LookupMixIn.lookup, 

493 _cache_normalize_path_, 

494 _has_init, 

495 cached_os_path_isfile, 

496 util.is_namespace, 

497 ObjectModel.attributes, 

498 ClassDef._metaclass_lookup_attribute, 

499 _find_spec, 

500 _is_setuptools_namespace, 

501 ): 

502 lru_cache.cache_clear() # type: ignore[attr-defined] 

503 

504 for finder in spec._SPEC_FINDERS: 

505 finder.find_module.cache_clear() 

506 

507 self.bootstrap() 

508 

509 # Reload brain plugins. During initialisation this is done in astroid.manager.py 

510 register_all_brains(self)