Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/astroid/manager.py: 31%

229 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-07 06:53 +0000

1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html 

2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE 

3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt 

4 

5"""astroid manager: avoid multiple astroid build of a same module when 

6possible by providing a class responsible to get astroid representation 

7from various source and using a cache of built modules) 

8""" 

9 

10from __future__ import annotations 

11 

12import collections 

13import os 

14import types 

15import zipimport 

16from collections.abc import Callable, Iterator, Sequence 

17from importlib.util import find_spec, module_from_spec 

18from typing import Any, ClassVar 

19 

20from astroid import nodes 

21from astroid.const import BRAIN_MODULES_DIRECTORY 

22from astroid.context import InferenceContext, _invalidate_cache 

23from astroid.exceptions import AstroidBuildingError, AstroidImportError 

24from astroid.interpreter._import import spec, util 

25from astroid.modutils import ( 

26 NoSourceFile, 

27 _cache_normalize_path_, 

28 file_info_from_modpath, 

29 get_source_file, 

30 is_module_name_part_of_extension_package_whitelist, 

31 is_python_source, 

32 is_stdlib_module, 

33 load_module_from_name, 

34 modpath_from_file, 

35) 

36from astroid.transforms import TransformVisitor 

37from astroid.typing import AstroidManagerBrain, InferenceResult 

38 

39ZIP_IMPORT_EXTS = (".zip", ".egg", ".whl", ".pyz", ".pyzw") 

40 

41 

42def safe_repr(obj: Any) -> str: 

43 try: 

44 return repr(obj) 

45 except Exception: # pylint: disable=broad-except 

46 return "???" 

47 

48 

49class AstroidManager: 

50 """Responsible to build astroid from files or modules. 

51 

52 Use the Borg (singleton) pattern. 

53 """ 

54 

55 name = "astroid loader" 

56 brain: AstroidManagerBrain = { 

57 "astroid_cache": {}, 

58 "_mod_file_cache": {}, 

59 "_failed_import_hooks": [], 

60 "always_load_extensions": False, 

61 "optimize_ast": False, 

62 "extension_package_whitelist": set(), 

63 "_transform": TransformVisitor(), 

64 } 

65 max_inferable_values: ClassVar[int] = 100 

66 

67 def __init__(self) -> None: 

68 # NOTE: cache entries are added by the [re]builder 

69 self.astroid_cache = AstroidManager.brain["astroid_cache"] 

70 self._mod_file_cache = AstroidManager.brain["_mod_file_cache"] 

71 self._failed_import_hooks = AstroidManager.brain["_failed_import_hooks"] 

72 self.always_load_extensions = AstroidManager.brain["always_load_extensions"] 

73 self.optimize_ast = AstroidManager.brain["optimize_ast"] 

74 self.extension_package_whitelist = AstroidManager.brain[ 

75 "extension_package_whitelist" 

76 ] 

77 self._transform = AstroidManager.brain["_transform"] 

78 

79 @property 

80 def register_transform(self): 

81 # This and unregister_transform below are exported for convenience 

82 return self._transform.register_transform 

83 

84 @property 

85 def unregister_transform(self): 

86 return self._transform.unregister_transform 

87 

88 @property 

89 def builtins_module(self) -> nodes.Module: 

90 return self.astroid_cache["builtins"] 

91 

92 def visit_transforms(self, node: nodes.NodeNG) -> InferenceResult: 

93 """Visit the transforms and apply them to the given *node*.""" 

94 return self._transform.visit(node) 

95 

96 def ast_from_file( 

97 self, 

98 filepath: str, 

99 modname: str | None = None, 

100 fallback: bool = True, 

101 source: bool = False, 

102 ) -> nodes.Module: 

103 """Given a module name, return the astroid object.""" 

104 if modname is None: 

105 try: 

106 modname = ".".join(modpath_from_file(filepath)) 

107 except ImportError: 

108 modname = filepath 

109 if ( 

110 modname in self.astroid_cache 

111 and self.astroid_cache[modname].file == filepath 

112 ): 

113 return self.astroid_cache[modname] 

114 # Call get_source_file() only after a cache miss, 

115 # since it calls os.path.exists(). 

116 try: 

117 filepath = get_source_file(filepath, include_no_ext=True) 

118 source = True 

119 except NoSourceFile: 

120 pass 

121 # Second attempt on the cache after get_source_file(). 

122 if ( 

123 modname in self.astroid_cache 

124 and self.astroid_cache[modname].file == filepath 

125 ): 

126 return self.astroid_cache[modname] 

127 if source: 

128 # pylint: disable=import-outside-toplevel; circular import 

129 from astroid.builder import AstroidBuilder 

130 

131 return AstroidBuilder(self).file_build(filepath, modname) 

132 if fallback and modname: 

133 return self.ast_from_module_name(modname) 

134 raise AstroidBuildingError("Unable to build an AST for {path}.", path=filepath) 

135 

136 def ast_from_string( 

137 self, data: str, modname: str = "", filepath: str | None = None 

138 ) -> nodes.Module: 

139 """Given some source code as a string, return its corresponding astroid 

140 object. 

141 """ 

142 # pylint: disable=import-outside-toplevel; circular import 

143 from astroid.builder import AstroidBuilder 

144 

145 return AstroidBuilder(self).string_build(data, modname, filepath) 

146 

147 def _build_stub_module(self, modname: str) -> nodes.Module: 

148 # pylint: disable=import-outside-toplevel; circular import 

149 from astroid.builder import AstroidBuilder 

150 

151 return AstroidBuilder(self).string_build("", modname) 

152 

153 def _build_namespace_module( 

154 self, modname: str, path: Sequence[str] 

155 ) -> nodes.Module: 

156 # pylint: disable=import-outside-toplevel; circular import 

157 from astroid.builder import build_namespace_package_module 

158 

159 return build_namespace_package_module(modname, path) 

160 

161 def _can_load_extension(self, modname: str) -> bool: 

162 if self.always_load_extensions: 

163 return True 

164 if is_stdlib_module(modname): 

165 return True 

166 return is_module_name_part_of_extension_package_whitelist( 

167 modname, self.extension_package_whitelist 

168 ) 

169 

170 def ast_from_module_name( # noqa: C901 

171 self, 

172 modname: str | None, 

173 context_file: str | None = None, 

174 use_cache: bool = True, 

175 ) -> nodes.Module: 

176 """Given a module name, return the astroid object.""" 

177 if modname is None: 

178 raise AstroidBuildingError("No module name given.") 

179 # Sometimes we don't want to use the cache. For example, when we're 

180 # importing a module with the same name as the file that is importing 

181 # we want to fallback on the import system to make sure we get the correct 

182 # module. 

183 if modname in self.astroid_cache and use_cache: 

184 return self.astroid_cache[modname] 

185 if modname == "__main__": 

186 return self._build_stub_module(modname) 

187 if context_file: 

188 old_cwd = os.getcwd() 

189 os.chdir(os.path.dirname(context_file)) 

190 try: 

191 found_spec = self.file_from_module_name(modname, context_file) 

192 if found_spec.type == spec.ModuleType.PY_ZIPMODULE: 

193 module = self.zip_import_data(found_spec.location) 

194 if module is not None: 

195 return module 

196 

197 elif found_spec.type in ( 

198 spec.ModuleType.C_BUILTIN, 

199 spec.ModuleType.C_EXTENSION, 

200 ): 

201 if ( 

202 found_spec.type == spec.ModuleType.C_EXTENSION 

203 and not self._can_load_extension(modname) 

204 ): 

205 return self._build_stub_module(modname) 

206 try: 

207 named_module = load_module_from_name(modname) 

208 except Exception as e: 

209 raise AstroidImportError( 

210 "Loading {modname} failed with:\n{error}", 

211 modname=modname, 

212 path=found_spec.location, 

213 ) from e 

214 return self.ast_from_module(named_module, modname) 

215 

216 elif found_spec.type == spec.ModuleType.PY_COMPILED: 

217 raise AstroidImportError( 

218 "Unable to load compiled module {modname}.", 

219 modname=modname, 

220 path=found_spec.location, 

221 ) 

222 

223 elif found_spec.type == spec.ModuleType.PY_NAMESPACE: 

224 return self._build_namespace_module( 

225 modname, found_spec.submodule_search_locations or [] 

226 ) 

227 elif found_spec.type == spec.ModuleType.PY_FROZEN: 

228 if found_spec.location is None: 

229 return self._build_stub_module(modname) 

230 # For stdlib frozen modules we can determine the location and 

231 # can therefore create a module from the source file 

232 return self.ast_from_file(found_spec.location, modname, fallback=False) 

233 

234 if found_spec.location is None: 

235 raise AstroidImportError( 

236 "Can't find a file for module {modname}.", modname=modname 

237 ) 

238 

239 return self.ast_from_file(found_spec.location, modname, fallback=False) 

240 except AstroidBuildingError as e: 

241 for hook in self._failed_import_hooks: 

242 try: 

243 return hook(modname) 

244 except AstroidBuildingError: 

245 pass 

246 raise e 

247 finally: 

248 if context_file: 

249 os.chdir(old_cwd) 

250 

251 def zip_import_data(self, filepath: str) -> nodes.Module | None: 

252 if zipimport is None: 

253 return None 

254 

255 # pylint: disable=import-outside-toplevel; circular import 

256 from astroid.builder import AstroidBuilder 

257 

258 builder = AstroidBuilder(self) 

259 for ext in ZIP_IMPORT_EXTS: 

260 try: 

261 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1) 

262 except ValueError: 

263 continue 

264 try: 

265 importer = zipimport.zipimporter(eggpath + ext) 

266 zmodname = resource.replace(os.path.sep, ".") 

267 if importer.is_package(resource): 

268 zmodname = zmodname + ".__init__" 

269 module = builder.string_build( 

270 importer.get_source(resource), zmodname, filepath 

271 ) 

272 return module 

273 except Exception: # pylint: disable=broad-except 

274 continue 

275 return None 

276 

277 def file_from_module_name( 

278 self, modname: str, contextfile: str | None 

279 ) -> spec.ModuleSpec: 

280 try: 

281 value = self._mod_file_cache[(modname, contextfile)] 

282 except KeyError: 

283 try: 

284 value = file_info_from_modpath( 

285 modname.split("."), context_file=contextfile 

286 ) 

287 except ImportError as e: 

288 # pylint: disable-next=redefined-variable-type 

289 value = AstroidImportError( 

290 "Failed to import module {modname} with error:\n{error}.", 

291 modname=modname, 

292 # we remove the traceback here to save on memory usage (since these exceptions are cached) 

293 error=e.with_traceback(None), 

294 ) 

295 self._mod_file_cache[(modname, contextfile)] = value 

296 if isinstance(value, AstroidBuildingError): 

297 # we remove the traceback here to save on memory usage (since these exceptions are cached) 

298 raise value.with_traceback(None) # pylint: disable=no-member 

299 return value 

300 

301 def ast_from_module( 

302 self, module: types.ModuleType, modname: str | None = None 

303 ) -> nodes.Module: 

304 """Given an imported module, return the astroid object.""" 

305 modname = modname or module.__name__ 

306 if modname in self.astroid_cache: 

307 return self.astroid_cache[modname] 

308 try: 

309 # some builtin modules don't have __file__ attribute 

310 filepath = module.__file__ 

311 if is_python_source(filepath): 

312 # Type is checked in is_python_source 

313 return self.ast_from_file(filepath, modname) # type: ignore[arg-type] 

314 except AttributeError: 

315 pass 

316 

317 # pylint: disable=import-outside-toplevel; circular import 

318 from astroid.builder import AstroidBuilder 

319 

320 return AstroidBuilder(self).module_build(module, modname) 

321 

322 def ast_from_class(self, klass: type, modname: str | None = None) -> nodes.ClassDef: 

323 """Get astroid for the given class.""" 

324 if modname is None: 

325 try: 

326 modname = klass.__module__ 

327 except AttributeError as exc: 

328 raise AstroidBuildingError( 

329 "Unable to get module for class {class_name}.", 

330 cls=klass, 

331 class_repr=safe_repr(klass), 

332 modname=modname, 

333 ) from exc 

334 modastroid = self.ast_from_module_name(modname) 

335 ret = modastroid.getattr(klass.__name__)[0] 

336 assert isinstance(ret, nodes.ClassDef) 

337 return ret 

338 

339 def infer_ast_from_something( 

340 self, obj: object, context: InferenceContext | None = None 

341 ) -> Iterator[InferenceResult]: 

342 """Infer astroid for the given class.""" 

343 if hasattr(obj, "__class__") and not isinstance(obj, type): 

344 klass = obj.__class__ 

345 elif isinstance(obj, type): 

346 klass = obj 

347 else: 

348 raise AstroidBuildingError( # pragma: no cover 

349 "Unable to get type for {class_repr}.", 

350 cls=None, 

351 class_repr=safe_repr(obj), 

352 ) 

353 try: 

354 modname = klass.__module__ 

355 except AttributeError as exc: 

356 raise AstroidBuildingError( 

357 "Unable to get module for {class_repr}.", 

358 cls=klass, 

359 class_repr=safe_repr(klass), 

360 ) from exc 

361 except Exception as exc: 

362 raise AstroidImportError( 

363 "Unexpected error while retrieving module for {class_repr}:\n" 

364 "{error}", 

365 cls=klass, 

366 class_repr=safe_repr(klass), 

367 ) from exc 

368 try: 

369 name = klass.__name__ 

370 except AttributeError as exc: 

371 raise AstroidBuildingError( 

372 "Unable to get name for {class_repr}:\n", 

373 cls=klass, 

374 class_repr=safe_repr(klass), 

375 ) from exc 

376 except Exception as exc: 

377 raise AstroidImportError( 

378 "Unexpected error while retrieving name for {class_repr}:\n{error}", 

379 cls=klass, 

380 class_repr=safe_repr(klass), 

381 ) from exc 

382 # take care, on living object __module__ is regularly wrong :( 

383 modastroid = self.ast_from_module_name(modname) 

384 if klass is obj: 

385 for inferred in modastroid.igetattr(name, context): 

386 yield inferred 

387 else: 

388 for inferred in modastroid.igetattr(name, context): 

389 yield inferred.instantiate_class() 

390 

391 def register_failed_import_hook(self, hook: Callable[[str], nodes.Module]) -> None: 

392 """Registers a hook to resolve imports that cannot be found otherwise. 

393 

394 `hook` must be a function that accepts a single argument `modname` which 

395 contains the name of the module or package that could not be imported. 

396 If `hook` can resolve the import, must return a node of type `astroid.Module`, 

397 otherwise, it must raise `AstroidBuildingError`. 

398 """ 

399 self._failed_import_hooks.append(hook) 

400 

401 def cache_module(self, module: nodes.Module) -> None: 

402 """Cache a module if no module with the same name is known yet.""" 

403 self.astroid_cache.setdefault(module.name, module) 

404 

405 def bootstrap(self) -> None: 

406 """Bootstrap the required AST modules needed for the manager to work. 

407 

408 The bootstrap usually involves building the AST for the builtins 

409 module, which is required by the rest of astroid to work correctly. 

410 """ 

411 from astroid import raw_building # pylint: disable=import-outside-toplevel 

412 

413 raw_building._astroid_bootstrapping() 

414 

415 def clear_cache(self) -> None: 

416 """Clear the underlying caches, bootstrap the builtins module and 

417 re-register transforms. 

418 """ 

419 # import here because of cyclic imports 

420 # pylint: disable=import-outside-toplevel 

421 from astroid.inference_tip import clear_inference_tip_cache 

422 from astroid.interpreter.objectmodel import ObjectModel 

423 from astroid.nodes.node_classes import LookupMixIn 

424 from astroid.nodes.scoped_nodes import ClassDef 

425 

426 clear_inference_tip_cache() 

427 _invalidate_cache() # inference context cache 

428 

429 self.astroid_cache.clear() 

430 # NB: not a new TransformVisitor() 

431 AstroidManager.brain["_transform"].transforms = collections.defaultdict(list) 

432 

433 for lru_cache in ( 

434 LookupMixIn.lookup, 

435 _cache_normalize_path_, 

436 util.is_namespace, 

437 ObjectModel.attributes, 

438 ClassDef._metaclass_lookup_attribute, 

439 ): 

440 lru_cache.cache_clear() # type: ignore[attr-defined] 

441 

442 self.bootstrap() 

443 

444 # Reload brain plugins. During initialisation this is done in astroid.__init__.py 

445 for module in BRAIN_MODULES_DIRECTORY.iterdir(): 

446 if module.suffix == ".py": 

447 module_spec = find_spec(f"astroid.brain.{module.stem}") 

448 assert module_spec 

449 module_object = module_from_spec(module_spec) 

450 assert module_spec.loader 

451 module_spec.loader.exec_module(module_object)