Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/manager.py: 31%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""astroid manager: avoid multiple astroid build of a same module when
6possible by providing a class responsible to get astroid representation
7from various source and using a cache of built modules)
8"""
10from __future__ import annotations
12import collections
13import os
14import types
15import zipimport
16from collections.abc import Callable, Iterator, Sequence
17from typing import Any, ClassVar
19from astroid import nodes
20from astroid.builder import AstroidBuilder, build_namespace_package_module
21from astroid.context import InferenceContext, _invalidate_cache
22from astroid.exceptions import AstroidBuildingError, AstroidImportError
23from astroid.interpreter._import import spec, util
24from astroid.modutils import (
25 NoSourceFile,
26 _cache_normalize_path_,
27 _has_init,
28 cached_os_path_isfile,
29 file_info_from_modpath,
30 get_source_file,
31 is_module_name_part_of_extension_package_whitelist,
32 is_python_source,
33 is_stdlib_module,
34 is_stdlib_path,
35 load_module_from_name,
36 modpath_from_file,
37)
38from astroid.transforms import TransformVisitor
39from astroid.typing import AstroidManagerBrain, InferenceResult
41ZIP_IMPORT_EXTS = (".zip", ".egg", ".whl", ".pyz", ".pyzw")
44def safe_repr(obj: Any) -> str:
45 try:
46 return repr(obj)
47 except Exception: # pylint: disable=broad-except
48 return "???"
51class AstroidManager:
52 """Responsible to build astroid from files or modules.
54 Use the Borg (singleton) pattern.
55 """
57 name = "astroid loader"
58 brain: ClassVar[AstroidManagerBrain] = {
59 "astroid_cache": {},
60 "_mod_file_cache": {},
61 "_failed_import_hooks": [],
62 "always_load_extensions": False,
63 "optimize_ast": False,
64 "max_inferable_values": 100,
65 "extension_package_whitelist": set(),
66 "module_denylist": set(),
67 "_transform": TransformVisitor(),
68 "prefer_stubs": False,
69 }
71 def __init__(self) -> None:
72 # NOTE: cache entries are added by the [re]builder
73 self.astroid_cache = AstroidManager.brain["astroid_cache"]
74 self._mod_file_cache = AstroidManager.brain["_mod_file_cache"]
75 self._failed_import_hooks = AstroidManager.brain["_failed_import_hooks"]
76 self.extension_package_whitelist = AstroidManager.brain[
77 "extension_package_whitelist"
78 ]
79 self.module_denylist = AstroidManager.brain["module_denylist"]
80 self._transform = AstroidManager.brain["_transform"]
81 self.prefer_stubs = AstroidManager.brain["prefer_stubs"]
83 @property
84 def always_load_extensions(self) -> bool:
85 return AstroidManager.brain["always_load_extensions"]
87 @always_load_extensions.setter
88 def always_load_extensions(self, value: bool) -> None:
89 AstroidManager.brain["always_load_extensions"] = value
91 @property
92 def optimize_ast(self) -> bool:
93 return AstroidManager.brain["optimize_ast"]
95 @optimize_ast.setter
96 def optimize_ast(self, value: bool) -> None:
97 AstroidManager.brain["optimize_ast"] = value
99 @property
100 def max_inferable_values(self) -> int:
101 return AstroidManager.brain["max_inferable_values"]
103 @max_inferable_values.setter
104 def max_inferable_values(self, value: int) -> None:
105 AstroidManager.brain["max_inferable_values"] = value
107 @property
108 def register_transform(self):
109 # This and unregister_transform below are exported for convenience
110 return self._transform.register_transform
112 @property
113 def unregister_transform(self):
114 return self._transform.unregister_transform
116 @property
117 def builtins_module(self) -> nodes.Module:
118 return self.astroid_cache["builtins"]
120 @property
121 def prefer_stubs(self) -> bool:
122 return AstroidManager.brain["prefer_stubs"]
124 @prefer_stubs.setter
125 def prefer_stubs(self, value: bool) -> None:
126 AstroidManager.brain["prefer_stubs"] = value
128 def visit_transforms(self, node: nodes.NodeNG) -> InferenceResult:
129 """Visit the transforms and apply them to the given *node*."""
130 return self._transform.visit(node)
132 def ast_from_file(
133 self,
134 filepath: str,
135 modname: str | None = None,
136 fallback: bool = True,
137 source: bool = False,
138 ) -> nodes.Module:
139 """Given a module name, return the astroid object."""
140 if modname is None:
141 try:
142 modname = ".".join(modpath_from_file(filepath))
143 except ImportError:
144 modname = filepath
145 if (
146 modname in self.astroid_cache
147 and self.astroid_cache[modname].file == filepath
148 ):
149 return self.astroid_cache[modname]
150 # Call get_source_file() only after a cache miss,
151 # since it calls os.path.exists().
152 try:
153 filepath = get_source_file(
154 filepath, include_no_ext=True, prefer_stubs=self.prefer_stubs
155 )
156 source = True
157 except NoSourceFile:
158 pass
159 # Second attempt on the cache after get_source_file().
160 if (
161 modname in self.astroid_cache
162 and self.astroid_cache[modname].file == filepath
163 ):
164 return self.astroid_cache[modname]
165 if source:
166 try:
167 return AstroidBuilder(self).file_build(filepath, modname)
168 except RecursionError as e:
169 e.add_note(
170 f"Astroid was unable to parse {filepath}.\n"
171 "From pylint, try: --init-hook='import sys; sys.setrecursionlimit(2000)' or higher.",
172 )
173 raise e
174 if fallback and modname:
175 return self.ast_from_module_name(modname)
176 raise AstroidBuildingError("Unable to build an AST for {path}.", path=filepath)
178 def ast_from_string(
179 self, data: str, modname: str = "", filepath: str | None = None
180 ) -> nodes.Module:
181 """Given some source code as a string, return its corresponding astroid
182 object.
183 """
184 return AstroidBuilder(self).string_build(data, modname, filepath)
186 def _build_stub_module(self, modname: str) -> nodes.Module:
187 return AstroidBuilder(self).string_build("", modname)
189 def _build_namespace_module(
190 self, modname: str, path: Sequence[str]
191 ) -> nodes.Module:
192 return build_namespace_package_module(modname, path)
194 def _can_load_extension(self, modname: str, location: str | None) -> bool:
195 if self.always_load_extensions:
196 return True
197 # Only trust a stdlib name that resolves to the stdlib itself.
198 if (
199 is_stdlib_module(modname)
200 and location is not None
201 and is_stdlib_path(location)
202 ):
203 return True
204 return is_module_name_part_of_extension_package_whitelist(
205 modname, self.extension_package_whitelist
206 )
208 def _ast_from_c_module(
209 self, modname: str, found_spec: spec.ModuleSpec
210 ) -> nodes.Module:
211 if (
212 found_spec.type == spec.ModuleType.C_EXTENSION
213 and not self._can_load_extension(modname, found_spec.location)
214 ):
215 if not self.prefer_stubs or found_spec.location is None:
216 return self._build_stub_module(modname)
217 try:
218 source = get_source_file(
219 found_spec.location, prefer_stubs=self.prefer_stubs
220 )
221 except NoSourceFile:
222 return self._build_stub_module(modname)
223 return AstroidBuilder(self).file_build(source, modname)
224 try:
225 named_module = load_module_from_name(modname)
226 except Exception as e:
227 raise AstroidImportError(
228 "Loading {modname} failed with:\n{error}",
229 modname=modname,
230 path=found_spec.location,
231 ) from e
232 return self.ast_from_module(named_module, modname)
234 def ast_from_module_name( # noqa: C901
235 self,
236 modname: str | None,
237 context_file: str | None = None,
238 use_cache: bool = True,
239 ) -> nodes.Module:
240 """Given a module name, return the astroid object."""
241 if modname is None:
242 raise AstroidBuildingError("No module name given.")
243 # Sometimes we don't want to use the cache. For example, when we're
244 # importing a module with the same name as the file that is importing
245 # we want to fallback on the import system to make sure we get the correct
246 # module.
247 if modname in self.module_denylist:
248 raise AstroidImportError(f"Skipping ignored module {modname!r}")
249 if modname in self.astroid_cache and use_cache:
250 return self.astroid_cache[modname]
251 if modname == "__main__":
252 return self._build_stub_module(modname)
253 if context_file:
254 old_cwd = os.getcwd()
255 os.chdir(os.path.dirname(context_file))
256 try:
257 found_spec = self.file_from_module_name(modname, context_file)
258 if found_spec.type == spec.ModuleType.PY_ZIPMODULE:
259 module = self.zip_import_data(found_spec.location)
260 if module is not None:
261 return module
263 elif found_spec.type in (
264 spec.ModuleType.C_BUILTIN,
265 spec.ModuleType.C_EXTENSION,
266 ):
267 return self._ast_from_c_module(modname, found_spec)
269 elif found_spec.type == spec.ModuleType.PY_COMPILED:
270 raise AstroidImportError(
271 "Unable to load compiled module {modname}.",
272 modname=modname,
273 path=found_spec.location,
274 )
276 elif found_spec.type == spec.ModuleType.PY_NAMESPACE:
277 return self._build_namespace_module(
278 modname, found_spec.submodule_search_locations or []
279 )
280 elif found_spec.type == spec.ModuleType.PY_FROZEN:
281 if found_spec.location is None:
282 return self._build_stub_module(modname)
283 # For stdlib frozen modules we can determine the location and
284 # can therefore create a module from the source file
285 return self.ast_from_file(found_spec.location, modname, fallback=False)
287 if found_spec.location is None:
288 raise AstroidImportError(
289 "Can't find a file for module {modname}.", modname=modname
290 )
292 return self.ast_from_file(found_spec.location, modname, fallback=False)
293 except AstroidBuildingError as e:
294 for hook in self._failed_import_hooks:
295 try:
296 return hook(modname)
297 except AstroidBuildingError:
298 pass
299 raise e
300 finally:
301 if context_file:
302 os.chdir(old_cwd)
304 def zip_import_data(self, filepath: str) -> nodes.Module | None:
305 if zipimport is None:
306 return None
308 builder = AstroidBuilder(self)
309 for ext in ZIP_IMPORT_EXTS:
310 try:
311 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
312 except ValueError:
313 continue
314 try:
315 importer = zipimport.zipimporter(eggpath + ext)
316 zmodname = resource.replace(os.path.sep, ".")
317 if importer.is_package(resource):
318 zmodname = zmodname + ".__init__"
319 module = builder.string_build(
320 importer.get_source(resource), zmodname, filepath
321 )
322 return module
323 except Exception: # pylint: disable=broad-except
324 continue
325 return None
327 def file_from_module_name(
328 self, modname: str, contextfile: str | None
329 ) -> spec.ModuleSpec:
330 cache_key = (modname, contextfile)
331 try:
332 cached = self._mod_file_cache[cache_key]
333 except KeyError:
334 try:
335 module_spec = file_info_from_modpath(
336 modname.split("."), context_file=contextfile
337 )
338 except ImportError as e:
339 error = AstroidImportError(
340 "Failed to import module {modname} with error:\n{error}.",
341 modname=modname,
342 # we remove the traceback here to save on memory usage (since these exceptions are cached)
343 error=e.with_traceback(None),
344 )
345 self._mod_file_cache[cache_key] = error
346 raise error from None
347 self._mod_file_cache[cache_key] = module_spec
348 return module_spec
349 if isinstance(cached, AstroidImportError):
350 # we remove the traceback here to save on memory usage (since these exceptions are cached)
351 raise cached.with_traceback(None)
352 return cached
354 def ast_from_module(
355 self, module: types.ModuleType, modname: str | None = None
356 ) -> nodes.Module:
357 """Given an imported module, return the astroid object."""
358 modname = modname or module.__name__
359 if modname in self.astroid_cache:
360 return self.astroid_cache[modname]
361 try:
362 # some builtin modules don't have __file__ attribute
363 filepath = module.__file__
364 if is_python_source(filepath):
365 # Type is checked in is_python_source
366 return self.ast_from_file(filepath, modname) # type: ignore[arg-type]
367 except AttributeError:
368 pass
370 return AstroidBuilder(self).module_build(module, modname)
372 def ast_from_class(self, klass: type, modname: str | None = None) -> nodes.ClassDef:
373 """Get astroid for the given class."""
374 if modname is None:
375 try:
376 modname = klass.__module__
377 except AttributeError as exc:
378 raise AstroidBuildingError(
379 "Unable to get module for class {class_name}.",
380 cls=klass,
381 class_repr=safe_repr(klass),
382 modname=modname,
383 ) from exc
384 modastroid = self.ast_from_module_name(modname)
385 ret = modastroid.getattr(klass.__name__)[0]
386 assert isinstance(ret, nodes.ClassDef)
387 return ret
389 def infer_ast_from_something(
390 self, obj: object, context: InferenceContext | None = None
391 ) -> Iterator[InferenceResult]:
392 """Infer astroid for the given class."""
393 if hasattr(obj, "__class__") and not isinstance(obj, type):
394 klass = obj.__class__
395 elif isinstance(obj, type):
396 klass = obj
397 else:
398 raise AstroidBuildingError( # pragma: no cover
399 "Unable to get type for {class_repr}.",
400 cls=None,
401 class_repr=safe_repr(obj),
402 )
403 try:
404 modname = klass.__module__
405 except AttributeError as exc:
406 raise AstroidBuildingError(
407 "Unable to get module for {class_repr}.",
408 cls=klass,
409 class_repr=safe_repr(klass),
410 ) from exc
411 except Exception as exc:
412 raise AstroidImportError(
413 "Unexpected error while retrieving module for {class_repr}:\n"
414 "{error}",
415 cls=klass,
416 class_repr=safe_repr(klass),
417 ) from exc
418 try:
419 name = klass.__name__
420 except AttributeError as exc:
421 raise AstroidBuildingError(
422 "Unable to get name for {class_repr}:\n",
423 cls=klass,
424 class_repr=safe_repr(klass),
425 ) from exc
426 except Exception as exc:
427 raise AstroidImportError(
428 "Unexpected error while retrieving name for {class_repr}:\n{error}",
429 cls=klass,
430 class_repr=safe_repr(klass),
431 ) from exc
432 # take care, on living object __module__ is regularly wrong :(
433 modastroid = self.ast_from_module_name(modname)
434 if klass is obj:
435 yield from modastroid.igetattr(name, context)
436 else:
437 for inferred in modastroid.igetattr(name, context):
438 yield inferred.instantiate_class()
440 def register_failed_import_hook(self, hook: Callable[[str], nodes.Module]) -> None:
441 """Registers a hook to resolve imports that cannot be found otherwise.
443 `hook` must be a function that accepts a single argument `modname` which
444 contains the name of the module or package that could not be imported.
445 If `hook` can resolve the import, must return a node of type `nodes.Module`,
446 otherwise, it must raise `AstroidBuildingError`.
447 """
448 self._failed_import_hooks.append(hook)
450 def cache_module(self, module: nodes.Module) -> None:
451 """Cache a module if no module with the same name is known yet."""
452 self.astroid_cache.setdefault(module.name, module)
454 def bootstrap(self) -> None:
455 """Bootstrap the required AST modules needed for the manager to work.
457 The bootstrap usually involves building the AST for the builtins
458 module, which is required by the rest of astroid to work correctly.
459 """
460 from astroid import raw_building # pylint: disable=import-outside-toplevel
462 raw_building._astroid_bootstrapping()
464 def clear_cache(self) -> None:
465 """Clear the underlying caches, bootstrap the builtins module and
466 re-register transforms.
467 """
468 # import here because of cyclic imports
469 # pylint: disable=import-outside-toplevel
470 from astroid.brain.helpers import register_all_brains
471 from astroid.constraint import clear_preceding_guards_cache
472 from astroid.inference_tip import clear_inference_tip_cache
473 from astroid.interpreter._import.spec import (
474 _find_spec,
475 _is_setuptools_namespace,
476 )
477 from astroid.interpreter.objectmodel import ObjectModel
478 from astroid.nodes._base_nodes import LookupMixIn
479 from astroid.nodes.scoped_nodes import ClassDef
481 clear_inference_tip_cache()
482 clear_preceding_guards_cache()
483 _invalidate_cache() # inference context cache
485 self.astroid_cache.clear()
486 self._mod_file_cache.clear()
488 # NB: not a new TransformVisitor()
489 AstroidManager.brain["_transform"].transforms = collections.defaultdict(list)
491 for lru_cache in (
492 LookupMixIn.lookup,
493 _cache_normalize_path_,
494 _has_init,
495 cached_os_path_isfile,
496 util.is_namespace,
497 ObjectModel.attributes,
498 ClassDef._metaclass_lookup_attribute,
499 _find_spec,
500 _is_setuptools_namespace,
501 ):
502 lru_cache.cache_clear() # type: ignore[attr-defined]
504 for finder in spec._SPEC_FINDERS:
505 finder.find_module.cache_clear()
507 self.bootstrap()
509 # Reload brain plugins. During initialisation this is done in astroid.manager.py
510 register_all_brains(self)