Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/manager.py: 24%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""astroid manager: avoid multiple astroid build of a same module when
6possible by providing a class responsible to get astroid representation
7from various source and using a cache of built modules)
8"""
10from __future__ import annotations
12import collections
13import os
14import types
15import zipimport
16from collections.abc import Callable, Iterator, Sequence
17from typing import Any, ClassVar
19from astroid import nodes
20from astroid.builder import AstroidBuilder, build_namespace_package_module
21from astroid.context import InferenceContext, _invalidate_cache
22from astroid.exceptions import AstroidBuildingError, AstroidImportError
23from astroid.interpreter._import import spec, util
24from astroid.modutils import (
25 NoSourceFile,
26 _cache_normalize_path_,
27 _has_init,
28 cached_os_path_isfile,
29 file_info_from_modpath,
30 get_source_file,
31 is_module_name_part_of_extension_package_whitelist,
32 is_python_source,
33 is_stdlib_module,
34 load_module_from_name,
35 modpath_from_file,
36)
37from astroid.transforms import TransformVisitor
38from astroid.typing import AstroidManagerBrain, InferenceResult
40ZIP_IMPORT_EXTS = (".zip", ".egg", ".whl", ".pyz", ".pyzw")
43def safe_repr(obj: Any) -> str:
44 try:
45 return repr(obj)
46 except Exception: # pylint: disable=broad-except
47 return "???"
50class AstroidManager:
51 """Responsible to build astroid from files or modules.
53 Use the Borg (singleton) pattern.
54 """
56 name = "astroid loader"
57 brain: ClassVar[AstroidManagerBrain] = {
58 "astroid_cache": {},
59 "_mod_file_cache": {},
60 "_failed_import_hooks": [],
61 "always_load_extensions": False,
62 "optimize_ast": False,
63 "max_inferable_values": 100,
64 "extension_package_whitelist": set(),
65 "module_denylist": set(),
66 "_transform": TransformVisitor(),
67 "prefer_stubs": False,
68 }
70 def __init__(self) -> None:
71 # NOTE: cache entries are added by the [re]builder
72 self.astroid_cache = AstroidManager.brain["astroid_cache"]
73 self._mod_file_cache = AstroidManager.brain["_mod_file_cache"]
74 self._failed_import_hooks = AstroidManager.brain["_failed_import_hooks"]
75 self.extension_package_whitelist = AstroidManager.brain[
76 "extension_package_whitelist"
77 ]
78 self.module_denylist = AstroidManager.brain["module_denylist"]
79 self._transform = AstroidManager.brain["_transform"]
80 self.prefer_stubs = AstroidManager.brain["prefer_stubs"]
82 @property
83 def always_load_extensions(self) -> bool:
84 return AstroidManager.brain["always_load_extensions"]
86 @always_load_extensions.setter
87 def always_load_extensions(self, value: bool) -> None:
88 AstroidManager.brain["always_load_extensions"] = value
90 @property
91 def optimize_ast(self) -> bool:
92 return AstroidManager.brain["optimize_ast"]
94 @optimize_ast.setter
95 def optimize_ast(self, value: bool) -> None:
96 AstroidManager.brain["optimize_ast"] = value
98 @property
99 def max_inferable_values(self) -> int:
100 return AstroidManager.brain["max_inferable_values"]
102 @max_inferable_values.setter
103 def max_inferable_values(self, value: int) -> None:
104 AstroidManager.brain["max_inferable_values"] = value
106 @property
107 def register_transform(self):
108 # This and unregister_transform below are exported for convenience
109 return self._transform.register_transform
111 @property
112 def unregister_transform(self):
113 return self._transform.unregister_transform
115 @property
116 def builtins_module(self) -> nodes.Module:
117 return self.astroid_cache["builtins"]
119 @property
120 def prefer_stubs(self) -> bool:
121 return AstroidManager.brain["prefer_stubs"]
123 @prefer_stubs.setter
124 def prefer_stubs(self, value: bool) -> None:
125 AstroidManager.brain["prefer_stubs"] = value
127 def visit_transforms(self, node: nodes.NodeNG) -> InferenceResult:
128 """Visit the transforms and apply them to the given *node*."""
129 return self._transform.visit(node)
131 def ast_from_file(
132 self,
133 filepath: str,
134 modname: str | None = None,
135 fallback: bool = True,
136 source: bool = False,
137 ) -> nodes.Module:
138 """Given a module name, return the astroid object."""
139 if modname is None:
140 try:
141 modname = ".".join(modpath_from_file(filepath))
142 except ImportError:
143 modname = filepath
144 if (
145 modname in self.astroid_cache
146 and self.astroid_cache[modname].file == filepath
147 ):
148 return self.astroid_cache[modname]
149 # Call get_source_file() only after a cache miss,
150 # since it calls os.path.exists().
151 try:
152 filepath = get_source_file(
153 filepath, include_no_ext=True, prefer_stubs=self.prefer_stubs
154 )
155 source = True
156 except NoSourceFile:
157 pass
158 # Second attempt on the cache after get_source_file().
159 if (
160 modname in self.astroid_cache
161 and self.astroid_cache[modname].file == filepath
162 ):
163 return self.astroid_cache[modname]
164 if source:
165 return AstroidBuilder(self).file_build(filepath, modname)
166 if fallback and modname:
167 return self.ast_from_module_name(modname)
168 raise AstroidBuildingError("Unable to build an AST for {path}.", path=filepath)
170 def ast_from_string(
171 self, data: str, modname: str = "", filepath: str | None = None
172 ) -> nodes.Module:
173 """Given some source code as a string, return its corresponding astroid
174 object.
175 """
176 return AstroidBuilder(self).string_build(data, modname, filepath)
178 def _build_stub_module(self, modname: str) -> nodes.Module:
179 return AstroidBuilder(self).string_build("", modname)
181 def _build_namespace_module(
182 self, modname: str, path: Sequence[str]
183 ) -> nodes.Module:
184 return build_namespace_package_module(modname, path)
186 def _can_load_extension(self, modname: str) -> bool:
187 if self.always_load_extensions:
188 return True
189 if is_stdlib_module(modname):
190 return True
191 return is_module_name_part_of_extension_package_whitelist(
192 modname, self.extension_package_whitelist
193 )
195 def ast_from_module_name( # noqa: C901
196 self,
197 modname: str | None,
198 context_file: str | None = None,
199 use_cache: bool = True,
200 ) -> nodes.Module:
201 """Given a module name, return the astroid object."""
202 if modname is None:
203 raise AstroidBuildingError("No module name given.")
204 # Sometimes we don't want to use the cache. For example, when we're
205 # importing a module with the same name as the file that is importing
206 # we want to fallback on the import system to make sure we get the correct
207 # module.
208 if modname in self.module_denylist:
209 raise AstroidImportError(f"Skipping ignored module {modname!r}")
210 if modname in self.astroid_cache and use_cache:
211 return self.astroid_cache[modname]
212 if modname == "__main__":
213 return self._build_stub_module(modname)
214 if context_file:
215 old_cwd = os.getcwd()
216 os.chdir(os.path.dirname(context_file))
217 try:
218 found_spec = self.file_from_module_name(modname, context_file)
219 if found_spec.type == spec.ModuleType.PY_ZIPMODULE:
220 module = self.zip_import_data(found_spec.location)
221 if module is not None:
222 return module
224 elif found_spec.type in (
225 spec.ModuleType.C_BUILTIN,
226 spec.ModuleType.C_EXTENSION,
227 ):
228 if (
229 found_spec.type == spec.ModuleType.C_EXTENSION
230 and not self._can_load_extension(modname)
231 ):
232 return self._build_stub_module(modname)
233 try:
234 named_module = load_module_from_name(modname)
235 except Exception as e:
236 raise AstroidImportError(
237 "Loading {modname} failed with:\n{error}",
238 modname=modname,
239 path=found_spec.location,
240 ) from e
241 return self.ast_from_module(named_module, modname)
243 elif found_spec.type == spec.ModuleType.PY_COMPILED:
244 raise AstroidImportError(
245 "Unable to load compiled module {modname}.",
246 modname=modname,
247 path=found_spec.location,
248 )
250 elif found_spec.type == spec.ModuleType.PY_NAMESPACE:
251 return self._build_namespace_module(
252 modname, found_spec.submodule_search_locations or []
253 )
254 elif found_spec.type == spec.ModuleType.PY_FROZEN:
255 if found_spec.location is None:
256 return self._build_stub_module(modname)
257 # For stdlib frozen modules we can determine the location and
258 # can therefore create a module from the source file
259 return self.ast_from_file(found_spec.location, modname, fallback=False)
261 if found_spec.location is None:
262 raise AstroidImportError(
263 "Can't find a file for module {modname}.", modname=modname
264 )
266 return self.ast_from_file(found_spec.location, modname, fallback=False)
267 except AstroidBuildingError as e:
268 for hook in self._failed_import_hooks:
269 try:
270 return hook(modname)
271 except AstroidBuildingError:
272 pass
273 raise e
274 finally:
275 if context_file:
276 os.chdir(old_cwd)
278 def zip_import_data(self, filepath: str) -> nodes.Module | None:
279 if zipimport is None:
280 return None
282 builder = AstroidBuilder(self)
283 for ext in ZIP_IMPORT_EXTS:
284 try:
285 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
286 except ValueError:
287 continue
288 try:
289 importer = zipimport.zipimporter(eggpath + ext)
290 zmodname = resource.replace(os.path.sep, ".")
291 if importer.is_package(resource):
292 zmodname = zmodname + ".__init__"
293 module = builder.string_build(
294 importer.get_source(resource), zmodname, filepath
295 )
296 return module
297 except Exception: # pylint: disable=broad-except
298 continue
299 return None
301 def file_from_module_name(
302 self, modname: str, contextfile: str | None
303 ) -> spec.ModuleSpec:
304 try:
305 value = self._mod_file_cache[(modname, contextfile)]
306 except KeyError:
307 try:
308 value = file_info_from_modpath(
309 modname.split("."), context_file=contextfile
310 )
311 except ImportError as e:
312 value = AstroidImportError(
313 "Failed to import module {modname} with error:\n{error}.",
314 modname=modname,
315 # we remove the traceback here to save on memory usage (since these exceptions are cached)
316 error=e.with_traceback(None),
317 )
318 self._mod_file_cache[(modname, contextfile)] = value
319 if isinstance(value, AstroidBuildingError):
320 # we remove the traceback here to save on memory usage (since these exceptions are cached)
321 raise value.with_traceback(None) # pylint: disable=no-member
322 return value
324 def ast_from_module(
325 self, module: types.ModuleType, modname: str | None = None
326 ) -> nodes.Module:
327 """Given an imported module, return the astroid object."""
328 modname = modname or module.__name__
329 if modname in self.astroid_cache:
330 return self.astroid_cache[modname]
331 try:
332 # some builtin modules don't have __file__ attribute
333 filepath = module.__file__
334 if is_python_source(filepath):
335 # Type is checked in is_python_source
336 return self.ast_from_file(filepath, modname) # type: ignore[arg-type]
337 except AttributeError:
338 pass
340 return AstroidBuilder(self).module_build(module, modname)
342 def ast_from_class(self, klass: type, modname: str | None = None) -> nodes.ClassDef:
343 """Get astroid for the given class."""
344 if modname is None:
345 try:
346 modname = klass.__module__
347 except AttributeError as exc:
348 raise AstroidBuildingError(
349 "Unable to get module for class {class_name}.",
350 cls=klass,
351 class_repr=safe_repr(klass),
352 modname=modname,
353 ) from exc
354 modastroid = self.ast_from_module_name(modname)
355 ret = modastroid.getattr(klass.__name__)[0]
356 assert isinstance(ret, nodes.ClassDef)
357 return ret
359 def infer_ast_from_something(
360 self, obj: object, context: InferenceContext | None = None
361 ) -> Iterator[InferenceResult]:
362 """Infer astroid for the given class."""
363 if hasattr(obj, "__class__") and not isinstance(obj, type):
364 klass = obj.__class__
365 elif isinstance(obj, type):
366 klass = obj
367 else:
368 raise AstroidBuildingError( # pragma: no cover
369 "Unable to get type for {class_repr}.",
370 cls=None,
371 class_repr=safe_repr(obj),
372 )
373 try:
374 modname = klass.__module__
375 except AttributeError as exc:
376 raise AstroidBuildingError(
377 "Unable to get module for {class_repr}.",
378 cls=klass,
379 class_repr=safe_repr(klass),
380 ) from exc
381 except Exception as exc:
382 raise AstroidImportError(
383 "Unexpected error while retrieving module for {class_repr}:\n"
384 "{error}",
385 cls=klass,
386 class_repr=safe_repr(klass),
387 ) from exc
388 try:
389 name = klass.__name__
390 except AttributeError as exc:
391 raise AstroidBuildingError(
392 "Unable to get name for {class_repr}:\n",
393 cls=klass,
394 class_repr=safe_repr(klass),
395 ) from exc
396 except Exception as exc:
397 raise AstroidImportError(
398 "Unexpected error while retrieving name for {class_repr}:\n{error}",
399 cls=klass,
400 class_repr=safe_repr(klass),
401 ) from exc
402 # take care, on living object __module__ is regularly wrong :(
403 modastroid = self.ast_from_module_name(modname)
404 if klass is obj:
405 yield from modastroid.igetattr(name, context)
406 else:
407 for inferred in modastroid.igetattr(name, context):
408 yield inferred.instantiate_class()
410 def register_failed_import_hook(self, hook: Callable[[str], nodes.Module]) -> None:
411 """Registers a hook to resolve imports that cannot be found otherwise.
413 `hook` must be a function that accepts a single argument `modname` which
414 contains the name of the module or package that could not be imported.
415 If `hook` can resolve the import, must return a node of type `nodes.Module`,
416 otherwise, it must raise `AstroidBuildingError`.
417 """
418 self._failed_import_hooks.append(hook)
420 def cache_module(self, module: nodes.Module) -> None:
421 """Cache a module if no module with the same name is known yet."""
422 self.astroid_cache.setdefault(module.name, module)
424 def bootstrap(self) -> None:
425 """Bootstrap the required AST modules needed for the manager to work.
427 The bootstrap usually involves building the AST for the builtins
428 module, which is required by the rest of astroid to work correctly.
429 """
430 from astroid import raw_building # pylint: disable=import-outside-toplevel
432 raw_building._astroid_bootstrapping()
434 def clear_cache(self) -> None:
435 """Clear the underlying caches, bootstrap the builtins module and
436 re-register transforms.
437 """
438 # import here because of cyclic imports
439 # pylint: disable=import-outside-toplevel
440 from astroid.brain.helpers import register_all_brains
441 from astroid.inference_tip import clear_inference_tip_cache
442 from astroid.interpreter._import.spec import (
443 _find_spec,
444 _is_setuptools_namespace,
445 )
446 from astroid.interpreter.objectmodel import ObjectModel
447 from astroid.nodes._base_nodes import LookupMixIn
448 from astroid.nodes.scoped_nodes import ClassDef
450 clear_inference_tip_cache()
451 _invalidate_cache() # inference context cache
453 self.astroid_cache.clear()
454 self._mod_file_cache.clear()
456 # NB: not a new TransformVisitor()
457 AstroidManager.brain["_transform"].transforms = collections.defaultdict(list)
459 for lru_cache in (
460 LookupMixIn.lookup,
461 _cache_normalize_path_,
462 _has_init,
463 cached_os_path_isfile,
464 util.is_namespace,
465 ObjectModel.attributes,
466 ClassDef._metaclass_lookup_attribute,
467 _find_spec,
468 _is_setuptools_namespace,
469 ):
470 lru_cache.cache_clear() # type: ignore[attr-defined]
472 for finder in spec._SPEC_FINDERS:
473 finder.find_module.cache_clear()
475 self.bootstrap()
477 # Reload brain plugins. During initialisation this is done in astroid.manager.py
478 register_all_brains(self)