Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/astroid/manager.py: 54%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""astroid manager: avoid multiple astroid build of a same module when
6possible by providing a class responsible to get astroid representation
7from various source and using a cache of built modules)
8"""
10from __future__ import annotations
12import collections
13import os
14import types
15import zipimport
16from collections.abc import Callable, Iterator, Sequence
17from typing import Any, ClassVar
19from astroid import nodes
20from astroid.context import InferenceContext, _invalidate_cache
21from astroid.exceptions import AstroidBuildingError, AstroidImportError
22from astroid.interpreter._import import spec, util
23from astroid.modutils import (
24 NoSourceFile,
25 _cache_normalize_path_,
26 _has_init,
27 cached_os_path_isfile,
28 file_info_from_modpath,
29 get_source_file,
30 is_module_name_part_of_extension_package_whitelist,
31 is_python_source,
32 is_stdlib_module,
33 load_module_from_name,
34 modpath_from_file,
35)
36from astroid.transforms import TransformVisitor
37from astroid.typing import AstroidManagerBrain, InferenceResult
39ZIP_IMPORT_EXTS = (".zip", ".egg", ".whl", ".pyz", ".pyzw")
42def safe_repr(obj: Any) -> str:
43 try:
44 return repr(obj)
45 except Exception: # pylint: disable=broad-except
46 return "???"
49class AstroidManager:
50 """Responsible to build astroid from files or modules.
52 Use the Borg (singleton) pattern.
53 """
55 name = "astroid loader"
56 brain: ClassVar[AstroidManagerBrain] = {
57 "astroid_cache": {},
58 "_mod_file_cache": {},
59 "_failed_import_hooks": [],
60 "always_load_extensions": False,
61 "optimize_ast": False,
62 "max_inferable_values": 100,
63 "extension_package_whitelist": set(),
64 "module_denylist": set(),
65 "_transform": TransformVisitor(),
66 "prefer_stubs": False,
67 }
69 def __init__(self) -> None:
70 # NOTE: cache entries are added by the [re]builder
71 self.astroid_cache = AstroidManager.brain["astroid_cache"]
72 self._mod_file_cache = AstroidManager.brain["_mod_file_cache"]
73 self._failed_import_hooks = AstroidManager.brain["_failed_import_hooks"]
74 self.extension_package_whitelist = AstroidManager.brain[
75 "extension_package_whitelist"
76 ]
77 self.module_denylist = AstroidManager.brain["module_denylist"]
78 self._transform = AstroidManager.brain["_transform"]
79 self.prefer_stubs = AstroidManager.brain["prefer_stubs"]
81 @property
82 def always_load_extensions(self) -> bool:
83 return AstroidManager.brain["always_load_extensions"]
85 @always_load_extensions.setter
86 def always_load_extensions(self, value: bool) -> None:
87 AstroidManager.brain["always_load_extensions"] = value
89 @property
90 def optimize_ast(self) -> bool:
91 return AstroidManager.brain["optimize_ast"]
93 @optimize_ast.setter
94 def optimize_ast(self, value: bool) -> None:
95 AstroidManager.brain["optimize_ast"] = value
97 @property
98 def max_inferable_values(self) -> int:
99 return AstroidManager.brain["max_inferable_values"]
101 @max_inferable_values.setter
102 def max_inferable_values(self, value: int) -> None:
103 AstroidManager.brain["max_inferable_values"] = value
105 @property
106 def register_transform(self):
107 # This and unregister_transform below are exported for convenience
108 return self._transform.register_transform
110 @property
111 def unregister_transform(self):
112 return self._transform.unregister_transform
114 @property
115 def builtins_module(self) -> nodes.Module:
116 return self.astroid_cache["builtins"]
118 @property
119 def prefer_stubs(self) -> bool:
120 return AstroidManager.brain["prefer_stubs"]
122 @prefer_stubs.setter
123 def prefer_stubs(self, value: bool) -> None:
124 AstroidManager.brain["prefer_stubs"] = value
126 def visit_transforms(self, node: nodes.NodeNG) -> InferenceResult:
127 """Visit the transforms and apply them to the given *node*."""
128 return self._transform.visit(node)
130 def ast_from_file(
131 self,
132 filepath: str,
133 modname: str | None = None,
134 fallback: bool = True,
135 source: bool = False,
136 ) -> nodes.Module:
137 """Given a module name, return the astroid object."""
138 if modname is None:
139 try:
140 modname = ".".join(modpath_from_file(filepath))
141 except ImportError:
142 modname = filepath
143 if (
144 modname in self.astroid_cache
145 and self.astroid_cache[modname].file == filepath
146 ):
147 return self.astroid_cache[modname]
148 # Call get_source_file() only after a cache miss,
149 # since it calls os.path.exists().
150 try:
151 filepath = get_source_file(
152 filepath, include_no_ext=True, prefer_stubs=self.prefer_stubs
153 )
154 source = True
155 except NoSourceFile:
156 pass
157 # Second attempt on the cache after get_source_file().
158 if (
159 modname in self.astroid_cache
160 and self.astroid_cache[modname].file == filepath
161 ):
162 return self.astroid_cache[modname]
163 if source:
164 # pylint: disable=import-outside-toplevel; circular import
165 from astroid.builder import AstroidBuilder
167 return AstroidBuilder(self).file_build(filepath, modname)
168 if fallback and modname:
169 return self.ast_from_module_name(modname)
170 raise AstroidBuildingError("Unable to build an AST for {path}.", path=filepath)
172 def ast_from_string(
173 self, data: str, modname: str = "", filepath: str | None = None
174 ) -> nodes.Module:
175 """Given some source code as a string, return its corresponding astroid
176 object.
177 """
178 # pylint: disable=import-outside-toplevel; circular import
179 from astroid.builder import AstroidBuilder
181 return AstroidBuilder(self).string_build(data, modname, filepath)
183 def _build_stub_module(self, modname: str) -> nodes.Module:
184 # pylint: disable=import-outside-toplevel; circular import
185 from astroid.builder import AstroidBuilder
187 return AstroidBuilder(self).string_build("", modname)
189 def _build_namespace_module(
190 self, modname: str, path: Sequence[str]
191 ) -> nodes.Module:
192 # pylint: disable=import-outside-toplevel; circular import
193 from astroid.builder import build_namespace_package_module
195 return build_namespace_package_module(modname, path)
197 def _can_load_extension(self, modname: str) -> bool:
198 if self.always_load_extensions:
199 return True
200 if is_stdlib_module(modname):
201 return True
202 return is_module_name_part_of_extension_package_whitelist(
203 modname, self.extension_package_whitelist
204 )
206 def ast_from_module_name( # noqa: C901
207 self,
208 modname: str | None,
209 context_file: str | None = None,
210 use_cache: bool = True,
211 ) -> nodes.Module:
212 """Given a module name, return the astroid object."""
213 if modname is None:
214 raise AstroidBuildingError("No module name given.")
215 # Sometimes we don't want to use the cache. For example, when we're
216 # importing a module with the same name as the file that is importing
217 # we want to fallback on the import system to make sure we get the correct
218 # module.
219 if modname in self.module_denylist:
220 raise AstroidImportError(f"Skipping ignored module {modname!r}")
221 if modname in self.astroid_cache and use_cache:
222 return self.astroid_cache[modname]
223 if modname == "__main__":
224 return self._build_stub_module(modname)
225 if context_file:
226 old_cwd = os.getcwd()
227 os.chdir(os.path.dirname(context_file))
228 try:
229 found_spec = self.file_from_module_name(modname, context_file)
230 if found_spec.type == spec.ModuleType.PY_ZIPMODULE:
231 module = self.zip_import_data(found_spec.location)
232 if module is not None:
233 return module
235 elif found_spec.type in (
236 spec.ModuleType.C_BUILTIN,
237 spec.ModuleType.C_EXTENSION,
238 ):
239 if (
240 found_spec.type == spec.ModuleType.C_EXTENSION
241 and not self._can_load_extension(modname)
242 ):
243 return self._build_stub_module(modname)
244 try:
245 named_module = load_module_from_name(modname)
246 except Exception as e:
247 raise AstroidImportError(
248 "Loading {modname} failed with:\n{error}",
249 modname=modname,
250 path=found_spec.location,
251 ) from e
252 return self.ast_from_module(named_module, modname)
254 elif found_spec.type == spec.ModuleType.PY_COMPILED:
255 raise AstroidImportError(
256 "Unable to load compiled module {modname}.",
257 modname=modname,
258 path=found_spec.location,
259 )
261 elif found_spec.type == spec.ModuleType.PY_NAMESPACE:
262 return self._build_namespace_module(
263 modname, found_spec.submodule_search_locations or []
264 )
265 elif found_spec.type == spec.ModuleType.PY_FROZEN:
266 if found_spec.location is None:
267 return self._build_stub_module(modname)
268 # For stdlib frozen modules we can determine the location and
269 # can therefore create a module from the source file
270 return self.ast_from_file(found_spec.location, modname, fallback=False)
272 if found_spec.location is None:
273 raise AstroidImportError(
274 "Can't find a file for module {modname}.", modname=modname
275 )
277 return self.ast_from_file(found_spec.location, modname, fallback=False)
278 except AstroidBuildingError as e:
279 for hook in self._failed_import_hooks:
280 try:
281 return hook(modname)
282 except AstroidBuildingError:
283 pass
284 raise e
285 finally:
286 if context_file:
287 os.chdir(old_cwd)
289 def zip_import_data(self, filepath: str) -> nodes.Module | None:
290 if zipimport is None:
291 return None
293 # pylint: disable=import-outside-toplevel; circular import
294 from astroid.builder import AstroidBuilder
296 builder = AstroidBuilder(self)
297 for ext in ZIP_IMPORT_EXTS:
298 try:
299 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
300 except ValueError:
301 continue
302 try:
303 importer = zipimport.zipimporter(eggpath + ext)
304 zmodname = resource.replace(os.path.sep, ".")
305 if importer.is_package(resource):
306 zmodname = zmodname + ".__init__"
307 module = builder.string_build(
308 importer.get_source(resource), zmodname, filepath
309 )
310 return module
311 except Exception: # pylint: disable=broad-except
312 continue
313 return None
315 def file_from_module_name(
316 self, modname: str, contextfile: str | None
317 ) -> spec.ModuleSpec:
318 try:
319 value = self._mod_file_cache[(modname, contextfile)]
320 except KeyError:
321 try:
322 value = file_info_from_modpath(
323 modname.split("."), context_file=contextfile
324 )
325 except ImportError as e:
326 value = AstroidImportError(
327 "Failed to import module {modname} with error:\n{error}.",
328 modname=modname,
329 # we remove the traceback here to save on memory usage (since these exceptions are cached)
330 error=e.with_traceback(None),
331 )
332 self._mod_file_cache[(modname, contextfile)] = value
333 if isinstance(value, AstroidBuildingError):
334 # we remove the traceback here to save on memory usage (since these exceptions are cached)
335 raise value.with_traceback(None) # pylint: disable=no-member
336 return value
338 def ast_from_module(
339 self, module: types.ModuleType, modname: str | None = None
340 ) -> nodes.Module:
341 """Given an imported module, return the astroid object."""
342 modname = modname or module.__name__
343 if modname in self.astroid_cache:
344 return self.astroid_cache[modname]
345 try:
346 # some builtin modules don't have __file__ attribute
347 filepath = module.__file__
348 if is_python_source(filepath):
349 # Type is checked in is_python_source
350 return self.ast_from_file(filepath, modname) # type: ignore[arg-type]
351 except AttributeError:
352 pass
354 # pylint: disable=import-outside-toplevel; circular import
355 from astroid.builder import AstroidBuilder
357 return AstroidBuilder(self).module_build(module, modname)
359 def ast_from_class(self, klass: type, modname: str | None = None) -> nodes.ClassDef:
360 """Get astroid for the given class."""
361 if modname is None:
362 try:
363 modname = klass.__module__
364 except AttributeError as exc:
365 raise AstroidBuildingError(
366 "Unable to get module for class {class_name}.",
367 cls=klass,
368 class_repr=safe_repr(klass),
369 modname=modname,
370 ) from exc
371 modastroid = self.ast_from_module_name(modname)
372 ret = modastroid.getattr(klass.__name__)[0]
373 assert isinstance(ret, nodes.ClassDef)
374 return ret
376 def infer_ast_from_something(
377 self, obj: object, context: InferenceContext | None = None
378 ) -> Iterator[InferenceResult]:
379 """Infer astroid for the given class."""
380 if hasattr(obj, "__class__") and not isinstance(obj, type):
381 klass = obj.__class__
382 elif isinstance(obj, type):
383 klass = obj
384 else:
385 raise AstroidBuildingError( # pragma: no cover
386 "Unable to get type for {class_repr}.",
387 cls=None,
388 class_repr=safe_repr(obj),
389 )
390 try:
391 modname = klass.__module__
392 except AttributeError as exc:
393 raise AstroidBuildingError(
394 "Unable to get module for {class_repr}.",
395 cls=klass,
396 class_repr=safe_repr(klass),
397 ) from exc
398 except Exception as exc:
399 raise AstroidImportError(
400 "Unexpected error while retrieving module for {class_repr}:\n"
401 "{error}",
402 cls=klass,
403 class_repr=safe_repr(klass),
404 ) from exc
405 try:
406 name = klass.__name__
407 except AttributeError as exc:
408 raise AstroidBuildingError(
409 "Unable to get name for {class_repr}:\n",
410 cls=klass,
411 class_repr=safe_repr(klass),
412 ) from exc
413 except Exception as exc:
414 raise AstroidImportError(
415 "Unexpected error while retrieving name for {class_repr}:\n{error}",
416 cls=klass,
417 class_repr=safe_repr(klass),
418 ) from exc
419 # take care, on living object __module__ is regularly wrong :(
420 modastroid = self.ast_from_module_name(modname)
421 if klass is obj:
422 yield from modastroid.igetattr(name, context)
423 else:
424 for inferred in modastroid.igetattr(name, context):
425 yield inferred.instantiate_class()
427 def register_failed_import_hook(self, hook: Callable[[str], nodes.Module]) -> None:
428 """Registers a hook to resolve imports that cannot be found otherwise.
430 `hook` must be a function that accepts a single argument `modname` which
431 contains the name of the module or package that could not be imported.
432 If `hook` can resolve the import, must return a node of type `astroid.Module`,
433 otherwise, it must raise `AstroidBuildingError`.
434 """
435 self._failed_import_hooks.append(hook)
437 def cache_module(self, module: nodes.Module) -> None:
438 """Cache a module if no module with the same name is known yet."""
439 self.astroid_cache.setdefault(module.name, module)
441 def bootstrap(self) -> None:
442 """Bootstrap the required AST modules needed for the manager to work.
444 The bootstrap usually involves building the AST for the builtins
445 module, which is required by the rest of astroid to work correctly.
446 """
447 from astroid import raw_building # pylint: disable=import-outside-toplevel
449 raw_building._astroid_bootstrapping()
451 def clear_cache(self) -> None:
452 """Clear the underlying caches, bootstrap the builtins module and
453 re-register transforms.
454 """
455 # import here because of cyclic imports
456 # pylint: disable=import-outside-toplevel
457 from astroid.brain.helpers import register_all_brains
458 from astroid.inference_tip import clear_inference_tip_cache
459 from astroid.interpreter._import.spec import _find_spec
460 from astroid.interpreter.objectmodel import ObjectModel
461 from astroid.nodes._base_nodes import LookupMixIn
462 from astroid.nodes.scoped_nodes import ClassDef
464 clear_inference_tip_cache()
465 _invalidate_cache() # inference context cache
467 self.astroid_cache.clear()
468 self._mod_file_cache.clear()
470 # NB: not a new TransformVisitor()
471 AstroidManager.brain["_transform"].transforms = collections.defaultdict(list)
473 for lru_cache in (
474 LookupMixIn.lookup,
475 _cache_normalize_path_,
476 _has_init,
477 cached_os_path_isfile,
478 util.is_namespace,
479 ObjectModel.attributes,
480 ClassDef._metaclass_lookup_attribute,
481 _find_spec,
482 ):
483 lru_cache.cache_clear() # type: ignore[attr-defined]
485 for finder in spec._SPEC_FINDERS:
486 finder.find_module.cache_clear()
488 self.bootstrap()
490 # Reload brain plugins. During initialisation this is done in astroid.manager.py
491 register_all_brains(self)