Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/astroid/manager.py: 66%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""astroid manager: avoid multiple astroid build of a same module when
6possible by providing a class responsible to get astroid representation
7from various source and using a cache of built modules)
8"""
10from __future__ import annotations
12import collections
13import os
14import types
15import zipimport
16from collections.abc import Callable, Iterator, Sequence
17from typing import Any, ClassVar
19from astroid import nodes
20from astroid.context import InferenceContext, _invalidate_cache
21from astroid.exceptions import AstroidBuildingError, AstroidImportError
22from astroid.interpreter._import import spec, util
23from astroid.modutils import (
24 NoSourceFile,
25 _cache_normalize_path_,
26 file_info_from_modpath,
27 get_source_file,
28 is_module_name_part_of_extension_package_whitelist,
29 is_python_source,
30 is_stdlib_module,
31 load_module_from_name,
32 modpath_from_file,
33)
34from astroid.transforms import TransformVisitor
35from astroid.typing import AstroidManagerBrain, InferenceResult
37ZIP_IMPORT_EXTS = (".zip", ".egg", ".whl", ".pyz", ".pyzw")
40def safe_repr(obj: Any) -> str:
41 try:
42 return repr(obj)
43 except Exception: # pylint: disable=broad-except
44 return "???"
47class AstroidManager:
48 """Responsible to build astroid from files or modules.
50 Use the Borg (singleton) pattern.
51 """
53 name = "astroid loader"
54 brain: ClassVar[AstroidManagerBrain] = {
55 "astroid_cache": {},
56 "_mod_file_cache": {},
57 "_failed_import_hooks": [],
58 "always_load_extensions": False,
59 "optimize_ast": False,
60 "max_inferable_values": 100,
61 "extension_package_whitelist": set(),
62 "module_denylist": set(),
63 "_transform": TransformVisitor(),
64 "prefer_stubs": False,
65 }
67 def __init__(self) -> None:
68 # NOTE: cache entries are added by the [re]builder
69 self.astroid_cache = AstroidManager.brain["astroid_cache"]
70 self._mod_file_cache = AstroidManager.brain["_mod_file_cache"]
71 self._failed_import_hooks = AstroidManager.brain["_failed_import_hooks"]
72 self.extension_package_whitelist = AstroidManager.brain[
73 "extension_package_whitelist"
74 ]
75 self.module_denylist = AstroidManager.brain["module_denylist"]
76 self._transform = AstroidManager.brain["_transform"]
77 self.prefer_stubs = AstroidManager.brain["prefer_stubs"]
79 @property
80 def always_load_extensions(self) -> bool:
81 return AstroidManager.brain["always_load_extensions"]
83 @always_load_extensions.setter
84 def always_load_extensions(self, value: bool) -> None:
85 AstroidManager.brain["always_load_extensions"] = value
87 @property
88 def optimize_ast(self) -> bool:
89 return AstroidManager.brain["optimize_ast"]
91 @optimize_ast.setter
92 def optimize_ast(self, value: bool) -> None:
93 AstroidManager.brain["optimize_ast"] = value
95 @property
96 def max_inferable_values(self) -> int:
97 return AstroidManager.brain["max_inferable_values"]
99 @max_inferable_values.setter
100 def max_inferable_values(self, value: int) -> None:
101 AstroidManager.brain["max_inferable_values"] = value
103 @property
104 def register_transform(self):
105 # This and unregister_transform below are exported for convenience
106 return self._transform.register_transform
108 @property
109 def unregister_transform(self):
110 return self._transform.unregister_transform
112 @property
113 def builtins_module(self) -> nodes.Module:
114 return self.astroid_cache["builtins"]
116 @property
117 def prefer_stubs(self) -> bool:
118 return AstroidManager.brain["prefer_stubs"]
120 @prefer_stubs.setter
121 def prefer_stubs(self, value: bool) -> None:
122 AstroidManager.brain["prefer_stubs"] = value
124 def visit_transforms(self, node: nodes.NodeNG) -> InferenceResult:
125 """Visit the transforms and apply them to the given *node*."""
126 return self._transform.visit(node)
128 def ast_from_file(
129 self,
130 filepath: str,
131 modname: str | None = None,
132 fallback: bool = True,
133 source: bool = False,
134 ) -> nodes.Module:
135 """Given a module name, return the astroid object."""
136 if modname is None:
137 try:
138 modname = ".".join(modpath_from_file(filepath))
139 except ImportError:
140 modname = filepath
141 if (
142 modname in self.astroid_cache
143 and self.astroid_cache[modname].file == filepath
144 ):
145 return self.astroid_cache[modname]
146 # Call get_source_file() only after a cache miss,
147 # since it calls os.path.exists().
148 try:
149 filepath = get_source_file(
150 filepath, include_no_ext=True, prefer_stubs=self.prefer_stubs
151 )
152 source = True
153 except NoSourceFile:
154 pass
155 # Second attempt on the cache after get_source_file().
156 if (
157 modname in self.astroid_cache
158 and self.astroid_cache[modname].file == filepath
159 ):
160 return self.astroid_cache[modname]
161 if source:
162 # pylint: disable=import-outside-toplevel; circular import
163 from astroid.builder import AstroidBuilder
165 return AstroidBuilder(self).file_build(filepath, modname)
166 if fallback and modname:
167 return self.ast_from_module_name(modname)
168 raise AstroidBuildingError("Unable to build an AST for {path}.", path=filepath)
170 def ast_from_string(
171 self, data: str, modname: str = "", filepath: str | None = None
172 ) -> nodes.Module:
173 """Given some source code as a string, return its corresponding astroid
174 object.
175 """
176 # pylint: disable=import-outside-toplevel; circular import
177 from astroid.builder import AstroidBuilder
179 return AstroidBuilder(self).string_build(data, modname, filepath)
181 def _build_stub_module(self, modname: str) -> nodes.Module:
182 # pylint: disable=import-outside-toplevel; circular import
183 from astroid.builder import AstroidBuilder
185 return AstroidBuilder(self).string_build("", modname)
187 def _build_namespace_module(
188 self, modname: str, path: Sequence[str]
189 ) -> nodes.Module:
190 # pylint: disable=import-outside-toplevel; circular import
191 from astroid.builder import build_namespace_package_module
193 return build_namespace_package_module(modname, path)
195 def _can_load_extension(self, modname: str) -> bool:
196 if self.always_load_extensions:
197 return True
198 if is_stdlib_module(modname):
199 return True
200 return is_module_name_part_of_extension_package_whitelist(
201 modname, self.extension_package_whitelist
202 )
204 def ast_from_module_name( # noqa: C901
205 self,
206 modname: str | None,
207 context_file: str | None = None,
208 use_cache: bool = True,
209 ) -> nodes.Module:
210 """Given a module name, return the astroid object."""
211 if modname is None:
212 raise AstroidBuildingError("No module name given.")
213 # Sometimes we don't want to use the cache. For example, when we're
214 # importing a module with the same name as the file that is importing
215 # we want to fallback on the import system to make sure we get the correct
216 # module.
217 if modname in self.module_denylist:
218 raise AstroidImportError(f"Skipping ignored module {modname!r}")
219 if modname in self.astroid_cache and use_cache:
220 return self.astroid_cache[modname]
221 if modname == "__main__":
222 return self._build_stub_module(modname)
223 if context_file:
224 old_cwd = os.getcwd()
225 os.chdir(os.path.dirname(context_file))
226 try:
227 found_spec = self.file_from_module_name(modname, context_file)
228 if found_spec.type == spec.ModuleType.PY_ZIPMODULE:
229 module = self.zip_import_data(found_spec.location)
230 if module is not None:
231 return module
233 elif found_spec.type in (
234 spec.ModuleType.C_BUILTIN,
235 spec.ModuleType.C_EXTENSION,
236 ):
237 if (
238 found_spec.type == spec.ModuleType.C_EXTENSION
239 and not self._can_load_extension(modname)
240 ):
241 return self._build_stub_module(modname)
242 try:
243 named_module = load_module_from_name(modname)
244 except Exception as e:
245 raise AstroidImportError(
246 "Loading {modname} failed with:\n{error}",
247 modname=modname,
248 path=found_spec.location,
249 ) from e
250 return self.ast_from_module(named_module, modname)
252 elif found_spec.type == spec.ModuleType.PY_COMPILED:
253 raise AstroidImportError(
254 "Unable to load compiled module {modname}.",
255 modname=modname,
256 path=found_spec.location,
257 )
259 elif found_spec.type == spec.ModuleType.PY_NAMESPACE:
260 return self._build_namespace_module(
261 modname, found_spec.submodule_search_locations or []
262 )
263 elif found_spec.type == spec.ModuleType.PY_FROZEN:
264 if found_spec.location is None:
265 return self._build_stub_module(modname)
266 # For stdlib frozen modules we can determine the location and
267 # can therefore create a module from the source file
268 return self.ast_from_file(found_spec.location, modname, fallback=False)
270 if found_spec.location is None:
271 raise AstroidImportError(
272 "Can't find a file for module {modname}.", modname=modname
273 )
275 return self.ast_from_file(found_spec.location, modname, fallback=False)
276 except AstroidBuildingError as e:
277 for hook in self._failed_import_hooks:
278 try:
279 return hook(modname)
280 except AstroidBuildingError:
281 pass
282 raise e
283 finally:
284 if context_file:
285 os.chdir(old_cwd)
287 def zip_import_data(self, filepath: str) -> nodes.Module | None:
288 if zipimport is None:
289 return None
291 # pylint: disable=import-outside-toplevel; circular import
292 from astroid.builder import AstroidBuilder
294 builder = AstroidBuilder(self)
295 for ext in ZIP_IMPORT_EXTS:
296 try:
297 eggpath, resource = filepath.rsplit(ext + os.path.sep, 1)
298 except ValueError:
299 continue
300 try:
301 importer = zipimport.zipimporter(eggpath + ext)
302 zmodname = resource.replace(os.path.sep, ".")
303 if importer.is_package(resource):
304 zmodname = zmodname + ".__init__"
305 module = builder.string_build(
306 importer.get_source(resource), zmodname, filepath
307 )
308 return module
309 except Exception: # pylint: disable=broad-except
310 continue
311 return None
313 def file_from_module_name(
314 self, modname: str, contextfile: str | None
315 ) -> spec.ModuleSpec:
316 try:
317 value = self._mod_file_cache[(modname, contextfile)]
318 except KeyError:
319 try:
320 value = file_info_from_modpath(
321 modname.split("."), context_file=contextfile
322 )
323 except ImportError as e:
324 value = AstroidImportError(
325 "Failed to import module {modname} with error:\n{error}.",
326 modname=modname,
327 # we remove the traceback here to save on memory usage (since these exceptions are cached)
328 error=e.with_traceback(None),
329 )
330 self._mod_file_cache[(modname, contextfile)] = value
331 if isinstance(value, AstroidBuildingError):
332 # we remove the traceback here to save on memory usage (since these exceptions are cached)
333 raise value.with_traceback(None) # pylint: disable=no-member
334 return value
336 def ast_from_module(
337 self, module: types.ModuleType, modname: str | None = None
338 ) -> nodes.Module:
339 """Given an imported module, return the astroid object."""
340 modname = modname or module.__name__
341 if modname in self.astroid_cache:
342 return self.astroid_cache[modname]
343 try:
344 # some builtin modules don't have __file__ attribute
345 filepath = module.__file__
346 if is_python_source(filepath):
347 # Type is checked in is_python_source
348 return self.ast_from_file(filepath, modname) # type: ignore[arg-type]
349 except AttributeError:
350 pass
352 # pylint: disable=import-outside-toplevel; circular import
353 from astroid.builder import AstroidBuilder
355 return AstroidBuilder(self).module_build(module, modname)
357 def ast_from_class(self, klass: type, modname: str | None = None) -> nodes.ClassDef:
358 """Get astroid for the given class."""
359 if modname is None:
360 try:
361 modname = klass.__module__
362 except AttributeError as exc:
363 raise AstroidBuildingError(
364 "Unable to get module for class {class_name}.",
365 cls=klass,
366 class_repr=safe_repr(klass),
367 modname=modname,
368 ) from exc
369 modastroid = self.ast_from_module_name(modname)
370 ret = modastroid.getattr(klass.__name__)[0]
371 assert isinstance(ret, nodes.ClassDef)
372 return ret
374 def infer_ast_from_something(
375 self, obj: object, context: InferenceContext | None = None
376 ) -> Iterator[InferenceResult]:
377 """Infer astroid for the given class."""
378 if hasattr(obj, "__class__") and not isinstance(obj, type):
379 klass = obj.__class__
380 elif isinstance(obj, type):
381 klass = obj
382 else:
383 raise AstroidBuildingError( # pragma: no cover
384 "Unable to get type for {class_repr}.",
385 cls=None,
386 class_repr=safe_repr(obj),
387 )
388 try:
389 modname = klass.__module__
390 except AttributeError as exc:
391 raise AstroidBuildingError(
392 "Unable to get module for {class_repr}.",
393 cls=klass,
394 class_repr=safe_repr(klass),
395 ) from exc
396 except Exception as exc:
397 raise AstroidImportError(
398 "Unexpected error while retrieving module for {class_repr}:\n"
399 "{error}",
400 cls=klass,
401 class_repr=safe_repr(klass),
402 ) from exc
403 try:
404 name = klass.__name__
405 except AttributeError as exc:
406 raise AstroidBuildingError(
407 "Unable to get name for {class_repr}:\n",
408 cls=klass,
409 class_repr=safe_repr(klass),
410 ) from exc
411 except Exception as exc:
412 raise AstroidImportError(
413 "Unexpected error while retrieving name for {class_repr}:\n{error}",
414 cls=klass,
415 class_repr=safe_repr(klass),
416 ) from exc
417 # take care, on living object __module__ is regularly wrong :(
418 modastroid = self.ast_from_module_name(modname)
419 if klass is obj:
420 yield from modastroid.igetattr(name, context)
421 else:
422 for inferred in modastroid.igetattr(name, context):
423 yield inferred.instantiate_class()
425 def register_failed_import_hook(self, hook: Callable[[str], nodes.Module]) -> None:
426 """Registers a hook to resolve imports that cannot be found otherwise.
428 `hook` must be a function that accepts a single argument `modname` which
429 contains the name of the module or package that could not be imported.
430 If `hook` can resolve the import, must return a node of type `astroid.Module`,
431 otherwise, it must raise `AstroidBuildingError`.
432 """
433 self._failed_import_hooks.append(hook)
435 def cache_module(self, module: nodes.Module) -> None:
436 """Cache a module if no module with the same name is known yet."""
437 self.astroid_cache.setdefault(module.name, module)
439 def bootstrap(self) -> None:
440 """Bootstrap the required AST modules needed for the manager to work.
442 The bootstrap usually involves building the AST for the builtins
443 module, which is required by the rest of astroid to work correctly.
444 """
445 from astroid import raw_building # pylint: disable=import-outside-toplevel
447 raw_building._astroid_bootstrapping()
449 def clear_cache(self) -> None:
450 """Clear the underlying caches, bootstrap the builtins module and
451 re-register transforms.
452 """
453 # import here because of cyclic imports
454 # pylint: disable=import-outside-toplevel
455 from astroid.brain.helpers import register_all_brains
456 from astroid.inference_tip import clear_inference_tip_cache
457 from astroid.interpreter._import.spec import _find_spec
458 from astroid.interpreter.objectmodel import ObjectModel
459 from astroid.nodes._base_nodes import LookupMixIn
460 from astroid.nodes.scoped_nodes import ClassDef
462 clear_inference_tip_cache()
463 _invalidate_cache() # inference context cache
465 self.astroid_cache.clear()
466 # NB: not a new TransformVisitor()
467 AstroidManager.brain["_transform"].transforms = collections.defaultdict(list)
469 for lru_cache in (
470 LookupMixIn.lookup,
471 _cache_normalize_path_,
472 util.is_namespace,
473 ObjectModel.attributes,
474 ClassDef._metaclass_lookup_attribute,
475 _find_spec,
476 ):
477 lru_cache.cache_clear() # type: ignore[attr-defined]
479 self.bootstrap()
481 # Reload brain plugins. During initialisation this is done in astroid.manager.py
482 register_all_brains(self)