Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/builder.py: 51%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""The AstroidBuilder makes astroid from living object and / or from _ast.
7The builder is not thread safe and can't be used to parse different sources
8at the same time.
9"""
11from __future__ import annotations
13import ast
14import os
15import re
16import textwrap
17import types
18import warnings
19from collections.abc import Collection, Iterator, Sequence
20from io import TextIOWrapper
21from tokenize import detect_encoding
22from typing import TYPE_CHECKING, cast
24from astroid import bases, modutils, nodes, raw_building, rebuilder, util
25from astroid.const import PY312_PLUS, PY314_PLUS
26from astroid.exceptions import AstroidBuildingError, AstroidSyntaxError, InferenceError
28if TYPE_CHECKING:
29 from astroid.manager import AstroidManager
31# The name of the transient function that is used to
32# wrap expressions to be extracted when calling
33# extract_node.
34_TRANSIENT_FUNCTION = "__"
36# The comment used to select a statement to be extracted
37# when calling extract_node.
38_STATEMENT_SELECTOR = "#@"
40if PY312_PLUS:
41 warnings.filterwarnings("ignore", ".*invalid escape sequence", SyntaxWarning)
42if PY314_PLUS:
43 warnings.filterwarnings(
44 "ignore", "'(return|continue|break)' in a 'finally'", SyntaxWarning
45 )
48def open_source_file(filename: str) -> tuple[TextIOWrapper, str, str]:
49 # pylint: disable=consider-using-with
50 with open(filename, "rb") as byte_stream:
51 encoding = detect_encoding(byte_stream.readline)[0]
52 stream = open(filename, newline=None, encoding=encoding)
53 data = stream.read()
54 return stream, encoding, data
57def _can_assign_attr(node: nodes.ClassDef, attrname: str | None) -> bool:
58 try:
59 slots = node.slots()
60 except NotImplementedError:
61 pass
62 else:
63 if slots and attrname not in {slot.value for slot in slots}:
64 return False
65 return node.qname() != "builtins.object"
68class AstroidBuilder(raw_building.InspectBuilder):
69 """Class for building an astroid tree from source code or from a live module.
71 The param *manager* specifies the manager class which should be used. The
72 param *apply_transforms* determines if the transforms should be
73 applied after the tree was built from source or from a live object,
74 by default being True.
75 """
77 def __init__(self, manager: AstroidManager, apply_transforms: bool = True) -> None:
78 super().__init__(manager)
79 self._apply_transforms = apply_transforms
80 if not raw_building.InspectBuilder.bootstrapped:
81 manager.bootstrap()
83 def module_build(
84 self, module: types.ModuleType, modname: str | None = None
85 ) -> nodes.Module:
86 """Build an astroid from a living module instance."""
87 node = None
88 path = getattr(module, "__file__", None)
89 loader = getattr(module, "__loader__", None)
90 # Prefer the loader to get the source rather than assuming we have a
91 # filesystem to read the source file from ourselves.
92 if loader:
93 modname = modname or module.__name__
94 source = loader.get_source(modname)
95 if source:
96 node = self.string_build(source, modname, path=path)
97 if node is None and path is not None:
98 path_, ext = os.path.splitext(modutils._path_from_filename(path))
99 if ext in {".py", ".pyc", ".pyo"} and os.path.exists(path_ + ".py"):
100 node = self.file_build(path_ + ".py", modname)
101 if node is None:
102 # this is a built-in module
103 # get a partial representation by introspection
104 node = self.inspect_build(module, modname=modname, path=path)
105 if self._apply_transforms:
106 # We have to handle transformation by ourselves since the
107 # rebuilder isn't called for builtin nodes
108 node = self._manager.visit_transforms(node)
109 assert isinstance(node, nodes.Module)
110 return node
112 def file_build(self, path: str, modname: str | None = None) -> nodes.Module:
113 """Build astroid from a source code file (i.e. from an ast).
115 *path* is expected to be a python source file
116 """
117 try:
118 stream, encoding, data = open_source_file(path)
119 except OSError as exc:
120 raise AstroidBuildingError(
121 "Unable to load file {path}:\n{error}",
122 modname=modname,
123 path=path,
124 error=exc,
125 ) from exc
126 except (SyntaxError, LookupError) as exc:
127 raise AstroidSyntaxError(
128 "Python 3 encoding specification error or unknown encoding:\n"
129 "{error}",
130 modname=modname,
131 path=path,
132 error=exc,
133 ) from exc
134 except UnicodeError as exc: # wrong encoding
135 # detect_encoding returns utf-8 if no encoding specified
136 raise AstroidBuildingError(
137 "Wrong or no encoding specified for {filename}.", filename=path
138 ) from exc
139 with stream:
140 # get module name if necessary
141 if modname is None:
142 try:
143 modname = ".".join(modutils.modpath_from_file(path))
144 except ImportError:
145 modname = os.path.splitext(os.path.basename(path))[0]
146 # build astroid representation
147 module, builder = self._data_build(data, modname, path)
148 return self._post_build(module, builder, encoding)
150 def string_build(
151 self, data: str, modname: str = "", path: str | None = None
152 ) -> nodes.Module:
153 """Build astroid from source code string."""
154 module, builder = self._data_build(data, modname, path)
155 module.file_bytes = data.encode("utf-8")
156 return self._post_build(module, builder, "utf-8")
158 def _post_build(
159 self, module: nodes.Module, builder: rebuilder.TreeRebuilder, encoding: str
160 ) -> nodes.Module:
161 """Handles encoding and delayed nodes after a module has been built."""
162 module.file_encoding = encoding
163 self._manager.cache_module(module)
164 # post tree building steps after we stored the module in the cache:
165 for from_node, global_names in builder._import_from_nodes:
166 if from_node.modname == "__future__":
167 for symbol, _ in from_node.names:
168 module.future_imports.add(symbol)
169 self.add_from_names_to_locals(from_node, global_names)
170 # handle delayed assattr nodes
171 for delayed in builder._delayed_assattr:
172 self.delayed_assattr(delayed)
174 # Visit the transforms
175 if self._apply_transforms:
176 module = self._manager.visit_transforms(module)
177 return module
179 def _data_build(
180 self, data: str, modname: str, path: str | None
181 ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
182 """Build tree node from data and add some informations."""
183 try:
184 node = _parse_string(data, type_comments=True, modname=modname)
185 except (TypeError, ValueError, SyntaxError, MemoryError) as exc:
186 raise AstroidSyntaxError(
187 "Parsing Python code failed:\n{error}",
188 source=data,
189 modname=modname,
190 path=path,
191 error=exc,
192 ) from exc
194 if path is not None:
195 node_file = os.path.abspath(path)
196 else:
197 node_file = "<?>"
198 if modname.endswith(".__init__"):
199 modname = modname[:-9]
200 package = True
201 else:
202 package = (
203 path is not None
204 and os.path.splitext(os.path.basename(path))[0] == "__init__"
205 )
206 builder = rebuilder.TreeRebuilder(self._manager, data)
207 module = builder.visit_module(node, modname, node_file, package)
208 return module, builder
210 def add_from_names_to_locals(
211 self, node: nodes.ImportFrom, global_name: Collection[str]
212 ) -> None:
213 """Store imported names to the locals.
215 Resort the locals if coming from a delayed node
216 """
218 def add_local(parent_or_root: nodes.NodeNG, name: str) -> None:
219 parent_or_root.set_local(name, node)
220 my_list = parent_or_root.scope().locals[name]
221 if TYPE_CHECKING:
222 my_list = cast(list[nodes.NodeNG], my_list)
223 my_list.sort(key=lambda n: n.fromlineno or 0)
225 assert node.parent # It should always default to the module
226 module = node.root()
227 for name, asname in node.names:
228 if name == "*":
229 try:
230 imported = node.do_import_module()
231 except AstroidBuildingError:
232 continue
233 for name in imported.public_names():
234 if name in global_name:
235 add_local(module, name)
236 else:
237 add_local(node.parent, name)
238 else:
239 name = asname or name
240 if name in global_name:
241 add_local(module, name)
242 else:
243 add_local(node.parent, name)
245 def delayed_assattr(self, node: nodes.AssignAttr) -> None:
246 """Visit an AssignAttr node.
248 This adds name to locals and handle members definition.
249 """
250 from astroid import objects # pylint: disable=import-outside-toplevel
252 try:
253 for inferred in node.expr.infer():
254 if isinstance(inferred, util.UninferableBase):
255 continue
256 try:
257 # We want a narrow check on the parent type, not all of its subclasses
258 if type(inferred) in {bases.Instance, objects.ExceptionInstance}:
259 inferred = inferred._proxied
260 iattrs = inferred.instance_attrs
261 if not _can_assign_attr(inferred, node.attrname):
262 continue
263 elif isinstance(inferred, bases.Instance):
264 # Const, Tuple or other containers that inherit from
265 # `Instance`
266 continue
267 elif isinstance(inferred, (bases.Proxy, util.UninferableBase)):
268 continue
269 elif inferred.is_function:
270 iattrs = inferred.instance_attrs
271 else:
272 iattrs = inferred.locals
273 except AttributeError:
274 # XXX log error
275 continue
276 values = iattrs.setdefault(node.attrname, [])
277 if node in values:
278 continue
279 values.append(node)
280 except InferenceError:
281 pass
284def build_namespace_package_module(name: str, path: Sequence[str]) -> nodes.Module:
285 module = nodes.Module(name, path=path, package=True)
286 module.postinit(body=[], doc_node=None)
287 return module
290def parse(
291 code: str,
292 module_name: str = "",
293 path: str | None = None,
294 apply_transforms: bool = True,
295) -> nodes.Module:
296 """Parses a source string in order to obtain an astroid AST from it.
298 :param str code: The code for the module.
299 :param str module_name: The name for the module, if any
300 :param str path: The path for the module
301 :param bool apply_transforms:
302 Apply the transforms for the give code. Use it if you
303 don't want the default transforms to be applied.
304 """
305 # pylint: disable-next=import-outside-toplevel
306 from astroid.manager import AstroidManager
308 code = textwrap.dedent(code)
309 builder = AstroidBuilder(AstroidManager(), apply_transforms=apply_transforms)
310 return builder.string_build(code, modname=module_name, path=path)
313def _extract_expressions(node: nodes.NodeNG) -> Iterator[nodes.NodeNG]:
314 """Find expressions in a call to _TRANSIENT_FUNCTION and extract them.
316 The function walks the AST recursively to search for expressions that
317 are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an
318 expression, it completely removes the function call node from the tree,
319 replacing it by the wrapped expression inside the parent.
321 :param node: An astroid node.
322 :type node: astroid.bases.NodeNG
323 :yields: The sequence of wrapped expressions on the modified tree
324 expression can be found.
325 """
326 if (
327 isinstance(node, nodes.Call)
328 and isinstance(node.func, nodes.Name)
329 and node.func.name == _TRANSIENT_FUNCTION
330 and node.args
331 ):
332 real_expr = node.args[0]
333 assert node.parent
334 real_expr.parent = node.parent
335 # Search for node in all _astng_fields (the fields checked when
336 # get_children is called) of its parent. Some of those fields may
337 # be lists or tuples, in which case the elements need to be checked.
338 # When we find it, replace it by real_expr, so that the AST looks
339 # like no call to _TRANSIENT_FUNCTION ever took place.
340 for name in node.parent._astroid_fields:
341 child = getattr(node.parent, name)
342 if isinstance(child, list):
343 for idx, compound_child in enumerate(child):
344 if compound_child is node:
345 child[idx] = real_expr
346 elif child is node:
347 setattr(node.parent, name, real_expr)
348 yield real_expr
349 else:
350 for child in node.get_children():
351 yield from _extract_expressions(child)
354def _find_statement_by_line(node: nodes.NodeNG, line: int) -> nodes.NodeNG | None:
355 """Extracts the statement on a specific line from an AST.
357 If the line number of node matches line, it will be returned;
358 otherwise its children are iterated and the function is called
359 recursively.
361 :param node: An astroid node.
362 :type node: astroid.bases.NodeNG
363 :param line: The line number of the statement to extract.
364 :type line: int
365 :returns: The statement on the line, or None if no statement for the line
366 can be found.
367 :rtype: astroid.bases.NodeNG or None
368 """
369 if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.MatchCase)):
370 # This is an inaccuracy in the AST: the nodes that can be
371 # decorated do not carry explicit information on which line
372 # the actual definition (class/def), but .fromline seems to
373 # be close enough.
374 node_line = node.fromlineno
375 else:
376 node_line = node.lineno
378 if node_line == line:
379 return node
381 for child in node.get_children():
382 result = _find_statement_by_line(child, line)
383 if result:
384 return result
386 return None
389def extract_node(code: str, module_name: str = "") -> nodes.NodeNG | list[nodes.NodeNG]:
390 """Parses some Python code as a module and extracts a designated AST node.
392 Statements:
393 To extract one or more statement nodes, append #@ to the end of the line
395 Examples::
397 def x():
398 def y():
399 return 1 #@
401 The return statement will be extracted.
403 ::
405 class X(object):
406 def meth(self): #@
407 pass
409 The function object 'meth' will be extracted.
411 Expressions:
412 To extract arbitrary expressions, surround them with the fake
413 function call __(...). After parsing, the surrounded expression
414 will be returned and the whole AST (accessible via the returned
415 node's parent attribute) will look like the function call was
416 never there in the first place.
418 Examples::
420 a = __(1)
422 The const node will be extracted.
424 ::
426 def x(d=__(foo.bar)): pass
428 The node containing the default argument will be extracted.
430 ::
432 def foo(a, b):
433 return 0 < __(len(a)) < b
435 The node containing the function call 'len' will be extracted.
437 If no statements or expressions are selected, the last toplevel
438 statement will be returned.
440 If the selected statement is a discard statement, (i.e. an expression
441 turned into a statement), the wrapped expression is returned instead.
443 For convenience, singleton lists are unpacked.
445 :param str code: A piece of Python code that is parsed as
446 a module. Will be passed through textwrap.dedent first.
447 :param str module_name: The name of the module.
448 :returns: The designated node from the parse tree, or a list of nodes.
449 """
451 def _extract(node: nodes.NodeNG | None) -> nodes.NodeNG | None:
452 if isinstance(node, nodes.Expr):
453 return node.value
455 return node
457 requested_lines: list[int] = []
458 for idx, line in enumerate(code.splitlines()):
459 if line.strip().endswith(_STATEMENT_SELECTOR):
460 requested_lines.append(idx + 1)
462 tree = parse(code, module_name=module_name)
463 if not tree.body:
464 raise ValueError("Empty tree, cannot extract from it")
466 extracted: list[nodes.NodeNG | None] = []
467 if requested_lines:
468 extracted = [_find_statement_by_line(tree, line) for line in requested_lines]
470 # Modifies the tree.
471 extracted.extend(_extract_expressions(tree))
473 if not extracted:
474 extracted.append(tree.body[-1])
476 extracted = [_extract(node) for node in extracted]
477 extracted_without_none = [node for node in extracted if node is not None]
478 if len(extracted_without_none) == 1:
479 return extracted_without_none[0]
480 return extracted_without_none
483def _extract_single_node(code: str, module_name: str = "") -> nodes.NodeNG:
484 """Call extract_node while making sure that only one value is returned."""
485 ret = extract_node(code, module_name)
486 if isinstance(ret, list):
487 return ret[0]
488 return ret
491def _parse_string(
492 data: str, type_comments: bool = True, modname: str | None = None
493) -> ast.Module:
494 try:
495 parsed = ast.parse(
496 data + "\n", filename=modname or "<unknown>", type_comments=type_comments
497 )
498 except SyntaxError as exc:
499 # If the type annotations are misplaced for some reason, we do not want
500 # to fail the entire parsing of the file, so we need to retry the
501 # parsing without type comment support. We use a heuristic for
502 # determining if the error is due to type annotations.
503 type_annot_related = re.search(r"#\s+type:", exc.text or "")
504 if not (type_annot_related and type_comments):
505 raise
507 parsed = ast.parse(data + "\n", type_comments=False)
508 return parsed