Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/astroid/builder.py: 68%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""The AstroidBuilder makes astroid from living object and / or from _ast.
7The builder is not thread safe and can't be used to parse different sources
8at the same time.
9"""
11from __future__ import annotations
13import ast
14import os
15import textwrap
16import types
17import warnings
18from collections.abc import Iterator, Sequence
19from io import TextIOWrapper
20from tokenize import detect_encoding
22from astroid import bases, modutils, nodes, raw_building, rebuilder, util
23from astroid._ast import ParserModule, get_parser_module
24from astroid.const import PY312_PLUS
25from astroid.exceptions import AstroidBuildingError, AstroidSyntaxError, InferenceError
26from astroid.manager import AstroidManager
28# The name of the transient function that is used to
29# wrap expressions to be extracted when calling
30# extract_node.
31_TRANSIENT_FUNCTION = "__"
33# The comment used to select a statement to be extracted
34# when calling extract_node.
35_STATEMENT_SELECTOR = "#@"
36MISPLACED_TYPE_ANNOTATION_ERROR = "misplaced type annotation"
38if PY312_PLUS:
39 warnings.filterwarnings("ignore", "invalid escape sequence", SyntaxWarning)
42def open_source_file(filename: str) -> tuple[TextIOWrapper, str, str]:
43 # pylint: disable=consider-using-with
44 with open(filename, "rb") as byte_stream:
45 encoding = detect_encoding(byte_stream.readline)[0]
46 stream = open(filename, newline=None, encoding=encoding)
47 data = stream.read()
48 return stream, encoding, data
51def _can_assign_attr(node: nodes.ClassDef, attrname: str | None) -> bool:
52 try:
53 slots = node.slots()
54 except NotImplementedError:
55 pass
56 else:
57 if slots and attrname not in {slot.value for slot in slots}:
58 return False
59 return node.qname() != "builtins.object"
62class AstroidBuilder(raw_building.InspectBuilder):
63 """Class for building an astroid tree from source code or from a live module.
65 The param *manager* specifies the manager class which should be used.
66 If no manager is given, then the default one will be used. The
67 param *apply_transforms* determines if the transforms should be
68 applied after the tree was built from source or from a live object,
69 by default being True.
70 """
72 def __init__(
73 self, manager: AstroidManager | None = None, apply_transforms: bool = True
74 ) -> None:
75 super().__init__(manager)
76 self._apply_transforms = apply_transforms
77 if not raw_building.InspectBuilder.bootstrapped:
78 raw_building._astroid_bootstrapping()
80 def module_build(
81 self, module: types.ModuleType, modname: str | None = None
82 ) -> nodes.Module:
83 """Build an astroid from a living module instance."""
84 node = None
85 path = getattr(module, "__file__", None)
86 loader = getattr(module, "__loader__", None)
87 # Prefer the loader to get the source rather than assuming we have a
88 # filesystem to read the source file from ourselves.
89 if loader:
90 modname = modname or module.__name__
91 source = loader.get_source(modname)
92 if source:
93 node = self.string_build(source, modname, path=path)
94 if node is None and path is not None:
95 path_, ext = os.path.splitext(modutils._path_from_filename(path))
96 if ext in {".py", ".pyc", ".pyo"} and os.path.exists(path_ + ".py"):
97 node = self.file_build(path_ + ".py", modname)
98 if node is None:
99 # this is a built-in module
100 # get a partial representation by introspection
101 node = self.inspect_build(module, modname=modname, path=path)
102 if self._apply_transforms:
103 # We have to handle transformation by ourselves since the
104 # rebuilder isn't called for builtin nodes
105 node = self._manager.visit_transforms(node)
106 assert isinstance(node, nodes.Module)
107 return node
109 def file_build(self, path: str, modname: str | None = None) -> nodes.Module:
110 """Build astroid from a source code file (i.e. from an ast).
112 *path* is expected to be a python source file
113 """
114 try:
115 stream, encoding, data = open_source_file(path)
116 except OSError as exc:
117 raise AstroidBuildingError(
118 "Unable to load file {path}:\n{error}",
119 modname=modname,
120 path=path,
121 error=exc,
122 ) from exc
123 except (SyntaxError, LookupError) as exc:
124 raise AstroidSyntaxError(
125 "Python 3 encoding specification error or unknown encoding:\n"
126 "{error}",
127 modname=modname,
128 path=path,
129 error=exc,
130 ) from exc
131 except UnicodeError as exc: # wrong encoding
132 # detect_encoding returns utf-8 if no encoding specified
133 raise AstroidBuildingError(
134 "Wrong or no encoding specified for {filename}.", filename=path
135 ) from exc
136 with stream:
137 # get module name if necessary
138 if modname is None:
139 try:
140 modname = ".".join(modutils.modpath_from_file(path))
141 except ImportError:
142 modname = os.path.splitext(os.path.basename(path))[0]
143 # build astroid representation
144 module, builder = self._data_build(data, modname, path)
145 return self._post_build(module, builder, encoding)
147 def string_build(
148 self, data: str, modname: str = "", path: str | None = None
149 ) -> nodes.Module:
150 """Build astroid from source code string."""
151 module, builder = self._data_build(data, modname, path)
152 module.file_bytes = data.encode("utf-8")
153 return self._post_build(module, builder, "utf-8")
155 def _post_build(
156 self, module: nodes.Module, builder: rebuilder.TreeRebuilder, encoding: str
157 ) -> nodes.Module:
158 """Handles encoding and delayed nodes after a module has been built."""
159 module.file_encoding = encoding
160 self._manager.cache_module(module)
161 # post tree building steps after we stored the module in the cache:
162 for from_node in builder._import_from_nodes:
163 if from_node.modname == "__future__":
164 for symbol, _ in from_node.names:
165 module.future_imports.add(symbol)
166 self.add_from_names_to_locals(from_node)
167 # handle delayed assattr nodes
168 for delayed in builder._delayed_assattr:
169 self.delayed_assattr(delayed)
171 # Visit the transforms
172 if self._apply_transforms:
173 module = self._manager.visit_transforms(module)
174 return module
176 def _data_build(
177 self, data: str, modname: str, path: str | None
178 ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
179 """Build tree node from data and add some informations."""
180 try:
181 node, parser_module = _parse_string(
182 data, type_comments=True, modname=modname
183 )
184 except (TypeError, ValueError, SyntaxError) as exc:
185 raise AstroidSyntaxError(
186 "Parsing Python code failed:\n{error}",
187 source=data,
188 modname=modname,
189 path=path,
190 error=exc,
191 ) from exc
193 if path is not None:
194 node_file = os.path.abspath(path)
195 else:
196 node_file = "<?>"
197 if modname.endswith(".__init__"):
198 modname = modname[:-9]
199 package = True
200 else:
201 package = (
202 path is not None
203 and os.path.splitext(os.path.basename(path))[0] == "__init__"
204 )
205 builder = rebuilder.TreeRebuilder(self._manager, parser_module, data)
206 module = builder.visit_module(node, modname, node_file, package)
207 return module, builder
209 def add_from_names_to_locals(self, node: nodes.ImportFrom) -> None:
210 """Store imported names to the locals.
212 Resort the locals if coming from a delayed node
213 """
215 def _key_func(node: nodes.NodeNG) -> int:
216 return node.fromlineno or 0
218 def sort_locals(my_list: list[nodes.NodeNG]) -> None:
219 my_list.sort(key=_key_func)
221 assert node.parent # It should always default to the module
222 for name, asname in node.names:
223 if name == "*":
224 try:
225 imported = node.do_import_module()
226 except AstroidBuildingError:
227 continue
228 for name in imported.public_names():
229 node.parent.set_local(name, node)
230 sort_locals(node.parent.scope().locals[name]) # type: ignore[arg-type]
231 else:
232 node.parent.set_local(asname or name, node)
233 sort_locals(node.parent.scope().locals[asname or name]) # type: ignore[arg-type]
235 def delayed_assattr(self, node: nodes.AssignAttr) -> None:
236 """Visit a AssAttr node.
238 This adds name to locals and handle members definition.
239 """
240 from astroid import objects # pylint: disable=import-outside-toplevel
242 try:
243 for inferred in node.expr.infer():
244 if isinstance(inferred, util.UninferableBase):
245 continue
246 try:
247 # pylint: disable=unidiomatic-typecheck # We want a narrow check on the
248 # parent type, not all of its subclasses
249 if (
250 type(inferred) == bases.Instance
251 or type(inferred) == objects.ExceptionInstance
252 ):
253 inferred = inferred._proxied
254 iattrs = inferred.instance_attrs
255 if not _can_assign_attr(inferred, node.attrname):
256 continue
257 elif isinstance(inferred, bases.Instance):
258 # Const, Tuple or other containers that inherit from
259 # `Instance`
260 continue
261 elif isinstance(inferred, (bases.Proxy, util.UninferableBase)):
262 continue
263 elif inferred.is_function:
264 iattrs = inferred.instance_attrs
265 else:
266 iattrs = inferred.locals
267 except AttributeError:
268 # XXX log error
269 continue
270 values = iattrs.setdefault(node.attrname, [])
271 if node in values:
272 continue
273 values.append(node)
274 except InferenceError:
275 pass
278def build_namespace_package_module(name: str, path: Sequence[str]) -> nodes.Module:
279 module = nodes.Module(name, path=path, package=True)
280 module.postinit(body=[], doc_node=None)
281 return module
284def parse(
285 code: str,
286 module_name: str = "",
287 path: str | None = None,
288 apply_transforms: bool = True,
289) -> nodes.Module:
290 """Parses a source string in order to obtain an astroid AST from it.
292 :param str code: The code for the module.
293 :param str module_name: The name for the module, if any
294 :param str path: The path for the module
295 :param bool apply_transforms:
296 Apply the transforms for the give code. Use it if you
297 don't want the default transforms to be applied.
298 """
299 code = textwrap.dedent(code)
300 builder = AstroidBuilder(
301 manager=AstroidManager(), apply_transforms=apply_transforms
302 )
303 return builder.string_build(code, modname=module_name, path=path)
306def _extract_expressions(node: nodes.NodeNG) -> Iterator[nodes.NodeNG]:
307 """Find expressions in a call to _TRANSIENT_FUNCTION and extract them.
309 The function walks the AST recursively to search for expressions that
310 are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an
311 expression, it completely removes the function call node from the tree,
312 replacing it by the wrapped expression inside the parent.
314 :param node: An astroid node.
315 :type node: astroid.bases.NodeNG
316 :yields: The sequence of wrapped expressions on the modified tree
317 expression can be found.
318 """
319 if (
320 isinstance(node, nodes.Call)
321 and isinstance(node.func, nodes.Name)
322 and node.func.name == _TRANSIENT_FUNCTION
323 ):
324 real_expr = node.args[0]
325 assert node.parent
326 real_expr.parent = node.parent
327 # Search for node in all _astng_fields (the fields checked when
328 # get_children is called) of its parent. Some of those fields may
329 # be lists or tuples, in which case the elements need to be checked.
330 # When we find it, replace it by real_expr, so that the AST looks
331 # like no call to _TRANSIENT_FUNCTION ever took place.
332 for name in node.parent._astroid_fields:
333 child = getattr(node.parent, name)
334 if isinstance(child, list):
335 for idx, compound_child in enumerate(child):
336 if compound_child is node:
337 child[idx] = real_expr
338 elif child is node:
339 setattr(node.parent, name, real_expr)
340 yield real_expr
341 else:
342 for child in node.get_children():
343 yield from _extract_expressions(child)
346def _find_statement_by_line(node: nodes.NodeNG, line: int) -> nodes.NodeNG | None:
347 """Extracts the statement on a specific line from an AST.
349 If the line number of node matches line, it will be returned;
350 otherwise its children are iterated and the function is called
351 recursively.
353 :param node: An astroid node.
354 :type node: astroid.bases.NodeNG
355 :param line: The line number of the statement to extract.
356 :type line: int
357 :returns: The statement on the line, or None if no statement for the line
358 can be found.
359 :rtype: astroid.bases.NodeNG or None
360 """
361 if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.MatchCase)):
362 # This is an inaccuracy in the AST: the nodes that can be
363 # decorated do not carry explicit information on which line
364 # the actual definition (class/def), but .fromline seems to
365 # be close enough.
366 node_line = node.fromlineno
367 else:
368 node_line = node.lineno
370 if node_line == line:
371 return node
373 for child in node.get_children():
374 result = _find_statement_by_line(child, line)
375 if result:
376 return result
378 return None
381def extract_node(code: str, module_name: str = "") -> nodes.NodeNG | list[nodes.NodeNG]:
382 """Parses some Python code as a module and extracts a designated AST node.
384 Statements:
385 To extract one or more statement nodes, append #@ to the end of the line
387 Examples:
388 >>> def x():
389 >>> def y():
390 >>> return 1 #@
392 The return statement will be extracted.
394 >>> class X(object):
395 >>> def meth(self): #@
396 >>> pass
398 The function object 'meth' will be extracted.
400 Expressions:
401 To extract arbitrary expressions, surround them with the fake
402 function call __(...). After parsing, the surrounded expression
403 will be returned and the whole AST (accessible via the returned
404 node's parent attribute) will look like the function call was
405 never there in the first place.
407 Examples:
408 >>> a = __(1)
410 The const node will be extracted.
412 >>> def x(d=__(foo.bar)): pass
414 The node containing the default argument will be extracted.
416 >>> def foo(a, b):
417 >>> return 0 < __(len(a)) < b
419 The node containing the function call 'len' will be extracted.
421 If no statements or expressions are selected, the last toplevel
422 statement will be returned.
424 If the selected statement is a discard statement, (i.e. an expression
425 turned into a statement), the wrapped expression is returned instead.
427 For convenience, singleton lists are unpacked.
429 :param str code: A piece of Python code that is parsed as
430 a module. Will be passed through textwrap.dedent first.
431 :param str module_name: The name of the module.
432 :returns: The designated node from the parse tree, or a list of nodes.
433 """
435 def _extract(node: nodes.NodeNG | None) -> nodes.NodeNG | None:
436 if isinstance(node, nodes.Expr):
437 return node.value
439 return node
441 requested_lines: list[int] = []
442 for idx, line in enumerate(code.splitlines()):
443 if line.strip().endswith(_STATEMENT_SELECTOR):
444 requested_lines.append(idx + 1)
446 tree = parse(code, module_name=module_name)
447 if not tree.body:
448 raise ValueError("Empty tree, cannot extract from it")
450 extracted: list[nodes.NodeNG | None] = []
451 if requested_lines:
452 extracted = [_find_statement_by_line(tree, line) for line in requested_lines]
454 # Modifies the tree.
455 extracted.extend(_extract_expressions(tree))
457 if not extracted:
458 extracted.append(tree.body[-1])
460 extracted = [_extract(node) for node in extracted]
461 extracted_without_none = [node for node in extracted if node is not None]
462 if len(extracted_without_none) == 1:
463 return extracted_without_none[0]
464 return extracted_without_none
467def _extract_single_node(code: str, module_name: str = "") -> nodes.NodeNG:
468 """Call extract_node while making sure that only one value is returned."""
469 ret = extract_node(code, module_name)
470 if isinstance(ret, list):
471 return ret[0]
472 return ret
475def _parse_string(
476 data: str, type_comments: bool = True, modname: str | None = None
477) -> tuple[ast.Module, ParserModule]:
478 parser_module = get_parser_module(type_comments=type_comments)
479 try:
480 parsed = parser_module.parse(
481 data + "\n", type_comments=type_comments, filename=modname
482 )
483 except SyntaxError as exc:
484 # If the type annotations are misplaced for some reason, we do not want
485 # to fail the entire parsing of the file, so we need to retry the parsing without
486 # type comment support.
487 if exc.args[0] != MISPLACED_TYPE_ANNOTATION_ERROR or not type_comments:
488 raise
490 parser_module = get_parser_module(type_comments=False)
491 parsed = parser_module.parse(data + "\n", type_comments=False)
492 return parsed, parser_module