Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/builder.py: 50%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

241 statements  

1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html 

2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE 

3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt 

4 

5"""The AstroidBuilder makes astroid from living object and / or from _ast. 

6 

7The builder is not thread safe and can't be used to parse different sources 

8at the same time. 

9""" 

10 

11from __future__ import annotations 

12 

13import ast 

14import os 

15import re 

16import textwrap 

17import types 

18import warnings 

19from collections.abc import Collection, Iterator, Sequence 

20from io import TextIOWrapper 

21from tokenize import detect_encoding 

22from typing import TYPE_CHECKING, cast 

23 

24from astroid import bases, modutils, nodes, raw_building, rebuilder, util 

25from astroid.const import PY312_PLUS, PY314_PLUS 

26from astroid.exceptions import AstroidBuildingError, AstroidSyntaxError, InferenceError 

27 

28if TYPE_CHECKING: 

29 from astroid.manager import AstroidManager 

30 

31# The name of the transient function that is used to 

32# wrap expressions to be extracted when calling 

33# extract_node. 

34_TRANSIENT_FUNCTION = "__" 

35 

36# The comment used to select a statement to be extracted 

37# when calling extract_node. 

38_STATEMENT_SELECTOR = "#@" 

39 

40if PY312_PLUS: 

41 warnings.filterwarnings("ignore", ".*invalid escape sequence", SyntaxWarning) 

42if PY314_PLUS: 

43 warnings.filterwarnings( 

44 "ignore", "'(return|continue|break)' in a 'finally'", SyntaxWarning 

45 ) 

46 

47 

48def open_source_file(filename: str) -> tuple[TextIOWrapper, str, str]: 

49 # pylint: disable=consider-using-with 

50 with open(filename, "rb") as byte_stream: 

51 encoding = detect_encoding(byte_stream.readline)[0] 

52 stream = open(filename, newline=None, encoding=encoding) 

53 try: 

54 data = stream.read() 

55 except BaseException: 

56 stream.close() 

57 raise 

58 return stream, encoding, data 

59 

60 

61def _can_assign_attr(node: nodes.ClassDef, attrname: str | None) -> bool: 

62 try: 

63 slots = node.slots() 

64 except NotImplementedError: 

65 pass 

66 else: 

67 if slots and attrname not in {slot.value for slot in slots}: 

68 return False 

69 return node.qname() != "builtins.object" 

70 

71 

72class AstroidBuilder(raw_building.InspectBuilder): 

73 """Class for building an astroid tree from source code or from a live module. 

74 

75 The param *manager* specifies the manager class which should be used. The 

76 param *apply_transforms* determines if the transforms should be 

77 applied after the tree was built from source or from a live object, 

78 by default being True. 

79 """ 

80 

81 def __init__(self, manager: AstroidManager, apply_transforms: bool = True) -> None: 

82 super().__init__(manager) 

83 self._apply_transforms = apply_transforms 

84 if not raw_building.InspectBuilder.bootstrapped: 

85 manager.bootstrap() 

86 

87 def module_build( 

88 self, module: types.ModuleType, modname: str | None = None 

89 ) -> nodes.Module: 

90 """Build an astroid from a living module instance.""" 

91 node = None 

92 path = getattr(module, "__file__", None) 

93 loader = getattr(module, "__loader__", None) 

94 # Prefer the loader to get the source rather than assuming we have a 

95 # filesystem to read the source file from ourselves. 

96 if loader: 

97 modname = modname or module.__name__ 

98 source = loader.get_source(modname) 

99 if source: 

100 node = self.string_build(source, modname, path=path) 

101 if node is None and path is not None: 

102 path_, ext = os.path.splitext(modutils._path_from_filename(path)) 

103 if ext in {".py", ".pyc", ".pyo"} and os.path.exists(path_ + ".py"): 

104 node = self.file_build(path_ + ".py", modname) 

105 if node is None: 

106 # this is a built-in module 

107 # get a partial representation by introspection 

108 node = self.inspect_build(module, modname=modname, path=path) 

109 if self._apply_transforms: 

110 # We have to handle transformation by ourselves since the 

111 # rebuilder isn't called for builtin nodes 

112 node = self._manager.visit_transforms(node) 

113 assert isinstance(node, nodes.Module) 

114 return node 

115 

116 def file_build(self, path: str, modname: str | None = None) -> nodes.Module: 

117 """Build astroid from a source code file (i.e. from an ast). 

118 

119 *path* is expected to be a python source file 

120 """ 

121 try: 

122 stream, encoding, data = open_source_file(path) 

123 except OSError as exc: 

124 raise AstroidBuildingError( 

125 "Unable to load file {path}:\n{error}", 

126 modname=modname, 

127 path=path, 

128 error=exc, 

129 ) from exc 

130 except (SyntaxError, LookupError) as exc: 

131 raise AstroidSyntaxError( 

132 "Python 3 encoding specification error or unknown encoding:\n" 

133 "{error}", 

134 modname=modname, 

135 path=path, 

136 error=exc, 

137 ) from exc 

138 except UnicodeError as exc: # wrong encoding 

139 # detect_encoding returns utf-8 if no encoding specified 

140 raise AstroidBuildingError( 

141 "Wrong or no encoding specified for {filename}.", filename=path 

142 ) from exc 

143 with stream: 

144 # get module name if necessary 

145 if modname is None: 

146 try: 

147 modname = ".".join(modutils.modpath_from_file(path)) 

148 except ImportError: 

149 modname = os.path.splitext(os.path.basename(path))[0] 

150 # build astroid representation 

151 module, builder = self._data_build(data, modname, path) 

152 return self._post_build(module, builder, encoding) 

153 

154 def string_build( 

155 self, 

156 data: str, 

157 modname: str = "", 

158 path: str | None = None, 

159 is_stub: bool | None = None, 

160 ) -> nodes.Module: 

161 """Build astroid from source code string.""" 

162 module, builder = self._data_build(data, modname, path, is_stub=is_stub) 

163 module.file_bytes = data.encode("utf-8") 

164 return self._post_build(module, builder, "utf-8") 

165 

166 def _post_build( 

167 self, module: nodes.Module, builder: rebuilder.TreeRebuilder, encoding: str 

168 ) -> nodes.Module: 

169 """Handles encoding and delayed nodes after a module has been built.""" 

170 module.file_encoding = encoding 

171 self._manager.cache_module(module) 

172 # post tree building steps after we stored the module in the cache: 

173 for from_node, global_names in builder._import_from_nodes: 

174 if from_node.modname == "__future__": 

175 for symbol, _ in from_node.names: 

176 module.future_imports.add(symbol) 

177 self.add_from_names_to_locals(from_node, global_names) 

178 # handle delayed assattr nodes 

179 for delayed in builder._delayed_assattr: 

180 self.delayed_assattr(delayed) 

181 

182 # Visit the transforms 

183 if self._apply_transforms: 

184 module = self._manager.visit_transforms(module) 

185 return module 

186 

187 def _data_build( 

188 self, 

189 data: str, 

190 modname: str, 

191 path: str | None, 

192 is_stub: bool | None = None, 

193 ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]: 

194 """Build tree node from data and add some informations.""" 

195 try: 

196 node = _parse_string(data, type_comments=True, modname=modname) 

197 except (TypeError, ValueError, SyntaxError, MemoryError) as exc: 

198 raise AstroidSyntaxError( 

199 "Parsing Python code failed:\n{error}", 

200 source=data, 

201 modname=modname, 

202 path=path, 

203 error=exc, 

204 ) from exc 

205 

206 if path is not None: 

207 node_file = os.path.abspath(path) 

208 else: 

209 node_file = "<?>" 

210 if modname.endswith(".__init__"): 

211 modname = modname[:-9] 

212 package = True 

213 else: 

214 package = ( 

215 path is not None 

216 and os.path.splitext(os.path.basename(path))[0] == "__init__" 

217 ) 

218 if is_stub is None: 

219 is_stub = path is not None and path.endswith(".pyi") 

220 builder = rebuilder.TreeRebuilder(self._manager, data, is_stub=is_stub) 

221 module = builder.visit_module(node, modname, node_file, package) 

222 return module, builder 

223 

224 def add_from_names_to_locals( 

225 self, node: nodes.ImportFrom, global_name: Collection[str] 

226 ) -> None: 

227 """Store imported names to the locals. 

228 

229 Resort the locals if coming from a delayed node 

230 """ 

231 

232 def add_local(parent_or_root: nodes.NodeNG, name: str) -> None: 

233 parent_or_root.set_local(name, node) 

234 my_list = parent_or_root.scope().locals[name] 

235 if TYPE_CHECKING: 

236 my_list = cast(list[nodes.NodeNG], my_list) 

237 my_list.sort(key=lambda n: n.fromlineno or 0) 

238 

239 assert node.parent # It should always default to the module 

240 module = node.root() 

241 for name, asname in node.names: 

242 if name == "*": 

243 try: 

244 imported = node.do_import_module() 

245 except AstroidBuildingError: 

246 continue 

247 for name in imported.public_names(): 

248 if name in global_name: 

249 add_local(module, name) 

250 else: 

251 add_local(node.parent, name) 

252 else: 

253 name = asname or name 

254 if name in global_name: 

255 add_local(module, name) 

256 else: 

257 add_local(node.parent, name) 

258 

259 def delayed_assattr(self, node: nodes.AssignAttr) -> None: 

260 """Visit an AssignAttr node. 

261 

262 This adds name to locals and handle members definition. 

263 """ 

264 from astroid import objects # pylint: disable=import-outside-toplevel 

265 

266 try: 

267 for inferred in node.expr.infer(): 

268 if isinstance(inferred, util.UninferableBase): 

269 continue 

270 try: 

271 # We want a narrow check on the parent type, not all of its subclasses 

272 if type(inferred) in {bases.Instance, objects.ExceptionInstance}: 

273 inferred = inferred._proxied 

274 iattrs = inferred.instance_attrs 

275 if not _can_assign_attr(inferred, node.attrname): 

276 continue 

277 elif isinstance(inferred, bases.Instance): 

278 # Const, Tuple or other containers that inherit from 

279 # `Instance` 

280 continue 

281 elif isinstance(inferred, (bases.Proxy, util.UninferableBase)): 

282 continue 

283 elif inferred.is_function: 

284 iattrs = inferred.instance_attrs 

285 else: 

286 iattrs = inferred.locals 

287 except AttributeError: 

288 # XXX log error 

289 continue 

290 values = iattrs.setdefault(node.attrname, []) 

291 if node in values: 

292 continue 

293 values.append(node) 

294 except InferenceError: 

295 pass 

296 

297 

298def build_namespace_package_module(name: str, path: Sequence[str]) -> nodes.Module: 

299 module = nodes.Module(name, path=path, package=True) 

300 module.postinit(body=[], doc_node=None) 

301 return module 

302 

303 

304def parse( 

305 code: str, 

306 module_name: str = "", 

307 path: str | None = None, 

308 apply_transforms: bool = True, 

309 is_stub: bool | None = None, 

310) -> nodes.Module: 

311 """Parses a source string in order to obtain an astroid AST from it. 

312 

313 :param code: The code for the module. 

314 :param module_name: The name for the module, if any 

315 :param path: The path for the module 

316 :param apply_transforms: 

317 Apply the transforms for the give code. Use it if you 

318 don't want the default transforms to be applied. 

319 :param is_stub: Whether to treat the code as a ``.pyi`` stub. 

320 When ``None``, auto-detected from *path*. 

321 """ 

322 # pylint: disable-next=import-outside-toplevel 

323 from astroid.manager import AstroidManager 

324 

325 code = textwrap.dedent(code) 

326 builder = AstroidBuilder(AstroidManager(), apply_transforms=apply_transforms) 

327 return builder.string_build(code, modname=module_name, path=path, is_stub=is_stub) 

328 

329 

330def _extract_expressions(node: nodes.NodeNG) -> Iterator[nodes.NodeNG]: 

331 """Find expressions in a call to _TRANSIENT_FUNCTION and extract them. 

332 

333 The function walks the AST recursively to search for expressions that 

334 are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an 

335 expression, it completely removes the function call node from the tree, 

336 replacing it by the wrapped expression inside the parent. 

337 

338 :param node: An astroid node. 

339 :yields: The sequence of wrapped expressions on the modified tree 

340 expression can be found. 

341 """ 

342 if ( 

343 isinstance(node, nodes.Call) 

344 and isinstance(node.func, nodes.Name) 

345 and node.func.name == _TRANSIENT_FUNCTION 

346 and node.args 

347 ): 

348 real_expr = node.args[0] 

349 assert node.parent 

350 real_expr.parent = node.parent 

351 # Search for node in all _astng_fields (the fields checked when 

352 # get_children is called) of its parent. Some of those fields may 

353 # be lists or tuples, in which case the elements need to be checked. 

354 # When we find it, replace it by real_expr, so that the AST looks 

355 # like no call to _TRANSIENT_FUNCTION ever took place. 

356 for name in node.parent._astroid_fields: 

357 child = getattr(node.parent, name) 

358 if isinstance(child, list): 

359 for idx, compound_child in enumerate(child): 

360 if compound_child is node: 

361 child[idx] = real_expr 

362 elif child is node: 

363 setattr(node.parent, name, real_expr) 

364 yield real_expr 

365 else: 

366 for child in node.get_children(): 

367 yield from _extract_expressions(child) 

368 

369 

370def _find_statement_by_line(node: nodes.NodeNG, line: int) -> nodes.NodeNG | None: 

371 """Extracts the statement on a specific line from an AST. 

372 

373 If the line number of node matches line, it will be returned; 

374 otherwise its children are iterated and the function is called 

375 recursively. 

376 

377 :param node: An astroid node. 

378 :param line: The line number of the statement to extract. 

379 :returns: The statement on the line, or None if no statement for the line 

380 can be found. 

381 """ 

382 if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.MatchCase)): 

383 # This is an inaccuracy in the AST: the nodes that can be 

384 # decorated do not carry explicit information on which line 

385 # the actual definition (class/def), but .fromline seems to 

386 # be close enough. 

387 node_line = node.fromlineno 

388 else: 

389 node_line = node.lineno 

390 

391 if node_line == line: 

392 return node 

393 

394 for child in node.get_children(): 

395 result = _find_statement_by_line(child, line) 

396 if result: 

397 return result 

398 

399 return None 

400 

401 

402def extract_node(code: str, module_name: str = "") -> nodes.NodeNG | list[nodes.NodeNG]: 

403 """Parses some Python code as a module and extracts a designated AST node. 

404 

405 Statements: 

406 To extract one or more statement nodes, append #@ to the end of the line 

407 

408 Examples:: 

409 

410 def x(): 

411 def y(): 

412 return 1 #@ 

413 

414 The return statement will be extracted. 

415 

416 :: 

417 

418 class X(object): 

419 def meth(self): #@ 

420 pass 

421 

422 The function object 'meth' will be extracted. 

423 

424 Expressions: 

425 To extract arbitrary expressions, surround them with the fake 

426 function call __(...). After parsing, the surrounded expression 

427 will be returned and the whole AST (accessible via the returned 

428 node's parent attribute) will look like the function call was 

429 never there in the first place. 

430 

431 Examples:: 

432 

433 a = __(1) 

434 

435 The const node will be extracted. 

436 

437 :: 

438 

439 def x(d=__(foo.bar)): pass 

440 

441 The node containing the default argument will be extracted. 

442 

443 :: 

444 

445 def foo(a, b): 

446 return 0 < __(len(a)) < b 

447 

448 The node containing the function call 'len' will be extracted. 

449 

450 If no statements or expressions are selected, the last toplevel 

451 statement will be returned. 

452 

453 If the selected statement is a discard statement, (i.e. an expression 

454 turned into a statement), the wrapped expression is returned instead. 

455 

456 For convenience, singleton lists are unpacked. 

457 

458 :param code: A piece of Python code that is parsed as 

459 a module. Will be passed through textwrap.dedent first. 

460 :param module_name: The name of the module. 

461 :returns: The designated node from the parse tree, or a list of nodes. 

462 """ 

463 

464 def _extract(node: nodes.NodeNG | None) -> nodes.NodeNG | None: 

465 if isinstance(node, nodes.Expr): 

466 return node.value 

467 

468 return node 

469 

470 requested_lines: list[int] = [] 

471 for idx, line in enumerate(code.splitlines()): 

472 if line.strip().endswith(_STATEMENT_SELECTOR): 

473 requested_lines.append(idx + 1) 

474 

475 tree = parse(code, module_name=module_name) 

476 if not tree.body: 

477 raise ValueError("Empty tree, cannot extract from it") 

478 

479 extracted: list[nodes.NodeNG | None] = [] 

480 if requested_lines: 

481 extracted = [_find_statement_by_line(tree, line) for line in requested_lines] 

482 

483 # Modifies the tree. 

484 extracted.extend(_extract_expressions(tree)) 

485 

486 if not extracted: 

487 extracted.append(tree.body[-1]) 

488 

489 extracted = [_extract(node) for node in extracted] 

490 extracted_without_none = [node for node in extracted if node is not None] 

491 if len(extracted_without_none) == 1: 

492 return extracted_without_none[0] 

493 return extracted_without_none 

494 

495 

496def _extract_single_node(code: str, module_name: str = "") -> nodes.NodeNG: 

497 """Call extract_node while making sure that only one value is returned.""" 

498 ret = extract_node(code, module_name) 

499 if isinstance(ret, list): 

500 return ret[0] 

501 return ret 

502 

503 

504def _parse_string( 

505 data: str, type_comments: bool = True, modname: str | None = None 

506) -> ast.Module: 

507 try: 

508 parsed = ast.parse( 

509 data + "\n", filename=modname or "<unknown>", type_comments=type_comments 

510 ) 

511 except SyntaxError as exc: 

512 # If the type annotations are misplaced for some reason, we do not want 

513 # to fail the entire parsing of the file, so we need to retry the 

514 # parsing without type comment support. We use a heuristic for 

515 # determining if the error is due to type annotations. 

516 type_annot_related = re.search(r"#\s+type:", exc.text or "") 

517 if not (type_annot_related and type_comments): 

518 raise 

519 

520 parsed = ast.parse(data + "\n", type_comments=False) 

521 return parsed