Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/asttokens/asttokens.py: 59%

1# Copyright 2016 Grist Labs, Inc. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# https://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15import abc 

16import ast 

17import bisect 

18import sys 

19import token 

20from ast import Module 

21from typing import Iterable, Iterator, List, Optional, Tuple, Any, cast 

22 

23from .line_numbers import LineNumbers 

24from .util import ( 

25 AstNode, Token, TokenInfo, match_token, is_non_coding_token, patched_generate_tokens, last_stmt, 

26 annotate_fstring_nodes, generate_tokens, is_module, is_stmt 

27) 

28 

29 

30class ASTTextBase(metaclass=abc.ABCMeta): 

31 def __init__(self, source_text: str, filename: str) -> None: 

32 self._filename = filename 

33 

34 # Decode source after parsing to let Python 2 handle coding declarations. 

35 # (If the encoding was not utf-8 compatible, then even if it parses correctly, 

36 # we'll fail with a unicode error here.) 

37 source_text = str(source_text) 

38 

39 self._text = source_text 

40 self._line_numbers = LineNumbers(source_text) 

41 

42 @abc.abstractmethod 

43 def get_text_positions( 

44 self, node: AstNode, padded: bool 

45 ) -> Tuple[Tuple[int, int], Tuple[int, int]]: 

46 """ 

47 Returns two ``(lineno, col_offset)`` tuples for the start and end of the given node. 

48 If the positions can't be determined, or the nodes don't correspond to any particular text, 

49 returns ``(1, 0)`` for both. 

50 

51 ``padded`` corresponds to the ``padded`` argument to ``ast.get_source_segment()``. 

52 This means that if ``padded`` is True, the start position will be adjusted to include 

53 leading whitespace if ``node`` is a multiline statement. 

54 """ 

55 raise NotImplementedError # pragma: no cover 

56 

57 def get_text_range(self, node: AstNode, padded: bool = True) -> Tuple[int, int]: 

58 """ 

59 Returns the (startpos, endpos) positions in source text corresponding to the given node. 

60 Returns (0, 0) for nodes (like `Load`) that don't correspond to any particular text. 

61 

62 See ``get_text_positions()`` for details on the ``padded`` argument. 

63 """ 

64 start, end = self.get_text_positions(node, padded) 

65 return ( 

66 self._line_numbers.line_to_offset(*start), 

67 self._line_numbers.line_to_offset(*end), 

68 ) 

69 

70 def get_text(self, node: AstNode, padded: bool = True) -> str: 

71 """ 

72 Returns the text corresponding to the given node. 

73 Returns '' for nodes (like `Load`) that don't correspond to any particular text. 

74 

75 See ``get_text_positions()`` for details on the ``padded`` argument. 

76 """ 

77 start, end = self.get_text_range(node, padded) 

78 return self._text[start: end] 

79 

80 

81class ASTTokens(ASTTextBase): 

82 """ 

83 ASTTokens maintains the text of Python code in several forms: as a string, as line numbers, and 

84 as tokens, and is used to mark and access token and position information. 

85 

86 ``source_text`` must be a unicode or UTF8-encoded string. If you pass in UTF8 bytes, remember 

87 that all offsets you'll get are to the unicode text, which is available as the ``.text`` 

88 property. 

89 

90 If ``parse`` is set, the ``source_text`` will be parsed with ``ast.parse()``, and the resulting 

91 tree marked with token info and made available as the ``.tree`` property. 

92 

93 If ``tree`` is given, it will be marked and made available as the ``.tree`` property. In 

94 addition to the trees produced by the ``ast`` module, ASTTokens will also mark trees produced 

95 using ``astroid`` library <https://www.astroid.org>. 

96 

97 If only ``source_text`` is given, you may use ``.mark_tokens(tree)`` to mark the nodes of an AST 

98 tree created separately. 

99 """ 

100 

101 def __init__( 

102 self, 

103 source_text: Any, 

104 parse: bool = False, 

105 tree: Optional[Module] = None, 

106 filename: str = '<unknown>', 

107 tokens: Optional[Iterable[TokenInfo]] = None 

108 ) -> None: 

109 super().__init__(source_text, filename) 

110 

111 self._tree = ast.parse(source_text, filename) if parse else tree 

112 

113 # Tokenize the code. 

114 if tokens is None: 

115 tokens = generate_tokens(self._text) 

116 self._tokens = list(self._translate_tokens(tokens)) 

117 

118 # Extract the start positions of all tokens, so that we can quickly map positions to tokens. 

119 self._token_offsets = [tok.startpos for tok in self._tokens] 

120 

121 if self._tree: 

122 self.mark_tokens(self._tree) 

123 

124 def mark_tokens(self, root_node: Module) -> None: 

125 """ 

126 Given the root of the AST or Astroid tree produced from source_text, visits all nodes marking 

127 them with token and position information by adding ``.first_token`` and 

128 ``.last_token`` attributes. This is done automatically in the constructor when ``parse`` or 

129 ``tree`` arguments are set, but may be used manually with a separate AST or Astroid tree. 

130 """ 

131 # The hard work of this class is done by MarkTokens 

132 from .mark_tokens import MarkTokens # to avoid import loops 

133 MarkTokens(self).visit_tree(root_node) 

134 

135 def _translate_tokens(self, original_tokens: Iterable[TokenInfo]) -> Iterator[Token]: 

136 """ 

137 Translates the given standard library tokens into our own representation. 

138 """ 

139 for index, tok in enumerate(patched_generate_tokens(original_tokens)): 

140 tok_type, tok_str, start, end, line = tok 

141 yield Token(tok_type, tok_str, start, end, line, index, 

142 self._line_numbers.line_to_offset(start[0], start[1]), 

143 self._line_numbers.line_to_offset(end[0], end[1])) 

144 

145 @property 

146 def text(self) -> str: 

147 """The source code passed into the constructor.""" 

148 return self._text 

149 

150 @property 

151 def tokens(self) -> List[Token]: 

152 """The list of tokens corresponding to the source code from the constructor.""" 

153 return self._tokens 

154 

155 @property 

156 def tree(self) -> Optional[Module]: 

157 """The root of the AST tree passed into the constructor or parsed from the source code.""" 

158 return self._tree 

159 

160 @property 

161 def filename(self) -> str: 

162 """The filename that was parsed""" 

163 return self._filename 

164 

165 def get_token_from_offset(self, offset: int) -> Token: 

166 """ 

167 Returns the token containing the given character offset (0-based position in source text), 

168 or the preceeding token if the position is between tokens. 

169 """ 

170 return self._tokens[bisect.bisect(self._token_offsets, offset) - 1] 

171 

172 def get_token(self, lineno: int, col_offset: int) -> Token: 

173 """ 

174 Returns the token containing the given (lineno, col_offset) position, or the preceeding token 

175 if the position is between tokens. 

176 """ 

177 # TODO: add test for multibyte unicode. We need to translate offsets from ast module (which 

178 # are in utf8) to offsets into the unicode text. tokenize module seems to use unicode offsets 

179 # but isn't explicit. 

180 return self.get_token_from_offset(self._line_numbers.line_to_offset(lineno, col_offset)) 

181 

182 def get_token_from_utf8(self, lineno: int, col_offset: int) -> Token: 

183 """ 

184 Same as get_token(), but interprets col_offset as a UTF8 offset, which is what `ast` uses. 

185 """ 

186 return self.get_token(lineno, self._line_numbers.from_utf8_col(lineno, col_offset)) 

187 

188 def next_token(self, tok: Token, include_extra: bool = False) -> Token: 

189 """ 

190 Returns the next token after the given one. If include_extra is True, includes non-coding 

191 tokens from the tokenize module, such as NL and COMMENT. 

192 """ 

193 i = tok.index + 1 

194 if not include_extra: 

195 while is_non_coding_token(self._tokens[i].type): 

196 i += 1 

197 return self._tokens[i] 

198 

199 def prev_token(self, tok: Token, include_extra: bool = False) -> Token: 

200 """ 

201 Returns the previous token before the given one. If include_extra is True, includes non-coding 

202 tokens from the tokenize module, such as NL and COMMENT. 

203 """ 

204 i = tok.index - 1 

205 if not include_extra: 

206 while is_non_coding_token(self._tokens[i].type): 

207 i -= 1 

208 return self._tokens[i] 

209 

210 def find_token( 

211 self, start_token: Token, tok_type: int, tok_str: Optional[str] = None, reverse: bool = False 

212 ) -> Token: 

213 """ 

214 Looks for the first token, starting at start_token, that matches tok_type and, if given, the 

215 token string. Searches backwards if reverse is True. Returns ENDMARKER token if not found (you 

216 can check it with `token.ISEOF(t.type)`). 

217 """ 

218 t = start_token 

219 advance = self.prev_token if reverse else self.next_token 

220 while not match_token(t, tok_type, tok_str) and not token.ISEOF(t.type): 

221 t = advance(t, include_extra=True) 

222 return t 

223 

224 def token_range( 

225 self, 

226 first_token: Token, 

227 last_token: Token, 

228 include_extra: bool = False, 

229 ) -> Iterator[Token]: 

230 """ 

231 Yields all tokens in order from first_token through and including last_token. If 

232 include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT. 

233 """ 

234 for i in range(first_token.index, last_token.index + 1): 

235 if include_extra or not is_non_coding_token(self._tokens[i].type): 

236 yield self._tokens[i] 

237 

238 def get_tokens(self, node: AstNode, include_extra: bool = False) -> Iterator[Token]: 

239 """ 

240 Yields all tokens making up the given node. If include_extra is True, includes non-coding 

241 tokens such as tokenize.NL and .COMMENT. 

242 """ 

243 return self.token_range(node.first_token, node.last_token, include_extra=include_extra) 

244 

245 def get_text_positions(self, node: AstNode, padded: bool) -> Tuple[Tuple[int, int], Tuple[int, int]]: 

246 """ 

247 Returns two ``(lineno, col_offset)`` tuples for the start and end of the given node. 

248 If the positions can't be determined, or the nodes don't correspond to any particular text, 

249 returns ``(1, 0)`` for both. 

250 

251 ``padded`` corresponds to the ``padded`` argument to ``ast.get_source_segment()``. 

252 This means that if ``padded`` is True, the start position will be adjusted to include 

253 leading whitespace if ``node`` is a multiline statement. 

254 """ 

255 if not hasattr(node, 'first_token'): 

256 return (1, 0), (1, 0) 

257 

258 start = node.first_token.start 

259 end = node.last_token.end 

260 if padded and any(match_token(t, token.NEWLINE) for t in self.get_tokens(node)): 

261 # Set col_offset to 0 to include leading indentation for multiline statements. 

262 start = (start[0], 0) 

263 

264 return start, end 

265 

266 

267class ASTText(ASTTextBase): 

268 """ 

269 Supports the same ``get_text*`` methods as ``ASTTokens``, 

270 but uses the AST to determine the text positions instead of tokens. 

271 This is faster than ``ASTTokens`` as it requires less setup work. 

272 

273 It also (sometimes) supports nodes inside f-strings, which ``ASTTokens`` doesn't. 

274 

275 Some node types and/or Python versions are not supported. 

276 In these cases the ``get_text*`` methods will fall back to using ``ASTTokens`` 

277 which incurs the usual setup cost the first time. 

278 If you want to avoid this, check ``supports_tokenless(node)`` before calling ``get_text*`` methods. 

279 """ 

280 def __init__(self, source_text: Any, tree: Optional[Module] = None, filename: str = '<unknown>') -> None: 

281 super().__init__(source_text, filename) 

282 

283 self._tree = tree 

284 if self._tree is not None: 

285 annotate_fstring_nodes(self._tree) 

286 

287 self._asttokens: Optional[ASTTokens] = None 

288 

289 @property 

290 def tree(self) -> Module: 

291 if self._tree is None: 

292 self._tree = ast.parse(self._text, self._filename) 

293 annotate_fstring_nodes(self._tree) 

294 return self._tree 

295 

296 @property 

297 def asttokens(self) -> ASTTokens: 

298 if self._asttokens is None: 

299 self._asttokens = ASTTokens( 

300 self._text, 

301 tree=self.tree, 

302 filename=self._filename, 

303 ) 

304 return self._asttokens 

305 

306 def _get_text_positions_tokenless( 

307 self, node: AstNode, padded: bool 

308 ) -> Tuple[Tuple[int, int], Tuple[int, int]]: 

309 """ 

310 Version of ``get_text_positions()`` that doesn't use tokens. 

311 """ 

312 if is_module(node): 

313 # Modules don't have position info, so just return the range of the whole text. 

314 # The token-using method does something different, but its behavior seems weird and inconsistent. 

315 # For example, in a file with only comments, it only returns the first line. 

316 # It's hard to imagine a case when this matters. 

317 return (1, 0), self._line_numbers.offset_to_line(len(self._text)) 

318 

319 if getattr(node, 'lineno', None) is None: 

320 return (1, 0), (1, 0) 

321 

322 assert node # tell mypy that node is not None, which we allowed up to here for compatibility 

323 

324 decorators = getattr(node, 'decorator_list', []) 

325 if not decorators: 

326 # Astroid uses node.decorators.nodes instead of node.decorator_list. 

327 decorators_node = getattr(node, 'decorators', None) 

328 decorators = getattr(decorators_node, 'nodes', []) 

329 if decorators: 

330 # Function/Class definition nodes are marked by AST as starting at def/class, 

331 # not the first decorator. This doesn't match the token-using behavior, 

332 # or inspect.getsource(), and just seems weird. 

333 start_node = decorators[0] 

334 else: 

335 start_node = node 

336 

337 start_lineno = start_node.lineno 

338 end_node = last_stmt(node) 

339 

340 # Include leading indentation for multiline statements. 

341 # This doesn't mean simple statements that happen to be on multiple lines, 

342 # but compound statements where inner indentation matters. 

343 # So we don't just compare node.lineno and node.end_lineno, 

344 # we check for a contained statement starting on a different line. 

345 if padded and ( 

346 start_lineno != end_node.lineno 

347 or ( 

348 # Astroid docstrings aren't treated as separate statements. 

349 # So to handle function/class definitions with a docstring but no other body, 

350 # we just check that the node is a statement with a docstring 

351 # and spanning multiple lines in the simple, literal sense. 

352 start_lineno != node.end_lineno 

353 and getattr(node, "doc_node", None) 

354 and is_stmt(node) 

355 ) 

356 ): 

357 start_col_offset = 0 

358 else: 

359 start_col_offset = self._line_numbers.from_utf8_col(start_lineno, start_node.col_offset) 

360 

361 start = (start_lineno, start_col_offset) 

362 

363 # To match the token-using behaviour, we exclude trailing semicolons and comments. 

364 # This means that for blocks containing multiple statements, we have to use the last one 

365 # instead of the actual node for end_lineno and end_col_offset. 

366 end_lineno = cast(int, end_node.end_lineno) 

367 end_col_offset = cast(int, end_node.end_col_offset) 

368 end_col_offset = self._line_numbers.from_utf8_col(end_lineno, end_col_offset) 

369 end = (end_lineno, end_col_offset) 

370 

371 return start, end 

372 

373 def get_text_positions(self, node: AstNode, padded: bool) -> Tuple[Tuple[int, int], Tuple[int, int]]: 

374 """ 

375 Returns two ``(lineno, col_offset)`` tuples for the start and end of the given node. 

376 If the positions can't be determined, or the nodes don't correspond to any particular text, 

377 returns ``(1, 0)`` for both. 

378 

379 ``padded`` corresponds to the ``padded`` argument to ``ast.get_source_segment()``. 

380 This means that if ``padded`` is True, the start position will be adjusted to include 

381 leading whitespace if ``node`` is a multiline statement. 

382 """ 

383 if getattr(node, "_broken_positions", None): 

384 # This node was marked in util.annotate_fstring_nodes as having untrustworthy lineno/col_offset. 

385 return (1, 0), (1, 0) 

386 

387 if supports_tokenless(node): 

388 return self._get_text_positions_tokenless(node, padded) 

389 

390 return self.asttokens.get_text_positions(node, padded) 

391 

392 

393# Node types that _get_text_positions_tokenless doesn't support. 

394# These initial values are missing lineno. 

395_unsupported_tokenless_types: Tuple[str, ...] = ("arguments", "Arguments", "withitem") 

396if sys.version_info[:2] == (3, 8): 

397 # _get_text_positions_tokenless works incorrectly for these types due to bugs in Python 3.8. 

398 _unsupported_tokenless_types += ("arg", "Starred") 

399 # no lineno in 3.8 

400 _unsupported_tokenless_types += ("Slice", "ExtSlice", "Index", "keyword") 

401 

402 

403def supports_tokenless(node: Any = None) -> bool: 

404 """ 

405 Returns True if the Python version and the node (if given) are supported by 

406 the ``get_text*`` methods of ``ASTText`` without falling back to ``ASTTokens``. 

407 See ``ASTText`` for why this matters. 

408 

409 The following cases are not supported: 

410 

411 - PyPy 

412 - ``ast.arguments`` / ``astroid.Arguments`` 

413 - ``ast.withitem`` 

414 - ``astroid.Comprehension`` 

415 - ``astroid.AssignName`` inside ``astroid.Arguments`` or ``astroid.ExceptHandler`` 

416 - The following nodes in Python 3.8 only: 

417 - ``ast.arg`` 

418 - ``ast.Starred`` 

419 - ``ast.Slice`` 

420 - ``ast.ExtSlice`` 

421 - ``ast.Index`` 

422 - ``ast.keyword`` 

423 """ 

424 return ( 

425 type(node).__name__ not in _unsupported_tokenless_types 

426 and not ( 

427 # astroid nodes 

428 not isinstance(node, ast.AST) and node is not None and ( 

429 type(node).__name__ == "AssignName" 

430 and type(node.parent).__name__ in ("Arguments", "ExceptHandler") 

431 ) 

432 ) 

433 and 'pypy' not in sys.version.lower() 

434 )