Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/black/parsing.py: 53%

1""" 

2Parse Python code and perform AST validation. 

3""" 

4 

5import ast 

6import sys 

7import warnings 

8from collections.abc import Collection, Iterator 

9 

10from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature 

11from black.nodes import syms 

12from blib2to3 import pygram 

13from blib2to3.pgen2 import driver 

14from blib2to3.pgen2.grammar import Grammar 

15from blib2to3.pgen2.parse import ParseError 

16from blib2to3.pgen2.tokenize import TokenError 

17from blib2to3.pytree import Leaf, Node 

18 

19 

20class InvalidInput(ValueError): 

21 """Raised when input source code fails all parse attempts.""" 

22 

23 lineno: int | None = None 

24 column: int | None = None 

25 context: str | None = None 

26 details: str | None = None 

27 

28 def __init__( 

29 self, 

30 message: str, 

31 lineno: int | None = None, 

32 column: int | None = None, 

33 context: str | None = None, 

34 details: str | None = None, 

35 ) -> None: 

36 super().__init__(message, lineno, column, context, details) 

37 self.lineno = lineno 

38 self.column = column 

39 self.context = context 

40 self.details = details 

41 

42 def __str__(self) -> str: 

43 return str(self.args[0]) 

44 

45 

46def get_grammars(target_versions: set[TargetVersion]) -> list[Grammar]: 

47 if not target_versions: 

48 # No target_version specified, so try all grammars. 

49 return [ 

50 # Python 3.7-3.9 

51 pygram.python_grammar_async_keywords, 

52 # Python 3.0-3.6 

53 pygram.python_grammar, 

54 # Python 3.10+ 

55 pygram.python_grammar_soft_keywords, 

56 ] 

57 

58 grammars = [] 

59 # If we have to parse both, try to parse async as a keyword first 

60 if not supports_feature( 

61 target_versions, Feature.ASYNC_IDENTIFIERS 

62 ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING): 

63 # Python 3.7-3.9 

64 grammars.append(pygram.python_grammar_async_keywords) 

65 if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): 

66 # Python 3.0-3.6 

67 grammars.append(pygram.python_grammar) 

68 if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions): 

69 # Python 3.10+ 

70 grammars.append(pygram.python_grammar_soft_keywords) 

71 

72 # At least one of the above branches must have been taken, because every Python 

73 # version has exactly one of the two 'ASYNC_*' flags 

74 return grammars 

75 

76 

77def lib2to3_parse( 

78 src_txt: str, target_versions: Collection[TargetVersion] = () 

79) -> Node: 

80 """Given a string with source, return the lib2to3 Node.""" 

81 if not src_txt.endswith("\n"): 

82 src_txt += "\n" 

83 

84 grammars = get_grammars(set(target_versions)) 

85 if target_versions: 

86 max_tv = max(target_versions, key=lambda tv: tv.value) 

87 tv_str = f" for target version {max_tv.pretty()}" 

88 else: 

89 tv_str = "" 

90 

91 errors = {} 

92 for grammar in grammars: 

93 drv = driver.Driver(grammar) 

94 try: 

95 result = drv.parse_string(src_txt, False) 

96 break 

97 

98 except ParseError as pe: 

99 lineno, column = pe.context[1] 

100 lines = src_txt.splitlines() 

101 try: 

102 faulty_line = lines[lineno - 1] 

103 except IndexError: 

104 faulty_line = "<line number missing in source>" 

105 context = f"cannot parse{tv_str}" 

106 details = "\n".join(( 

107 "", 

108 f" {faulty_line}", 

109 f" {' ' * (column - 1)}^", 

110 f"ParseError: {pe.msg}", 

111 )) 

112 error_msg = f"{context}: {lineno}:{column}{details}" 

113 errors[grammar.version] = InvalidInput( 

114 error_msg, lineno, column, context, details 

115 ) 

116 

117 except TokenError as te: 

118 lineno, column = te.args[1] 

119 lines = src_txt.splitlines() 

120 try: 

121 faulty_line = lines[lineno - 1] 

122 except IndexError: 

123 faulty_line = "<line number missing in source>" 

124 context = f"cannot parse{tv_str}" 

125 details = "\n".join(( 

126 "", 

127 f" {faulty_line}", 

128 f" {' ' * (column - 1)}^", 

129 f"TokenError: {te.args[0]}", 

130 )) 

131 error_msg = f"{context}: {lineno}:{column}{details}" 

132 errors[grammar.version] = InvalidInput( 

133 error_msg, lineno, column, context, details 

134 ) 

135 

136 else: 

137 # Choose the latest version when raising the actual parsing error. 

138 assert len(errors) >= 1 

139 exc = errors[max(errors)] 

140 raise exc from None 

141 

142 if isinstance(result, Leaf): 

143 result = Node(syms.file_input, [result]) 

144 return result 

145 

146 

147class ASTSafetyError(Exception): 

148 """Raised when Black's generated code is not equivalent to the old AST.""" 

149 

150 

151class SourceASTParseError(Exception): 

152 """Raised when the source file cannot be parsed by ast.parse(). 

153 

154 This is not a bug in Black — Black's lib2to3-based parser is more lenient 

155 than Python's ast.parse(), so it may accept code that ast.parse() rejects. 

156 In blackd, this should be reported as a 400 Bad Request. 

157 """ 

158 

159 

160def _parse_single_version( 

161 src: str, version: tuple[int, int], *, type_comments: bool 

162) -> ast.AST: 

163 filename = "<unknown>" 

164 with warnings.catch_warnings(): 

165 warnings.simplefilter("ignore", SyntaxWarning) 

166 warnings.simplefilter("ignore", DeprecationWarning) 

167 return ast.parse( 

168 src, filename, feature_version=version, type_comments=type_comments 

169 ) 

170 

171 

172def parse_ast(src: str) -> ast.AST: 

173 # TODO: support Python 4+ ;) 

174 versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)] 

175 

176 first_error = "" 

177 for version in sorted(versions, reverse=True): 

178 try: 

179 return _parse_single_version(src, version, type_comments=True) 

180 except SyntaxError as e: 

181 if not first_error: 

182 first_error = str(e) 

183 

184 # Try to parse without type comments 

185 for version in sorted(versions, reverse=True): 

186 try: 

187 return _parse_single_version(src, version, type_comments=False) 

188 except SyntaxError: 

189 pass 

190 

191 raise SyntaxError(first_error) 

192 

193 

194def _normalize(lineend: str, value: str) -> str: 

195 # To normalize, we strip any leading and trailing space from 

196 # each line... 

197 stripped: list[str] = [i.strip() for i in value.splitlines()] 

198 normalized = lineend.join(stripped) 

199 # ...and remove any blank lines at the beginning and end of 

200 # the whole string 

201 return normalized.strip() 

202 

203 

204def stringify_ast(node: ast.AST) -> Iterator[str]: 

205 """Simple visitor generating strings to compare ASTs by content.""" 

206 return _stringify_ast(node, []) 

207 

208 

209def _stringify_ast_with_new_parent( 

210 node: ast.AST, parent_stack: list[ast.AST], new_parent: ast.AST 

211) -> Iterator[str]: 

212 parent_stack.append(new_parent) 

213 yield from _stringify_ast(node, parent_stack) 

214 parent_stack.pop() 

215 

216 

217def _stringify_ast(node: ast.AST, parent_stack: list[ast.AST]) -> Iterator[str]: 

218 if ( 

219 isinstance(node, ast.Constant) 

220 and isinstance(node.value, str) 

221 and node.kind == "u" 

222 ): 

223 # It's a quirk of history that we strip the u prefix over here. We used to 

224 # rewrite the AST nodes for Python version compatibility and we never copied 

225 # over the kind 

226 node.kind = None 

227 

228 yield f"{' ' * len(parent_stack)}{node.__class__.__name__}(" 

229 

230 for field in sorted(node._fields): 

231 # TypeIgnore has only one field 'lineno' which breaks this comparison 

232 if isinstance(node, ast.TypeIgnore): 

233 break 

234 

235 try: 

236 value: object = getattr(node, field) 

237 except AttributeError: 

238 continue 

239 

240 yield f"{' ' * (len(parent_stack) + 1)}{field}=" 

241 

242 if isinstance(value, list): 

243 for item in value: 

244 # Ignore nested tuples within del statements, because we may insert 

245 # parentheses and they change the AST. 

246 if ( 

247 field == "targets" 

248 and isinstance(node, ast.Delete) 

249 and isinstance(item, ast.Tuple) 

250 ): 

251 for elt in _unwrap_tuples(item): 

252 yield from _stringify_ast_with_new_parent( 

253 elt, parent_stack, node 

254 ) 

255 

256 elif isinstance(item, ast.AST): 

257 yield from _stringify_ast_with_new_parent(item, parent_stack, node) 

258 

259 elif isinstance(value, ast.AST): 

260 yield from _stringify_ast_with_new_parent(value, parent_stack, node) 

261 

262 else: 

263 normalized: object 

264 if ( 

265 isinstance(node, ast.Constant) 

266 and field == "value" 

267 and isinstance(value, str) 

268 and len(parent_stack) >= 2 

269 # Any standalone string, ideally this would 

270 # exactly match black.nodes.is_docstring 

271 and isinstance(parent_stack[-1], ast.Expr) 

272 ): 

273 # Constant strings may be indented across newlines, if they are 

274 # docstrings; fold spaces after newlines when comparing. Similarly, 

275 # trailing and leading space may be removed. 

276 normalized = _normalize("\n", value) 

277 elif field == "type_comment" and isinstance(value, str): 

278 # Trailing whitespace in type comments is removed. 

279 normalized = value.rstrip() 

280 else: 

281 normalized = value 

282 yield ( 

283 f"{' ' * (len(parent_stack) + 1)}{normalized!r}, #" 

284 f" {value.__class__.__name__}" 

285 ) 

286 

287 yield f"{' ' * len(parent_stack)}) # /{node.__class__.__name__}" 

288 

289 

290def _unwrap_tuples(node: ast.Tuple) -> Iterator[ast.AST]: 

291 for elt in node.elts: 

292 if isinstance(elt, ast.Tuple): 

293 yield from _unwrap_tuples(elt) 

294 else: 

295 yield elt