Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pip/_vendor/packaging/_parser.py: 24%

1"""Handwritten parser of dependency specifiers. 

2 

3The docstring for each __parse_* function contains EBNF-inspired grammar representing 

4the implementation. 

5""" 

6 

7from __future__ import annotations 

8 

9import ast 

10from collections.abc import Sequence 

11from typing import Literal, NamedTuple, Union 

12 

13from ._tokenizer import DEFAULT_RULES, ParserSyntaxError, Tokenizer 

14 

15 

16class Node: 

17 __slots__ = ("value",) 

18 

19 def __init__(self, value: str) -> None: 

20 self.value = value 

21 

22 def __str__(self) -> str: 

23 return self.value 

24 

25 def __repr__(self) -> str: 

26 return f"<{self.__class__.__name__}({self.value!r})>" 

27 

28 def serialize(self) -> str: 

29 raise NotImplementedError 

30 

31 def __getstate__(self) -> str: 

32 # Return just the value string for compactness and stability. 

33 return self.value 

34 

35 def _restore_value(self, value: object) -> None: 

36 if not isinstance(value, str): 

37 raise TypeError( 

38 f"Cannot restore {self.__class__.__name__} value from {value!r}" 

39 ) 

40 self.value = value 

41 

42 def __setstate__(self, state: object) -> None: 

43 if isinstance(state, str): 

44 # New format (26.2+): just the value string. 

45 self._restore_value(state) 

46 return 

47 if isinstance(state, tuple) and len(state) == 2: 

48 # Old format (packaging <= 26.0, __slots__): (None, {slot: value}). 

49 _, slot_dict = state 

50 if isinstance(slot_dict, dict) and "value" in slot_dict: 

51 self._restore_value(slot_dict["value"]) 

52 return 

53 if isinstance(state, dict) and "value" in state: 

54 # Old format (packaging <= 25.0, no __slots__): plain __dict__. 

55 self._restore_value(state["value"]) 

56 return 

57 raise TypeError(f"Cannot restore {self.__class__.__name__} from {state!r}") 

58 

59 

60class Variable(Node): 

61 __slots__ = () 

62 

63 def serialize(self) -> str: 

64 return str(self) 

65 

66 

67class Value(Node): 

68 __slots__ = () 

69 

70 def serialize(self) -> str: 

71 value = str(self) 

72 if '"' not in value: 

73 return f'"{value}"' 

74 if "'" not in value: 

75 return f"'{value}'" 

76 raise ValueError( 

77 "Cannot serialize marker value containing both quote characters" 

78 ) 

79 

80 

81class Op(Node): 

82 __slots__ = () 

83 

84 def serialize(self) -> str: 

85 return str(self) 

86 

87 

88MarkerLogical = Literal["and", "or"] 

89MarkerVar = Union[Variable, Value] 

90MarkerItem = tuple[MarkerVar, Op, MarkerVar] 

91MarkerAtom = Union[MarkerItem, Sequence["MarkerAtom"]] 

92MarkerList = list[Union["MarkerList", MarkerAtom, MarkerLogical]] 

93 

94 

95class ParsedRequirement(NamedTuple): 

96 name: str 

97 url: str 

98 extras: list[str] 

99 specifier: str 

100 marker: MarkerList | None 

101 

102 

103# -------------------------------------------------------------------------------------- 

104# Recursive descent parser for dependency specifier 

105# -------------------------------------------------------------------------------------- 

106def parse_requirement(source: str) -> ParsedRequirement: 

107 return _parse_requirement(Tokenizer(source, rules=DEFAULT_RULES)) 

108 

109 

110def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: 

111 """ 

112 requirement = WS? IDENTIFIER WS? extras WS? requirement_details 

113 """ 

114 tokenizer.consume("WS") 

115 

116 name_token = tokenizer.expect( 

117 "IDENTIFIER", expected="package name at the start of dependency specifier" 

118 ) 

119 name = name_token.text 

120 tokenizer.consume("WS") 

121 

122 extras = _parse_extras(tokenizer) 

123 tokenizer.consume("WS") 

124 

125 url, specifier, marker = _parse_requirement_details(tokenizer) 

126 tokenizer.expect("END", expected="end of dependency specifier") 

127 

128 return ParsedRequirement(name, url, extras, specifier, marker) 

129 

130 

131def _parse_requirement_details( 

132 tokenizer: Tokenizer, 

133) -> tuple[str, str, MarkerList | None]: 

134 """ 

135 requirement_details = AT URL (WS requirement_marker?)? 

136 | specifier WS? (requirement_marker)? 

137 """ 

138 

139 specifier = "" 

140 url = "" 

141 marker = None 

142 

143 if tokenizer.check("AT"): 

144 tokenizer.read() 

145 tokenizer.consume("WS") 

146 

147 url_start = tokenizer.position 

148 url = tokenizer.expect("URL", expected="URL after @").text 

149 if tokenizer.check("END", peek=True): 

150 return (url, specifier, marker) 

151 

152 tokenizer.expect("WS", expected="whitespace after URL") 

153 

154 # The input might end after whitespace. 

155 if tokenizer.check("END", peek=True): 

156 return (url, specifier, marker) 

157 

158 marker = _parse_requirement_marker( 

159 tokenizer, 

160 span_start=url_start, 

161 expected="semicolon (after URL and whitespace)", 

162 ) 

163 else: 

164 specifier_start = tokenizer.position 

165 specifier = _parse_specifier(tokenizer) 

166 tokenizer.consume("WS") 

167 

168 if tokenizer.check("END", peek=True): 

169 return (url, specifier, marker) 

170 

171 marker = _parse_requirement_marker( 

172 tokenizer, 

173 span_start=specifier_start, 

174 expected=( 

175 "comma (within version specifier), semicolon (after version specifier)" 

176 if specifier 

177 else "semicolon (after name with no version specifier)" 

178 ), 

179 ) 

180 

181 return (url, specifier, marker) 

182 

183 

184def _parse_requirement_marker( 

185 tokenizer: Tokenizer, *, span_start: int, expected: str 

186) -> MarkerList: 

187 """ 

188 requirement_marker = SEMICOLON marker WS? 

189 """ 

190 

191 if not tokenizer.check("SEMICOLON"): 

192 tokenizer.raise_syntax_error( 

193 f"Expected {expected} or end", 

194 span_start=span_start, 

195 span_end=None, 

196 ) 

197 tokenizer.read() 

198 

199 marker = _parse_marker(tokenizer) 

200 tokenizer.consume("WS") 

201 

202 return marker 

203 

204 

205def _parse_extras(tokenizer: Tokenizer) -> list[str]: 

206 """ 

207 extras = (LEFT_BRACKET wsp* extras_list? wsp* RIGHT_BRACKET)? 

208 """ 

209 if not tokenizer.check("LEFT_BRACKET", peek=True): 

210 return [] 

211 

212 with tokenizer.enclosing_tokens( 

213 "LEFT_BRACKET", 

214 "RIGHT_BRACKET", 

215 around="extras", 

216 ): 

217 tokenizer.consume("WS") 

218 extras = _parse_extras_list(tokenizer) 

219 tokenizer.consume("WS") 

220 

221 return extras 

222 

223 

224def _parse_extras_list(tokenizer: Tokenizer) -> list[str]: 

225 """ 

226 extras_list = identifier (wsp* ',' wsp* identifier)* 

227 """ 

228 extras: list[str] = [] 

229 

230 if not tokenizer.check("IDENTIFIER"): 

231 return extras 

232 

233 extras.append(tokenizer.read().text) 

234 

235 while True: 

236 tokenizer.consume("WS") 

237 if tokenizer.check("IDENTIFIER", peek=True): 

238 tokenizer.raise_syntax_error("Expected comma between extra names") 

239 elif not tokenizer.check("COMMA"): 

240 break 

241 

242 tokenizer.read() 

243 tokenizer.consume("WS") 

244 

245 extra_token = tokenizer.expect("IDENTIFIER", expected="extra name after comma") 

246 extras.append(extra_token.text) 

247 

248 return extras 

249 

250 

251def _parse_specifier(tokenizer: Tokenizer) -> str: 

252 """ 

253 specifier = LEFT_PARENTHESIS WS? version_many WS? RIGHT_PARENTHESIS 

254 | WS? version_many WS? 

255 """ 

256 with tokenizer.enclosing_tokens( 

257 "LEFT_PARENTHESIS", 

258 "RIGHT_PARENTHESIS", 

259 around="version specifier", 

260 ): 

261 tokenizer.consume("WS") 

262 parsed_specifiers = _parse_version_many(tokenizer) 

263 tokenizer.consume("WS") 

264 

265 return parsed_specifiers 

266 

267 

268def _parse_version_many(tokenizer: Tokenizer) -> str: 

269 """ 

270 version_many = (SPECIFIER (WS? COMMA WS? SPECIFIER)*)? 

271 """ 

272 parsed_specifiers = "" 

273 while tokenizer.check("SPECIFIER"): 

274 span_start = tokenizer.position 

275 specifier = tokenizer.read().text 

276 parsed_specifiers += specifier 

277 if tokenizer.check("VERSION_PREFIX_TRAIL", peek=True): 

278 message = ".* suffix can only be used with `==` or `!=` operators" 

279 if specifier.startswith("!=") or ( 

280 specifier.startswith("==") and not specifier.startswith("===") 

281 ): 

282 message = ( 

283 ".* suffix cannot be used with pre-release, post-release, " 

284 "dev or local versions" 

285 ) 

286 tokenizer.raise_syntax_error( 

287 message, 

288 span_start=span_start, 

289 span_end=tokenizer.position + 1, 

290 ) 

291 if tokenizer.check("VERSION_LOCAL_LABEL_TRAIL", peek=True): 

292 tokenizer.raise_syntax_error( 

293 "Local version label can only be used with `==` or `!=` operators", 

294 span_start=span_start, 

295 span_end=tokenizer.position, 

296 ) 

297 tokenizer.consume("WS") 

298 if not tokenizer.check("COMMA"): 

299 break 

300 parsed_specifiers += tokenizer.read().text 

301 tokenizer.consume("WS") 

302 

303 return parsed_specifiers 

304 

305 

306# -------------------------------------------------------------------------------------- 

307# Recursive descent parser for marker expression 

308# -------------------------------------------------------------------------------------- 

309def parse_marker(source: str) -> MarkerList: 

310 return _parse_full_marker(Tokenizer(source, rules=DEFAULT_RULES)) 

311 

312 

313def _parse_full_marker(tokenizer: Tokenizer) -> MarkerList: 

314 retval = _parse_marker(tokenizer) 

315 tokenizer.expect("END", expected="end of marker expression") 

316 return retval 

317 

318 

319def _parse_marker(tokenizer: Tokenizer) -> MarkerList: 

320 """ 

321 marker = marker_atom (BOOLOP marker_atom)+ 

322 """ 

323 expression = [_parse_marker_atom(tokenizer)] 

324 while tokenizer.check("BOOLOP"): 

325 token = tokenizer.read() 

326 expr_right = _parse_marker_atom(tokenizer) 

327 expression.extend((token.text, expr_right)) 

328 return expression 

329 

330 

331def _parse_marker_atom(tokenizer: Tokenizer) -> MarkerAtom: 

332 """ 

333 marker_atom = WS? LEFT_PARENTHESIS WS? marker WS? RIGHT_PARENTHESIS WS? 

334 | WS? marker_item WS? 

335 """ 

336 

337 tokenizer.consume("WS") 

338 if tokenizer.check("LEFT_PARENTHESIS", peek=True): 

339 with tokenizer.enclosing_tokens( 

340 "LEFT_PARENTHESIS", 

341 "RIGHT_PARENTHESIS", 

342 around="marker expression", 

343 ): 

344 tokenizer.consume("WS") 

345 marker: MarkerAtom = _parse_marker(tokenizer) 

346 tokenizer.consume("WS") 

347 else: 

348 marker = _parse_marker_item(tokenizer) 

349 tokenizer.consume("WS") 

350 return marker 

351 

352 

353def _parse_marker_item(tokenizer: Tokenizer) -> MarkerItem: 

354 """ 

355 marker_item = WS? marker_var WS? marker_op WS? marker_var WS? 

356 """ 

357 tokenizer.consume("WS") 

358 marker_var_left = _parse_marker_var(tokenizer) 

359 tokenizer.consume("WS") 

360 marker_op = _parse_marker_op(tokenizer) 

361 tokenizer.consume("WS") 

362 marker_var_right = _parse_marker_var(tokenizer) 

363 tokenizer.consume("WS") 

364 return (marker_var_left, marker_op, marker_var_right) 

365 

366 

367def _parse_marker_var(tokenizer: Tokenizer) -> MarkerVar: # noqa: RET503 

368 """ 

369 marker_var = VARIABLE | QUOTED_STRING 

370 """ 

371 if tokenizer.check("VARIABLE"): 

372 return process_env_var(tokenizer.read().text.replace(".", "_")) 

373 elif tokenizer.check("QUOTED_STRING"): 

374 token = tokenizer.read() 

375 try: 

376 return process_python_str(token.text) 

377 except (SyntaxError, ValueError) as exc: 

378 raise ParserSyntaxError( 

379 "Invalid quoted string", 

380 source=tokenizer.source, 

381 span=(token.position, token.position + len(token.text)), 

382 ) from exc 

383 else: 

384 tokenizer.raise_syntax_error( 

385 message="Expected a marker variable or quoted string" 

386 ) 

387 

388 

389def process_env_var(env_var: str) -> Variable: 

390 if env_var in ("platform_python_implementation", "python_implementation"): 

391 return Variable("platform_python_implementation") 

392 else: 

393 return Variable(env_var) 

394 

395 

396def process_python_str(python_str: str) -> Value: 

397 value = ast.literal_eval(python_str) 

398 return Value(str(value)) 

399 

400 

401def _parse_marker_op(tokenizer: Tokenizer) -> Op: 

402 """ 

403 marker_op = IN | NOT IN | OP 

404 """ 

405 if tokenizer.check("IN"): 

406 tokenizer.read() 

407 return Op("in") 

408 elif tokenizer.check("NOT"): 

409 tokenizer.read() 

410 tokenizer.expect("WS", expected="whitespace after 'not'") 

411 tokenizer.expect("IN", expected="'in' after 'not'") 

412 return Op("not in") 

413 elif tokenizer.check("OP"): 

414 return Op(tokenizer.read().text) 

415 else: 

416 return tokenizer.raise_syntax_error( 

417 "Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in" 

418 )