Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/cssselect/xpath.py: 78%

1""" 

2cssselect.xpath 

3=============== 

4 

5Translation of parsed CSS selectors to XPath expressions. 

6 

7 

8:copyright: (c) 2007-2012 Ian Bicking and contributors. 

9See AUTHORS for more details. 

10:license: BSD, see LICENSE for more details. 

11 

12""" 

13 

14from __future__ import annotations 

15 

16import re 

17from string import ascii_lowercase, ascii_uppercase 

18from typing import TYPE_CHECKING, Self, cast 

19 

20from cssselect.parser import ( 

21 Attrib, 

22 Class, 

23 CombinedSelector, 

24 Element, 

25 Function, 

26 Hash, 

27 Matching, 

28 Negation, 

29 Pseudo, 

30 PseudoElement, 

31 Relation, 

32 Selector, 

33 SelectorError, 

34 SpecificityAdjustment, 

35 Tree, 

36 ascii_lower, 

37 parse, 

38 parse_series, 

39) 

40 

41if TYPE_CHECKING: 

42 from collections.abc import Callable, Iterable 

43 

44 

45class ExpressionError(SelectorError, RuntimeError): 

46 """Unknown or unsupported selector (eg. pseudo-class).""" 

47 

48 

49#### XPath Helpers 

50 

51 

52class XPathExpr: 

53 def __init__( 

54 self, 

55 path: str = "", 

56 element: str = "*", 

57 condition: str = "", 

58 star_prefix: bool = False, 

59 ) -> None: 

60 self.path = path 

61 self.element = element 

62 self.condition = condition 

63 

64 def __str__(self) -> str: 

65 path = str(self.path) + str(self.element) 

66 if self.condition: 

67 path += f"[{self.condition}]" 

68 return path 

69 

70 def __repr__(self) -> str: 

71 return f"{self.__class__.__name__}[{self}]" 

72 

73 def add_condition(self, condition: str, conjuction: str = "and") -> Self: 

74 if self.condition: 

75 self.condition = f"({self.condition}) {conjuction} ({condition})" 

76 else: 

77 self.condition = condition 

78 return self 

79 

80 def add_name_test(self) -> None: 

81 if self.element == "*": 

82 # We weren't doing a test anyway 

83 return 

84 prefix, colon, local = self.element.partition(":") 

85 if is_safe_name(prefix) and (not colon or local == "*" or is_safe_name(local)): 

86 # A node test, not a name() comparison: name() returns the 

87 # qualified name as written in the document, which would bypass 

88 # the XPath prefix mapping for a prefixed name like "ns:f" or 

89 # "ns:*" (from the CSS "ns|f" or "ns|*"), and would match 

90 # elements in a default namespace for an unprefixed name (which 

91 # the node test emitted for a bare "f" selector does not). 

92 self.add_condition(f"self::{self.element}") 

93 else: 

94 self.add_condition( 

95 f"name() = {GenericTranslator.xpath_literal(self.element)}" 

96 ) 

97 self.element = "*" 

98 

99 def add_star_prefix(self) -> None: 

100 """ 

101 Append '*/' to the path to keep the context constrained 

102 to a single parent. 

103 """ 

104 self.path += "*/" 

105 

106 def join(self, combiner: str, other: XPathExpr) -> Self: 

107 path = str(self) + combiner 

108 # Any "star prefix" is redundant when joining. 

109 if other.path != "*/": 

110 path += other.path 

111 self.path = path 

112 self.element = other.element 

113 self.condition = other.condition 

114 return self 

115 

116 

117split_at_single_quotes = re.compile("('+)").split 

118 

119# The spec is actually more permissive than that, but don’t bother. 

120# This is just for the fast path. 

121# http://www.w3.org/TR/REC-xml/#NT-NameStartChar 

122is_safe_name = re.compile("[a-zA-Z_][a-zA-Z0-9_.-]*").fullmatch 

123 

124# Test that the string is not empty and does not contain whitespace 

125is_non_whitespace = re.compile(r"[^ \t\r\n\f]+").fullmatch 

126 

127 

128def _or_conditions(conditions: list[str]) -> str: 

129 """Join conditions with "or".""" 

130 if len(conditions) == 1: 

131 return conditions[0] 

132 return " or ".join(f"({condition})" for condition in conditions) 

133 

134 

135#### Translation 

136 

137 

138class GenericTranslator: 

139 """ 

140 Translator for "generic" XML documents. 

141 

142 Everything is case-sensitive, no assumption is made on the meaning 

143 of element names and attribute names. 

144 

145 """ 

146 

147 #### 

148 #### HERE BE DRAGONS 

149 #### 

150 #### You are welcome to hook into this to change some behavior, 

151 #### but do so at your own risks. 

152 #### Until it has received a lot more work and review, 

153 #### I reserve the right to change this API in backward-incompatible ways 

154 #### with any minor version of cssselect. 

155 #### See https://github.com/scrapy/cssselect/pull/22 

156 #### -- Simon Sapin. 

157 #### 

158 

159 combinator_mapping = { 

160 " ": "descendant", 

161 ">": "child", 

162 "+": "direct_adjacent", 

163 "~": "indirect_adjacent", 

164 } 

165 

166 # Used to match a combinator against the context node, in :not(). 

167 _reverse_combinator_mapping = { 

168 " ": "ancestor::*", 

169 ">": "parent::*", 

170 "+": "preceding-sibling::*[1]", 

171 "~": "preceding-sibling::*", 

172 } 

173 

174 attribute_operator_mapping = { 

175 "exists": "exists", 

176 "=": "equals", 

177 "~=": "includes", 

178 "|=": "dashmatch", 

179 "^=": "prefixmatch", 

180 "$=": "suffixmatch", 

181 "*=": "substringmatch", 

182 "!=": "different", # not part of Selectors Level 3, but widely supported 

183 } 

184 

185 #: The attribute used for ID selectors depends on the document language: 

186 #: http://www.w3.org/TR/selectors/#id-selectors 

187 id_attribute = "id" 

188 

189 #: The attribute used for ``:lang()`` depends on the document language: 

190 #: http://www.w3.org/TR/selectors/#lang-pseudo 

191 lang_attribute = "xml:lang" 

192 

193 #: The case sensitivity of document language element names, 

194 #: attribute names, and attribute values in selectors depends 

195 #: on the document language. 

196 #: http://www.w3.org/TR/selectors/#casesens 

197 #: 

198 #: When a document language defines one of these as case-insensitive, 

199 #: cssselect assumes that the document parser makes the parsed values 

200 #: lower-case. Making the selector lower-case too makes the comparaison 

201 #: case-insensitive. 

202 #: 

203 #: In HTML, element names and attributes names (but not attribute values) 

204 #: are case-insensitive. All of lxml.html, html5lib, BeautifulSoup4 

205 #: and HTMLParser make them lower-case in their parse result, so 

206 #: the assumption holds. 

207 lower_case_element_names = False 

208 lower_case_attribute_names = False 

209 lower_case_attribute_values = False 

210 

211 # class used to represent and xpath expression 

212 xpathexpr_cls = XPathExpr 

213 

214 def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: 

215 """Translate a *group of selectors* to XPath. 

216 

217 Pseudo-elements are not supported here since XPath only knows 

218 about "real" elements. 

219 

220 :param css: 

221 A *group of selectors* as a string. 

222 :param prefix: 

223 This string is prepended to the XPath expression for each selector. 

224 The default makes selectors scoped to the context node’s subtree. 

225 :raises: 

226 :class:`~cssselect.SelectorSyntaxError` on invalid selectors, 

227 :class:`ExpressionError` on unknown/unsupported selectors, 

228 including pseudo-elements. 

229 :returns: 

230 The equivalent XPath 1.0 expression as a string. 

231 

232 """ 

233 return " | ".join( 

234 self.selector_to_xpath(selector, prefix, translate_pseudo_elements=True) 

235 for selector in parse(css) 

236 ) 

237 

238 def selector_to_xpath( 

239 self, 

240 selector: Selector, 

241 prefix: str = "descendant-or-self::", 

242 translate_pseudo_elements: bool = False, 

243 ) -> str: 

244 """Translate a parsed selector to XPath. 

245 

246 

247 :param selector: 

248 A parsed :class:`Selector` object. 

249 :param prefix: 

250 This string is prepended to the resulting XPath expression. 

251 The default makes selectors scoped to the context node’s subtree. 

252 :param translate_pseudo_elements: 

253 Unless this is set to ``True`` (as :meth:`css_to_xpath` does), 

254 the :attr:`~Selector.pseudo_element` attribute of the selector 

255 is ignored. 

256 It is the caller's responsibility to reject selectors 

257 with pseudo-elements, or to account for them somehow. 

258 :raises: 

259 :class:`ExpressionError` on unknown/unsupported selectors. 

260 :returns: 

261 The equivalent XPath 1.0 expression as a string. 

262 

263 """ 

264 tree = getattr(selector, "parsed_tree", None) 

265 if not tree: 

266 raise TypeError(f"Expected a parsed selector, got {selector!r}") 

267 xpath = self.xpath(tree) 

268 assert isinstance(xpath, self.xpathexpr_cls) # help debug a missing 'return' 

269 if translate_pseudo_elements and selector.pseudo_element: 

270 xpath = self.xpath_pseudo_element(xpath, selector.pseudo_element) 

271 return (prefix or "") + str(xpath) 

272 

273 def xpath_pseudo_element( 

274 self, xpath: XPathExpr, pseudo_element: PseudoElement 

275 ) -> XPathExpr: 

276 """Translate a pseudo-element. 

277 

278 Defaults to not supporting pseudo-elements at all, 

279 but can be overridden by sub-classes. 

280 

281 """ 

282 raise ExpressionError("Pseudo-elements are not supported.") 

283 

284 @staticmethod 

285 def xpath_literal(s: str) -> str: 

286 s = str(s) 

287 if "'" not in s: 

288 s = f"'{s}'" 

289 elif '"' not in s: 

290 s = f'"{s}"' 

291 else: 

292 parts_quoted = [ 

293 f'"{part}"' if "'" in part else f"'{part}'" 

294 for part in split_at_single_quotes(s) 

295 if part 

296 ] 

297 s = "concat({})".format(",".join(parts_quoted)) 

298 return s 

299 

300 def xpath(self, parsed_selector: Tree) -> XPathExpr: 

301 """Translate any parsed selector object.""" 

302 type_name = type(parsed_selector).__name__ 

303 method = cast( 

304 "Callable[[Tree], XPathExpr] | None", 

305 getattr(self, f"xpath_{type_name.lower()}", None), 

306 ) 

307 if method is None: 

308 raise ExpressionError(f"{type_name} is not supported.") 

309 return method(parsed_selector) 

310 

311 # Dispatched by parsed object type 

312 

313 def xpath_combinedselector(self, combined: CombinedSelector) -> XPathExpr: 

314 """Translate a combined selector.""" 

315 combinator = self.combinator_mapping[combined.combinator] 

316 method = cast( 

317 "Callable[[XPathExpr, XPathExpr], XPathExpr]", 

318 getattr(self, f"xpath_{combinator}_combinator"), 

319 ) 

320 return method(self.xpath(combined.selector), self.xpath(combined.subselector)) 

321 

322 def xpath_negation(self, negation: Negation) -> XPathExpr: 

323 xpath = self.xpath(negation.selector) 

324 condition = self._xpath_match_condition(negation.subselector) 

325 if condition is None: 

326 # The argument matches every element, so :not() matches none. 

327 return xpath.add_condition("0") 

328 return xpath.add_condition(f"not({condition})") 

329 

330 def _xpath_match_condition(self, selector: Tree) -> str | None: 

331 """Return a condition that holds for the elements matching *selector*, 

332 or None if that is every element. 

333 

334 Unlike xpath(), which walks a selector from left to right, this 

335 matches the whole selector against the context node, using reverse 

336 axes for combinators. 

337 """ 

338 if isinstance(selector, CombinedSelector): 

339 steps = [] 

340 left: Tree = selector 

341 while isinstance(left, CombinedSelector): 

342 axis = self._reverse_combinator_mapping[left.combinator] 

343 left = left.selector 

344 condition = self._xpath_match_condition( 

345 left.subselector if isinstance(left, CombinedSelector) else left 

346 ) 

347 steps.append(axis if condition is None else f"{axis}[{condition}]") 

348 path = "/".join(steps) 

349 right = self._xpath_match_condition(selector.subselector) 

350 return path if right is None else f"({right}) and {path}" 

351 sub_xpath = self.xpath(selector) 

352 sub_xpath.add_name_test() 

353 return sub_xpath.condition or None 

354 

355 def xpath_relation(self, relation: Relation) -> XPathExpr: 

356 conditions: list[str] = [] 

357 for combinator, subselector in relation.arguments: 

358 method = cast( 

359 "Callable[[XPathExpr], str]", 

360 getattr( 

361 self, 

362 f"xpath_relation_{self.combinator_mapping[cast('str', combinator.value)]}_combinator", 

363 ), 

364 ) 

365 conditions.append(method(self.xpath(subselector.parsed_tree))) 

366 return self.xpath(relation.selector).add_condition(_or_conditions(conditions)) 

367 

368 def xpath_matching(self, matching: Matching) -> XPathExpr: 

369 return self._xpath_add_selector_list_condition( 

370 self.xpath(matching.selector), matching.selector_list 

371 ) 

372 

373 def xpath_specificityadjustment(self, matching: SpecificityAdjustment) -> XPathExpr: 

374 return self._xpath_add_selector_list_condition( 

375 self.xpath(matching.selector), matching.selector_list 

376 ) 

377 

378 def _xpath_add_selector_list_condition( 

379 self, xpath: XPathExpr, selector_list: Iterable[Tree] 

380 ) -> XPathExpr: 

381 """Add a condition matching any selector of the list 

382 (for :is() and :where()).""" 

383 conditions: list[str] = [] 

384 for e in (self.xpath(selector) for selector in selector_list): 

385 if e.path: 

386 # Only a combined selector (e.g. "a b") translates to a path, 

387 # which cannot be embedded into a predicate of the outer 

388 # expression. The parser rejects combinators in these arguments, 

389 # so this is only reachable through a hand-built Matching or 

390 # SpecificityAdjustment node. 

391 raise ExpressionError( 

392 "Combined selectors are not supported inside " 

393 ":is(), :where() and :matches()" 

394 ) 

395 e.add_name_test() 

396 if not e.condition: 

397 # This argument matches any element, so the whole selector 

398 # list does too: it adds no condition. 

399 return xpath 

400 conditions.append(e.condition) 

401 return xpath.add_condition(_or_conditions(conditions)) 

402 

403 def xpath_function(self, function: Function) -> XPathExpr: 

404 """Translate a functional pseudo-class.""" 

405 method_name = "xpath_{}_function".format(function.name.replace("-", "_")) 

406 method = cast( 

407 "Callable[[XPathExpr, Function], XPathExpr] | None", 

408 getattr(self, method_name, None), 

409 ) 

410 if not method: 

411 raise ExpressionError(f"The pseudo-class :{function.name}() is unknown") 

412 return method(self.xpath(function.selector), function) 

413 

414 def xpath_pseudo(self, pseudo: Pseudo) -> XPathExpr: 

415 """Translate a pseudo-class.""" 

416 method_name = "xpath_{}_pseudo".format(pseudo.ident.replace("-", "_")) 

417 method = cast( 

418 "Callable[[XPathExpr], XPathExpr] | None", 

419 getattr(self, method_name, None), 

420 ) 

421 if not method: 

422 raise ExpressionError(f"The pseudo-class :{pseudo.ident} is unknown") 

423 return method(self.xpath(pseudo.selector)) 

424 

425 def xpath_attrib(self, selector: Attrib) -> XPathExpr: 

426 """Translate an attribute selector.""" 

427 operator = self.attribute_operator_mapping[selector.operator] 

428 method = cast( 

429 "Callable[[XPathExpr, str, str | None], XPathExpr]", 

430 getattr(self, f"xpath_attrib_{operator}"), 

431 ) 

432 if self.lower_case_attribute_names: 

433 name = ascii_lower(selector.attrib) 

434 else: 

435 name = selector.attrib 

436 if selector.namespace == "*": 

437 # Namespace wildcard, e.g. "[*|href]": any namespace, including 

438 # none. XPath 1.0 has no "*:name" name test for this, so match 

439 # by local name instead. 

440 attrib = f"attribute::*[local-name() = {self.xpath_literal(name)}]" 

441 else: 

442 safe = is_safe_name(name) 

443 if selector.namespace: 

444 name = f"{selector.namespace}:{name}" 

445 safe = safe and is_safe_name(selector.namespace) 

446 if safe: 

447 attrib = "@" + name 

448 else: 

449 attrib = f"attribute::*[name() = {self.xpath_literal(name)}]" 

450 if selector.value is None: 

451 value = None 

452 elif self.lower_case_attribute_values: 

453 value = ascii_lower(cast("str", selector.value.value)) 

454 else: 

455 value = selector.value.value 

456 attribute = None 

457 if selector.namespace == "*" and selector.operator not in ("exists", "!="): 

458 # String functions convert a node-set to its first node's value. 

459 # Apply the operator to each attribute before testing existence. 

460 attribute = attrib 

461 attrib = "." 

462 if selector.flag == "i" and value: 

463 # ASCII-lowering both sides is what the specification defines a 

464 # case-insensitive match as, and all XPath 1.0 can express. 

465 attrib = f"translate({attrib}, {self.xpath_literal(ascii_uppercase)}, {self.xpath_literal(ascii_lowercase)})" 

466 value = ascii_lower(value) 

467 if attribute is not None: 

468 condition = method(self.xpathexpr_cls(), attrib, value).condition 

469 return self.xpath(selector.selector).add_condition( 

470 f"{attribute}[{condition}]" 

471 ) 

472 return method(self.xpath(selector.selector), attrib, value) 

473 

474 def xpath_class(self, class_selector: Class) -> XPathExpr: 

475 """Translate a class selector.""" 

476 # .foo is defined as [class~=foo] in the spec. 

477 xpath = self.xpath(class_selector.selector) 

478 return self.xpath_attrib_includes(xpath, "@class", class_selector.class_name) 

479 

480 def xpath_hash(self, id_selector: Hash) -> XPathExpr: 

481 """Translate an ID selector.""" 

482 xpath = self.xpath(id_selector.selector) 

483 return self.xpath_attrib_equals(xpath, "@id", id_selector.id) 

484 

485 def xpath_element(self, selector: Element) -> XPathExpr: 

486 """Translate a type or universal selector.""" 

487 element = selector.element 

488 if not element: 

489 if selector.namespace == "": 

490 # "|*": any element with no namespace. 

491 xpath = self.xpathexpr_cls(element="*") 

492 xpath.add_condition("namespace-uri() = ''") 

493 return xpath 

494 element = "*" 

495 safe = True 

496 else: 

497 safe = bool(is_safe_name(element)) 

498 if self.lower_case_element_names: 

499 element = ascii_lower(element) 

500 if selector.namespace == "*": 

501 # Namespace wildcard, e.g. "*|div": any namespace, 

502 # including none. XPath 1.0 has no "*:name" name test 

503 # for this, so match by local name instead. 

504 xpath = self.xpathexpr_cls(element="*") 

505 xpath.add_condition(f"local-name() = {self.xpath_literal(element)}") 

506 return xpath 

507 if selector.namespace and selector.namespace != "*": 

508 # Namespace prefixes are case-sensitive. 

509 # http://www.w3.org/TR/css3-namespace/#prefixes 

510 element = f"{selector.namespace}:{element}" 

511 safe = safe and bool(is_safe_name(selector.namespace)) 

512 xpath = self.xpathexpr_cls(element=element) 

513 if not safe: 

514 # Not usable as an XPath name test (e.g. an escaped identifier 

515 # like di\a0 v): compare the serialized name instead. Done here 

516 # rather than through add_name_test(), which would mistake a ":" 

517 # inside such a name for a namespace prefix separator. 

518 xpath.add_condition(f"name() = {self.xpath_literal(element)}") 

519 xpath.element = "*" 

520 return xpath 

521 

522 # CombinedSelector: dispatch by combinator 

523 

524 def xpath_descendant_combinator( 

525 self, left: XPathExpr, right: XPathExpr 

526 ) -> XPathExpr: 

527 """right is a child, grand-child or further descendant of left""" 

528 if right.element == "*" and not right.condition and not right.path: 

529 # A subclass that maps a pseudo-element to a different node test 

530 # rewrites the trailing "descendant-or-self::*/*" of the 

531 # expression, so an unconstrained right-hand side keeps that shape. 

532 return left.join("/descendant-or-self::*/", right) 

533 return left.join("/descendant::", right) 

534 

535 def xpath_child_combinator(self, left: XPathExpr, right: XPathExpr) -> XPathExpr: 

536 """right is an immediate child of left""" 

537 return left.join("/", right) 

538 

539 def xpath_direct_adjacent_combinator( 

540 self, left: XPathExpr, right: XPathExpr 

541 ) -> XPathExpr: 

542 """right is a sibling immediately after left""" 

543 xpath = left.join("/following-sibling::", right) 

544 xpath.add_name_test() 

545 return xpath.add_condition("position() = 1") 

546 

547 def xpath_indirect_adjacent_combinator( 

548 self, left: XPathExpr, right: XPathExpr 

549 ) -> XPathExpr: 

550 """right is a sibling after left, immediately or not""" 

551 return left.join("/following-sibling::", right) 

552 

553 # The relative selector is kept in `condition` (instead of being folded 

554 # into `path`/`element`) so that `element` stays a plain element name: 

555 # later steps such as :first-of-type or :not() read and rewrite it. 

556 

557 def xpath_relation_descendant_combinator(self, right: XPathExpr) -> str: 

558 """right is a child, grand-child or further descendant of the element""" 

559 return f"descendant::{right}" 

560 

561 def xpath_relation_child_combinator(self, right: XPathExpr) -> str: 

562 """right is an immediate child of the element""" 

563 return f"./{right}" 

564 

565 def xpath_relation_direct_adjacent_combinator(self, right: XPathExpr) -> str: 

566 """right is a sibling immediately after the element""" 

567 # Test the first step of right against that sibling itself, so that 

568 # any further step of right is walked from there. 

569 if right.path: 

570 head, sep, tail = right.path.partition("/") 

571 right.path = f"self::{head}{sep}{tail}" 

572 else: 

573 right.element = f"self::{right.element}" 

574 return f"following-sibling::*[1]/{right}" 

575 

576 def xpath_relation_indirect_adjacent_combinator(self, right: XPathExpr) -> str: 

577 """right is a sibling after the element, immediately or not""" 

578 return f"following-sibling::{right}" 

579 

580 # Function: dispatch by function/pseudo-class name 

581 

582 def xpath_nth_child_function( 

583 self, 

584 xpath: XPathExpr, 

585 function: Function, 

586 last: bool = False, 

587 add_name_test: bool = True, 

588 ) -> XPathExpr: 

589 try: 

590 a, b = parse_series(function.arguments) 

591 except ValueError as ex: 

592 raise ExpressionError(f"Invalid series: '{function.arguments!r}'") from ex 

593 

594 # From https://www.w3.org/TR/css3-selectors/#structural-pseudos: 

595 # 

596 # :nth-child(an+b) 

597 # an+b-1 siblings before 

598 # 

599 # :nth-last-child(an+b) 

600 # an+b-1 siblings after 

601 # 

602 # :nth-of-type(an+b) 

603 # an+b-1 siblings with the same expanded element name before 

604 # 

605 # :nth-last-of-type(an+b) 

606 # an+b-1 siblings with the same expanded element name after 

607 # 

608 # So, 

609 # for :nth-child and :nth-of-type 

610 # 

611 # count(preceding-sibling::<nodetest>) = an+b-1 

612 # 

613 # for :nth-last-child and :nth-last-of-type 

614 # 

615 # count(following-sibling::<nodetest>) = an+b-1 

616 # 

617 # therefore, 

618 # count(...) - (b-1) ≡ 0 (mod a) 

619 # 

620 # if a == 0: 

621 # ~~~~~~~~~~ 

622 # count(...) = b-1 

623 # 

624 # if a < 0: 

625 # ~~~~~~~~~ 

626 # count(...) - b +1 <= 0 

627 # -> count(...) <= b-1 

628 # 

629 # if a > 0: 

630 # ~~~~~~~~~ 

631 # count(...) - b +1 >= 0 

632 # -> count(...) >= b-1 

633 

634 # work with b-1 instead 

635 b_min_1 = b - 1 

636 

637 # early-exit condition 1: 

638 # ~~~~~~~~~~~~~~~~~~~~~~~ 

639 # for a == 1, nth-*(an+b) means n+b-1 siblings before/after, 

640 # and since n ∈ {0, 1, 2, ...}, if b-1<=0, 

641 # there is always an "n" matching any number of siblings (maybe none) 

642 if a == 1 and b_min_1 <= 0: 

643 return xpath 

644 

645 # early-exit condition 2: 

646 # ~~~~~~~~~~~~~~~~~~~~~~~ 

647 # an+b-1 siblings with a<=0 and (b-1)<0 is not possible 

648 if a <= 0 and b_min_1 < 0: 

649 return xpath.add_condition("0") 

650 

651 # `add_name_test` boolean is inverted and somewhat counter-intuitive: 

652 # 

653 # nth_of_type() calls nth_child(add_name_test=False) 

654 nodetest = "*" if add_name_test else f"{xpath.element}" 

655 

656 # siblings before or after the element 

657 if not last: 

658 siblings = f"preceding-sibling::{nodetest}" 

659 else: 

660 siblings = f"following-sibling::{nodetest}" 

661 siblings_count = f"count({siblings})" 

662 

663 # count(***-sibling::***) >= k is tested as ***-sibling::***[k], 

664 # which only walks the k nearest siblings instead of all of them. 

665 

666 # special case of fixed position: nth-*(0n+b) 

667 # if a == 0: 

668 # ~~~~~~~~~~ 

669 # count(***-sibling::***) = b-1 

670 if a == 0: 

671 if b_min_1 == 0: 

672 return xpath.add_condition(f"not({siblings}[1])") 

673 return xpath.add_condition( 

674 f"{siblings}[{b_min_1}] and not({siblings}[{b_min_1 + 1}])" 

675 ) 

676 

677 expressions = [] 

678 

679 if a > 0: 

680 # siblings count, an+b-1, is always >= 0, 

681 # so if a>0, and (b-1)<=0, an "n" exists to satisfy this, 

682 # therefore, the predicate is only interesting if (b-1)>0 

683 if b_min_1 > 0: 

684 expressions.append(f"{siblings}[{b_min_1}]") 

685 else: 

686 # if a<0, and (b-1)<0, no "n" satisfies this, 

687 # this is tested above as an early exit condition 

688 # otherwise, 

689 expressions.append(f"not({siblings}[{b_min_1 + 1}])") 

690 

691 # operations modulo 1 or -1 are simpler, one only needs to verify: 

692 # 

693 # - either: 

694 # count(***-sibling::***) - (b-1) = n = 0, 1, 2, 3, etc., 

695 # i.e. count(***-sibling::***) >= (b-1) 

696 # 

697 # - or: 

698 # count(***-sibling::***) - (b-1) = -n = 0, -1, -2, -3, etc., 

699 # i.e. count(***-sibling::***) <= (b-1) 

700 # we just did above. 

701 # 

702 if abs(a) != 1: 

703 # count(***-sibling::***) - (b-1) ≡ 0 (mod a) 

704 left = siblings_count 

705 

706 # apply "modulo a" on 2nd term, -(b-1), 

707 # to simplify things like "(... +6) % -3", 

708 # and also make it positive with |a| 

709 b_neg = (-b_min_1) % abs(a) 

710 

711 if b_neg != 0: 

712 left = f"({left} +{b_neg})" 

713 

714 expressions.append(f"{left} mod {a} = 0") 

715 

716 template = "(%s)" if len(expressions) > 1 else "%s" 

717 xpath.add_condition( 

718 " and ".join(template % expression for expression in expressions) 

719 ) 

720 return xpath 

721 

722 def xpath_nth_last_child_function( 

723 self, xpath: XPathExpr, function: Function 

724 ) -> XPathExpr: 

725 return self.xpath_nth_child_function(xpath, function, last=True) 

726 

727 @staticmethod 

728 def _check_of_type_element(xpath: XPathExpr, pseudo: str) -> None: 

729 """Raise an exception if an -of-type pseudo-class can't be used with 

730 the given element. 

731 

732 For "*" and namespace wildcards like "ns:*" the type of the element is 

733 not known, and counting same-type siblings cannot be expressed as an 

734 XPath 1.0 node test. 

735 """ 

736 element = xpath.element 

737 if element == "*" or element.endswith(":*"): 

738 css_element = element.replace(":", "|") 

739 raise ExpressionError(f"{css_element}:{pseudo} is not implemented") 

740 

741 def xpath_nth_of_type_function( 

742 self, xpath: XPathExpr, function: Function 

743 ) -> XPathExpr: 

744 self._check_of_type_element(xpath, "nth-of-type()") 

745 return self.xpath_nth_child_function(xpath, function, add_name_test=False) 

746 

747 def xpath_nth_last_of_type_function( 

748 self, xpath: XPathExpr, function: Function 

749 ) -> XPathExpr: 

750 self._check_of_type_element(xpath, "nth-last-of-type()") 

751 return self.xpath_nth_child_function( 

752 xpath, function, last=True, add_name_test=False 

753 ) 

754 

755 def xpath_contains_function( 

756 self, xpath: XPathExpr, function: Function 

757 ) -> XPathExpr: 

758 # Defined there, removed in later drafts: 

759 # http://www.w3.org/TR/2001/CR-css3-selectors-20011113/#content-selectors 

760 if function.argument_types() not in (["STRING"], ["IDENT"]): 

761 raise ExpressionError( 

762 f"Expected a single string or ident for :contains(), got {function.arguments!r}" 

763 ) 

764 value = cast("str", function.arguments[0].value) 

765 return xpath.add_condition(f"contains(., {self.xpath_literal(value)})") 

766 

767 def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr: 

768 if function.argument_types() not in (["STRING"], ["IDENT"]): 

769 raise ExpressionError( 

770 f"Expected a single string or ident for :lang(), got {function.arguments!r}" 

771 ) 

772 value = cast("str", function.arguments[0].value) 

773 return xpath.add_condition(f"lang({self.xpath_literal(value)})") 

774 

775 # Pseudo: dispatch by pseudo-class name 

776 

777 def xpath_root_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

778 return xpath.add_condition("not(parent::*)") 

779 

780 # CSS immediate children (CSS ":scope > div" to XPath "child::div" or "./div") 

781 # Works only at the start of a selector 

782 # Needed to get immediate children of a processed selector in Scrapy 

783 # for product in response.css('.product'): 

784 # description = product.css(':scope > div::text').get() 

785 def xpath_scope_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

786 xpath.add_name_test() 

787 return xpath.add_condition("position() = 1") 

788 

789 def xpath_first_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

790 return xpath.add_condition("not(preceding-sibling::*[1])") 

791 

792 def xpath_last_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

793 return xpath.add_condition("not(following-sibling::*[1])") 

794 

795 def xpath_first_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

796 self._check_of_type_element(xpath, "first-of-type") 

797 return xpath.add_condition(f"not(preceding-sibling::{xpath.element}[1])") 

798 

799 def xpath_last_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

800 self._check_of_type_element(xpath, "last-of-type") 

801 return xpath.add_condition(f"not(following-sibling::{xpath.element}[1])") 

802 

803 def xpath_only_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

804 # Test siblings, not the parent's children: the root element has 

805 # no parent, but it has no siblings either, so it must match. 

806 return xpath.add_condition( 

807 "not(preceding-sibling::*[1]) and not(following-sibling::*[1])" 

808 ) 

809 

810 def xpath_only_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

811 self._check_of_type_element(xpath, "only-of-type") 

812 return xpath.add_condition( 

813 f"not(preceding-sibling::{xpath.element}[1]) " 

814 f"and not(following-sibling::{xpath.element}[1])" 

815 ) 

816 

817 def xpath_empty_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

818 return xpath.add_condition("not(*) and not(string-length())") 

819 

820 def pseudo_never_matches(self, xpath: XPathExpr) -> XPathExpr: 

821 """Common implementation for pseudo-classes that never match.""" 

822 return xpath.add_condition("0") 

823 

824 xpath_link_pseudo = pseudo_never_matches 

825 xpath_visited_pseudo = pseudo_never_matches 

826 xpath_hover_pseudo = pseudo_never_matches 

827 xpath_active_pseudo = pseudo_never_matches 

828 xpath_focus_pseudo = pseudo_never_matches 

829 xpath_target_pseudo = pseudo_never_matches 

830 xpath_enabled_pseudo = pseudo_never_matches 

831 xpath_disabled_pseudo = pseudo_never_matches 

832 xpath_checked_pseudo = pseudo_never_matches 

833 

834 # Attrib: dispatch by attribute operator 

835 

836 def xpath_attrib_exists( 

837 self, xpath: XPathExpr, name: str, value: str | None 

838 ) -> XPathExpr: 

839 assert not value 

840 xpath.add_condition(name) 

841 return xpath 

842 

843 def xpath_attrib_equals( 

844 self, xpath: XPathExpr, name: str, value: str | None 

845 ) -> XPathExpr: 

846 assert value is not None 

847 xpath.add_condition(f"{name} = {self.xpath_literal(value)}") 

848 return xpath 

849 

850 def xpath_attrib_different( 

851 self, xpath: XPathExpr, name: str, value: str | None 

852 ) -> XPathExpr: 

853 assert value is not None 

854 # FIXME: this seems like a weird hack... 

855 if value: 

856 xpath.add_condition(f"not({name}) or {name} != {self.xpath_literal(value)}") 

857 else: 

858 xpath.add_condition(f"{name} != {self.xpath_literal(value)}") 

859 return xpath 

860 

861 def xpath_attrib_includes( 

862 self, xpath: XPathExpr, name: str, value: str | None 

863 ) -> XPathExpr: 

864 if value and is_non_whitespace(value): 

865 arg = self.xpath_literal(" " + value + " ") 

866 xpath.add_condition( 

867 f"{name} and contains({name}, {self.xpath_literal(value)}) " 

868 f"and contains(concat(' ', normalize-space({name}), ' '), {arg})" 

869 ) 

870 else: 

871 xpath.add_condition("0") 

872 return xpath 

873 

874 def xpath_attrib_dashmatch( 

875 self, xpath: XPathExpr, name: str, value: str | None 

876 ) -> XPathExpr: 

877 assert value is not None 

878 arg = self.xpath_literal(value) 

879 arg_dash = self.xpath_literal(value + "-") 

880 # Weird, but true... 

881 xpath.add_condition( 

882 f"{name} and ({name} = {arg} or starts-with({name}, {arg_dash}))" 

883 ) 

884 return xpath 

885 

886 def xpath_attrib_prefixmatch( 

887 self, xpath: XPathExpr, name: str, value: str | None 

888 ) -> XPathExpr: 

889 if value: 

890 xpath.add_condition( 

891 f"{name} and starts-with({name}, {self.xpath_literal(value)})" 

892 ) 

893 else: 

894 xpath.add_condition("0") 

895 return xpath 

896 

897 def xpath_attrib_suffixmatch( 

898 self, xpath: XPathExpr, name: str, value: str | None 

899 ) -> XPathExpr: 

900 if value: 

901 # Oddly there is a starts-with in XPath 1.0, but not ends-with 

902 xpath.add_condition( 

903 f"{name} and substring({name}, string-length({name})-{len(value) - 1}) = {self.xpath_literal(value)}" 

904 ) 

905 else: 

906 xpath.add_condition("0") 

907 return xpath 

908 

909 def xpath_attrib_substringmatch( 

910 self, xpath: XPathExpr, name: str, value: str | None 

911 ) -> XPathExpr: 

912 if value: 

913 # Attribute selectors are case sensitive 

914 xpath.add_condition(