Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/cssselect/xpath.py: 77%

1""" 

2cssselect.xpath 

3=============== 

4 

5Translation of parsed CSS selectors to XPath expressions. 

6 

7 

8:copyright: (c) 2007-2012 Ian Bicking and contributors. 

9See AUTHORS for more details. 

10:license: BSD, see LICENSE for more details. 

11 

12""" 

13 

14from __future__ import annotations 

15 

16import re 

17from typing import TYPE_CHECKING, cast 

18 

19from cssselect.parser import ( 

20 Attrib, 

21 Class, 

22 CombinedSelector, 

23 Element, 

24 Function, 

25 Hash, 

26 Matching, 

27 Negation, 

28 Pseudo, 

29 PseudoElement, 

30 Relation, 

31 Selector, 

32 SelectorError, 

33 SpecificityAdjustment, 

34 Tree, 

35 parse, 

36 parse_series, 

37) 

38 

39if TYPE_CHECKING: 

40 from collections.abc import Callable, Iterable 

41 

42 # typing.Self requires Python 3.11 

43 from typing_extensions import Self 

44 

45 

46class ExpressionError(SelectorError, RuntimeError): 

47 """Unknown or unsupported selector (eg. pseudo-class).""" 

48 

49 

50#### XPath Helpers 

51 

52 

53class XPathExpr: 

54 def __init__( 

55 self, 

56 path: str = "", 

57 element: str = "*", 

58 condition: str = "", 

59 star_prefix: bool = False, 

60 ) -> None: 

61 self.path = path 

62 self.element = element 

63 self.condition = condition 

64 

65 def __str__(self) -> str: 

66 path = str(self.path) + str(self.element) 

67 if self.condition: 

68 path += f"[{self.condition}]" 

69 return path 

70 

71 def __repr__(self) -> str: 

72 return f"{self.__class__.__name__}[{self}]" 

73 

74 def add_condition(self, condition: str, conjuction: str = "and") -> Self: 

75 if self.condition: 

76 self.condition = f"({self.condition}) {conjuction} ({condition})" 

77 else: 

78 self.condition = condition 

79 return self 

80 

81 def add_name_test(self) -> None: 

82 if self.element == "*": 

83 # We weren't doing a test anyway 

84 return 

85 prefix, colon, local = self.element.partition(":") 

86 if is_safe_name(prefix) and (not colon or local == "*" or is_safe_name(local)): 

87 # A node test, not a name() comparison: name() returns the 

88 # qualified name as written in the document, which would bypass 

89 # the XPath prefix mapping for a prefixed name like "ns:f" or 

90 # "ns:*" (from the CSS "ns|f" or "ns|*"), and would match 

91 # elements in a default namespace for an unprefixed name (which 

92 # the node test emitted for a bare "f" selector does not). 

93 self.add_condition(f"self::{self.element}") 

94 else: 

95 self.add_condition( 

96 f"name() = {GenericTranslator.xpath_literal(self.element)}" 

97 ) 

98 self.element = "*" 

99 

100 def add_star_prefix(self) -> None: 

101 """ 

102 Append '*/' to the path to keep the context constrained 

103 to a single parent. 

104 """ 

105 self.path += "*/" 

106 

107 def join(self, combiner: str, other: XPathExpr) -> Self: 

108 path = str(self) + combiner 

109 # Any "star prefix" is redundant when joining. 

110 if other.path != "*/": 

111 path += other.path 

112 self.path = path 

113 self.element = other.element 

114 self.condition = other.condition 

115 return self 

116 

117 

118split_at_single_quotes = re.compile("('+)").split 

119 

120# The spec is actually more permissive than that, but don’t bother. 

121# This is just for the fast path. 

122# http://www.w3.org/TR/REC-xml/#NT-NameStartChar 

123is_safe_name = re.compile("^[a-zA-Z_][a-zA-Z0-9_.-]*$").match 

124 

125# Test that the string is not empty and does not contain whitespace 

126is_non_whitespace = re.compile(r"^[^ \t\r\n\f]+$").match 

127 

128 

129#### Translation 

130 

131 

132class GenericTranslator: 

133 """ 

134 Translator for "generic" XML documents. 

135 

136 Everything is case-sensitive, no assumption is made on the meaning 

137 of element names and attribute names. 

138 

139 """ 

140 

141 #### 

142 #### HERE BE DRAGONS 

143 #### 

144 #### You are welcome to hook into this to change some behavior, 

145 #### but do so at your own risks. 

146 #### Until it has received a lot more work and review, 

147 #### I reserve the right to change this API in backward-incompatible ways 

148 #### with any minor version of cssselect. 

149 #### See https://github.com/scrapy/cssselect/pull/22 

150 #### -- Simon Sapin. 

151 #### 

152 

153 combinator_mapping = { 

154 " ": "descendant", 

155 ">": "child", 

156 "+": "direct_adjacent", 

157 "~": "indirect_adjacent", 

158 } 

159 

160 # Used to match a combinator against the context node, in :not(). 

161 _reverse_combinator_mapping = { 

162 " ": "ancestor::*", 

163 ">": "parent::*", 

164 "+": "preceding-sibling::*[1]", 

165 "~": "preceding-sibling::*", 

166 } 

167 

168 attribute_operator_mapping = { 

169 "exists": "exists", 

170 "=": "equals", 

171 "~=": "includes", 

172 "|=": "dashmatch", 

173 "^=": "prefixmatch", 

174 "$=": "suffixmatch", 

175 "*=": "substringmatch", 

176 "!=": "different", # XXX Not in Level 3 but meh 

177 } 

178 

179 #: The attribute used for ID selectors depends on the document language: 

180 #: http://www.w3.org/TR/selectors/#id-selectors 

181 id_attribute = "id" 

182 

183 #: The attribute used for ``:lang()`` depends on the document language: 

184 #: http://www.w3.org/TR/selectors/#lang-pseudo 

185 lang_attribute = "xml:lang" 

186 

187 #: The case sensitivity of document language element names, 

188 #: attribute names, and attribute values in selectors depends 

189 #: on the document language. 

190 #: http://www.w3.org/TR/selectors/#casesens 

191 #: 

192 #: When a document language defines one of these as case-insensitive, 

193 #: cssselect assumes that the document parser makes the parsed values 

194 #: lower-case. Making the selector lower-case too makes the comparaison 

195 #: case-insensitive. 

196 #: 

197 #: In HTML, element names and attributes names (but not attribute values) 

198 #: are case-insensitive. All of lxml.html, html5lib, BeautifulSoup4 

199 #: and HTMLParser make them lower-case in their parse result, so 

200 #: the assumption holds. 

201 lower_case_element_names = False 

202 lower_case_attribute_names = False 

203 lower_case_attribute_values = False 

204 

205 # class used to represent and xpath expression 

206 xpathexpr_cls = XPathExpr 

207 

208 def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: 

209 """Translate a *group of selectors* to XPath. 

210 

211 Pseudo-elements are not supported here since XPath only knows 

212 about "real" elements. 

213 

214 :param css: 

215 A *group of selectors* as a string. 

216 :param prefix: 

217 This string is prepended to the XPath expression for each selector. 

218 The default makes selectors scoped to the context node’s subtree. 

219 :raises: 

220 :class:`~cssselect.SelectorSyntaxError` on invalid selectors, 

221 :class:`ExpressionError` on unknown/unsupported selectors, 

222 including pseudo-elements. 

223 :returns: 

224 The equivalent XPath 1.0 expression as a string. 

225 

226 """ 

227 return " | ".join( 

228 self.selector_to_xpath(selector, prefix, translate_pseudo_elements=True) 

229 for selector in parse(css) 

230 ) 

231 

232 def selector_to_xpath( 

233 self, 

234 selector: Selector, 

235 prefix: str = "descendant-or-self::", 

236 translate_pseudo_elements: bool = False, 

237 ) -> str: 

238 """Translate a parsed selector to XPath. 

239 

240 

241 :param selector: 

242 A parsed :class:`Selector` object. 

243 :param prefix: 

244 This string is prepended to the resulting XPath expression. 

245 The default makes selectors scoped to the context node’s subtree. 

246 :param translate_pseudo_elements: 

247 Unless this is set to ``True`` (as :meth:`css_to_xpath` does), 

248 the :attr:`~Selector.pseudo_element` attribute of the selector 

249 is ignored. 

250 It is the caller's responsibility to reject selectors 

251 with pseudo-elements, or to account for them somehow. 

252 :raises: 

253 :class:`ExpressionError` on unknown/unsupported selectors. 

254 :returns: 

255 The equivalent XPath 1.0 expression as a string. 

256 

257 """ 

258 tree = getattr(selector, "parsed_tree", None) 

259 if not tree: 

260 raise TypeError(f"Expected a parsed selector, got {selector!r}") 

261 xpath = self.xpath(tree) 

262 assert isinstance(xpath, self.xpathexpr_cls) # help debug a missing 'return' 

263 if translate_pseudo_elements and selector.pseudo_element: 

264 xpath = self.xpath_pseudo_element(xpath, selector.pseudo_element) 

265 return (prefix or "") + str(xpath) 

266 

267 def xpath_pseudo_element( 

268 self, xpath: XPathExpr, pseudo_element: PseudoElement 

269 ) -> XPathExpr: 

270 """Translate a pseudo-element. 

271 

272 Defaults to not supporting pseudo-elements at all, 

273 but can be overridden by sub-classes. 

274 

275 """ 

276 raise ExpressionError("Pseudo-elements are not supported.") 

277 

278 @staticmethod 

279 def xpath_literal(s: str) -> str: 

280 s = str(s) 

281 if "'" not in s: 

282 s = f"'{s}'" 

283 elif '"' not in s: 

284 s = f'"{s}"' 

285 else: 

286 parts_quoted = [ 

287 f'"{part}"' if "'" in part else f"'{part}'" 

288 for part in split_at_single_quotes(s) 

289 if part 

290 ] 

291 s = "concat({})".format(",".join(parts_quoted)) 

292 return s 

293 

294 def xpath(self, parsed_selector: Tree) -> XPathExpr: 

295 """Translate any parsed selector object.""" 

296 type_name = type(parsed_selector).__name__ 

297 method = cast( 

298 "Callable[[Tree], XPathExpr] | None", 

299 getattr(self, f"xpath_{type_name.lower()}", None), 

300 ) 

301 if method is None: 

302 raise ExpressionError(f"{type_name} is not supported.") 

303 return method(parsed_selector) 

304 

305 # Dispatched by parsed object type 

306 

307 def xpath_combinedselector(self, combined: CombinedSelector) -> XPathExpr: 

308 """Translate a combined selector.""" 

309 combinator = self.combinator_mapping[combined.combinator] 

310 method = cast( 

311 "Callable[[XPathExpr, XPathExpr], XPathExpr]", 

312 getattr(self, f"xpath_{combinator}_combinator"), 

313 ) 

314 return method(self.xpath(combined.selector), self.xpath(combined.subselector)) 

315 

316 def xpath_negation(self, negation: Negation) -> XPathExpr: 

317 xpath = self.xpath(negation.selector) 

318 condition = self._xpath_match_condition(negation.subselector) 

319 if condition is None: 

320 # The argument matches every element, so :not() matches none. 

321 return xpath.add_condition("0") 

322 return xpath.add_condition(f"not({condition})") 

323 

324 def _xpath_match_condition(self, selector: Tree) -> str | None: 

325 """Return a condition that holds for the elements matching *selector*, 

326 or None if that is every element. 

327 

328 Unlike xpath(), which walks a selector from left to right, this 

329 matches the whole selector against the context node, using reverse 

330 axes for combinators. 

331 """ 

332 if isinstance(selector, CombinedSelector): 

333 axis = self._reverse_combinator_mapping[selector.combinator] 

334 left = self._xpath_match_condition(selector.selector) 

335 if left is not None: 

336 axis = f"{axis}[{left}]" 

337 right = self._xpath_match_condition(selector.subselector) 

338 return axis if right is None else f"{right} and {axis}" 

339 sub_xpath = self.xpath(selector) 

340 sub_xpath.add_name_test() 

341 return sub_xpath.condition or None 

342 

343 def xpath_relation(self, relation: Relation) -> XPathExpr: 

344 xpath = self.xpath(relation.selector) 

345 combinator = relation.combinator 

346 subselector = relation.subselector 

347 right = self.xpath(subselector.parsed_tree) 

348 method = cast( 

349 "Callable[[XPathExpr, XPathExpr], XPathExpr]", 

350 getattr( 

351 self, 

352 f"xpath_relation_{self.combinator_mapping[cast('str', combinator.value)]}_combinator", 

353 ), 

354 ) 

355 return method(xpath, right) 

356 

357 def xpath_matching(self, matching: Matching) -> XPathExpr: 

358 return self._xpath_add_selector_list_condition( 

359 self.xpath(matching.selector), matching.selector_list 

360 ) 

361 

362 def xpath_specificityadjustment(self, matching: SpecificityAdjustment) -> XPathExpr: 

363 return self._xpath_add_selector_list_condition( 

364 self.xpath(matching.selector), matching.selector_list 

365 ) 

366 

367 def _xpath_add_selector_list_condition( 

368 self, xpath: XPathExpr, selector_list: Iterable[Tree] 

369 ) -> XPathExpr: 

370 """Add a condition matching any selector of the list 

371 (for :is() and :where()).""" 

372 condition = "" 

373 for e in (self.xpath(selector) for selector in selector_list): 

374 if e.path: 

375 # Only a combined selector (e.g. "a b") translates to a path, 

376 # which cannot be embedded into a predicate of the outer 

377 # expression. The parser rejects combinators in these arguments, 

378 # so this is only reachable through a hand-built Matching or 

379 # SpecificityAdjustment node. 

380 raise ExpressionError( 

381 "Combined selectors are not supported inside " 

382 ":is(), :where() and :matches()" 

383 ) 

384 e.add_name_test() 

385 if not e.condition: 

386 # This argument matches any element, so the whole selector 

387 # list does too: it adds no condition. 

388 return xpath 

389 condition = ( 

390 f"({condition}) or ({e.condition})" if condition else e.condition 

391 ) 

392 return xpath.add_condition(condition) 

393 

394 def xpath_function(self, function: Function) -> XPathExpr: 

395 """Translate a functional pseudo-class.""" 

396 method_name = "xpath_{}_function".format(function.name.replace("-", "_")) 

397 method = cast( 

398 "Callable[[XPathExpr, Function], XPathExpr] | None", 

399 getattr(self, method_name, None), 

400 ) 

401 if not method: 

402 raise ExpressionError(f"The pseudo-class :{function.name}() is unknown") 

403 return method(self.xpath(function.selector), function) 

404 

405 def xpath_pseudo(self, pseudo: Pseudo) -> XPathExpr: 

406 """Translate a pseudo-class.""" 

407 method_name = "xpath_{}_pseudo".format(pseudo.ident.replace("-", "_")) 

408 method = cast( 

409 "Callable[[XPathExpr], XPathExpr] | None", 

410 getattr(self, method_name, None), 

411 ) 

412 if not method: 

413 # TODO: better error message for pseudo-elements? 

414 raise ExpressionError(f"The pseudo-class :{pseudo.ident} is unknown") 

415 return method(self.xpath(pseudo.selector)) 

416 

417 def xpath_attrib(self, selector: Attrib) -> XPathExpr: 

418 """Translate an attribute selector.""" 

419 operator = self.attribute_operator_mapping[selector.operator] 

420 method = cast( 

421 "Callable[[XPathExpr, str, str | None], XPathExpr]", 

422 getattr(self, f"xpath_attrib_{operator}"), 

423 ) 

424 if self.lower_case_attribute_names: 

425 name = selector.attrib.lower() 

426 else: 

427 name = selector.attrib 

428 safe = is_safe_name(name) 

429 if selector.namespace: 

430 name = f"{selector.namespace}:{name}" 

431 safe = safe and is_safe_name(selector.namespace) 

432 if safe: 

433 attrib = "@" + name 

434 else: 

435 attrib = f"attribute::*[name() = {self.xpath_literal(name)}]" 

436 if selector.value is None: 

437 value = None 

438 elif self.lower_case_attribute_values: 

439 value = cast("str", selector.value.value).lower() 

440 else: 

441 value = selector.value.value 

442 return method(self.xpath(selector.selector), attrib, value) 

443 

444 def xpath_class(self, class_selector: Class) -> XPathExpr: 

445 """Translate a class selector.""" 

446 # .foo is defined as [class~=foo] in the spec. 

447 xpath = self.xpath(class_selector.selector) 

448 return self.xpath_attrib_includes(xpath, "@class", class_selector.class_name) 

449 

450 def xpath_hash(self, id_selector: Hash) -> XPathExpr: 

451 """Translate an ID selector.""" 

452 xpath = self.xpath(id_selector.selector) 

453 return self.xpath_attrib_equals(xpath, "@id", id_selector.id) 

454 

455 def xpath_element(self, selector: Element) -> XPathExpr: 

456 """Translate a type or universal selector.""" 

457 element = selector.element 

458 if not element: 

459 element = "*" 

460 safe = True 

461 else: 

462 safe = bool(is_safe_name(element)) 

463 if self.lower_case_element_names: 

464 element = element.lower() 

465 if selector.namespace: 

466 # Namespace prefixes are case-sensitive. 

467 # http://www.w3.org/TR/css3-namespace/#prefixes 

468 element = f"{selector.namespace}:{element}" 

469 safe = safe and bool(is_safe_name(selector.namespace)) 

470 xpath = self.xpathexpr_cls(element=element) 

471 if not safe: 

472 # Not usable as an XPath name test (e.g. an escaped identifier 

473 # like di\a0 v): compare the serialized name instead. Done here 

474 # rather than through add_name_test(), which would mistake a ":" 

475 # inside such a name for a namespace prefix separator. 

476 xpath.add_condition(f"name() = {self.xpath_literal(element)}") 

477 xpath.element = "*" 

478 return xpath 

479 

480 # CombinedSelector: dispatch by combinator 

481 

482 def xpath_descendant_combinator( 

483 self, left: XPathExpr, right: XPathExpr 

484 ) -> XPathExpr: 

485 """right is a child, grand-child or further descendant of left""" 

486 return left.join("/descendant-or-self::*/", right) 

487 

488 def xpath_child_combinator(self, left: XPathExpr, right: XPathExpr) -> XPathExpr: 

489 """right is an immediate child of left""" 

490 return left.join("/", right) 

491 

492 def xpath_direct_adjacent_combinator( 

493 self, left: XPathExpr, right: XPathExpr 

494 ) -> XPathExpr: 

495 """right is a sibling immediately after left""" 

496 xpath = left.join("/following-sibling::", right) 

497 xpath.add_name_test() 

498 return xpath.add_condition("position() = 1") 

499 

500 def xpath_indirect_adjacent_combinator( 

501 self, left: XPathExpr, right: XPathExpr 

502 ) -> XPathExpr: 

503 """right is a sibling after left, immediately or not""" 

504 return left.join("/following-sibling::", right) 

505 

506 # The relative selector is kept in `condition` (instead of being folded 

507 # into `path`/`element`) so that `element` stays a plain element name: 

508 # later steps such as :first-of-type or :not() read and rewrite it. 

509 

510 def xpath_relation_descendant_combinator( 

511 self, left: XPathExpr, right: XPathExpr 

512 ) -> XPathExpr: 

513 """right is a child, grand-child or further descendant of left; select left""" 

514 return left.add_condition(f"descendant::{right}") 

515 

516 def xpath_relation_child_combinator( 

517 self, left: XPathExpr, right: XPathExpr 

518 ) -> XPathExpr: 

519 """right is an immediate child of left; select left""" 

520 return left.add_condition(f"./{right}") 

521 

522 def xpath_relation_direct_adjacent_combinator( 

523 self, left: XPathExpr, right: XPathExpr 

524 ) -> XPathExpr: 

525 """right is a sibling immediately after left; select left""" 

526 right.add_name_test() 

527 right.add_condition("position() = 1") 

528 return left.add_condition(f"following-sibling::{right}") 

529 

530 def xpath_relation_indirect_adjacent_combinator( 

531 self, left: XPathExpr, right: XPathExpr 

532 ) -> XPathExpr: 

533 """right is a sibling after left, immediately or not; select left""" 

534 return left.add_condition(f"following-sibling::{right}") 

535 

536 # Function: dispatch by function/pseudo-class name 

537 

538 def xpath_nth_child_function( 

539 self, 

540 xpath: XPathExpr, 

541 function: Function, 

542 last: bool = False, 

543 add_name_test: bool = True, 

544 ) -> XPathExpr: 

545 try: 

546 a, b = parse_series(function.arguments) 

547 except ValueError as ex: 

548 raise ExpressionError(f"Invalid series: '{function.arguments!r}'") from ex 

549 

550 # From https://www.w3.org/TR/css3-selectors/#structural-pseudos: 

551 # 

552 # :nth-child(an+b) 

553 # an+b-1 siblings before 

554 # 

555 # :nth-last-child(an+b) 

556 # an+b-1 siblings after 

557 # 

558 # :nth-of-type(an+b) 

559 # an+b-1 siblings with the same expanded element name before 

560 # 

561 # :nth-last-of-type(an+b) 

562 # an+b-1 siblings with the same expanded element name after 

563 # 

564 # So, 

565 # for :nth-child and :nth-of-type 

566 # 

567 # count(preceding-sibling::<nodetest>) = an+b-1 

568 # 

569 # for :nth-last-child and :nth-last-of-type 

570 # 

571 # count(following-sibling::<nodetest>) = an+b-1 

572 # 

573 # therefore, 

574 # count(...) - (b-1) ≡ 0 (mod a) 

575 # 

576 # if a == 0: 

577 # ~~~~~~~~~~ 

578 # count(...) = b-1 

579 # 

580 # if a < 0: 

581 # ~~~~~~~~~ 

582 # count(...) - b +1 <= 0 

583 # -> count(...) <= b-1 

584 # 

585 # if a > 0: 

586 # ~~~~~~~~~ 

587 # count(...) - b +1 >= 0 

588 # -> count(...) >= b-1 

589 

590 # work with b-1 instead 

591 b_min_1 = b - 1 

592 

593 # early-exit condition 1: 

594 # ~~~~~~~~~~~~~~~~~~~~~~~ 

595 # for a == 1, nth-*(an+b) means n+b-1 siblings before/after, 

596 # and since n ∈ {0, 1, 2, ...}, if b-1<=0, 

597 # there is always an "n" matching any number of siblings (maybe none) 

598 if a == 1 and b_min_1 <= 0: 

599 return xpath 

600 

601 # early-exit condition 2: 

602 # ~~~~~~~~~~~~~~~~~~~~~~~ 

603 # an+b-1 siblings with a<0 and (b-1)<0 is not possible 

604 if a < 0 and b_min_1 < 0: 

605 return xpath.add_condition("0") 

606 

607 # `add_name_test` boolean is inverted and somewhat counter-intuitive: 

608 # 

609 # nth_of_type() calls nth_child(add_name_test=False) 

610 nodetest = "*" if add_name_test else f"{xpath.element}" 

611 

612 # count siblings before or after the element 

613 if not last: 

614 siblings_count = f"count(preceding-sibling::{nodetest})" 

615 else: 

616 siblings_count = f"count(following-sibling::{nodetest})" 

617 

618 # special case of fixed position: nth-*(0n+b) 

619 # if a == 0: 

620 # ~~~~~~~~~~ 

621 # count(***-sibling::***) = b-1 

622 if a == 0: 

623 return xpath.add_condition(f"{siblings_count} = {b_min_1}") 

624 

625 expressions = [] 

626 

627 if a > 0: 

628 # siblings count, an+b-1, is always >= 0, 

629 # so if a>0, and (b-1)<=0, an "n" exists to satisfy this, 

630 # therefore, the predicate is only interesting if (b-1)>0 

631 if b_min_1 > 0: 

632 expressions.append(f"{siblings_count} >= {b_min_1}") 

633 else: 

634 # if a<0, and (b-1)<0, no "n" satisfies this, 

635 # this is tested above as an early exit condition 

636 # otherwise, 

637 expressions.append(f"{siblings_count} <= {b_min_1}") 

638 

639 # operations modulo 1 or -1 are simpler, one only needs to verify: 

640 # 

641 # - either: 

642 # count(***-sibling::***) - (b-1) = n = 0, 1, 2, 3, etc., 

643 # i.e. count(***-sibling::***) >= (b-1) 

644 # 

645 # - or: 

646 # count(***-sibling::***) - (b-1) = -n = 0, -1, -2, -3, etc., 

647 # i.e. count(***-sibling::***) <= (b-1) 

648 # we just did above. 

649 # 

650 if abs(a) != 1: 

651 # count(***-sibling::***) - (b-1) ≡ 0 (mod a) 

652 left = siblings_count 

653 

654 # apply "modulo a" on 2nd term, -(b-1), 

655 # to simplify things like "(... +6) % -3", 

656 # and also make it positive with |a| 

657 b_neg = (-b_min_1) % abs(a) 

658 

659 if b_neg != 0: 

660 left = f"({left} +{b_neg})" 

661 

662 expressions.append(f"{left} mod {a} = 0") 

663 

664 template = "(%s)" if len(expressions) > 1 else "%s" 

665 xpath.add_condition( 

666 " and ".join(template % expression for expression in expressions) 

667 ) 

668 return xpath 

669 

670 def xpath_nth_last_child_function( 

671 self, xpath: XPathExpr, function: Function 

672 ) -> XPathExpr: 

673 return self.xpath_nth_child_function(xpath, function, last=True) 

674 

675 @staticmethod 

676 def _check_of_type_element(xpath: XPathExpr, pseudo: str) -> None: 

677 """Raise an exception if an -of-type pseudo-class can't be used with 

678 the given element. 

679 

680 For "*" and namespace wildcards like "ns:*" the type of the element is 

681 not known, and counting same-type siblings cannot be expressed as an 

682 XPath 1.0 node test. 

683 """ 

684 element = xpath.element 

685 if element == "*" or element.endswith(":*"): 

686 css_element = element.replace(":", "|") 

687 raise ExpressionError(f"{css_element}:{pseudo} is not implemented") 

688 

689 def xpath_nth_of_type_function( 

690 self, xpath: XPathExpr, function: Function 

691 ) -> XPathExpr: 

692 self._check_of_type_element(xpath, "nth-of-type()") 

693 return self.xpath_nth_child_function(xpath, function, add_name_test=False) 

694 

695 def xpath_nth_last_of_type_function( 

696 self, xpath: XPathExpr, function: Function 

697 ) -> XPathExpr: 

698 self._check_of_type_element(xpath, "nth-last-of-type()") 

699 return self.xpath_nth_child_function( 

700 xpath, function, last=True, add_name_test=False 

701 ) 

702 

703 def xpath_contains_function( 

704 self, xpath: XPathExpr, function: Function 

705 ) -> XPathExpr: 

706 # Defined there, removed in later drafts: 

707 # http://www.w3.org/TR/2001/CR-css3-selectors-20011113/#content-selectors 

708 if function.argument_types() not in (["STRING"], ["IDENT"]): 

709 raise ExpressionError( 

710 f"Expected a single string or ident for :contains(), got {function.arguments!r}" 

711 ) 

712 value = cast("str", function.arguments[0].value) 

713 return xpath.add_condition(f"contains(., {self.xpath_literal(value)})") 

714 

715 def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr: 

716 if function.argument_types() not in (["STRING"], ["IDENT"]): 

717 raise ExpressionError( 

718 f"Expected a single string or ident for :lang(), got {function.arguments!r}" 

719 ) 

720 value = cast("str", function.arguments[0].value) 

721 return xpath.add_condition(f"lang({self.xpath_literal(value)})") 

722 

723 # Pseudo: dispatch by pseudo-class name 

724 

725 def xpath_root_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

726 return xpath.add_condition("not(parent::*)") 

727 

728 # CSS immediate children (CSS ":scope > div" to XPath "child::div" or "./div") 

729 # Works only at the start of a selector 

730 # Needed to get immediate children of a processed selector in Scrapy 

731 # for product in response.css('.product'): 

732 # description = product.css(':scope > div::text').get() 

733 def xpath_scope_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

734 xpath.add_name_test() 

735 return xpath.add_condition("position() = 1") 

736 

737 def xpath_first_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

738 return xpath.add_condition("count(preceding-sibling::*) = 0") 

739 

740 def xpath_last_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

741 return xpath.add_condition("count(following-sibling::*) = 0") 

742 

743 def xpath_first_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

744 self._check_of_type_element(xpath, "first-of-type") 

745 return xpath.add_condition(f"count(preceding-sibling::{xpath.element}) = 0") 

746 

747 def xpath_last_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

748 self._check_of_type_element(xpath, "last-of-type") 

749 return xpath.add_condition(f"count(following-sibling::{xpath.element}) = 0") 

750 

751 def xpath_only_child_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

752 # Count siblings, not the parent's children: the root element has 

753 # no parent, but it has no siblings either, so it must match. 

754 return xpath.add_condition( 

755 "count(preceding-sibling::*) = 0 and count(following-sibling::*) = 0" 

756 ) 

757 

758 def xpath_only_of_type_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

759 self._check_of_type_element(xpath, "only-of-type") 

760 return xpath.add_condition( 

761 f"count(preceding-sibling::{xpath.element}) = 0 " 

762 f"and count(following-sibling::{xpath.element}) = 0" 

763 ) 

764 

765 def xpath_empty_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

766 return xpath.add_condition("not(*) and not(string-length())") 

767 

768 def pseudo_never_matches(self, xpath: XPathExpr) -> XPathExpr: 

769 """Common implementation for pseudo-classes that never match.""" 

770 return xpath.add_condition("0") 

771 

772 xpath_link_pseudo = pseudo_never_matches 

773 xpath_visited_pseudo = pseudo_never_matches 

774 xpath_hover_pseudo = pseudo_never_matches 

775 xpath_active_pseudo = pseudo_never_matches 

776 xpath_focus_pseudo = pseudo_never_matches 

777 xpath_target_pseudo = pseudo_never_matches 

778 xpath_enabled_pseudo = pseudo_never_matches 

779 xpath_disabled_pseudo = pseudo_never_matches 

780 xpath_checked_pseudo = pseudo_never_matches 

781 

782 # Attrib: dispatch by attribute operator 

783 

784 def xpath_attrib_exists( 

785 self, xpath: XPathExpr, name: str, value: str | None 

786 ) -> XPathExpr: 

787 assert not value 

788 xpath.add_condition(name) 

789 return xpath 

790 

791 def xpath_attrib_equals( 

792 self, xpath: XPathExpr, name: str, value: str | None 

793 ) -> XPathExpr: 

794 assert value is not None 

795 xpath.add_condition(f"{name} = {self.xpath_literal(value)}") 

796 return xpath 

797 

798 def xpath_attrib_different( 

799 self, xpath: XPathExpr, name: str, value: str | None 

800 ) -> XPathExpr: 

801 assert value is not None 

802 # FIXME: this seems like a weird hack... 

803 if value: 

804 xpath.add_condition(f"not({name}) or {name} != {self.xpath_literal(value)}") 

805 else: 

806 xpath.add_condition(f"{name} != {self.xpath_literal(value)}") 

807 return xpath 

808 

809 def xpath_attrib_includes( 

810 self, xpath: XPathExpr, name: str, value: str | None 

811 ) -> XPathExpr: 

812 if value and is_non_whitespace(value): 

813 arg = self.xpath_literal(" " + value + " ") 

814 xpath.add_condition( 

815 f"{name} and contains(concat(' ', normalize-space({name}), ' '), {arg})" 

816 ) 

817 else: 

818 xpath.add_condition("0") 

819 return xpath 

820 

821 def xpath_attrib_dashmatch( 

822 self, xpath: XPathExpr, name: str, value: str | None 

823 ) -> XPathExpr: 

824 assert value is not None 

825 arg = self.xpath_literal(value) 

826 arg_dash = self.xpath_literal(value + "-") 

827 # Weird, but true... 

828 xpath.add_condition( 

829 f"{name} and ({name} = {arg} or starts-with({name}, {arg_dash}))" 

830 ) 

831 return xpath 

832 

833 def xpath_attrib_prefixmatch( 

834 self, xpath: XPathExpr, name: str, value: str | None 

835 ) -> XPathExpr: 

836 if value: 

837 xpath.add_condition( 

838 f"{name} and starts-with({name}, {self.xpath_literal(value)})" 

839 ) 

840 else: 

841 xpath.add_condition("0") 

842 return xpath 

843 

844 def xpath_attrib_suffixmatch( 

845 self, xpath: XPathExpr, name: str, value: str | None 

846 ) -> XPathExpr: 

847 if value: 

848 # Oddly there is a starts-with in XPath 1.0, but not ends-with 

849 xpath.add_condition( 

850 f"{name} and substring({name}, string-length({name})-{len(value) - 1}) = {self.xpath_literal(value)}" 

851 ) 

852 else: 

853 xpath.add_condition("0") 

854 return xpath 

855 

856 def xpath_attrib_substringmatch( 

857 self, xpath: XPathExpr, name: str, value: str | None 

858 ) -> XPathExpr: 

859 if value: 

860 # Attribute selectors are case sensitive 

861 xpath.add_condition( 

862 f"{name} and contains({name}, {self.xpath_literal(value)})" 

863 ) 

864 else: 

865 xpath.add_condition("0") 

866 return xpath 

867 

868 

869class HTMLTranslator(GenericTranslator): 

870 """ 

871 Translator for (X)HTML documents. 

872 

873 Has a more useful implementation of some pseudo-classes based on 

874 HTML-specific element names and attribute names, as described in 

875 the `HTML5 specification`_. It assumes no-quirks mode. 

876 The API is the same as :class:`GenericTranslator`. 

877 

878 .. _HTML5 specification: http://www.w3.org/TR/html5/links.html#selectors 

879 

880 :param xhtml: 

881 If false (the default), element names and attribute names 

882 are case-insensitive. 

883 

884 """ 

885 

886 lang_attribute = "lang" 

887 

888 def __init__(self, xhtml: bool = False) -> None: 

889 self.xhtml = xhtml # Might be useful for sub-classes? 

890 if not xhtml: 

891 # See their definition in GenericTranslator. 

892 self.lower_case_element_names = True 

893 self.lower_case_attribute_names = True 

894 

895 def xpath_checked_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

896 # FIXME: is this really all the elements? 

897 return xpath.add_condition( 

898 "(@selected and name(.) = 'option') or " 

899 "(@checked " 

900 "and (name(.) = 'input' or name(.) = 'command')" 

901 "and (@type = 'checkbox' or @type = 'radio'))" 

902 ) 

903 

904 def xpath_lang_function(self, xpath: XPathExpr, function: Function) -> XPathExpr: 

905 if function.argument_types() not in (["STRING"], ["IDENT"]): 

906 raise ExpressionError( 

907 f"Expected a single string or ident for :lang(), got {function.arguments!r}" 

908 ) 

909 value = function.arguments[0].value 

910 assert value 

911 arg = self.xpath_literal(value.lower() + "-") 

912 return xpath.add_condition( 

913 "ancestor-or-self::*[@lang][1][starts-with(concat(" 

914 # XPath 1.0 has no lower-case function... 

915 f"translate(@{self.lang_attribute}, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', " 

916 "'abcdefghijklmnopqrstuvwxyz'), " 

917 f"'-'), {arg})]" 

918 ) 

919 

920 def xpath_link_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

921 return xpath.add_condition( 

922 "@href and (name(.) = 'a' or name(.) = 'link' or name(.) = 'area')" 

923 ) 

924 

925 # Links are never visited, the implementation for :visited is the same 

926 # as in GenericTranslator 

927 

928 def xpath_disabled_pseudo(self, xpath: XPathExpr) -> XPathExpr: 

929 # http://www.w3.org/TR/html5/section-index.html#attributes-1 

930 return xpath.add_condition( 

931 """