Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/_url.py: 74%

1from __future__ import annotations 

2 

3import dataclasses 

4import encodings.idna 

5import functools 

6import ipaddress 

7import os 

8import re 

9import string 

10import sys 

11import unicodedata 

12from typing import TYPE_CHECKING 

13from urllib.parse import ParseResult, scheme_chars, uses_netloc, uses_params 

14 

15from w3lib._infra import _ASCII_TAB_OR_NEWLINE, _C0_CONTROL_OR_SPACE 

16 

17if TYPE_CHECKING: 

18 from collections.abc import Generator 

19 from urllib.parse import _QueryType 

20 

21_IS_WINDOWS = os.name == "nt" 

22 

23 

24_FS_ENCODING = sys.getfilesystemencoding() 

25_FS_ERRORS = sys.getfilesystemencodeerrors() 

26 

27# https://url.spec.whatwg.org/ 

28# https://url.spec.whatwg.org/commit-snapshots/a46cb9188a48c2c9d80ba32a9b1891652d6b4900/#default-port 

29_DEFAULT_PORTS = { 

30 "ftp": 21, 

31 "file": None, 

32 "http": 80, 

33 "https": 443, 

34 "ws": 80, 

35 "wss": 443, 

36} 

37_SPECIAL_SCHEMES = set(_DEFAULT_PORTS.keys()) 

38 

39# constants from RFC 3986, Section 2.2 and 2.3 

40RFC3986_GEN_DELIMS = b":/?#[]@" 

41RFC3986_SUB_DELIMS = b"!$&'()*+,;=" 

42RFC3986_RESERVED = RFC3986_GEN_DELIMS + RFC3986_SUB_DELIMS 

43RFC3986_UNRESERVED = (string.ascii_letters + string.digits + "-._~").encode("ascii") 

44EXTRA_SAFE_CHARS = b"|" # see https://github.com/scrapy/w3lib/pull/25 

45 

46RFC3986_USERINFO_SAFE_CHARS = RFC3986_UNRESERVED + RFC3986_SUB_DELIMS + b":" 

47_SAFE_CHARS = RFC3986_RESERVED + RFC3986_UNRESERVED + EXTRA_SAFE_CHARS + b"%" 

48_PATH_SAFE_CHARS = _SAFE_CHARS.replace(b"#", b"") 

49_PATH_SAFE_CHARS_STR = _PATH_SAFE_CHARS.decode() 

50_USES_NETLOC = frozenset(uses_netloc) 

51_SCHEME_CHARS = frozenset(scheme_chars) 

52_USES_PARAMS = frozenset(uses_params) 

53_ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE = str.maketrans("", "", _ASCII_TAB_OR_NEWLINE) 

54_C0_CONTROL_OR_SPACE_RE = re.compile(rf"[{_C0_CONTROL_OR_SPACE}]") 

55_SCHEME_RE = re.compile(rf"^([a-zA-Z][{re.escape(scheme_chars)}]*):") 

56 

57_IPV_FUTURE_RE = re.compile(r"\Av[a-fA-F0-9]+\..+\Z") 

58# "\" terminates the authority of a special-scheme URL just like "/" under the 

59# URL living standard, so it belongs with the other authority delimiters here. 

60_NETLOC_DELIMS_RE = re.compile(r"[/?#@:\\]") 

61_NETLOC_STRIP_CHARS = str.maketrans("", "", "@:#?") 

62# https://url.spec.whatwg.org/#forbidden-host-code-point 

63_FORBIDDEN_HOST_CODE_POINT_RE = re.compile(r"[\x00\t\n\r #/:<>?@[\\\]^|]") 

64# https://url.spec.whatwg.org/#forbidden-domain-code-point 

65_FORBIDDEN_DOMAIN_CODE_POINT_RE = re.compile(r"[\x00-\x20#%/:<>?@[\\\]^|\x7f]") 

66_OPAQUE_HOST_SAFE_CHARS = RFC3986_SUB_DELIMS + b"%" 

67 

68 

69def _strip(input_string: str) -> str: 

70 if not input_string: 

71 return input_string 

72 

73 if not _C0_CONTROL_OR_SPACE_RE.search(input_string): 

74 return input_string 

75 

76 return input_string.strip(_C0_CONTROL_OR_SPACE).translate( 

77 _ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE 

78 ) 

79 

80 

81@functools.cache 

82def _hex_encode_table() -> bytes: 

83 """Build a lookup table for percent-encoded byte values. 

84 

85 | byte | encoding | 

86 |------|----------| 

87 | 0 | %00 | 

88 | 1 | %01 | 

89 | ... | ... | 

90 | 255 | %FF | 

91 

92 Each entry is exactly 3 bytes: b"%HH". 

93 

94 Returns: 

95 A bytes object of length 256 * 3 containing all percent encodings. 

96 """ 

97 return b"".join(f"%{i:02X}".encode() for i in range(256)) 

98 

99 

100@functools.cache 

101def _hex_decode_table() -> bytes: 

102 """Build a lookup table for decoding hex ASCII characters. 

103 

104 | ASCII | value | 

105 |--------|--------------| 

106 | '0'-'9'| 0-9 | 

107 | 'A'-'F'| 10-15 | 

108 | 'a'-'f'| 10-15 | 

109 | other | 255 (invalid)| 

110 

111 Returns: 

112 A bytes object of length 256 containing nibble values. 

113 """ 

114 table = bytearray([255]) * 256 

115 table[48:58] = bytes(range(10)) # '0'-'9' 

116 table[65:71] = bytes(range(10, 16)) # 'A'-'F' 

117 table[97:103] = bytes(range(10, 16)) # 'a'-'f' 

118 return bytes(table) 

119 

120 

121@functools.cache 

122def _safe_table(safe: bytes = RFC3986_UNRESERVED) -> bytes: 

123 """Build a lookup table marking safe (non-encoded) bytes. 

124 

125 | byte | is allowed? | 

126 |------|-------------| 

127 | 0 | 0 | 

128 | 32 | 1 (if safe) | 

129 | 65 | 1 | 

130 | 255 | 0 | 

131 

132 Returns: 

133 A bytes object of length 256 acting as a boolean mask (0/1). 

134 """ 

135 table = bytearray(256) 

136 for b in safe: 

137 table[b] = 1 

138 return bytes(table) 

139 

140 

141@functools.cache 

142def _quote_table(safe: bytes = b"", quote_plus: bool = False) -> tuple[bytes, ...]: 

143 """Precompute encoding rules for all 256 byte values. 

144 

145 Decision table: 

146 | condition | output | 

147 |-------------------------------|--------| 

148 | byte in safe | as-is | 

149 | byte == 32 and quote_plus | "+" | 

150 | otherwise | "%HH" | 

151 

152 Example mapping: 

153 | byte | char | output | 

154 |------|------|--------| 

155 | 65 | A | b"A" | 

156 | 32 | space| b"+" | 

157 | 255 | N/A | b"%FF" | 

158 

159 Returns: 

160 A 256-entry tuple mapping byte value (index) -> encoded bytes. 

161 """ 

162 hex_table = _hex_encode_table() 

163 allowed = _safe_table(RFC3986_UNRESERVED + safe) if safe else _safe_table() 

164 output: list[bytes] = [b""] * 256 

165 

166 for idx, byte in enumerate(range(256)): 

167 if allowed[byte]: 

168 output[idx] = chr(byte).encode() 

169 elif quote_plus and byte == 32: # ord(' ') 

170 output[idx] = b"+" 

171 else: 

172 offset = byte * 3 

173 output[idx] = hex_table[offset : offset + 3] 

174 

175 return tuple(output) 

176 

177 

178def _quote(data: bytes, safe: bytes = b"", quote_plus: bool = False) -> bytes: 

179 """Fast URL-style quoting using a precomputed table. 

180 

181 Args: 

182 data: Input bytes. 

183 safe: Additional unescaped bytes. 

184 quote_plus: Encode space as '+' if True. 

185 

186 Returns: 

187 Percent-encoded bytes. 

188 """ 

189 if not data: # pragma: no cover 

190 return b"" 

191 

192 transform_table = _quote_table(safe, quote_plus) 

193 return b"".join([transform_table[byte] for byte in data]) 

194 

195 

196def _quote_into( 

197 data: bytes, output: bytearray, safe: bytes = b"", quote_plus: bool = False 

198) -> None: 

199 if not data: # pragma: no cover 

200 return 

201 

202 transform_table = _quote_table(safe, quote_plus) 

203 output += b"".join([transform_table[byte] for byte in data]) 

204 

205 

206def _unquote(data: bytes | bytearray | str) -> bytes: 

207 if not data: 

208 return b"" 

209 

210 if isinstance(data, str): 

211 data = data.encode() 

212 

213 first_percent = data.find(b"%") 

214 

215 if first_percent < 0: 

216 return bytes(data) 

217 

218 hex_decode_table = _hex_decode_table() 

219 

220 data_length = len(data) 

221 # stop at len - 2 because "%HH" decoding reads 2 extra bytes after '%' 

222 decode_limit = data_length - 2 

223 

224 output = bytearray(data_length) 

225 output[:first_percent] = data[:first_percent] 

226 

227 input_index = first_percent 

228 output_index = first_percent 

229 

230 while input_index < decode_limit: 

231 current_byte = data[input_index] 

232 

233 if current_byte == 37: # ord('%') 

234 # Decoding "%HH" sequence 

235 # Step 1: read two hex characters after '%' 

236 # Example: "%4F" -> '4' and 'F' 

237 high_nibble = hex_decode_table[data[input_index + 1]] 

238 low_nibble = hex_decode_table[data[input_index + 2]] 

239 

240 # Step 2: validate both characters are valid hex digits 

241 # hex_decode_table returns 255 for invalid input 

242 # bitwise OR catches any invalid nibble quickly 

243 if (high_nibble | low_nibble) != 255: 

244 # Step 3: combine two 4-bit nibbles into one byte 

245 # (high_nibble << 4) + low_nibble 

246 # Example: 0x4 and 0xF -> 0x4F 

247 output[output_index] = (high_nibble << 4) | low_nibble 

248 input_index += 3 # skip past "%HH" in input 

249 output_index += 1 # advance output position by one decoded byte 

250 continue 

251 

252 output[output_index] = current_byte 

253 input_index += 1 

254 output_index += 1 

255 

256 while input_index < data_length: # tail 

257 output[output_index] = data[input_index] 

258 input_index += 1 

259 output_index += 1 

260 

261 return bytes(output[:output_index]) 

262 

263 

264def _unquote_plus( 

265 data: bytes | bytearray | str, 

266) -> bytes: 

267 # This function is intentionally duplicated from `_unquote` for performance. 

268 # The duplication avoids extra branching for '+' handling in hot loop. 

269 if not data: 

270 return b"" 

271 

272 if isinstance(data, str): # pragma: no cover 

273 data = data.encode() 

274 

275 first_percent = data.find(b"%") 

276 first_plus = data.find(b"+") 

277 

278 first_special = min(first_plus, first_percent) 

279 

280 if first_special < 0: 

281 first_special = max(first_percent, first_plus) 

282 

283 if first_special < 0: 

284 return bytes(data) 

285 

286 hex_decode_table = _hex_decode_table() 

287 

288 data_length = len(data) 

289 decode_limit = data_length - 2 

290 

291 output = bytearray(data_length) 

292 output[:first_special] = data[:first_special] 

293 

294 input_index = first_special 

295 output_index = first_special 

296 

297 while input_index < decode_limit: 

298 current_byte = data[input_index] 

299 

300 if current_byte == 43: # ord('+') 

301 output[output_index] = 32 # ord(' ') 

302 input_index += 1 

303 output_index += 1 

304 continue 

305 

306 if current_byte == 37: # ord('%') 

307 high_nibble = hex_decode_table[data[input_index + 1]] 

308 low_nibble = hex_decode_table[data[input_index + 2]] 

309 

310 if (high_nibble | low_nibble) != 255: 

311 output[output_index] = (high_nibble << 4) | low_nibble 

312 input_index += 3 

313 output_index += 1 

314 continue 

315 

316 output[output_index] = current_byte 

317 input_index += 1 

318 output_index += 1 

319 

320 while input_index < data_length: # tail 

321 current_byte = data[input_index] 

322 

323 if current_byte == 43: # ord('+') 

324 output[output_index] = 32 # ord(' ') 

325 else: 

326 output[output_index] = current_byte 

327 

328 input_index += 1 

329 output_index += 1 

330 

331 return bytes(output[:output_index]) 

332 

333 

334def _parse_qs( 

335 qs: str | bytes, 

336 keep_blank_values: bool = False, 

337 separator: bytes = b"&", 

338) -> dict[bytes, list[bytes]]: 

339 """Reimplementation of urllib.parse.parse_qs which: 

340 - Doesn't use _coerce_args or _coerce_result 

341 - Works directly on bytes internally (no type coercion layer) 

342 - Returns bytes keys/values only""" 

343 if not qs: # pragma: no cover 

344 return {} 

345 

346 if isinstance(qs, str): # pragma: no cover 

347 qs = qs.encode() 

348 

349 result: dict[bytes, list[bytes]] = {} 

350 

351 for field in qs.split(separator): 

352 if not field: 

353 continue 

354 

355 key, sep, value = field.partition(b"=") 

356 

357 if not keep_blank_values and (not sep or not value): 

358 continue 

359 

360 key = _unquote_plus(key) 

361 value = _unquote_plus(value) 

362 

363 if key in result: 

364 result[key].append(value) 

365 else: 

366 result[key] = [value] 

367 

368 return result 

369 

370 

371def _parse_qsl( 

372 qs: str | bytes, 

373 keep_blank_values: bool = False, 

374 separator: bytes = b"&", 

375) -> list[tuple[bytes, bytes]]: 

376 """Reimplementation of urllib.parse.parse_qsl which: 

377 - Doesn't use _coerce_args or _coerce_result 

378 - Works directly on bytes internally (no type coercion layer) 

379 - Returns only bytes tuples""" 

380 # This function is intentionally duplicated from `_parse_qs` for performance. 

381 if not qs: 

382 return [] 

383 

384 if isinstance(qs, str): 

385 qs = qs.encode() 

386 

387 result: list[tuple[bytes, bytes]] = [] 

388 

389 for field in qs.split(separator): 

390 if not field: 

391 continue 

392 

393 key, sep, value = field.partition(b"=") 

394 

395 if not keep_blank_values and (not sep or not value): 

396 continue 

397 

398 result.append((_unquote_plus(key), _unquote_plus(value))) 

399 

400 return result 

401 

402 

403def _urlencode(query: _QueryType, separator: bytes = b"&") -> bytes: 

404 if hasattr(query, "items"): # pragma: no cover 

405 query = query.items() # type: ignore[assignment] 

406 

407 if not query: # pragma: no cover 

408 return b"" 

409 

410 result: list[bytes] = [] 

411 tmp_buf = bytearray() 

412 

413 for key, value in query: # type: ignore[str-unpack] 

414 _quote_into( 

415 key if isinstance(key, bytes) else str(key).encode(), 

416 output=tmp_buf, 

417 quote_plus=True, 

418 ) 

419 tmp_buf.append(61) # ord('=') 

420 _quote_into( 

421 value if isinstance(value, bytes) else str(value).encode(), 

422 output=tmp_buf, 

423 quote_plus=True, 

424 ) 

425 result.append(bytes(tmp_buf)) 

426 tmp_buf.clear() 

427 

428 return separator.join(result) 

429 

430 

431def _split_params(scheme: str, url: str) -> tuple[str, str]: 

432 """Split the params from the path, as urlib.parse.urlparse does.""" 

433 if scheme in _USES_PARAMS: 

434 # Only a ";" in the last segment starts the params; one in an earlier 

435 # segment is an ordinary path character. 

436 semi_idx = url.find(";", url.rfind("/") + 1) 

437 

438 if semi_idx != -1: 

439 return url[:semi_idx], url[semi_idx + 1 :] 

440 

441 return url, "" 

442 

443 

444def _urlparse( 

445 url: str, 

446 scheme: str = "", 

447 allow_fragments: bool = True, 

448) -> ParseResult: 

449 """Reimplementation of urlib.parse.urlparse but without _coerce_args/_coerce_result.""" 

450 if not url: # pragma: no cover 

451 return ParseResult(scheme, "", "", "", "", "") 

452 

453 scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) 

454 url, params = _split_params(scheme, url) 

455 

456 return ParseResult(scheme, netloc, url, params, query, fragment) 

457 

458 

459def _urlunparse( 

460 scheme: str, 

461 netloc: str, 

462 url: str, 

463 params: str, 

464 query: str, 

465 fragment: str, 

466) -> str: 

467 """Reimplementation of urlib.parse.urlunparse but without _coerce_args/_coerce_result.""" 

468 if params: 

469 url = f"{url};{params}" 

470 return _urlunsplit(scheme, netloc, url, query, fragment) 

471 

472 

473def _urlunsplit(scheme: str, netloc: str, url: str, query: str, fragment: str) -> str: 

474 """Reimplementation of urlib.parse.urlunsplit but without _coerce_args/_coerce_result.""" 

475 

476 if netloc: 

477 if url and url[:1] != "/": 

478 url = f"/{url}" 

479 url = f"//{netloc}{url}" 

480 elif url[:2] == "//" or ( 

481 scheme and scheme in _USES_NETLOC and (not url or url[:1] == "/") 

482 ): 

483 url = f"//{url}" 

484 

485 if scheme: 

486 scheme = f"{scheme}:" 

487 

488 if query: 

489 query = f"?{query}" 

490 

491 if fragment: 

492 fragment = f"#{fragment}" 

493 

494 return f"{scheme}{url}{query}{fragment}" 

495 

496 

497@dataclasses.dataclass(slots=True, eq=False, repr=False) 

498class _SplitResult: # pylint: disable=too-many-instance-attributes 

499 scheme: str 

500 netloc: str 

501 path: str 

502 query: str 

503 fragment: str 

504 

505 username: str | None = None 

506 password: str | None = None 

507 hostname: str | None = None 

508 port: str | int | None = None 

509 

510 def __post_init__(self) -> None: 

511 if self.hostname is not None: 

512 hostname, delim, zone = self.hostname.partition("%") 

513 self.hostname = f"{hostname.lower()}{delim}{zone}" 

514 

515 if self.port is not None: 

516 if isinstance(self.port, str) and not ( 

517 self.port.isascii() and self.port.isdigit() 

518 ): 

519 raise ValueError( 

520 f"Port could not be cast to integer value as {self.port}" 

521 ) 

522 self.port = int(self.port) 

523 

524 if self.port not in range(65535 + 1): 

525 raise ValueError("Port out of range 0-65535") 

526 

527 def __iter__(self) -> Generator[str]: 

528 yield self.scheme 

529 yield self.netloc 

530 yield self.path 

531 yield self.query 

532 yield self.fragment 

533 

534 def __len__(self) -> int: 

535 return 5 # pragma: no cover 

536 

537 def __getitem__(self, index: int) -> str: # pragma: no cover 

538 match index: 

539 case 0: 

540 return self.scheme 

541 case 1: 

542 return self.netloc 

543 case 2: 

544 return self.path 

545 case 3: 

546 return self.query 

547 case 4: 

548 return self.fragment 

549 raise IndexError 

550 

551 

552def _checknetloc(netloc: str) -> None: 

553 """ 

554 Validate that NFKC normalization does not introduce reserved URL characters. 

555 

556 Raises: 

557 ValueError: If normalization introduces reserved delimiters. 

558 """ 

559 if not netloc or netloc.isascii(): 

560 return 

561 

562 # IDNA uses NFKC equivalence. Remove already-valid delimiters before 

563 # normalization so we only detect newly introduced ones. 

564 cleaned, normalized = _nfkc_netloc(netloc) 

565 

566 if cleaned == normalized: 

567 return 

568 

569 if _NETLOC_DELIMS_RE.search(normalized): 

570 raise ValueError( 

571 f"netloc {netloc!r} contains invalid characters under NFKC normalization" 

572 ) 

573 

574 

575def _check_bracketed_netloc(netloc: str) -> None: 

576 """ 

577 Validate bracket usage in a URL netloc. 

578 

579 Raises: 

580 ValueError: If bracket placement or host syntax is invalid. 

581 

582 This is basically a backport of https://github.com/python/cpython/issues/105704 

583 """ 

584 hostname_and_port = netloc.rpartition("@")[2] 

585 

586 before_bracket, has_open_bracket, bracketed = hostname_and_port.partition("[") 

587 

588 if not has_open_bracket: 

589 # Brackets confined to the userinfo leave a regular host. 

590 if "]" in hostname_and_port: 

591 raise ValueError("Invalid IPv6 URL") 

592 return 

593 

594 # No data is allowed before '['. 

595 if before_bracket: 

596 raise ValueError("Invalid IPv6 URL") 

597 

598 hostname, _, port = bracketed.partition("]") 

599 

600 # Only ':<port>' may follow ']'. 

601 if port and not port.startswith(":"): 

602 raise ValueError("Invalid IPv6 URL") 

603 # port validation done after, in `_SplitResult.__post_init__` 

604 

605 _check_bracketed_host(hostname) 

606 

607 

608def _check_bracketed_host(hostname: str) -> None: 

609 """ 

610 Validate a bracketed host according to RFC 3986 / WHATWG URL rules. 

611 

612 Raises: 

613 ValueError: If the host is invalid. 

614 """ 

615 # IPvFuture: v<HEXDIG>.<address> 

616 if hostname.startswith(("v", "V")): 

617 if not _IPV_FUTURE_RE.fullmatch(hostname): 

618 raise ValueError("IPvFuture address is invalid") 

619 return 

620 

621 # ip_address() raises ValueError if invalid. 

622 ip = ipaddress.ip_address(hostname) 

623 

624 # Bracketed IPv4 literals are forbidden. 

625 if isinstance(ip, ipaddress.IPv4Address): 

626 raise ValueError("An IPv4 address cannot be in brackets") # noqa: TRY004 

627 

628 

629@functools.lru_cache 

630def _urlsplit( # pylint: disable=too-many-locals,too-many-statements 

631 url: str, 

632 scheme: str = "", 

633 allow_fragments: bool = True, 

634) -> _SplitResult: 

635 """Reimplementation of urllib.parse.urlsplit which: 

636 - Doesn't use _coerce_args or _coerce_result 

637 - Does manual single-pass scanning instead of repeated .find/.split calls 

638 (where it's beneficial) 

639 - Have reduced string allocations by slicing once using computed indices 

640 - Avoids extra computations as much as possible 

641 """ 

642 if not url: 

643 return _SplitResult(scheme, "", "", "", "") 

644 

645 # urllib.parse.urlsplit removes every ASCII tab and newline from anywhere in 

646 # the URL before parsing (the WHATWG "remove all ASCII tab or newline" 

647 # step). This reimplementation only stripped leading C0/space, so a tab or 

648 # newline embedded in the authority survived: parse_url("http://exa\tmple.com") 

649 # reported host "exa\tmple.com" while urlsplit and browsers drop the tab and 

650 # resolve "example.com". The membership checks keep the common tab-free path 

651 # free of translate calls and string allocations. 

652 if "\t" in url or "\n" in url or "\r" in url: 

653 url = url.translate(_ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE) 

654 if "\t" in scheme or "\n" in scheme or "\r" in scheme: 

655 scheme = scheme.translate(_ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE) 

656 url, scheme = url.lstrip(_C0_CONTROL_OR_SPACE), scheme.strip(_C0_CONTROL_OR_SPACE) 

657 

658 netloc = query = fragment = "" 

659 

660 if m := _SCHEME_RE.match(url): 

661 scheme = m.group(1).lower() 

662 url = url[m.end() :] 

663 

664 # The URL living standard treats "\" like "/" for special-scheme URLs, but 

665 # only in the authority and path; a "\" in the query or fragment is left 

666 # alone. Without this, "http://evil.com\@good.com/" is read as userinfo 

667 # "evil.com\" plus host "good.com", while a browser ends the authority at 

668 # the "\" and connects to "evil.com". 

669 if scheme in _SPECIAL_SCHEMES and "\\" in url: 

670 cut = len(url) 

671 question_idx = url.find("?") 

672 if question_idx != -1: 

673 cut = question_idx 

674 hash_idx = url.find("#") 

675 if hash_idx != -1 and hash_idx < cut: 

676 cut = hash_idx 

677 url = url[:cut].replace("\\", "/") + url[cut:] 

678 

679 # The scan skips the leading "//" of an authority; for URLs without an 

680 # authority it must start at 0, otherwise a "?" or "#" at index 0 or 1 

681 # (e.g. relative URLs like "a?b" or "a#f") is never recorded. 

682 scan_start = 2 if url[:2] == "//" else 0 

683 # Multiple str.find() are faster than a single manual loop. 

684 slash_pos = url.find("/", scan_start) 

685 question_pos = url.find("?", scan_start) 

686 hash_pos = url.find("#", scan_start) 

687 open_br_pos = url.find("[", scan_start) 

688 closing_br_pos = url.find("]", scan_start) 

689 

690 if url[:2] == "//": 

691 delim = len(url) 

692 

693 if 0 < slash_pos < delim: 

694 delim = slash_pos 

695 if 0 < question_pos < delim: 

696 delim = question_pos 

697 if 0 < hash_pos < delim: 

698 delim = hash_pos 

699 

700 # Brackets only delimit an IPv6 host when they fall inside the 

701 # authority. A "[" or "]" in the path, query or fragment is an 

702 # ordinary character and must not drive IPv6 host parsing. 

703 if not 2 <= open_br_pos < delim: 

704 open_br_pos = -1 

705 if not 2 <= closing_br_pos < delim: 

706 closing_br_pos = -1 

707 

708 if (open_br_pos != -1) != (closing_br_pos != -1): 

709 raise ValueError("Invalid IPv6 URL") 

710 

711 netloc = url[2:delim] 

712 if open_br_pos != -1 and closing_br_pos != -1: 

713 _check_bracketed_netloc(netloc) 

714 

715 url = url[delim:] 

716 

717 if question_pos != -1: 

718 question_pos -= delim 

719 if hash_pos != -1: 

720 hash_pos -= delim 

721 _checknetloc(netloc) 

722 

723 if allow_fragments and hash_pos != -1: 

724 url, fragment = url[:hash_pos], url[hash_pos + 1 :] 

725 

726 if question_pos != -1: 

727 url, query = url[:question_pos], url[question_pos + 1 :] 

728 

729 username = password = hostname = port = None 

730 userinfo, have_info, hostinfo = netloc.rpartition("@") 

731 

732 if have_info: 

733 username, _, password = userinfo.partition(":") 

734 password = password if _ else None 

735 

736 if "[" in hostinfo: 

737 hostname, _, port = hostinfo.partition("[")[2].partition("]") 

738 port = port.partition(":")[2] 

739 else: 

740 hostname, _, port = hostinfo.partition(":") 

741 

742 return _SplitResult( 

743 scheme, 

744 netloc, 

745 url, 

746 query, 

747 fragment, 

748 username, 

749 password, 

750 hostname, 

751 port or None, 

752 ) 

753 

754 

755def _url2pathname(url: str) -> str: 

756 """Reimplementation of urllib.request.url2pathname but with faster _unquote""" 

757 if not url: 

758 return "" 

759 

760 # These branches are handled by `_urlparse` 

761 if url[:3] == "///": # pragma: no cover 

762 url = url[2:] 

763 elif url[:12] == "//localhost/": # pragma: no cover 

764 url = url[11:] 

765 

766 if not _IS_WINDOWS: 

767 if "%" not in url: 

768 return url 

769 

770 return _unquote(url).decode(_FS_ENCODING, _FS_ERRORS) 

771 

772 if url[:3] == "///": 

773 url = url[1:] 

774 url = url.replace(":", "|") 

775 if "|" not in url: 

776 return _unquote(url.replace("/", "\\").encode()).decode( 

777 _FS_ENCODING, _FS_ERRORS 

778 ) 

779 comp = url.split("|") 

780 if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: 

781 raise OSError(f"Bad URL: {url}") 

782 drive = comp[0][-1].upper() 

783 tail = _unquote(comp[1].replace("/", "\\")).decode(_FS_ENCODING, _FS_ERRORS) 

784 return f"{drive}:{tail}" 

785 

786 

787@functools.lru_cache 

788def _idna(input_string: str) -> tuple[bytes, str]: 

789 """Cached IDNA encoding using Python's built-in 'idna' codec. 

790 

791 IDNA processing in CPython is implemented in pure Python (not C), which 

792 makes it relatively slow and allocation-heavy. The only lower-level 

793 optimisation involved is Unicode normalization (NFKC), which may use 

794 optimized internal paths, but IDNA itself remains Python-level logic. 

795 """ 

796 if input_string.isascii(): 

797 return input_string.encode(), input_string 

798 

799 _, normalized = _nfkc_netloc(input_string) 

800 

801 if "ß" not in normalized and "ς" not in normalized: 

802 encoded = normalized.encode("idna") 

803 else: 

804 labels = encodings.idna.dots.split(normalized) 

805 trailing_dot = len(labels) > 1 and not labels[-1] 

806 if trailing_dot: 

807 labels.pop() 

808 encoded = b".".join(_idna_label(label) for label in labels) 

809 if trailing_dot: 

810 encoded += b"." 

811 return encoded, encoded.decode() 

812 

813 

814# The deviation characters "ß" and "ς" are mapped to "ss" and "σ" by IDNA 2003, 

815# i.e. by Python's idna codec, but kept by the nontransitional UTS #46 

816# processing of the URL living standard, so "faß.example" is "xn--fa-hia.example" 

817# in browsers, not the different domain "fass.example". 

818# https://www.unicode.org/reports/tr46/#Deviations 

819_IDNA_DEVIATION_RE = re.compile("([ßς])") 

820 

821 

822def _idna_label(label: str) -> bytes: 

823 if "ß" not in label and "ς" not in label: 

824 return encodings.idna.ToASCII(label) 

825 label = "".join( 

826 piece if piece in {"ß", "ς"} else encodings.idna.nameprep(piece) 

827 for piece in _IDNA_DEVIATION_RE.split(label) 

828 ) 

829 encoded = b"xn--" + label.encode("punycode") 

830 if len(encoded) > 63: 

831 raise UnicodeError("label too long") 

832 return encoded 

833 

834 

835def _idna_bytes(input_string: str) -> bytes: 

836 return _idna(input_string)[0] 

837 

838 

839@functools.lru_cache 

840def _opaque_host(host: str) -> bytes: 

841 """Return *host*, the host of a non-special URL, percent-encoded.""" 

842 if _FORBIDDEN_HOST_CODE_POINT_RE.search(host): 

843 raise ValueError(f"{host!r} contains a forbidden host code point") 

844 return _quote(host.encode(), _OPAQUE_HOST_SAFE_CHARS) 

845 

846 

847def _domain(host: str) -> str: 

848 """Return *host*, the host of a special URL, percent-decoded and 

849 lowercased, as the URL living standard does before IDNA and IPv4 parsing, 

850 e.g. "%31%32%37.0.0.1" as "127.0.0.1".""" 

851 if "%" in host: 

852 host = _unquote(host).decode("utf-8", "replace") 

853 if host.isascii(): 

854 return host.lower() 

855 # str.lower() turns a final "Σ" into "ς", which _idna() keeps, while 

856 # UTS #46 maps "Σ" to "σ". 

857 return host.replace("Σ", "σ").lower() 

858 

859 

860def _check_domain(host: str) -> None: 

861 """Raise ValueError if *host*, the IDNA-encoded host of a special URL, 

862 contains a forbidden domain code point. 

863 

864 A percent-decoded host can contain delimiters such as "/" or "@" that would 

865 change where the authority ends if serialized as is. 

866 """ 

867 if _FORBIDDEN_DOMAIN_CODE_POINT_RE.search(host):