Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/

1from __future__ import annotations

3import dataclasses

4import functools

5import ipaddress

6import os

7import re

8import string

9import sys

10import unicodedata

11from typing import TYPE_CHECKING

12from urllib.parse import ParseResult, scheme_chars, uses_netloc, uses_params

14from w3lib._infra import _ASCII_TAB_OR_NEWLINE, _C0_CONTROL_OR_SPACE

16if TYPE_CHECKING:

17 from collections.abc import Generator

18 from urllib.parse import _QueryType

20_IS_WINDOWS = os.name == "nt"

23_FS_ENCODING = sys.getfilesystemencoding()

24_FS_ERRORS = sys.getfilesystemencodeerrors()

26# https://url.spec.whatwg.org/

27# https://url.spec.whatwg.org/commit-snapshots/a46cb9188a48c2c9d80ba32a9b1891652d6b4900/#default-port

28_DEFAULT_PORTS = {

29 "ftp": 21,

30 "file": None,

31 "http": 80,

32 "https": 443,

33 "ws": 80,

34 "wss": 443,

35}

36_SPECIAL_SCHEMES = set(_DEFAULT_PORTS.keys())

38# constants from RFC 3986, Section 2.2 and 2.3

39RFC3986_GEN_DELIMS = b":/?#[]@"

40RFC3986_SUB_DELIMS = b"!$&'()*+,;="

41RFC3986_RESERVED = RFC3986_GEN_DELIMS + RFC3986_SUB_DELIMS

42RFC3986_UNRESERVED = (string.ascii_letters + string.digits + "-._~").encode("ascii")

43EXTRA_SAFE_CHARS = b"|" # see https://github.com/scrapy/w3lib/pull/25

45RFC3986_USERINFO_SAFE_CHARS = RFC3986_UNRESERVED + RFC3986_SUB_DELIMS + b":"

46_SAFE_CHARS = RFC3986_RESERVED + RFC3986_UNRESERVED + EXTRA_SAFE_CHARS + b"%"

47_PATH_SAFE_CHARS = _SAFE_CHARS.replace(b"#", b"")

48_PATH_SAFE_CHARS_STR = _PATH_SAFE_CHARS.decode()

49_USES_NETLOC = frozenset(uses_netloc)

50_SCHEME_CHARS = frozenset(scheme_chars)

51_USES_PARAMS = frozenset(uses_params)

52_ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE = str.maketrans("", "", _ASCII_TAB_OR_NEWLINE)

53_C0_CONTROL_OR_SPACE_RE = re.compile(rf"[{_C0_CONTROL_OR_SPACE}]")

54_SCHEME_RE = re.compile(rf"^([a-zA-Z][{scheme_chars}]*):")

56_IPV_FUTURE_RE = re.compile(r"\Av[a-fA-F0-9]+\..+\Z")

57# "\" terminates the authority of a special-scheme URL just like "/" under the

58# URL living standard, so it belongs with the other authority delimiters here.

59_NETLOC_DELIMS_RE = re.compile(r"[/?#@:\\]")

60_NETLOC_STRIP_CHARS = str.maketrans("", "", "@:#?")

63def _strip(input_string: str) -> str:

64 if not input_string:

65 return input_string

67 if not _C0_CONTROL_OR_SPACE_RE.search(input_string):

68 return input_string

70 return input_string.strip(_C0_CONTROL_OR_SPACE).translate(

71 _ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE

72 )

75@functools.cache

76def _hex_encode_table() -> bytes:

77 """Build a lookup table for percent-encoded byte values.

79 | byte | encoding |

80 |------|----------|

81 | 0 | %00 |

82 | 1 | %01 |

83 | ... | ... |

84 | 255 | %FF |

86 Each entry is exactly 3 bytes: b"%HH".

88 Returns:

89 A bytes object of length 256 * 3 containing all percent encodings.

90 """

91 return b"".join(f"%{i:02X}".encode() for i in range(256))

94@functools.cache

95def _hex_decode_table() -> bytes:

96 """Build a lookup table for decoding hex ASCII characters.

98 | ASCII | value |

99 |--------|--------------|

100 | '0'-'9'| 0-9 |

101 | 'A'-'F'| 10-15 |

102 | 'a'-'f'| 10-15 |

103 | other | 255 (invalid)|

104

105 Returns:

106 A bytes object of length 256 containing nibble values.

107 """

108 table = bytearray([255]) * 256

109 table[48:58] = bytes(range(10)) # '0'-'9'

110 table[65:71] = bytes(range(10, 16)) # 'A'-'F'

111 table[97:103] = bytes(range(10, 16)) # 'a'-'f'

112 return bytes(table)

113

114

115@functools.cache

116def _safe_table(safe: bytes = RFC3986_UNRESERVED) -> bytes:

117 """Build a lookup table marking safe (non-encoded) bytes.

118

119 | byte | is allowed? |

120 |------|-------------|

121 | 0 | 0 |

122 | 32 | 1 (if safe) |

123 | 65 | 1 |

124 | 255 | 0 |

125

126 Returns:

127 A bytes object of length 256 acting as a boolean mask (0/1).

128 """

129 table = bytearray(256)

130 for b in safe:

131 table[b] = 1

132 return bytes(table)

133

134

135@functools.cache

136def _quote_table(safe: bytes = b"", quote_plus: bool = False) -> tuple[bytes, ...]:

137 """Precompute encoding rules for all 256 byte values.

138

139 Decision table:

140 | condition | output |

141 |-------------------------------|--------|

142 | byte in safe | as-is |

143 | byte == 32 and quote_plus | "+" |

144 | otherwise | "%HH" |

145

146 Example mapping:

147 | byte | char | output |

148 |------|------|--------|

149 | 65 | A | b"A" |

150 | 32 | space| b"+" |

151 | 255 | N/A | b"%FF" |

152

153 Returns:

154 A 256-entry tuple mapping byte value (index) -> encoded bytes.

155 """

156 hex_table = _hex_encode_table()

157 allowed = _safe_table(RFC3986_UNRESERVED + safe) if safe else _safe_table()

158 output: list[bytes] = [b""] * 256

159

160 for idx, byte in enumerate(range(256)):

161 if allowed[byte]:

162 output[idx] = chr(byte).encode()

163 elif quote_plus and byte == 32: # ord(' ')

164 output[idx] = b"+"

165 else:

166 offset = byte * 3

167 output[idx] = hex_table[offset : offset + 3]

168

169 return tuple(output)

170

171

172def _quote(data: bytes, safe: bytes = b"", quote_plus: bool = False) -> bytes:

173 """Fast URL-style quoting using a precomputed table.

174

175 Args:

176 data: Input bytes.

177 safe: Additional unescaped bytes.

178 quote_plus: Encode space as '+' if True.

179

180 Returns:

181 Percent-encoded bytes.

182 """

183 if not data: # pragma: no cover

184 return b""

185

186 transform_table = _quote_table(safe, quote_plus)

187 return b"".join([transform_table[byte] for byte in data])

188

189

190def _quote_into(

191 data: bytes, output: bytearray, safe: bytes = b"", quote_plus: bool = False

192) -> None:

193 if not data: # pragma: no cover

194 return

195

196 transform_table = _quote_table(safe, quote_plus)

197 output += b"".join([transform_table[byte] for byte in data])

198

199

200def _unquote(

201 data: bytes | bytearray | str,

202 safe: bytes = b"",

203) -> bytes:

204 if not data:

205 return b""

206

207 if isinstance(data, str):

208 data = data.encode()

209

210 first_percent = data.find(b"%")

211

212 if first_percent < 0:

213 return bytes(data)

214

215 hex_decode_table = _hex_decode_table()

216 safe_table = _safe_table(safe)

217

218 data_length = len(data)

219 # stop at len - 2 because "%HH" decoding reads 2 extra bytes after '%'

220 decode_limit = data_length - 2

221

222 output = bytearray(data_length)

223 output[:first_percent] = data[:first_percent]

224

225 input_index = first_percent

226 output_index = first_percent

227

228 while input_index < decode_limit:

229 current_byte = data[input_index]

230

231 if current_byte == 37: # ord('%')

232 # Decoding "%HH" sequence

233 # Step 1: read two hex characters after '%'

234 # Example: "%4F" -> '4' and 'F'

235 high_nibble = hex_decode_table[data[input_index + 1]]

236 low_nibble = hex_decode_table[data[input_index + 2]]

237

238 # Step 2: validate both characters are valid hex digits

239 # hex_decode_table returns 255 for invalid input

240 # bitwise OR catches any invalid nibble quickly

241 if (high_nibble | low_nibble) != 255:

242 # Step 3: combine two 4-bit nibbles into one byte

243 # (high_nibble << 4) + low_nibble

244 # Example: 0x4 and 0xF -> 0x4F

245 decoded_byte = (high_nibble << 4) | low_nibble

246

247 # Step 4: check if decoded byte is NOT in safe set

248 # (only unsafe bytes are decoded; safe ones are left encoded

249 if not safe_table[decoded_byte]:

250 output[output_index] = decoded_byte

251 input_index += 3 # skip past "%HH" in input

252 output_index += 1 # advance output position by one decoded byte

253 continue

254

255 output[output_index] = current_byte

256 input_index += 1

257 output_index += 1

258

259 while input_index < data_length: # tail

260 output[output_index] = data[input_index]

261 input_index += 1

262 output_index += 1

263

264 return bytes(output[:output_index])

265

266

267def _unquote_plus(

268 data: bytes | bytearray | str,

269) -> bytes:

270 # This function is intentionally duplicated from `_unquote` for performance.

271 # The duplication avoids extra branching for '+' handling in hot loop.

272 if not data:

273 return b""

274

275 if isinstance(data, str): # pragma: no cover

276 data = data.encode()

277

278 first_percent = data.find(b"%")

279 first_plus = data.find(b"+")

280

281 first_special = min(first_plus, first_percent)

282

283 if first_special < 0:

284 first_special = max(first_percent, first_plus)

285

286 if first_special < 0:

287 return bytes(data)

288

289 hex_decode_table = _hex_decode_table()

290 safe_table = _safe_table(b"")

291

292 data_length = len(data)

293 decode_limit = data_length - 2

294

295 output = bytearray(data_length)

296 output[:first_special] = data[:first_special]

297

298 input_index = first_special

299 output_index = first_special

300

301 while input_index < decode_limit:

302 current_byte = data[input_index]

303

304 if current_byte == 43: # ord('+')

305 output[output_index] = 32 # ord(' ')

306 input_index += 1

307 output_index += 1

308 continue

309

310 if current_byte == 37: # ord('%')

311 high_nibble = hex_decode_table[data[input_index + 1]]

312 low_nibble = hex_decode_table[data[input_index + 2]]

313

314 if (high_nibble | low_nibble) != 255:

315 decoded_byte = (high_nibble << 4) | low_nibble

316

317 if not safe_table[decoded_byte]:

318 output[output_index] = decoded_byte

319 input_index += 3

320 output_index += 1

321 continue

322

323 output[output_index] = current_byte

324 input_index += 1

325 output_index += 1

326

327 while input_index < data_length: # tail

328 current_byte = data[input_index]

329

330 if current_byte == 43: # ord('+')

331 output[output_index] = 32 # ord(' ')

332 else:

333 output[output_index] = current_byte

334

335 input_index += 1

336 output_index += 1

337

338 return bytes(output[:output_index])

339

340

341def _parse_qs(

342 qs: str | bytes,

343 keep_blank_values: bool = False,

344) -> dict[bytes, list[bytes]]:

345 """Reimplementation of urllib.parse.parse_qs which:

346 - Doesn't use _coerce_args or _coerce_result

347 - Works directly on bytes internally (no type coercion layer)

348 - Returns bytes keys/values only"""

349 if not qs: # pragma: no cover

350 return {}

351

352 if isinstance(qs, str): # pragma: no cover

353 qs = qs.encode()

354

355 result: dict[bytes, list[bytes]] = {}

356

357 for field in qs.split(b"&"):

358 if not field:

359 continue

360

361 key, sep, value = field.partition(b"=")

362

363 if not keep_blank_values and (not sep or not value):

364 continue

365

366 key = _unquote_plus(key)

367 value = _unquote_plus(value)

368

369 if key in result:

370 result[key].append(value)

371 else:

372 result[key] = [value]

373

374 return result

375

376

377def _parse_qsl(

378 qs: str | bytes,

379 keep_blank_values: bool = False,

380) -> list[tuple[bytes, bytes]]:

381 """Reimplementation of urllib.parse.parse_qsl which:

382 - Doesn't use _coerce_args or _coerce_result

383 - Works directly on bytes internally (no type coercion layer)

384 - Returns only bytes tuples"""

385 # This function is intentionally duplicated from `_parse_qs` for performance.

386 if not qs:

387 return []

388

389 if isinstance(qs, str):

390 qs = qs.encode()

391

392 result: list[tuple[bytes, bytes]] = []

393

394 for field in qs.split(b"&"):

395 if not field:

396 continue

397

398 key, sep, value = field.partition(b"=")

399

400 if not keep_blank_values and (not sep or not value):

401 continue

402

403 result.append((_unquote_plus(key), _unquote_plus(value)))

404

405 return result

406

407

408def _urlencode(query: _QueryType) -> bytes:

409 if hasattr(query, "items"): # pragma: no cover

410 query = query.items() # type: ignore[assignment]

411

412 if not query: # pragma: no cover

413 return b""

414

415 result: list[bytes] = []

416 tmp_buf = bytearray()

417

418 for key, value in query: # type: ignore[str-unpack]

419 _quote_into(

420 key if isinstance(key, bytes) else str(key).encode(),

421 output=tmp_buf,

422 quote_plus=True,

423 )

424 tmp_buf.append(61) # ord('=')

425 _quote_into(

426 value if isinstance(value, bytes) else str(value).encode(),

427 output=tmp_buf,

428 quote_plus=True,

429 )

430 result.append(bytes(tmp_buf))

431 tmp_buf.clear()

432

433 return b"&".join(result)

434

435

436def _urlparse(

437 url: str,

438 scheme: str = "",

439 allow_fragments: bool = True,

440) -> ParseResult:

441 """Reimplementation of urlib.parse.urlparse but without _coerce_args/_coerce_result."""

442 if not url: # pragma: no cover

443 return ParseResult(scheme, "", "", "", "", "")

444

445 scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments)

446 params = ""

447

448 if scheme in _USES_PARAMS:

449 semi_idx = url.find(";")

450

451 if semi_idx != -1:

452 slash_idx = url.rfind("/")

453

454 if slash_idx != -1 and slash_idx < semi_idx:

455 semi_idx = url.find(";", slash_idx)

456

457 url, params = url[:semi_idx], url[semi_idx + 1 :]

458

459 return ParseResult(scheme, netloc, url, params, query, fragment)

460

461

462def _urlunparse(

463 scheme: str,

464 netloc: str,

465 url: str,

466 params: str,

467 query: str,

468 fragment: str,

469) -> str:

470 """Reimplementation of urlib.parse.urlunparse but without _coerce_args/_coerce_result."""

471 if params:

472 url = f"{url};{params}"

473 return _urlunsplit(scheme, netloc, url, query, fragment)

474

475

476def _urlunsplit(scheme: str, netloc: str, url: str, query: str, fragment: str) -> str:

477 """Reimplementation of urlib.parse.urlunsplit but without _coerce_args/_coerce_result."""

478

479 if netloc:

480 if url and url[:1] != "/":

481 url = f"/{url}"

482 url = f"//{netloc}{url}"

483 elif url[:2] == "//" or (

484 scheme and scheme in _USES_NETLOC and (not url or url[:1] == "/")

485 ):

486 url = f"//{url}"

487

488 if scheme:

489 scheme = f"{scheme}:"

490

491 if query:

492 query = f"?{query}"

493

494 if fragment:

495 fragment = f"#{fragment}"

496

497 return f"{scheme}{url}{query}{fragment}"

498

499

500@dataclasses.dataclass(slots=True, eq=False, repr=False)

501class _SplitResult: # pylint: disable=too-many-instance-attributes

502 scheme: str

503 netloc: str

504 path: str

505 query: str

506 fragment: str

507

508 username: str | None = None

509 password: str | None = None

510 hostname: str | None = None

511 port: str | int | None = None

512

513 def __post_init__(self) -> None:

514 if self.hostname is not None:

515 hostname, delim, zone = self.hostname.partition("%")

516 self.hostname = f"{hostname.lower()}{delim}{zone}"

517

518 if self.port is not None:

519 try:

520 self.port = int(self.port)

521 except ValueError:

522 raise ValueError(

523 f"Port could not be cast to integer value as {self.port}"

524 ) from None

525

526 if self.port not in range(65535 + 1):

527 raise ValueError("Port out of range 0-65535")

528

529 def __iter__(self) -> Generator[str]:

530 yield self.scheme

531 yield self.netloc

532 yield self.path

533 yield self.query

534 yield self.fragment

535

536 def __len__(self) -> int:

537 return 5 # pragma: no cover

538

539 def __getitem__(self, index: int) -> str: # pragma: no cover

540 match index:

541 case 0:

542 return self.scheme

543 case 1:

544 return self.netloc

545 case 2:

546 return self.path

547 case 3:

548 return self.query

549 case 4:

550 return self.fragment

551 raise IndexError

552

553

554def _checknetloc(netloc: str) -> None:

555 """

556 Validate that NFKC normalization does not introduce reserved URL characters.

557

558 Raises:

559 ValueError: If normalization introduces reserved delimiters.

560 """

561 if not netloc or netloc.isascii():

562 return

563

564 # IDNA uses NFKC equivalence. Remove already-valid delimiters before

565 # normalization so we only detect newly introduced ones.

566 cleaned, normalized = _nfkc_netloc(netloc)

567

568 if cleaned == normalized:

569 return

570

571 if _NETLOC_DELIMS_RE.search(normalized):

572 raise ValueError(

573 f"netloc {netloc!r} contains invalid characters under NFKC normalization"

574 )

575

576

577def _check_bracketed_netloc(netloc: str) -> None:

578 """

579 Validate bracket usage in a URL netloc.

580

581 Raises:

582 ValueError: If bracket placement or host syntax is invalid.

583

584 NOTE: this is basically a backport of https://github.com/python/cpython/issues/105704

585 """

586 hostname_and_port = netloc.rpartition("@")[2]

587

588 before_bracket, has_open_bracket, bracketed = hostname_and_port.partition("[")

589

590 if has_open_bracket:

591 # No data is allowed before '['.

592 if before_bracket:

593 raise ValueError("Invalid IPv6 URL")

594

595 hostname, _, port = bracketed.partition("]")

596

597 # Only ':<port>' may follow ']'.

598 if port and not port.startswith(":"):

599 raise ValueError("Invalid IPv6 URL")

600 # port validation done after, in `_SplitResult.__post_init__`

601 else:

602 hostname, _, _ = hostname_and_port.partition(":")

603

604 _check_bracketed_host(hostname)

605

606

607def _check_bracketed_host(hostname: str) -> None:

608 """

609 Validate a bracketed host according to RFC 3986 / WHATWG URL rules.

610

611 Raises:

612 ValueError: If the host is invalid.

613 """

614 # IPvFuture: v<HEXDIG>.<address>

615 if hostname.startswith(("v", "V")):

616 if not _IPV_FUTURE_RE.fullmatch(hostname):

617 raise ValueError("IPvFuture address is invalid")

618 return

619

620 # ip_address() raises ValueError if invalid.

621 ip = ipaddress.ip_address(hostname)

622

623 # Bracketed IPv4 literals are forbidden.

624 if isinstance(ip, ipaddress.IPv4Address):

625 raise ValueError("An IPv4 address cannot be in brackets")

626

627

628@functools.lru_cache

629def _urlsplit( # pylint: disable=too-many-locals,too-many-statements

630 url: str,

631 scheme: str = "",

632 allow_fragments: bool = True,

633) -> _SplitResult:

634 """Reimplementation of urllib.parse.urlsplit which:

635 - Doesn't use _coerce_args or _coerce_result

636 - Does manual single-pass scanning instead of repeated .find/.split calls

637 - Have reduced string allocations by slicing once using computed indices

638 - Avoids extra computations as much as possible

639 """

640 if not url:

641 return _SplitResult(scheme, "", "", "", "")

642

643 url, scheme = url.lstrip(_C0_CONTROL_OR_SPACE), scheme.strip(_C0_CONTROL_OR_SPACE)

644

645 netloc = query = fragment = ""

646

647 if m := _SCHEME_RE.match(url):

648 scheme = m.group(1).lower()

649 url = url[m.end() :]

650

651 # The URL living standard treats "\" like "/" for special-scheme URLs, but

652 # only in the authority and path; a "\" in the query or fragment is left

653 # alone. Without this, "http://evil.com\@good.com/" is read as userinfo

654 # "evil.com\" plus host "good.com", while a browser ends the authority at

655 # the "\" and connects to "evil.com".

656 if scheme in _SPECIAL_SCHEMES and "\\" in url:

657 cut = len(url)

658 question_idx = url.find("?")

659 if question_idx != -1:

660 cut = question_idx

661 hash_idx = url.find("#")

662 if hash_idx != -1 and hash_idx < cut:

663 cut = hash_idx

664 url = url[:cut].replace("\\", "/") + url[cut:]

665

666 # The scan skips the leading "//" of an authority; for URLs without an

667 # authority it must start at 0, otherwise a "?" or "#" at index 0 or 1

668 # (e.g. relative URLs like "a?b" or "a#f") is never recorded.

669 scan_start = 2 if url[:2] == "//" else 0

670 slash_pos = question_pos = hash_pos = open_br_pos = closing_br_pos = -1

671 for idx, char in enumerate(url[scan_start:], scan_start):

672 if char == "/" and slash_pos == -1:

673 slash_pos = idx

674 elif char == "?" and question_pos == -1:

675 question_pos = idx

676 elif char == "#" and hash_pos == -1:

677 hash_pos = idx

678 elif char == "[" and open_br_pos == -1:

679 open_br_pos = idx

680 elif char == "]" and closing_br_pos == -1:

681 closing_br_pos = idx

682 if -1 not in (

683 slash_pos,

684 question_pos,

685 hash_pos,

686 open_br_pos,

687 closing_br_pos,

688 ):

689 break

690

691 if url[:2] == "//":

692 delim = len(url)

693

694 if 0 < slash_pos < delim:

695 delim = slash_pos

696 if 0 < question_pos < delim:

697 delim = question_pos

698 if 0 < hash_pos < delim:

699 delim = hash_pos

700

701 # Brackets only delimit an IPv6 host when they fall inside the

702 # authority. A "[" or "]" in the path, query or fragment is an

703 # ordinary character and must not drive IPv6 host parsing.

704 if not 2 <= open_br_pos < delim:

705 open_br_pos = -1

706 if not 2 <= closing_br_pos < delim:

707 closing_br_pos = -1

708

709 if (open_br_pos != -1) != (closing_br_pos != -1):

710 raise ValueError("Invalid IPv6 URL")

711

712 netloc = url[2:delim]

713 if open_br_pos != -1 and closing_br_pos != -1:

714 _check_bracketed_netloc(netloc)

715

716 url = url[delim:]

717

718 if question_pos != -1:

719 question_pos -= delim

720 if hash_pos != -1:

721 hash_pos -= delim

722 _checknetloc(netloc)

723

724 if allow_fragments and hash_pos != -1:

725 url, fragment = url[:hash_pos], url[hash_pos + 1 :]

726

727 if question_pos != -1:

728 url, query = url[:question_pos], url[question_pos + 1 :]

729

730 username = password = hostname = port = None

731 userinfo, have_info, hostinfo = netloc.rpartition("@")

732

733 if have_info:

734 username, _, password = userinfo.partition(":")

735 password = password if _ else None

736

737 if open_br_pos != -1:

738 hostname, _, port = hostinfo.partition("[")[2].partition("]")

739 port = port.partition(":")[2]

740 else:

741 hostname, _, port = hostinfo.partition(":")

742

743 return _SplitResult(

744 scheme,

745 netloc,

746 url,

747 query,

748 fragment,

749 username,

750 password,

751 hostname,

752 port or None,

753 )

754

755

756def _url2pathname(url: str) -> str:

757 """Reimplementation of urllib.request.url2pathname but with faster _unquote"""

758 if not url:

759 return ""

760

761 # These branches are handled by `_urlparse`

762 if url[:3] == "///": # pragma: no cover

763 url = url[2:]

764 elif url[12:] == "//localhost/": # pragma: no cover

765 url = url[11:]

766

767 if not _IS_WINDOWS:

768 if "%" not in url:

769 return url

770

771 return _unquote(url, _PATH_SAFE_CHARS).decode(_FS_ENCODING, _FS_ERRORS)

772

773 if url[:3] == "///":

774 url = url[1:]

775 url = url.replace(":", "|")

776 if "|" not in url:

777 return _unquote(url.replace("/", "\\").encode(), _PATH_SAFE_CHARS).decode(

778 _FS_ENCODING, _FS_ERRORS

779 )

780 comp = url.split("|")

781 if len(comp) != 2 or comp[0][-1] not in string.ascii_letters:

782 raise OSError(f"Bad URL: {url}")

783 drive = comp[0][-1].upper()

784 tail = _unquote(comp[1].replace("/", "\\"), _PATH_SAFE_CHARS).decode(

785 _FS_ENCODING, _FS_ERRORS

786 )

787 return f"{drive}:{tail}"

788

789

790@functools.lru_cache

791def _idna(input_string: str) -> tuple[bytes, str]:

792 """Cached IDNA encoding using Python's built-in 'idna' codec.

793

794 NOTE: IDNA processing in CPython is implemented in pure Python (not C),

795 which makes it relatively slow and allocation-heavy. The only

796 lower-level optimisation involved is Unicode normalization

797 (NFKC), which may use optimized internal paths, but IDNA itself

798 remains Python-level logic.

799 """

800 if input_string.isascii():

801 return input_string.encode(), input_string

802

803 _, normalized = _nfkc_netloc(input_string)

804

805 encoded = normalized.encode("idna")

806 return encoded, encoded.decode()

807

808

809def _idna_bytes(input_string: str) -> bytes:

810 return _idna(input_string)[0]

811

812

813def _idna_str(input_string: str) -> str:

814 return _idna(input_string)[1]

815

816

817@functools.lru_cache

818def _nfkc_netloc(netloc: str) -> tuple[str, str]:

819 cleaned = netloc.translate(_NETLOC_STRIP_CHARS)

820 normalized = unicodedata.normalize("NFKC", cleaned)

821 return cleaned, normalized

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/_url.py: 66%

388 statements