Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/

1from __future__ import annotations

3import dataclasses

4import functools

5import ipaddress

6import os

7import re

8import string

9import sys

10import unicodedata

11from typing import TYPE_CHECKING

12from urllib.parse import ParseResult, scheme_chars, uses_netloc, uses_params

14from w3lib._infra import _ASCII_TAB_OR_NEWLINE, _C0_CONTROL_OR_SPACE

16if TYPE_CHECKING:

17 from collections.abc import Generator

18 from urllib.parse import _QueryType

20_IS_WINDOWS = os.name == "nt"

23_FS_ENCODING = sys.getfilesystemencoding()

24_FS_ERRORS = sys.getfilesystemencodeerrors()

26# https://url.spec.whatwg.org/

27# https://url.spec.whatwg.org/commit-snapshots/a46cb9188a48c2c9d80ba32a9b1891652d6b4900/#default-port

28_DEFAULT_PORTS = {

29 "ftp": 21,

30 "file": None,

31 "http": 80,

32 "https": 443,

33 "ws": 80,

34 "wss": 443,

35}

36_SPECIAL_SCHEMES = set(_DEFAULT_PORTS.keys())

38# constants from RFC 3986, Section 2.2 and 2.3

39RFC3986_GEN_DELIMS = b":/?#[]@"

40RFC3986_SUB_DELIMS = b"!$&'()*+,;="

41RFC3986_RESERVED = RFC3986_GEN_DELIMS + RFC3986_SUB_DELIMS

42RFC3986_UNRESERVED = (string.ascii_letters + string.digits + "-._~").encode("ascii")

43EXTRA_SAFE_CHARS = b"|" # see https://github.com/scrapy/w3lib/pull/25

45RFC3986_USERINFO_SAFE_CHARS = RFC3986_UNRESERVED + RFC3986_SUB_DELIMS + b":"

46_SAFE_CHARS = RFC3986_RESERVED + RFC3986_UNRESERVED + EXTRA_SAFE_CHARS + b"%"

47_PATH_SAFE_CHARS = _SAFE_CHARS.replace(b"#", b"")

48_PATH_SAFE_CHARS_STR = _PATH_SAFE_CHARS.decode()

49_USES_NETLOC = frozenset(uses_netloc)

50_SCHEME_CHARS = frozenset(scheme_chars)

51_USES_PARAMS = frozenset(uses_params)

52_ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE = str.maketrans("", "", _ASCII_TAB_OR_NEWLINE)

53_C0_CONTROL_OR_SPACE_RE = re.compile(rf"[{_C0_CONTROL_OR_SPACE}]")

54_SCHEME_RE = re.compile(rf"^([a-zA-Z][{scheme_chars}]*):")

56_IPV_FUTURE_RE = re.compile(r"\Av[a-fA-F0-9]+\..+\Z")

57# "\" terminates the authority of a special-scheme URL just like "/" under the

58# URL living standard, so it belongs with the other authority delimiters here.

59_NETLOC_DELIMS_RE = re.compile(r"[/?#@:\\]")

60_NETLOC_STRIP_CHARS = str.maketrans("", "", "@:#?")

63def _strip(input_string: str) -> str:

64 if not input_string:

65 return input_string

67 if not _C0_CONTROL_OR_SPACE_RE.search(input_string):

68 return input_string

70 return input_string.strip(_C0_CONTROL_OR_SPACE).translate(

71 _ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE

72 )

75@functools.cache

76def _hex_encode_table() -> bytes:

77 """Build a lookup table for percent-encoded byte values.

79 | byte | encoding |

80 |------|----------|

81 | 0 | %00 |

82 | 1 | %01 |

83 | ... | ... |

84 | 255 | %FF |

86 Each entry is exactly 3 bytes: b"%HH".

88 Returns:

89 A bytes object of length 256 * 3 containing all percent encodings.

90 """

91 return b"".join(f"%{i:02X}".encode() for i in range(256))

94@functools.cache

95def _hex_decode_table() -> bytes:

96 """Build a lookup table for decoding hex ASCII characters.

98 | ASCII | value |

99 |--------|--------------|

100 | '0'-'9'| 0-9 |

101 | 'A'-'F'| 10-15 |

102 | 'a'-'f'| 10-15 |

103 | other | 255 (invalid)|

104

105 Returns:

106 A bytes object of length 256 containing nibble values.

107 """

108 table = bytearray([255]) * 256

109 table[48:58] = bytes(range(10)) # '0'-'9'

110 table[65:71] = bytes(range(10, 16)) # 'A'-'F'

111 table[97:103] = bytes(range(10, 16)) # 'a'-'f'

112 return bytes(table)

113

114

115@functools.cache

116def _safe_table(safe: bytes = RFC3986_UNRESERVED) -> bytes:

117 """Build a lookup table marking safe (non-encoded) bytes.

118

119 | byte | is allowed? |

120 |------|-------------|

121 | 0 | 0 |

122 | 32 | 1 (if safe) |

123 | 65 | 1 |

124 | 255 | 0 |

125

126 Returns:

127 A bytes object of length 256 acting as a boolean mask (0/1).

128 """

129 table = bytearray(256)

130 for b in safe:

131 table[b] = 1

132 return bytes(table)

133

134

135@functools.cache

136def _quote_table(safe: bytes = b"", quote_plus: bool = False) -> tuple[bytes, ...]:

137 """Precompute encoding rules for all 256 byte values.

138

139 Decision table:

140 | condition | output |

141 |-------------------------------|--------|

142 | byte in safe | as-is |

143 | byte == 32 and quote_plus | "+" |

144 | otherwise | "%HH" |

145

146 Example mapping:

147 | byte | char | output |

148 |------|------|--------|

149 | 65 | A | b"A" |

150 | 32 | space| b"+" |

151 | 255 | N/A | b"%FF" |

152

153 Returns:

154 A 256-entry tuple mapping byte value (index) -> encoded bytes.

155 """

156 hex_table = _hex_encode_table()

157 allowed = _safe_table(RFC3986_UNRESERVED + safe) if safe else _safe_table()

158 output: list[bytes] = [b""] * 256

159

160 for idx, byte in enumerate(range(256)):

161 if allowed[byte]:

162 output[idx] = chr(byte).encode()

163 elif quote_plus and byte == 32: # ord(' ')

164 output[idx] = b"+"

165 else:

166 offset = byte * 3

167 output[idx] = hex_table[offset : offset + 3]

168

169 return tuple(output)

170

171

172def _quote(data: bytes, safe: bytes = b"", quote_plus: bool = False) -> bytes:

173 """Fast URL-style quoting using a precomputed table.

174

175 Args:

176 data: Input bytes.

177 safe: Additional unescaped bytes.

178 quote_plus: Encode space as '+' if True.

179

180 Returns:

181 Percent-encoded bytes.

182 """

183 if not data: # pragma: no cover

184 return b""

185

186 transform_table = _quote_table(safe, quote_plus)

187 return b"".join([transform_table[byte] for byte in data])

188

189

190def _quote_into(

191 data: bytes, output: bytearray, safe: bytes = b"", quote_plus: bool = False

192) -> None:

193 if not data: # pragma: no cover

194 return

195

196 transform_table = _quote_table(safe, quote_plus)

197 output += b"".join([transform_table[byte] for byte in data])

198

199

200def _unquote(

201 data: bytes | bytearray | str,

202 safe: bytes = b"",

203) -> bytes:

204 if not data:

205 return b""

206

207 if isinstance(data, str):

208 data = data.encode()

209

210 first_percent = data.find(b"%")

211

212 if first_percent < 0:

213 return bytes(data)

214

215 hex_decode_table = _hex_decode_table()

216 safe_table = _safe_table(safe)

217

218 data_length = len(data)

219 # stop at len - 2 because "%HH" decoding reads 2 extra bytes after '%'

220 decode_limit = data_length - 2

221

222 output = bytearray(data_length)

223 output[:first_percent] = data[:first_percent]

224

225 input_index = first_percent

226 output_index = first_percent

227

228 while input_index < decode_limit:

229 current_byte = data[input_index]

230

231 if current_byte == 37: # ord('%')

232 # Decoding "%HH" sequence

233 # Step 1: read two hex characters after '%'

234 # Example: "%4F" -> '4' and 'F'

235 high_nibble = hex_decode_table[data[input_index + 1]]

236 low_nibble = hex_decode_table[data[input_index + 2]]

237

238 # Step 2: validate both characters are valid hex digits

239 # hex_decode_table returns 255 for invalid input

240 # bitwise OR catches any invalid nibble quickly

241 if (high_nibble | low_nibble) != 255:

242 # Step 3: combine two 4-bit nibbles into one byte

243 # (high_nibble << 4) + low_nibble

244 # Example: 0x4 and 0xF -> 0x4F

245 decoded_byte = (high_nibble << 4) | low_nibble

246

247 # Step 4: check if decoded byte is NOT in safe set

248 # (only unsafe bytes are decoded; safe ones are left encoded

249 if not safe_table[decoded_byte]:

250 output[output_index] = decoded_byte

251 input_index += 3 # skip past "%HH" in input

252 output_index += 1 # advance output position by one decoded byte

253 continue

254

255 output[output_index] = current_byte

256 input_index += 1

257 output_index += 1

258

259 while input_index < data_length: # tail

260 output[output_index] = data[input_index]

261 input_index += 1

262 output_index += 1

263

264 return bytes(output[:output_index])

265

266

267def _unquote_plus(

268 data: bytes | bytearray | str,

269) -> bytes:

270 # This function is intentionally duplicated from `_unquote` for performance.

271 # The duplication avoids extra branching for '+' handling in hot loop.

272 if not data:

273 return b""

274

275 if isinstance(data, str): # pragma: no cover

276 data = data.encode()

277

278 first_percent = data.find(b"%")

279 first_plus = data.find(b"+")

280

281 first_special = min(first_plus, first_percent)

282

283 if first_special < 0:

284 first_special = max(first_percent, first_plus)

285

286 if first_special < 0:

287 return bytes(data)

288

289 hex_decode_table = _hex_decode_table()

290 safe_table = _safe_table(b"")

291

292 data_length = len(data)

293 decode_limit = data_length - 2

294

295 output = bytearray(data_length)

296 output[:first_special] = data[:first_special]

297

298 input_index = first_special

299 output_index = first_special

300

301 while input_index < decode_limit:

302 current_byte = data[input_index]

303

304 if current_byte == 43: # ord('+')

305 output[output_index] = 32 # ord(' ')

306 input_index += 1

307 output_index += 1

308 continue

309

310 if current_byte == 37: # ord('%')

311 high_nibble = hex_decode_table[data[input_index + 1]]

312 low_nibble = hex_decode_table[data[input_index + 2]]

313

314 if (high_nibble | low_nibble) != 255:

315 decoded_byte = (high_nibble << 4) | low_nibble

316

317 if not safe_table[decoded_byte]:

318 output[output_index] = decoded_byte

319 input_index += 3

320 output_index += 1

321 continue

322

323 output[output_index] = current_byte

324 input_index += 1

325 output_index += 1

326

327 while input_index < data_length: # tail

328 current_byte = data[input_index]

329

330 if current_byte == 43: # ord('+')

331 output[output_index] = 32 # ord(' ')

332 else:

333 output[output_index] = current_byte

334

335 input_index += 1

336 output_index += 1

337

338 return bytes(output[:output_index])

339

340

341def _parse_qs(

342 qs: str | bytes,

343 keep_blank_values: bool = False,

344 separator: bytes = b"&",

345) -> dict[bytes, list[bytes]]:

346 """Reimplementation of urllib.parse.parse_qs which:

347 - Doesn't use _coerce_args or _coerce_result

348 - Works directly on bytes internally (no type coercion layer)

349 - Returns bytes keys/values only"""

350 if not qs: # pragma: no cover

351 return {}

352

353 if isinstance(qs, str): # pragma: no cover

354 qs = qs.encode()

355

356 result: dict[bytes, list[bytes]] = {}

357

358 for field in qs.split(separator):

359 if not field:

360 continue

361

362 key, sep, value = field.partition(b"=")

363

364 if not keep_blank_values and (not sep or not value):

365 continue

366

367 key = _unquote_plus(key)

368 value = _unquote_plus(value)

369

370 if key in result:

371 result[key].append(value)

372 else:

373 result[key] = [value]

374

375 return result

376

377

378def _parse_qsl(

379 qs: str | bytes,

380 keep_blank_values: bool = False,

381 separator: bytes = b"&",

382) -> list[tuple[bytes, bytes]]:

383 """Reimplementation of urllib.parse.parse_qsl which:

384 - Doesn't use _coerce_args or _coerce_result

385 - Works directly on bytes internally (no type coercion layer)

386 - Returns only bytes tuples"""

387 # This function is intentionally duplicated from `_parse_qs` for performance.

388 if not qs:

389 return []

390

391 if isinstance(qs, str):

392 qs = qs.encode()

393

394 result: list[tuple[bytes, bytes]] = []

395

396 for field in qs.split(separator):

397 if not field:

398 continue

399

400 key, sep, value = field.partition(b"=")

401

402 if not keep_blank_values and (not sep or not value):

403 continue

404

405 result.append((_unquote_plus(key), _unquote_plus(value)))

406

407 return result

408

409

410def _urlencode(query: _QueryType, separator: bytes = b"&") -> bytes:

411 if hasattr(query, "items"): # pragma: no cover

412 query = query.items() # type: ignore[assignment]

413

414 if not query: # pragma: no cover

415 return b""

416

417 result: list[bytes] = []

418 tmp_buf = bytearray()

419

420 for key, value in query: # type: ignore[str-unpack]

421 _quote_into(

422 key if isinstance(key, bytes) else str(key).encode(),

423 output=tmp_buf,

424 quote_plus=True,

425 )

426 tmp_buf.append(61) # ord('=')

427 _quote_into(

428 value if isinstance(value, bytes) else str(value).encode(),

429 output=tmp_buf,

430 quote_plus=True,

431 )

432 result.append(bytes(tmp_buf))

433 tmp_buf.clear()

434

435 return separator.join(result)

436

437

438def _split_params(scheme: str, url: str) -> tuple[str, str]:

439 """Split the params from the path, as urlib.parse.urlparse does."""

440 if scheme in _USES_PARAMS:

441 # Only a ";" in the last segment starts the params; one in an earlier

442 # segment is an ordinary path character.

443 semi_idx = url.find(";", url.rfind("/") + 1)

444

445 if semi_idx != -1:

446 return url[:semi_idx], url[semi_idx + 1 :]

447

448 return url, ""

449

450

451def _urlparse(

452 url: str,

453 scheme: str = "",

454 allow_fragments: bool = True,

455) -> ParseResult:

456 """Reimplementation of urlib.parse.urlparse but without _coerce_args/_coerce_result."""

457 if not url: # pragma: no cover

458 return ParseResult(scheme, "", "", "", "", "")

459

460 scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments)

461 url, params = _split_params(scheme, url)

462

463 return ParseResult(scheme, netloc, url, params, query, fragment)

464

465

466def _urlunparse(

467 scheme: str,

468 netloc: str,

469 url: str,

470 params: str,

471 query: str,

472 fragment: str,

473) -> str:

474 """Reimplementation of urlib.parse.urlunparse but without _coerce_args/_coerce_result."""

475 if params:

476 url = f"{url};{params}"

477 return _urlunsplit(scheme, netloc, url, query, fragment)

478

479

480def _urlunsplit(scheme: str, netloc: str, url: str, query: str, fragment: str) -> str:

481 """Reimplementation of urlib.parse.urlunsplit but without _coerce_args/_coerce_result."""

482

483 if netloc:

484 if url and url[:1] != "/":

485 url = f"/{url}"

486 url = f"//{netloc}{url}"

487 elif url[:2] == "//" or (

488 scheme and scheme in _USES_NETLOC and (not url or url[:1] == "/")

489 ):

490 url = f"//{url}"

491

492 if scheme:

493 scheme = f"{scheme}:"

494

495 if query:

496 query = f"?{query}"

497

498 if fragment:

499 fragment = f"#{fragment}"

500

501 return f"{scheme}{url}{query}{fragment}"

502

503

504@dataclasses.dataclass(slots=True, eq=False, repr=False)

505class _SplitResult: # pylint: disable=too-many-instance-attributes

506 scheme: str

507 netloc: str

508 path: str

509 query: str

510 fragment: str

511

512 username: str | None = None

513 password: str | None = None

514 hostname: str | None = None

515 port: str | int | None = None

516

517 def __post_init__(self) -> None:

518 if self.hostname is not None:

519 hostname, delim, zone = self.hostname.partition("%")

520 self.hostname = f"{hostname.lower()}{delim}{zone}"

521

522 if self.port is not None:

523 try:

524 self.port = int(self.port)

525 except ValueError:

526 raise ValueError(

527 f"Port could not be cast to integer value as {self.port}"

528 ) from None

529

530 if self.port not in range(65535 + 1):

531 raise ValueError("Port out of range 0-65535")

532

533 def __iter__(self) -> Generator[str]:

534 yield self.scheme

535 yield self.netloc

536 yield self.path

537 yield self.query

538 yield self.fragment

539

540 def __len__(self) -> int:

541 return 5 # pragma: no cover

542

543 def __getitem__(self, index: int) -> str: # pragma: no cover

544 match index:

545 case 0:

546 return self.scheme

547 case 1:

548 return self.netloc

549 case 2:

550 return self.path

551 case 3:

552 return self.query

553 case 4:

554 return self.fragment

555 raise IndexError

556

557

558def _checknetloc(netloc: str) -> None:

559 """

560 Validate that NFKC normalization does not introduce reserved URL characters.

561

562 Raises:

563 ValueError: If normalization introduces reserved delimiters.

564 """

565 if not netloc or netloc.isascii():

566 return

567

568 # IDNA uses NFKC equivalence. Remove already-valid delimiters before

569 # normalization so we only detect newly introduced ones.

570 cleaned, normalized = _nfkc_netloc(netloc)

571

572 if cleaned == normalized:

573 return

574

575 if _NETLOC_DELIMS_RE.search(normalized):

576 raise ValueError(

577 f"netloc {netloc!r} contains invalid characters under NFKC normalization"

578 )

579

580

581def _check_bracketed_netloc(netloc: str) -> None:

582 """

583 Validate bracket usage in a URL netloc.

584

585 Raises:

586 ValueError: If bracket placement or host syntax is invalid.

587

588 NOTE: this is basically a backport of https://github.com/python/cpython/issues/105704

589 """

590 hostname_and_port = netloc.rpartition("@")[2]

591

592 before_bracket, has_open_bracket, bracketed = hostname_and_port.partition("[")

593

594 if has_open_bracket:

595 # No data is allowed before '['.

596 if before_bracket:

597 raise ValueError("Invalid IPv6 URL")

598

599 hostname, _, port = bracketed.partition("]")

600

601 # Only ':<port>' may follow ']'.

602 if port and not port.startswith(":"):

603 raise ValueError("Invalid IPv6 URL")

604 # port validation done after, in `_SplitResult.__post_init__`

605 else:

606 hostname, _, _ = hostname_and_port.partition(":")

607

608 _check_bracketed_host(hostname)

609

610

611def _check_bracketed_host(hostname: str) -> None:

612 """

613 Validate a bracketed host according to RFC 3986 / WHATWG URL rules.

614

615 Raises:

616 ValueError: If the host is invalid.

617 """

618 # IPvFuture: v<HEXDIG>.<address>

619 if hostname.startswith(("v", "V")):

620 if not _IPV_FUTURE_RE.fullmatch(hostname):

621 raise ValueError("IPvFuture address is invalid")

622 return

623

624 # ip_address() raises ValueError if invalid.

625 ip = ipaddress.ip_address(hostname)

626

627 # Bracketed IPv4 literals are forbidden.

628 if isinstance(ip, ipaddress.IPv4Address):

629 raise ValueError("An IPv4 address cannot be in brackets")

630

631

632@functools.lru_cache

633def _urlsplit( # pylint: disable=too-many-locals,too-many-statements

634 url: str,

635 scheme: str = "",

636 allow_fragments: bool = True,

637) -> _SplitResult:

638 """Reimplementation of urllib.parse.urlsplit which:

639 - Doesn't use _coerce_args or _coerce_result

640 - Does manual single-pass scanning instead of repeated .find/.split calls

641 - Have reduced string allocations by slicing once using computed indices

642 - Avoids extra computations as much as possible

643 """

644 if not url:

645 return _SplitResult(scheme, "", "", "", "")

646

647 # urllib.parse.urlsplit removes every ASCII tab and newline from anywhere in

648 # the URL before parsing (the WHATWG "remove all ASCII tab or newline"

649 # step). This reimplementation only stripped leading C0/space, so a tab or

650 # newline embedded in the authority survived: parse_url("http://exa\tmple.com")

651 # reported host "exa\tmple.com" while urlsplit and browsers drop the tab and

652 # resolve "example.com". The membership checks keep the common tab-free path

653 # free of translate calls and string allocations.

654 if "\t" in url or "\n" in url or "\r" in url:

655 url = url.translate(_ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE)

656 if "\t" in scheme or "\n" in scheme or "\r" in scheme:

657 scheme = scheme.translate(_ASCII_TAB_OR_NEWLINE_TRANSLATION_TABLE)

658 url, scheme = url.lstrip(_C0_CONTROL_OR_SPACE), scheme.strip(_C0_CONTROL_OR_SPACE)

659

660 netloc = query = fragment = ""

661

662 if m := _SCHEME_RE.match(url):

663 scheme = m.group(1).lower()

664 url = url[m.end() :]

665

666 # The URL living standard treats "\" like "/" for special-scheme URLs, but

667 # only in the authority and path; a "\" in the query or fragment is left

668 # alone. Without this, "http://evil.com\@good.com/" is read as userinfo

669 # "evil.com\" plus host "good.com", while a browser ends the authority at

670 # the "\" and connects to "evil.com".

671 if scheme in _SPECIAL_SCHEMES and "\\" in url:

672 cut = len(url)

673 question_idx = url.find("?")

674 if question_idx != -1:

675 cut = question_idx

676 hash_idx = url.find("#")

677 if hash_idx != -1 and hash_idx < cut:

678 cut = hash_idx

679 url = url[:cut].replace("\\", "/") + url[cut:]

680

681 # The scan skips the leading "//" of an authority; for URLs without an

682 # authority it must start at 0, otherwise a "?" or "#" at index 0 or 1

683 # (e.g. relative URLs like "a?b" or "a#f") is never recorded.

684 scan_start = 2 if url[:2] == "//" else 0

685 slash_pos = question_pos = hash_pos = open_br_pos = closing_br_pos = -1

686 for idx, char in enumerate(url[scan_start:], scan_start):

687 if char == "/" and slash_pos == -1:

688 slash_pos = idx

689 elif char == "?" and question_pos == -1:

690 question_pos = idx

691 elif char == "#" and hash_pos == -1:

692 hash_pos = idx

693 elif char == "[" and open_br_pos == -1:

694 open_br_pos = idx

695 elif char == "]" and closing_br_pos == -1:

696 closing_br_pos = idx

697 if -1 not in (

698 slash_pos,

699 question_pos,

700 hash_pos,

701 open_br_pos,

702 closing_br_pos,

703 ):

704 break

705

706 if url[:2] == "//":

707 delim = len(url)

708

709 if 0 < slash_pos < delim:

710 delim = slash_pos

711 if 0 < question_pos < delim:

712 delim = question_pos

713 if 0 < hash_pos < delim:

714 delim = hash_pos

715

716 # Brackets only delimit an IPv6 host when they fall inside the

717 # authority. A "[" or "]" in the path, query or fragment is an

718 # ordinary character and must not drive IPv6 host parsing.

719 if not 2 <= open_br_pos < delim:

720 open_br_pos = -1

721 if not 2 <= closing_br_pos < delim:

722 closing_br_pos = -1

723

724 if (open_br_pos != -1) != (closing_br_pos != -1):

725 raise ValueError("Invalid IPv6 URL")

726

727 netloc = url[2:delim]

728 if open_br_pos != -1 and closing_br_pos != -1:

729 _check_bracketed_netloc(netloc)

730

731 url = url[delim:]

732

733 if question_pos != -1:

734 question_pos -= delim

735 if hash_pos != -1:

736 hash_pos -= delim

737 _checknetloc(netloc)

738

739 if allow_fragments and hash_pos != -1:

740 url, fragment = url[:hash_pos], url[hash_pos + 1 :]

741

742 if question_pos != -1:

743 url, query = url[:question_pos], url[question_pos + 1 :]

744

745 username = password = hostname = port = None

746 userinfo, have_info, hostinfo = netloc.rpartition("@")

747

748 if have_info:

749 username, _, password = userinfo.partition(":")

750 password = password if _ else None

751

752 if open_br_pos != -1:

753 hostname, _, port = hostinfo.partition("[")[2].partition("]")

754 port = port.partition(":")[2]

755 else:

756 hostname, _, port = hostinfo.partition(":")

757

758 return _SplitResult(

759 scheme,

760 netloc,

761 url,

762 query,

763 fragment,

764 username,

765 password,

766 hostname,

767 port or None,

768 )

769

770

771def _url2pathname(url: str) -> str:

772 """Reimplementation of urllib.request.url2pathname but with faster _unquote"""

773 if not url:

774 return ""

775

776 # These branches are handled by `_urlparse`

777 if url[:3] == "///": # pragma: no cover

778 url = url[2:]

779 elif url[:12] == "//localhost/": # pragma: no cover

780 url = url[11:]

781

782 if not _IS_WINDOWS:

783 if "%" not in url:

784 return url

785

786 return _unquote(url, _PATH_SAFE_CHARS).decode(_FS_ENCODING, _FS_ERRORS)

787

788 if url[:3] == "///":

789 url = url[1:]

790 url = url.replace(":", "|")

791 if "|" not in url:

792 return _unquote(url.replace("/", "\\").encode(), _PATH_SAFE_CHARS).decode(

793 _FS_ENCODING, _FS_ERRORS

794 )

795 comp = url.split("|")

796 if len(comp) != 2 or comp[0][-1] not in string.ascii_letters:

797 raise OSError(f"Bad URL: {url}")

798 drive = comp[0][-1].upper()

799 tail = _unquote(comp[1].replace("/", "\\"), _PATH_SAFE_CHARS).decode(

800 _FS_ENCODING, _FS_ERRORS

801 )

802 return f"{drive}:{tail}"

803

804

805@functools.lru_cache

806def _idna(input_string: str) -> tuple[bytes, str]:

807 """Cached IDNA encoding using Python's built-in 'idna' codec.

808

809 NOTE: IDNA processing in CPython is implemented in pure Python (not C),

810 which makes it relatively slow and allocation-heavy. The only

811 lower-level optimisation involved is Unicode normalization

812 (NFKC), which may use optimized internal paths, but IDNA itself

813 remains Python-level logic.

814 """

815 if input_string.isascii():

816 return input_string.encode(), input_string

817

818 _, normalized = _nfkc_netloc(input_string)

819

820 encoded = normalized.encode("idna")

821 return encoded, encoded.decode()

822

823

824def _idna_bytes(input_string: str) -> bytes:

825 return _idna(input_string)[0]

826

827

828@functools.lru_cache

829def _nfkc_netloc(netloc: str) -> tuple[str, str]:

830 cleaned = netloc.translate(_NETLOC_STRIP_CHARS)

831 normalized = unicodedata.normalize("NFKC", cleaned)

832 return cleaned, normalized

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/_url.py: 69%

389 statements