Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/url.py: 53%

1""" 

2This module contains general purpose URL functions not found in the standard 

3library. 

4""" 

5 

6from __future__ import annotations 

7 

8import base64 

9import codecs 

10import os 

11import posixpath 

12import re 

13import warnings 

14from ipaddress import IPv6Address, ip_address 

15from pathlib import Path 

16from typing import TYPE_CHECKING, NamedTuple, cast, overload 

17from urllib.parse import ParseResult, urljoin 

18from urllib.request import pathname2url 

19 

20from ._infra import _ASCII_WHITESPACE 

21from ._url import ( 

22 _PATH_SAFE_CHARS, 

23 _SAFE_CHARS, 

24 _SCHEME_RE, 

25 _SPECIAL_SCHEMES, 

26 # reexports 

27 RFC3986_GEN_DELIMS as RFC3986_GEN_DELIMS, 

28 RFC3986_RESERVED as RFC3986_RESERVED, 

29 RFC3986_SUB_DELIMS as RFC3986_SUB_DELIMS, 

30 RFC3986_UNRESERVED as RFC3986_UNRESERVED, 

31 RFC3986_USERINFO_SAFE_CHARS as RFC3986_USERINFO_SAFE_CHARS, 

32 _check_domain, 

33 _domain, 

34 _idna_bytes, 

35 _ipv4_host, 

36 _opaque_host, 

37 _parse_qs, 

38 _parse_qsl, 

39 _quote, 

40 _quote_into, 

41 _split_params, 

42 _strip, 

43 _unquote, 

44 _url2pathname, 

45 _urlencode, 

46 _urlparse, 

47 _urlsplit, 

48 _urlunparse, 

49 _urlunsplit, 

50) 

51from ._util import to_unicode 

52 

53if TYPE_CHECKING: 

54 from collections.abc import Sequence 

55 

56 from ._types import AnyUnicodeError 

57 

58 

59# error handling function for bytes-to-Unicode decoding errors with URLs 

60def _quote_byte(error: UnicodeError) -> tuple[str, int]: 

61 error = cast("AnyUnicodeError", error) 

62 text = error.object[error.start : error.end] 

63 if isinstance(text, str): # pragma: no cover 

64 text = text.encode() 

65 return (to_unicode(_quote(text)), error.end) 

66 

67 

68codecs.register_error("percentencode", _quote_byte) 

69 

70# Characters that are safe in all of: 

71# 

72# - RFC 2396 + RFC 2732, as interpreted by Java 8’s java.net.URI class 

73# - RFC 3986 

74# - The URL living standard 

75# 

76# % is currently excluded from these lists of characters, due to limitations 

77# of the current safe_url_string implementation, but it should also be escaped 

78# as %25 when it is not already being used as part of an escape character. 

79_USERINFO_SAFEST_CHARS = RFC3986_USERINFO_SAFE_CHARS.translate(None, delete=b":;=") 

80_PATH_SAFEST_CHARS = _SAFE_CHARS.translate(None, delete=b"#[]|") 

81_QUERY_SAFEST_CHARS = _PATH_SAFEST_CHARS 

82_SPECIAL_QUERY_SAFEST_CHARS = _PATH_SAFEST_CHARS.translate(None, delete=b"'") 

83_FRAGMENT_SAFEST_CHARS = _PATH_SAFEST_CHARS 

84 

85 

86def _skip_special_authority_slashes(url: str) -> str: 

87 """Return *url* with exactly two slashes before the host if its scheme is 

88 special and not file.""" 

89 # Special schemes other than file skip any number of slashes and 

90 # backslashes before the host, so "https:example.com", "https:/example.com" 

91 # and "https:///example.com" all have the host example.com. 

92 # https://url.spec.whatwg.org/#special-authority-ignore-slashes-state 

93 colon = url.find(":") 

94 if colon < 1 or ( 

95 url[colon + 1 : colon + 3] == "//" 

96 and url[colon + 3 : colon + 4] not in {"/", "\\"} 

97 ): 

98 return url 

99 m = _SCHEME_RE.match(url) 

100 if not m or (scheme := m.group(1).lower()) not in _SPECIAL_SCHEMES - {"file"}: 

101 return url 

102 rest = url[colon + 1 :] 

103 authority = rest.lstrip("/\\") 

104 # A URL without a host is left as is. 

105 if authority[:1] in {"", "?", "#"}: 

106 return url 

107 return f"{scheme}://{authority}" 

108 

109 

110def _safe_url_split( # pylint: disable=too-many-statements 

111 url: str | bytes, 

112 encoding: str = "utf8", 

113 path_encoding: str = "utf8", 

114 quote_path: bool = True, 

115) -> tuple[str, str, str, str, str]: 

116 # urlsplit() chokes on bytes input with non-ASCII chars, 

117 # so let's decode (to Unicode) using page encoding: 

118 # - it is assumed that a raw bytes input comes from a document 

119 # encoded with the supplied encoding (or UTF8 by default) 

120 # - if the supplied (or default) encoding chokes, 

121 # percent-encode offending bytes 

122 url = _strip(to_unicode(url, encoding=encoding, errors="percentencode")) 

123 url = _skip_special_authority_slashes(url) 

124 parts = _urlsplit(url) 

125 tmp_buf = bytearray() # utf-8 bytes 

126 

127 if parts.username is not None: 

128 _quote_into( 

129 _unquote(parts.username), 

130 tmp_buf, 

131 _USERINFO_SAFEST_CHARS, 

132 ) 

133 

134 if parts.password is not None: 

135 tmp_buf.append(58) # ord(":") 

136 _quote_into( 

137 _unquote(parts.password), 

138 tmp_buf, 

139 _USERINFO_SAFEST_CHARS, 

140 ) 

141 

142 tmp_buf.append(64) # ord("@") 

143 

144 if parts.hostname is not None: 

145 if ":" in parts.hostname: 

146 # IPv6 address: urlsplit() strips the brackets from the hostname, 

147 # but they are required in the netloc when rebuilding the URL. 

148 tmp_buf.append(91) # ord("[") 

149 tmp_buf += parts.hostname.encode("ascii") 

150 tmp_buf.append(93) # ord("]") 

151 elif parts.scheme not in _SPECIAL_SCHEMES: 

152 # The host of a non-special URL is opaque: case-sensitive and 

153 # percent-encoded instead of IDNA-encoded. parts.hostname is 

154 # lowercased, so take the host from the netloc. 

155 host = parts.netloc.rpartition("@")[2].partition(":")[0] 

156 if not host and parts.netloc: 

157 raise ValueError(f"{url!r} has userinfo or a port but no host") 

158 tmp_buf += _opaque_host(host) 

159 else: 

160 # parts.hostname is lowercased with str.lower(), which turns a 

161 # final "Σ" into "ς", so take the host from the netloc and let 

162 # _domain() lowercase it. 

163 host = parts.netloc.rpartition("@")[2] 

164 host = _domain(parts.hostname if "[" in host else host.partition(":")[0]) 

165 try: 

166 host = _idna_bytes(host).decode() 

167 except UnicodeError: 

168 # IDNA encoding can fail for too long labels (>63 characters) or 

169 # missing labels (e.g. http://.example.com). 

170 host = parts.hostname 

171 _check_domain(host) 

172 tmp_buf += _ipv4_host(host).encode() 

173 

174 if parts.port is not None: 

175 tmp_buf.append(58) # ord(":") 

176 tmp_buf += str(parts.port).encode("ascii") 

177 

178 netloc = tmp_buf.decode() 

179 tmp_buf.clear() 

180 

181 if quote_path: 

182 _quote_into(parts.path.encode(path_encoding), tmp_buf, _PATH_SAFEST_CHARS) 

183 path = tmp_buf.decode() 

184 tmp_buf.clear() 

185 else: 

186 path = parts.path 

187 

188 if path[:1] == "/": 

189 path = _remove_dot_segments(path) 

190 # A path starting with "//" in a non-special URL without an authority 

191 # gets a "/." prefix, so that it does not reparse as an authority. 

192 if ( 

193 path[:2] == "//" 

194 and not netloc 

195 and parts.scheme not in _SPECIAL_SCHEMES 

196 and url[len(parts.scheme) + 1 if parts.scheme else 0 :][:2] != "//" 

197 ): 

198 path = f"/.{path}" 

199 

200 _quote_into( 

201 parts.query.encode(encoding), 

202 tmp_buf, 

203 _SPECIAL_QUERY_SAFEST_CHARS 

204 if parts.scheme in _SPECIAL_SCHEMES 

205 else _QUERY_SAFEST_CHARS, 

206 ) 

207 query = tmp_buf.decode() 

208 tmp_buf.clear() 

209 

210 if parts.fragment: 

211 _quote_into(parts.fragment.encode(encoding), tmp_buf, _FRAGMENT_SAFEST_CHARS) 

212 fragment = tmp_buf.decode() 

213 tmp_buf.clear() 

214 else: 

215 fragment = parts.fragment 

216 

217 # The path of a special URL is never empty, e.g. “https://example.com” 

218 # serializes as “https://example.com/”. 

219 if not path and parts.scheme in _SPECIAL_SCHEMES: 

220 path = "/" 

221 

222 return ( 

223 parts.scheme, 

224 netloc, 

225 path, 

226 query, 

227 fragment, 

228 ) 

229 

230 

231def safe_url_string( 

232 url: str | bytes, 

233 encoding: str = "utf8", 

234 path_encoding: str = "utf8", 

235 quote_path: bool = True, 

236) -> str: 

237 """Return *url* with its syntax fixed, ready to be sent in a request. 

238 

239 Use it on URLs found in HTML or given by users before requesting them. To 

240 compare or deduplicate URLs, use :func:`canonicalize_url` instead. 

241 

242 *url* is parsed the way web browsers parse it, following the `URL living 

243 standard`_, and during serialization additional characters are 

244 percent-encoded, so that the result is valid by *all* of the following URL 

245 standards known to be enforced by modern-day web browsers and web servers: 

246 

247 .. _URL living standard: https://url.spec.whatwg.org/ 

248 

249 - `URL living standard`_ 

250 

251 - `RFC 3986`_ 

252 

253 - `RFC 2396`_ and `RFC 2732`_, as interpreted by `Java 8’s java.net.URI 

254 class`_. 

255 

256 .. _Java 8’s java.net.URI class: https://docs.oracle.com/javase/8/docs/api/java/net/URI.html 

257 .. _RFC 2396: https://www.ietf.org/rfc/rfc2396.txt 

258 .. _RFC 2732: https://www.ietf.org/rfc/rfc2732.txt 

259 .. _RFC 3986: https://www.ietf.org/rfc/rfc3986.txt 

260 

261 Only the syntax is fixed: any scheme (e.g. ``file`` or ``javascript``), any 

262 host (e.g. ``localhost``, a private IP address or a domain name that 

263 resolves to one) and relative URLs are allowed. If *url* comes from an 

264 untrusted source, it is up to you to check those. 

265 

266 If *url* is :class:`bytes`, it is decoded with *encoding*. *encoding* is 

267 also used to percent-encode the query and the fragment, so it should be the 

268 encoding of the page where *url* was found. *path_encoding* is used to 

269 percent-encode the path, unless *quote_path* is ``False``, in which case 

270 the path is left as is. 

271 

272 Calling this function on an already safe URL returns the URL unmodified. 

273 """ 

274 return _urlunsplit(*_safe_url_split(url, encoding, path_encoding, quote_path)) 

275 

276 

277_parent_dirs = re.compile(r"/?(\.\./)+") 

278 

279# Percent-encoded forms of the single-dot and double-dot path segments of the 

280# URL living standard, which clients resolve like "." and "..". 

281_encoded_dot_segments = { 

282 "%2e": ".", 

283 ".%2e": "..", 

284 "%2e.": "..", 

285 "%2e%2e": "..", 

286} 

287 

288 

289def safe_download_url( 

290 url: str | bytes, encoding: str = "utf8", path_encoding: str = "utf8" 

291) -> str: 

292 warnings.warn( 

293 "w3lib.url.safe_download_url() is deprecated.", 

294 FutureWarning, 

295 stacklevel=2, 

296 ) 

297 safe_url = safe_url_string(url, encoding, path_encoding) 

298 scheme, netloc, path, query, _ = _urlsplit(safe_url) 

299 if path: 

300 if "%" in path: 

301 path = "/".join( 

302 _encoded_dot_segments.get(segment.lower(), segment) 

303 for segment in path.split("/") 

304 ) 

305 normalized_path = _parent_dirs.sub("", posixpath.normpath(path)) 

306 if path.endswith("/") and not normalized_path.endswith("/"): 

307 normalized_path = f"{normalized_path}/" 

308 path = normalized_path 

309 return _urlunsplit(scheme, netloc, path, query, "") 

310 

311 

312def is_url(text: str) -> bool: 

313 prefixes = ("file://", "http://", "https://") 

314 # Only lowercase the first 8 characters (len("https://")) when needed. 

315 return text.startswith(prefixes) or text[:8].lower().startswith(prefixes) 

316 

317 

318@overload 

319def url_query_parameter( 

320 url: str | bytes, 

321 parameter: str, 

322 default: None = None, 

323 keep_blank_values: bool | int = 0, 

324 *, 

325 separator: str = "&", 

326) -> str | None: ... 

327 

328 

329@overload 

330def url_query_parameter( 

331 url: str | bytes, 

332 parameter: str, 

333 default: str, 

334 keep_blank_values: bool | int = 0, 

335 *, 

336 separator: str = "&", 

337) -> str: ... 

338 

339 

340def url_query_parameter( 

341 url: str | bytes, 

342 parameter: str, 

343 default: str | None = None, 

344 keep_blank_values: bool | int = 0, 

345 *, 

346 separator: str = "&", 

347) -> str | None: 

348 """Return the value of a url parameter, given the url and parameter name 

349 

350 If url contains multiple parameters, the first leftmost one is returned. 

351 

352 General case: 

353 

354 >>> import w3lib.url 

355 >>> w3lib.url.url_query_parameter("product.html?id=200&foo=bar", "id") 

356 '200' 

357 >>> 

358 

359 Return a default value if the parameter is not found: 

360 

361 >>> w3lib.url.url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault") 

362 'mydefault' 

363 >>> 

364 

365 Returns None if `keep_blank_values` not set or 0 (default): 

366 

367 >>> w3lib.url.url_query_parameter("product.html?id=", "id") 

368 >>> 

369 

370 Returns an empty string if `keep_blank_values` set to 1: 

371 

372 >>> w3lib.url.url_query_parameter("product.html?id=", "id", keep_blank_values=1) 

373 '' 

374 >>> 

375 

376 """ 

377 

378 queryparams = _parse_qs( 

379 _urlsplit(to_unicode(url)).query, 

380 keep_blank_values=bool(keep_blank_values), 

381 separator=separator.encode(), 

382 ) 

383 parameter_bytes = parameter.encode() 

384 if parameter_bytes in queryparams: 

385 return queryparams[parameter_bytes][0].decode(errors="replace") 

386 return default 

387 

388 

389def url_query_cleaner( 

390 url: str | bytes, 

391 parameterlist: str | Sequence[str] = (), 

392 sep: str = "&", 

393 kvsep: str = "=", 

394 remove: bool = False, 

395 unique: bool = True, 

396 keep_fragments: bool = False, 

397) -> str: 

398 """Clean URL arguments leaving only those passed in the parameterlist keeping order 

399 

400 >>> import w3lib.url 

401 >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ('id',)) 

402 'product.html?id=200' 

403 >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']) 

404 'product.html?id=200&name=wired' 

405 >>> 

406 

407 If `unique` is ``False``, do not remove duplicated keys 

408 

409 >>> w3lib.url.url_query_cleaner("product.html?d=1&e=b&d=2&d=3&other=other", ['d'], unique=False) 

410 'product.html?d=1&d=2&d=3' 

411 >>> 

412 

413 If `remove` is ``True``, leave only those **not in parameterlist**. 

414 

415 >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id'], remove=True) 

416 'product.html?foo=bar&name=wired' 

417 >>> w3lib.url.url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'foo'], remove=True) 

418 'product.html?name=wired' 

419 >>> 

420 

421 By default, URL fragments are removed. If you need to preserve fragments, 

422 pass the ``keep_fragments`` argument as ``True``. 

423 

424 >>> w3lib.url.url_query_cleaner('http://domain.tld/?bla=123#123123', ['bla'], remove=True, keep_fragments=True) 

425 'http://domain.tld/#123123' 

426 

427 """ 

428 

429 if parameterlist and isinstance(parameterlist, (str, bytes)): 

430 parameterlist = (parameterlist,) 

431 

432 if isinstance(url, bytes): 

433 url = url.decode() 

434 

435 url, _, fragment = url.partition("#") 

436 base, _, query = url.partition("?") 

437 

438 if not query or (not parameterlist and not remove): 

439 return base if not (keep_fragments and fragment) else f"{base}#{fragment}" 

440 

441 param_lookup = frozenset(parameterlist) 

442 

443 seen: set[str] | None = set() if unique else None 

444 result: list[str] = [] 

445 

446 for ksv in query.split(sep): 

447 if not ksv: 

448 continue 

449 

450 k, _, _ = ksv.partition(kvsep) 

451 

452 if seen is not None: 

453 if k in seen: 

454 continue 

455 seen.add(k) 

456 

457 if remove: 

458 if k in param_lookup: 

459 continue 

460 elif k not in param_lookup: 

461 continue 

462 

463 result.append(ksv) 

464 del param_lookup, seen 

465 

466 url = base if not result else f"{base}?{sep.join(result)}" 

467 del result 

468 

469 if keep_fragments and fragment: 

470 url = f"{url}#{fragment}" 

471 

472 return url 

473 

474 

475def _add_or_replace_parameters( 

476 url: str, params: dict[bytes, bytes], *, separator: str = "&" 

477) -> str: 

478 parsed = _urlsplit(url) 

479 

480 current_args = _parse_qsl( 

481 parsed.query, keep_blank_values=True, separator=separator.encode() 

482 ) 

483 

484 new_args: list[tuple[bytes, bytes]] = [] 

485 seen_params: set[bytes] = set() 

486 

487 for name, value in current_args: 

488 if name in seen_params: 

489 continue 

490 replacement = params.get(name) 

491 if replacement is None: 

492 new_args.append((name, value)) 

493 else: 

494 new_args.append((name, replacement)) 

495 seen_params.add(name) 

496 

497 for name, value in params.items(): 

498 if name not in seen_params: 

499 new_args.append((name, value)) 

500 del seen_params, current_args 

501 

502 return _urlunsplit( 

503 parsed.scheme, 

504 parsed.netloc, 

505 parsed.path, 

506 _urlencode(new_args, separator.encode()).decode(), 

507 parsed.fragment, 

508 ) 

509 

510 

511def add_or_replace_parameter( 

512 url: str, name: str, new_value: str, *, separator: str = "&" 

513) -> str: 

514 """Add or remove a parameter to a given url 

515 

516 >>> import w3lib.url 

517 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php', 'arg', 'v') 

518 'http://www.example.com/index.php?arg=v' 

519 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg4', 'v4') 

520 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3&arg4=v4' 

521 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg3', 'v3new') 

522 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3new' 

523 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1;arg2=v2', 'arg3', 'v3', separator=';') 

524 'http://www.example.com/index.php?arg1=v1;arg2=v2;arg3=v3' 

525 >>> 

526 

527 ``separator`` is used both to split the existing query and to join the 

528 resulting one. 

529 

530 """ 

531 return _add_or_replace_parameters( 

532 url, {name.encode(): new_value.encode()}, separator=separator 

533 ) 

534 

535 

536def add_or_replace_parameters( 

537 url: str, new_parameters: dict[str, str], *, separator: str = "&" 

538) -> str: 

539 """Add or remove a parameters to a given url 

540 

541 >>> import w3lib.url 

542 >>> w3lib.url.add_or_replace_parameters('http://www.example.com/index.php', {'arg': 'v'}) 

543 'http://www.example.com/index.php?arg=v' 

544 >>> args = {'arg4': 'v4', 'arg3': 'v3new'} 

545 >>> w3lib.url.add_or_replace_parameters('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', args) 

546 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3new&arg4=v4' 

547 >>> 

548 

549 """ 

550 return _add_or_replace_parameters( 

551 url, 

552 {k.encode(): v.encode() for k, v in new_parameters.items()}, 

553 separator=separator, 

554 ) 

555 

556 

557def path_to_file_uri(path: str | os.PathLike[str]) -> str: 

558 """Convert local filesystem path to legal File URIs as described in: 

559 http://en.wikipedia.org/wiki/File_URI_scheme 

560 """ 

561 absolute_path = Path(path).absolute() 

562 if os.name == "nt" and absolute_path.drive.startswith("\\\\"): 

563 return absolute_path.as_uri() 

564 return f"file:///{pathname2url(str(absolute_path)).lstrip('/')}" 

565 

566 

567def file_uri_to_path(uri: str) -> str: 

568 """Convert File URI to local filesystem path according to: 

569 http://en.wikipedia.org/wiki/File_URI_scheme 

570 

571 A host other than ``localhost`` names a file on that host: on Windows the 

572 result is a UNC path, and elsewhere :exc:`OSError` is raised, as 

573 :func:`urllib.request.url2pathname` does since Python 3.14. 

574 """ 

575 parsed = _urlparse(uri) 

576 path = parsed.path 

577 if parsed.netloc and parsed.netloc.lower() != "localhost": 

578 if os.name != "nt": 

579 raise OSError(f"Bad URL: {uri} has a non-local host") 

580 path = f"//{parsed.netloc}{path}" 

581 return _url2pathname(path) 

582 

583 

584def any_to_uri(uri_or_path: str) -> str: 

585 """If given a path name, return its File URI, otherwise return it 

586 unmodified 

587 """ 

588 if os.path.splitdrive(uri_or_path)[0]: 

589 return path_to_file_uri(uri_or_path) 

590 return uri_or_path if _urlparse(uri_or_path)[0] else path_to_file_uri(uri_or_path) 

591 

592 

593def add_http_if_no_scheme(url: str) -> str: 

594 """Add ``http`` as the default scheme if it is missing from *url*.""" 

595 if not re.match(r"^\w+://", url, flags=re.IGNORECASE): 

596 scheme = "http:" if _urlparse(url).netloc else "http://" 

597 url = scheme + url 

598 return url 

599 

600 

601# ASCII characters. 

602_char = set(map(chr, range(127))) 

603 

604# RFC 2045 token. 

605_token = r"[{}]+".format( 

606 re.escape( 

607 "".join( 

608 _char 

609 - 

610 # Control characters. 

611 set(map(chr, range(32))) 

612 - 

613 # tspecials and space. 

614 set('()<>@,;:\\"/[]?= ') 

615 ) 

616 ) 

617) 

618 

619# RFC 822 quoted-string, without surrounding quotation marks. 

620_quoted_string = r"(?:[{}]|(?:\\[{}]))*".format( 

621 re.escape("".join(_char - {'"', "\\", "\r"})), re.escape("".join(_char)) 

622) 

623 

624# Encode the regular expression strings to make them into bytes, as Python 3 

625# bytes have no format() method, but bytes must be passed to re.compile() in 

626# order to make a pattern object that can be used to match on bytes. 

627 

628# RFC 2397 mediatype. 

629_mediatype_pattern = re.compile(rf"{_token}/{_token}".encode()) 

630_mediatype_parameter_pattern = re.compile( 

631 rf';({_token})=(?:({_token})|"({_quoted_string})")'.encode() 

632) 

633del _char, _token, _quoted_string 

634 

635_base64_pattern = re.compile(rb"[A-Za-z0-9+/]*") 

636_ASCII_WHITESPACE_BYTES = _ASCII_WHITESPACE.encode() 

637 

638 

639def _forgiving_base64_decode(data: bytes) -> bytes: 

640 """Decode *data* following the forgiving-base64 decode algorithm of the 

641 Infra Living Standard, which browsers use for data URIs. 

642 

643 https://infra.spec.whatwg.org/commit-snapshots/c67ae1fb1d161b23d0f8c9bf9917ea1993f7b19a/#forgiving-base64-decode 

644 

645 Raise :exc:`ValueError` if *data* is not valid base64. 

646 """ 

647 data = data.translate(None, _ASCII_WHITESPACE_BYTES) 

648 if len(data) % 4 == 0: 

649 if data.endswith(b"=="): 

650 data = data[:-2] 

651 elif data.endswith(b"="): 

652 data = data[:-1] 

653 if len(data) % 4 == 1 or not _base64_pattern.fullmatch(data): 

654 raise ValueError("invalid base64") 

655 return base64.b64decode(data + b"=" * (-len(data) % 4), validate=True) 

656 

657 

658class ParseDataURIResult(NamedTuple): 

659 """Named tuple returned by :func:`parse_data_uri`.""" 

660 

661 #: MIME type type and subtype, separated by / (e.g. ``"text/plain"``). 

662 media_type: str 

663 #: MIME type parameters (e.g. ``{"charset": "US-ASCII"}``). 

664 media_type_parameters: dict[str, str] 

665 #: Data, decoded if it was encoded in base64 format. 

666 data: bytes 

667 

668 

669def parse_data_uri(uri: str | bytes) -> ParseDataURIResult: 

670 """Parse a data: URI into :class:`ParseDataURIResult`. 

671 

672 Base64 data is decoded the way browsers do it, i.e. ignoring ASCII 

673 whitespace and allowing missing ``=`` padding. 

674 

675 Raise :exc:`ValueError` for invalid data URIs. 

676 """ 

677 if not isinstance(uri, bytes): 

678 uri = safe_url_string(uri).encode("ascii") 

679 

680 scheme, _, uri = uri.partition(b":") 

681 if not scheme or not uri: 

682 raise ValueError("invalid URI") 

683 if scheme.lower() != b"data": 

684 raise ValueError("not a data URI") 

685 

686 # RFC 3986 section 2.1 allows percent encoding to escape characters that 

687 # would be interpreted as delimiters, implying that actual delimiters 

688 # should not be percent-encoded. 

689 # Decoding before parsing will allow malformed URIs with percent-encoded 

690 # delimiters, but it makes parsing easier and should not affect 

691 # well-formed URIs, as the delimiters used in this URI scheme are not 

692 # allowed, percent-encoded or not, in tokens. 

693 uri = _unquote(uri) 

694 

695 media_type = "text/plain" 

696 media_type_params = {} 

697 

698 m = _mediatype_pattern.match(uri) 

699 if m: 

700 media_type = m.group().decode() 

701 uri = uri[m.end() :] 

702 else: 

703 media_type_params["charset"] = "US-ASCII" 

704 

705 while m := _mediatype_parameter_pattern.match(uri): 

706 attribute, value, value_quoted = m.groups() 

707 if value_quoted is not None: 

708 value = re.sub(rb"\\(.)", rb"\1", value_quoted) 

709 media_type_params[attribute.decode()] = value.decode() 

710 uri = uri[m.end() :] 

711 

712 is_base64, comma, data = uri.partition(b",") 

713 if not comma: 

714 raise ValueError("invalid data URI") 

715 if is_base64: 

716 if is_base64 != b";base64": 

717 raise ValueError("invalid data URI") 

718 try: 

719 data = _forgiving_base64_decode(data) 

720 except ValueError: 

721 raise ValueError("invalid data URI") from None 

722 

723 return ParseDataURIResult(media_type, media_type_params, data) 

724 

725 

726__all__ = [ 

727 "add_http_if_no_scheme", 

728 "add_or_replace_parameter", 

729 "add_or_replace_parameters", 

730 "any_to_uri", 

731 "canonicalize_url", 

732 "file_uri_to_path", 

733 "is_url", 

734 "parse_data_uri", 

735 "path_to_file_uri", 

736 "safe_url_string", 

737 "url_query_cleaner", 

738 "url_query_parameter", 

739] 

740 

741 

742_DOT_SEGMENTS = { 

743 ".": ".", 

744 "%2e": ".", 

745 "..": "..", 

746 ".%2e": "..", 

747 "%2e.": "..", 

748 "%2e%2e": "..", 

749} 

750 

751 

752def _remove_dot_segments(path: str) -> str: 

753 """Resolve dot segments in *path*, which starts with "/" (RFC 3986, 

754 section 5.2.4), including their percent-encoded spellings.""" 

755 # Every segment follows a "/", so a path without "/." or "/%2" has no dot 

756 # segment. 

757 if "/." not in path and "/%2" not in path: 

758 return path 

759 segments = path[1:].split("/") 

760 output: list[str] = [] 

761 for segment in segments[:-1]: 

762 dots = _DOT_SEGMENTS.get(segment.lower()) 

763 if dots == "..": 

764 if output: 

765 output.pop() 

766 elif dots is None: 

767 output.append(segment) 

768 # A trailing dot segment leaves the path ending in "/". 

769 last = segments[-1] 

770 dots = _DOT_SEGMENTS.get(last.lower()) 

771 if dots == "..": 

772 if output: 

773 output.pop() 

774 output.append("") 

775 elif dots == ".": 

776 output.append("") 

777 else: 

778 output.append(last) 

779 return "/" + "/".join(output) 

780 

781 

782def _urljoin(base: str, url: str) -> str: 

783 """Resolve *url* against *base* following the `URL living standard`_. 

784 

785 .. _URL living standard: https://url.spec.whatwg.org/#concept-basic-url-parser 

786 

787 If *base* has no scheme, :func:`urllib.parse.urljoin` resolves *url* 

788 instead. Raises :exc:`ValueError` if *url* cannot be resolved against 

789 *base*, i.e. if *base* has an opaque path (e.g. ``mailto:a``) and *url* is 

790 neither absolute nor a fragment. 

791 """ 

792 base, url = _strip(base), _strip(url) 

793 base_parts = _urlsplit(base) 

794 scheme = base_parts.scheme 

795 if not scheme: 

796 return urljoin(base, url) 

797 special = scheme in _SPECIAL_SCHEMES 

798 if m := _SCHEME_RE.match(url): 

799 # Only a special scheme matching that of the base keeps the URL 

800 # relative, e.g. "http:foo". 

801 if not special or m.group(1).lower() != scheme: 

802 return url 

803 url = url[m.end() :] 

804 

805 url, hash_mark, fragment = url.partition("#") 

806 path, question_mark, query = url.partition("?") 

807 base_has_authority = special or base[len(scheme) + 1 : len(scheme) + 3] == "//" 

808 if ( 

809 not base_has_authority 

810 and base_parts.path[:1] != "/" 

811 and (path or question_mark or not hash_mark) 

812 ): 

813 raise ValueError(f"Only a fragment can be resolved against {base!r}") 

814 

815 if not path: 

816 base = base.partition("#")[0] 

817 if question_mark: 

818 base = base.partition("?")[0] 

819 return f"{base}{question_mark}{query}{hash_mark}{fragment}" 

820 

821 tail = f"{question_mark}{query}{hash_mark}{fragment}" 

822 if special: 

823 path = path.replace("\\", "/") 

824 if path[:2] == "//": 

825 # Special schemes other than file ignore any extra slash before the 

826 # host, e.g. "///example.com" is "//example.com". 

827 if special and scheme != "file": 

828 path = f"//{path.lstrip('/')}" 

829 if (authority_end := path.find("/", 2)) != -1: 

830 path = path[:authority_end] + _remove_dot_segments(path[authority_end:]) 

831 return f"{scheme}:{path}{tail}" 

832 if path[0] != "/": 

833 base_path = base_parts.path 

834 path = f"{base_path[: base_path.rfind('/') + 1] or '/'}{path}" 

835 path = _remove_dot_segments(path) 

836 if base_has_authority: 

837 return f"{scheme}://{base_parts.netloc}{path}{tail}" 

838 if path[:2] == "//": 

839 path = f"/.{path}" 

840 return f"{scheme}:{path}{tail}" 

841 

842 

843def canonicalize_url( 

844 url: str | bytes | ParseResult, 

845 keep_blank_values: bool = True, 

846 keep_fragments: bool = False, 

847 encoding: str | None = None, 

848 *, 

849 query_separator: str = "&", 

850) -> str: 

851 r"""Return a normalized form of *url*, to compare or deduplicate URLs. 

852 

853 .. versionchanged:: 2.5.0 

854 Dot segments (``.`` and ``..``) in the path are now resolved, and 

855 IPv6 addresses in the host are now normalized. 

856 

857 URLs that differ only in ways that servers usually ignore get the same 

858 canonical form. The canonical form may point to a different resource than 

859 *url*, so to send a request, use :func:`safe_url_string` on *url* instead. 

860 

861 The following changes are applied: 

862 

863 - make the URL safe, as :func:`safe_url_string` does 

864 - lowercase the host 

865 - sort query arguments, first by key, then by value 

866 - normalize all spaces (in query arguments) '+' (plus symbol) 

867 - normalize percent encodings case (%2f -> %2F) 

868 - remove query arguments with blank values (unless `keep_blank_values` is True) 

869 - remove fragments (unless `keep_fragments` is True) 

870 - resolve dot segments (``.`` and ``..``) in the path 

871 - normalize IPv6 addresses in the host 

872 

873 >>> import w3lib.url 

874 >>> 

875 >>> # sorting query arguments 

876 >>> w3lib.url.canonicalize_url('http://www.example.com/do?c=3&b=5&b=2&a=50') 

877 'http://www.example.com/do?a=50&b=2&b=5&c=3' 

878 >>> 

879 >>> # UTF-8 conversion + percent-encoding of non-ASCII characters 

880 >>> w3lib.url.canonicalize_url('http://www.example.com/r\u00e9sum\u00e9') 

881 'http://www.example.com/r%C3%A9sum%C3%A9' 

882 >>> 

883 >>> # a query separator other than the default '&' 

884 >>> w3lib.url.canonicalize_url('http://www.example.com/do?c=3;a=50', query_separator=';') 

885 'http://www.example.com/do?a=50;c=3' 

886 >>> 

887 >>> # resolving dot segments and normalizing an IPv6 address 

888 >>> w3lib.url.canonicalize_url('http://[::0:1]/a/../b') 

889 'http://[::1]/b' 

890 >>> 

891 

892 For more examples, see the tests in `tests/test_url.py`. 

893 """ 

894 # If supplied `encoding` is not compatible with all characters in `url`, 

895 # fallback to UTF-8 as safety net. 

896 # UTF-8 can handle all Unicode characters, 

897 # so we should be covered regarding URL normalization, 

898 # if not for proper URL expected by remote website. 

899 if isinstance(url, ParseResult): 

900 url = _urlunparse(*url) 

901 try: 

902 scheme, netloc, path, query, fragment = _safe_url_split( 

903 url, encoding=encoding or "utf8"