Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/w3lib/url.py: 48%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2This module contains general purpose URL functions not found in the standard
3library.
4"""
6from __future__ import annotations
8import base64
9import codecs
10import os
11import posixpath
12import re
13from ipaddress import IPv6Address, ip_address
14from pathlib import Path
15from typing import TYPE_CHECKING, NamedTuple, cast, overload
16from urllib.parse import ParseResult
17from urllib.request import pathname2url
19from ._url import (
20 _PATH_SAFE_CHARS,
21 _SAFE_CHARS,
22 _SPECIAL_SCHEMES,
23 # reexports
24 RFC3986_GEN_DELIMS as RFC3986_GEN_DELIMS,
25 RFC3986_RESERVED as RFC3986_RESERVED,
26 RFC3986_SUB_DELIMS as RFC3986_SUB_DELIMS,
27 RFC3986_UNRESERVED as RFC3986_UNRESERVED,
28 RFC3986_USERINFO_SAFE_CHARS as RFC3986_USERINFO_SAFE_CHARS,
29 _idna_bytes,
30 _parse_qs,
31 _parse_qsl,
32 _quote,
33 _quote_into,
34 _split_params,
35 _strip,
36 _unquote,
37 _url2pathname,
38 _urlencode,
39 _urlparse,
40 _urlsplit,
41 _urlunparse,
42 _urlunsplit,
43)
44from ._util import to_unicode
46if TYPE_CHECKING:
47 from collections.abc import Sequence
49 from ._types import AnyUnicodeError
52# error handling function for bytes-to-Unicode decoding errors with URLs
53def _quote_byte(error: UnicodeError) -> tuple[str, int]:
54 error = cast("AnyUnicodeError", error)
55 text = error.object[error.start : error.end]
56 if isinstance(text, str): # pragma: no cover
57 text = text.encode()
58 return (to_unicode(_quote(text)), error.end)
61codecs.register_error("percentencode", _quote_byte)
63# Characters that are safe in all of:
64#
65# - RFC 2396 + RFC 2732, as interpreted by Java 8’s java.net.URI class
66# - RFC 3986
67# - The URL living standard
68#
69# NOTE: % is currently excluded from these lists of characters, due to
70# limitations of the current safe_url_string implementation, but it should also
71# be escaped as %25 when it is not already being used as part of an escape
72# character.
73_USERINFO_SAFEST_CHARS = RFC3986_USERINFO_SAFE_CHARS.translate(None, delete=b":;=")
74_PATH_SAFEST_CHARS = _SAFE_CHARS.translate(None, delete=b"#[]|")
75_QUERY_SAFEST_CHARS = _PATH_SAFEST_CHARS
76_SPECIAL_QUERY_SAFEST_CHARS = _PATH_SAFEST_CHARS.translate(None, delete=b"'")
77_FRAGMENT_SAFEST_CHARS = _PATH_SAFEST_CHARS
80def _safe_url_split(
81 url: str | bytes,
82 encoding: str = "utf8",
83 path_encoding: str = "utf8",
84 quote_path: bool = True,
85) -> tuple[str, str, str, str, str]:
86 # urlsplit() chokes on bytes input with non-ASCII chars,
87 # so let's decode (to Unicode) using page encoding:
88 # - it is assumed that a raw bytes input comes from a document
89 # encoded with the supplied encoding (or UTF8 by default)
90 # - if the supplied (or default) encoding chokes,
91 # percent-encode offending bytes
92 parts = _urlsplit(
93 _strip(to_unicode(url, encoding=encoding, errors="percentencode"))
94 )
95 tmp_buf = bytearray() # utf-8 bytes
97 if parts.username is not None or parts.password is not None:
98 if parts.username is not None:
99 _quote_into(
100 _unquote(parts.username),
101 tmp_buf,
102 _USERINFO_SAFEST_CHARS,
103 )
105 if parts.password is not None:
106 tmp_buf.append(58) # ord(":")
107 _quote_into(
108 _unquote(parts.password),
109 tmp_buf,
110 _USERINFO_SAFEST_CHARS,
111 )
113 tmp_buf.append(64) # ord("@")
115 if parts.hostname is not None:
116 if ":" in parts.hostname:
117 # IPv6 address: urlsplit() strips the brackets from the hostname,
118 # but they are required in the netloc when rebuilding the URL.
119 tmp_buf.append(91) # ord("[")
120 tmp_buf += parts.hostname.encode("ascii")
121 tmp_buf.append(93) # ord("]")
122 else:
123 try:
124 tmp_buf += _idna_bytes(parts.hostname)
125 except UnicodeError:
126 # IDNA encoding can fail for too long labels (>63 characters) or
127 # missing labels (e.g. http://.example.com).
128 tmp_buf += parts.hostname.encode()
130 if parts.port is not None:
131 tmp_buf.append(58) # ord(":")
132 tmp_buf += str(parts.port).encode("ascii")
134 netloc = tmp_buf.decode()
135 tmp_buf.clear()
137 if quote_path:
138 _quote_into(parts.path.encode(path_encoding), tmp_buf, _PATH_SAFEST_CHARS)
139 path = tmp_buf.decode()
140 tmp_buf.clear()
141 else:
142 path = parts.path
144 _quote_into(
145 parts.query.encode(encoding),
146 tmp_buf,
147 _SPECIAL_QUERY_SAFEST_CHARS
148 if parts.scheme in _SPECIAL_SCHEMES
149 else _QUERY_SAFEST_CHARS,
150 )
151 query = tmp_buf.decode()
152 tmp_buf.clear()
154 if parts.fragment:
155 _quote_into(parts.fragment.encode(encoding), tmp_buf, _FRAGMENT_SAFEST_CHARS)
156 fragment = tmp_buf.decode()
157 tmp_buf.clear()
158 else:
159 fragment = parts.fragment
161 # Without a path, the query follows the authority directly, e.g.
162 # “https://example.com?a=b”, and code that builds an HTTP request target
163 # out of the path and the query ends up sending “?a=b” as the target.
164 if not path and query and parts.scheme in _SPECIAL_SCHEMES:
165 path = "/"
167 return (
168 parts.scheme,
169 netloc,
170 path,
171 query,
172 fragment,
173 )
176def safe_url_string(
177 url: str | bytes,
178 encoding: str = "utf8",
179 path_encoding: str = "utf8",
180 quote_path: bool = True,
181) -> str:
182 """Return a URL equivalent to *url* that a wide range of web browsers and
183 web servers consider valid.
185 *url* is parsed according to the rules of the `URL living standard`_,
186 and during serialization additional characters are percent-encoded to make
187 the URL valid by additional URL standards.
189 .. _URL living standard: https://url.spec.whatwg.org/
191 The returned URL should be valid by *all* of the following URL standards
192 known to be enforced by modern-day web browsers and web servers:
194 - `URL living standard`_
196 - `RFC 3986`_
198 - `RFC 2396`_ and `RFC 2732`_, as interpreted by `Java 8’s java.net.URI
199 class`_.
201 .. _Java 8’s java.net.URI class: https://docs.oracle.com/javase/8/docs/api/java/net/URI.html
202 .. _RFC 2396: https://www.ietf.org/rfc/rfc2396.txt
203 .. _RFC 2732: https://www.ietf.org/rfc/rfc2732.txt
204 .. _RFC 3986: https://www.ietf.org/rfc/rfc3986.txt
206 If a bytes URL is given, it is first converted to `str` using the given
207 encoding (which defaults to 'utf-8'). If quote_path is True (default),
208 path_encoding ('utf-8' by default) is used to encode URL path component
209 which is then quoted. Otherwise, if quote_path is False, path component
210 is not encoded or quoted. Given encoding is used for query string
211 or form data.
213 When passing an encoding, you should use the encoding of the
214 original page (the page from which the URL was extracted from).
216 Calling this function on an already "safe" URL will return the URL
217 unmodified.
218 """
219 return _urlunsplit(*_safe_url_split(url, encoding, path_encoding, quote_path))
222_parent_dirs = re.compile(r"/?(\.\./)+")
224# Percent-encoded forms of the single-dot and double-dot path segments of the
225# URL living standard, which clients resolve like "." and "..".
226_encoded_dot_segments = {
227 "%2e": ".",
228 ".%2e": "..",
229 "%2e.": "..",
230 "%2e%2e": "..",
231}
234def safe_download_url(
235 url: str | bytes, encoding: str = "utf8", path_encoding: str = "utf8"
236) -> str:
237 """Make a url for download. This will call safe_url_string
238 and then strip the fragment, if one exists. The path will
239 be normalised.
241 If the path is outside the document root, it will be changed
242 to be within the document root.
243 """
244 safe_url = safe_url_string(url, encoding, path_encoding)
245 scheme, netloc, path, query, _ = _urlsplit(safe_url)
246 if path:
247 if "%" in path:
248 path = "/".join(
249 _encoded_dot_segments.get(segment.lower(), segment)
250 for segment in path.split("/")
251 )
252 normalized_path = _parent_dirs.sub("", posixpath.normpath(path))
253 if path.endswith("/") and not normalized_path.endswith("/"):
254 normalized_path = f"{normalized_path}/"
255 path = normalized_path
256 else:
257 path = "/"
258 return _urlunsplit(scheme, netloc, path, query, "")
261def is_url(text: str) -> bool:
262 return text.partition("://")[0] in {"file", "http", "https"}
265@overload
266def url_query_parameter(
267 url: str | bytes,
268 parameter: str,
269 default: None = None,
270 keep_blank_values: bool | int = 0,
271 *,
272 separator: str = "&",
273) -> str | None: ...
276@overload
277def url_query_parameter(
278 url: str | bytes,
279 parameter: str,
280 default: str,
281 keep_blank_values: bool | int = 0,
282 *,
283 separator: str = "&",
284) -> str: ...
287def url_query_parameter(
288 url: str | bytes,
289 parameter: str,
290 default: str | None = None,
291 keep_blank_values: bool | int = 0,
292 *,
293 separator: str = "&",
294) -> str | None:
295 """Return the value of a url parameter, given the url and parameter name
296 NOTE: If url contains multiple parameters, the first leftmost one is returned
298 General case:
300 >>> import w3lib.url
301 >>> w3lib.url.url_query_parameter("product.html?id=200&foo=bar", "id")
302 '200'
303 >>>
305 Return a default value if the parameter is not found:
307 >>> w3lib.url.url_query_parameter("product.html?id=200&foo=bar", "notthere", "mydefault")
308 'mydefault'
309 >>>
311 Returns None if `keep_blank_values` not set or 0 (default):
313 >>> w3lib.url.url_query_parameter("product.html?id=", "id")
314 >>>
316 Returns an empty string if `keep_blank_values` set to 1:
318 >>> w3lib.url.url_query_parameter("product.html?id=", "id", keep_blank_values=1)
319 ''
320 >>>
322 """
324 queryparams = _parse_qs(
325 _urlsplit(str(url)).query,
326 keep_blank_values=bool(keep_blank_values),
327 separator=separator.encode(),
328 )
329 parameter_bytes = parameter.encode()
330 if parameter_bytes in queryparams:
331 return queryparams[parameter_bytes][0].decode()
332 return default
335def url_query_cleaner(
336 url: str | bytes,
337 parameterlist: str | bytes | Sequence[str | bytes] = (),
338 sep: str = "&",
339 kvsep: str = "=",
340 remove: bool = False,
341 unique: bool = True,
342 keep_fragments: bool = False,
343) -> str:
344 """Clean URL arguments leaving only those passed in the parameterlist keeping order
346 >>> import w3lib.url
347 >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ('id',))
348 'product.html?id=200'
349 >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name'])
350 'product.html?id=200&name=wired'
351 >>>
353 If `unique` is ``False``, do not remove duplicated keys
355 >>> w3lib.url.url_query_cleaner("product.html?d=1&e=b&d=2&d=3&other=other", ['d'], unique=False)
356 'product.html?d=1&d=2&d=3'
357 >>>
359 If `remove` is ``True``, leave only those **not in parameterlist**.
361 >>> w3lib.url.url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id'], remove=True)
362 'product.html?foo=bar&name=wired'
363 >>> w3lib.url.url_query_cleaner("product.html?id=2&foo=bar&name=wired", ['id', 'foo'], remove=True)
364 'product.html?name=wired'
365 >>>
367 By default, URL fragments are removed. If you need to preserve fragments,
368 pass the ``keep_fragments`` argument as ``True``.
370 >>> w3lib.url.url_query_cleaner('http://domain.tld/?bla=123#123123', ['bla'], remove=True, keep_fragments=True)
371 'http://domain.tld/#123123'
373 """
375 if parameterlist and isinstance(parameterlist, (str, bytes)):
376 parameterlist = (parameterlist,)
378 if isinstance(url, bytes):
379 url = url.decode()
381 url, _, fragment = url.partition("#")
382 base, _, query = url.partition("?")
384 if not query or (not parameterlist and not remove):
385 return base if not keep_fragments else f"{base}#{fragment}"
387 param_lookup = frozenset(parameterlist)
389 seen: set[str] | None = set() if unique else None
390 result: list[str] = []
392 for ksv in query.split(sep):
393 if not ksv:
394 continue
396 k, _, _ = ksv.partition(kvsep)
398 if seen is not None:
399 if k in seen:
400 continue
401 seen.add(k)
403 if remove:
404 if k in param_lookup:
405 continue
406 elif k not in param_lookup:
407 continue
409 result.append(ksv)
410 del param_lookup, seen
412 url = base if not result else f"{base}?{sep.join(result)}"
413 del result
415 if keep_fragments and fragment:
416 url = f"{url}#{fragment}"
418 return url
421def _add_or_replace_parameters(
422 url: str, params: dict[bytes, bytes], *, separator: str = "&"
423) -> str:
424 parsed = _urlsplit(url)
426 current_args = _parse_qsl(
427 parsed.query, keep_blank_values=True, separator=separator.encode()
428 )
430 new_args: list[tuple[bytes, bytes]] = []
431 seen_params: set[bytes] = set()
433 for name, value in current_args:
434 if name in seen_params:
435 continue
436 replacement = params.get(name)
437 if replacement is None:
438 new_args.append((name, value))
439 else:
440 new_args.append((name, replacement))
441 seen_params.add(name)
443 for name, value in params.items():
444 if name not in seen_params:
445 new_args.append((name, value))
446 del seen_params, current_args
448 return _urlunsplit(
449 parsed.scheme,
450 parsed.netloc,
451 parsed.path,
452 _urlencode(new_args, separator.encode()).decode(),
453 parsed.fragment,
454 )
457def add_or_replace_parameter(
458 url: str, name: str, new_value: str, *, separator: str = "&"
459) -> str:
460 """Add or remove a parameter to a given url
462 >>> import w3lib.url
463 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php', 'arg', 'v')
464 'http://www.example.com/index.php?arg=v'
465 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg4', 'v4')
466 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3&arg4=v4'
467 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', 'arg3', 'v3new')
468 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3new'
469 >>> w3lib.url.add_or_replace_parameter('http://www.example.com/index.php?arg1=v1;arg2=v2', 'arg3', 'v3', separator=';')
470 'http://www.example.com/index.php?arg1=v1;arg2=v2;arg3=v3'
471 >>>
473 ``separator`` is used both to split the existing query and to join the
474 resulting one.
476 """
477 return _add_or_replace_parameters(
478 url, {name.encode(): new_value.encode()}, separator=separator
479 )
482def add_or_replace_parameters(
483 url: str, new_parameters: dict[str, str], *, separator: str = "&"
484) -> str:
485 """Add or remove a parameters to a given url
487 >>> import w3lib.url
488 >>> w3lib.url.add_or_replace_parameters('http://www.example.com/index.php', {'arg': 'v'})
489 'http://www.example.com/index.php?arg=v'
490 >>> args = {'arg4': 'v4', 'arg3': 'v3new'}
491 >>> w3lib.url.add_or_replace_parameters('http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3', args)
492 'http://www.example.com/index.php?arg1=v1&arg2=v2&arg3=v3new&arg4=v4'
493 >>>
495 """
496 return _add_or_replace_parameters(
497 url,
498 {k.encode(): v.encode() for k, v in new_parameters.items()},
499 separator=separator,
500 )
503def path_to_file_uri(path: str | os.PathLike[str]) -> str:
504 """Convert local filesystem path to legal File URIs as described in:
505 http://en.wikipedia.org/wiki/File_URI_scheme
506 """
507 absolute_path = Path(path).absolute()
508 if os.name == "nt" and absolute_path.drive.startswith("\\\\"):
509 return absolute_path.as_uri()
510 return f"file:///{pathname2url(str(absolute_path)).lstrip('/')}"
513def file_uri_to_path(uri: str) -> str:
514 """Convert File URI to local filesystem path according to:
515 http://en.wikipedia.org/wiki/File_URI_scheme
516 """
517 parsed = _urlparse(uri)
518 path = parsed.path
519 if os.name == "nt" and parsed.netloc and parsed.netloc.lower() != "localhost":
520 path = f"//{parsed.netloc}{path}"
521 return _url2pathname(path)
524def any_to_uri(uri_or_path: str) -> str:
525 """If given a path name, return its File URI, otherwise return it
526 unmodified
527 """
528 if os.path.splitdrive(uri_or_path)[0]:
529 return path_to_file_uri(uri_or_path)
530 return uri_or_path if _urlparse(uri_or_path)[0] else path_to_file_uri(uri_or_path)
533def add_http_if_no_scheme(url: str) -> str:
534 """Add ``http`` as the default scheme if it is missing from *url*."""
535 if not re.match(r"^\w+://", url, flags=re.IGNORECASE):
536 scheme = "http:" if _urlparse(url).netloc else "http://"
537 url = scheme + url
538 return url
541# ASCII characters.
542_char = set(map(chr, range(127)))
544# RFC 2045 token.
545_token = r"[{}]+".format(
546 re.escape(
547 "".join(
548 _char
549 -
550 # Control characters.
551 set(map(chr, range(32)))
552 -
553 # tspecials and space.
554 set('()<>@,;:\\"/[]?= ')
555 )
556 )
557)
559# RFC 822 quoted-string, without surrounding quotation marks.
560_quoted_string = r"(?:[{}]|(?:\\[{}]))*".format(
561 re.escape("".join(_char - {'"', "\\", "\r"})), re.escape("".join(_char))
562)
564# Encode the regular expression strings to make them into bytes, as Python 3
565# bytes have no format() method, but bytes must be passed to re.compile() in
566# order to make a pattern object that can be used to match on bytes.
568# RFC 2397 mediatype.
569_mediatype_pattern = re.compile(rf"{_token}/{_token}".encode())
570_mediatype_parameter_pattern = re.compile(
571 rf';({_token})=(?:({_token})|"({_quoted_string})")'.encode()
572)
573del _char, _token, _quoted_string
576class ParseDataURIResult(NamedTuple):
577 """Named tuple returned by :func:`parse_data_uri`."""
579 #: MIME type type and subtype, separated by / (e.g. ``"text/plain"``).
580 media_type: str
581 #: MIME type parameters (e.g. ``{"charset": "US-ASCII"}``).
582 media_type_parameters: dict[str, str]
583 #: Data, decoded if it was encoded in base64 format.
584 data: bytes
587def parse_data_uri(uri: str | bytes) -> ParseDataURIResult:
588 """Parse a data: URI into :class:`ParseDataURIResult`."""
589 if not isinstance(uri, bytes):
590 uri = safe_url_string(uri).encode("ascii")
592 scheme, _, uri = uri.partition(b":")
593 if not scheme or not uri:
594 raise ValueError("invalid URI")
595 if scheme.lower() != b"data":
596 raise ValueError("not a data URI")
598 # RFC 3986 section 2.1 allows percent encoding to escape characters that
599 # would be interpreted as delimiters, implying that actual delimiters
600 # should not be percent-encoded.
601 # Decoding before parsing will allow malformed URIs with percent-encoded
602 # delimiters, but it makes parsing easier and should not affect
603 # well-formed URIs, as the delimiters used in this URI scheme are not
604 # allowed, percent-encoded or not, in tokens.
605 uri = _unquote(uri)
607 media_type = "text/plain"
608 media_type_params = {}
610 m = _mediatype_pattern.match(uri)
611 if m:
612 media_type = m.group().decode()
613 uri = uri[m.end() :]
614 else:
615 media_type_params["charset"] = "US-ASCII"
617 while m := _mediatype_parameter_pattern.match(uri):
618 attribute, value, value_quoted = m.groups()
619 if value_quoted is not None:
620 value = re.sub(rb"\\(.)", rb"\1", value_quoted)
621 media_type_params[attribute.decode()] = value.decode()
622 uri = uri[m.end() :]
624 is_base64, _, data = uri.partition(b",")
625 if is_base64:
626 if is_base64 != b";base64":
627 raise ValueError("invalid data URI")
628 data = base64.b64decode(data)
630 return ParseDataURIResult(media_type, media_type_params, data)
633__all__ = [
634 "add_http_if_no_scheme",
635 "add_or_replace_parameter",
636 "add_or_replace_parameters",
637 "any_to_uri",
638 "canonicalize_url",
639 "file_uri_to_path",
640 "is_url",
641 "parse_data_uri",
642 "path_to_file_uri",
643 "safe_download_url",
644 "safe_url_string",
645 "url_query_cleaner",
646 "url_query_parameter",
647]
650def canonicalize_url(
651 url: str | bytes | ParseResult,
652 keep_blank_values: bool = True,
653 keep_fragments: bool = False,
654 encoding: str | None = None,
655 *,
656 query_separator: str = "&",
657) -> str:
658 r"""Canonicalize the given url by applying the following procedures:
660 .. versionchanged:: VERSION
661 Dot segments (``.`` and ``..``) in the path are now resolved, and
662 IPv6 addresses in the host are now normalized.
664 - make the URL safe
665 - sort query arguments, first by key, then by value
666 - normalize all spaces (in query arguments) '+' (plus symbol)
667 - normalize percent encodings case (%2f -> %2F)
668 - remove query arguments with blank values (unless `keep_blank_values` is True)
669 - remove fragments (unless `keep_fragments` is True)
670 - resolve dot segments (``.`` and ``..``) in the path
671 - normalize IPv6 addresses in the host
673 The url passed can be bytes or unicode, while the url returned is
674 always a native str (bytes in Python 2, unicode in Python 3).
676 >>> import w3lib.url
677 >>>
678 >>> # sorting query arguments
679 >>> w3lib.url.canonicalize_url('http://www.example.com/do?c=3&b=5&b=2&a=50')
680 'http://www.example.com/do?a=50&b=2&b=5&c=3'
681 >>>
682 >>> # UTF-8 conversion + percent-encoding of non-ASCII characters
683 >>> w3lib.url.canonicalize_url('http://www.example.com/r\u00e9sum\u00e9')
684 'http://www.example.com/r%C3%A9sum%C3%A9'
685 >>>
686 >>> # a query separator other than the default '&'
687 >>> w3lib.url.canonicalize_url('http://www.example.com/do?c=3;a=50', query_separator=';')
688 'http://www.example.com/do?a=50;c=3'
689 >>>
690 >>> # resolving dot segments and normalizing an IPv6 address
691 >>> w3lib.url.canonicalize_url('http://[::0:1]/a/../b')
692 'http://[::1]/b'
693 >>>
695 For more examples, see the tests in `tests/test_url.py`.
696 """
697 # If supplied `encoding` is not compatible with all characters in `url`,
698 # fallback to UTF-8 as safety net.
699 # UTF-8 can handle all Unicode characters,
700 # so we should be covered regarding URL normalization,
701 # if not for proper URL expected by remote website.
702 if isinstance(url, ParseResult):
703 url = _urlunparse(*url)
704 try:
705 scheme, netloc, path, query, fragment = _safe_url_split(
706 url, encoding=encoding or "utf8"
707 )
708 except UnicodeEncodeError:
709 scheme, netloc, path, query, fragment = _safe_url_split(url, encoding="utf8")
710 path, params = _split_params(scheme, path)
712 # 1. decode query-string as UTF-8 (or keep raw bytes),
713 # sort values,
714 # and percent-encode them back
716 # Python's urllib.parse.parse_qsl does not work as wanted
717 # for percent-encoded characters that do not match passed encoding,
718 # they get lost.
719 #
720 # e.g., 'q=b%a3' becomes [('q', 'b\ufffd')]
721 # (ie. with 'REPLACEMENT CHARACTER' (U+FFFD),
722 # instead of \xa3 that you get with Python2's parse_qsl)
723 #
724 # what we want here is to keep raw bytes, and percent encode them
725 # so as to preserve whatever encoding what originally used.
726 #
727 # See https://tools.ietf.org/html/rfc3987#section-6.4:
728 #
729 # For example, it is possible to have a URI reference of
730 # "http://www.example.org/r%E9sum%E9.xml#r%C3%A9sum%C3%A9", where the
731 # document name is encoded in iso-8859-1 based on server settings, but
732 # where the fragment identifier is encoded in UTF-8 according to
733 # [XPointer]. The IRI corresponding to the above URI would be (in XML
734 # notation)
735 # "http://www.example.org/r%E9sum%E9.xml#résumé".
736 # Similar considerations apply to query parts. The functionality of
737 # IRIs (namely, to be able to include non-ASCII characters) can only be
738 # used if the query part is encoded in UTF-8.
739 if query:
740 keyvals = _parse_qsl(
741 query, keep_blank_values, separator=query_separator.encode()
742 )
744 if len(keyvals) > 1:
745 keyvals.sort()
747 query = _urlencode(keyvals, query_separator.encode()).decode()
748 del keyvals
750 # 2. decode percent-encoded sequences in path as UTF-8 (or keep raw bytes)
751 # and percent-encode path again (this normalizes to upper-case %XX)
752 path = _quote(_unquotepath(path), _PATH_SAFE_CHARS).decode() if path else "/"
754 # 3. resolve dot segments (RFC 3986, section 5.2.4)
755 resolved_path = _parent_dirs.sub("", posixpath.normpath(path))
756 if not resolved_path.endswith("/") and path.endswith(("/", "/.", "/..")):
757 resolved_path += "/"
758 path = resolved_path
760 fragment = "" if not keep_fragments else fragment
762 # Apply lowercase to the domain, but not to the userinfo.
763 uinf_sep_idx = netloc.rfind("@")
764 host = (
765 (netloc[uinf_sep_idx + 1 :] if uinf_sep_idx != -1 else netloc)
766 .lower()
767 .removesuffix(":")
768 )
769 netloc = (netloc[: uinf_sep_idx + 1] + host) if uinf_sep_idx != -1 else host
771 netloc = _normalize_ipv6_host(netloc)
773 # every part should be safe already
774 return _urlunparse(scheme, netloc, path, params, query, fragment)
777def _normalize_ipv6_host(netloc: str) -> str:
778 """Normalize an IPv6 address in the host part of *netloc*, if any (RFC 5952)."""
779 if "[" not in netloc:
780 return netloc
781 bracket_start = netloc.index("[")
782 bracket_end = netloc.find("]", bracket_start)
783 if bracket_end == -1:
784 return netloc
785 try:
786 address = ip_address(netloc[bracket_start + 1 : bracket_end])
787 except ValueError:
788 return netloc
789 if not isinstance(address, IPv6Address):
790 return netloc
791 return f"{netloc[:bracket_start]}[{address}]{netloc[bracket_end + 1 :]}"
794def _unquotepath(path: str) -> bytes:
795 if "%" not in path:
796 return path.encode()
797 # standard lib's unquote() does not work for non-UTF-8
798 # percent-escaped characters, they get lost.
799 # e.g., '%a3' becomes 'REPLACEMENT CHARACTER' (U+FFFD)
800 return _unquote(
801 path.replace("%25", "%2525")
802 .replace("%2f", "%252F")
803 .replace("%2F", "%252F")
804 .replace("%3b", "%253B")
805 .replace("%3B", "%253B")
806 .replace("%3f", "%253F")
807 .replace("%3F", "%253F")
808 )
811def parse_url(
812 url: str | bytes | ParseResult, encoding: str | None = None
813) -> ParseResult:
814 """Return urlparsed url from the given argument (which could be an already
815 parsed url)
816 """
817 if isinstance(url, ParseResult):
818 return url
819 return _urlparse(to_unicode(url, encoding))
822def parse_qsl_to_bytes(
823 qs: str, keep_blank_values: bool = False, *, separator: str = "&"
824) -> list[tuple[bytes, bytes]]:
825 """Parse a query given as a string argument.
827 Data are returned as a list of name, value pairs as bytes.
829 Arguments:
831 qs: percent-encoded query string to be parsed
833 keep_blank_values: flag indicating whether blank values in
834 percent-encoded queries should be treated as blank strings. A
835 true value indicates that blanks should be retained as blank
836 strings. The default false value indicates that blank values
837 are to be ignored and treated as if they were not included.
839 separator: string used to separate query parameters, e.g. ``";"`` for
840 queries that use the legacy semicolon separator.
842 """
843 return _parse_qsl(qs, keep_blank_values, separator=separator.encode())