Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/yarl/_url.py: 39%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import re
2import warnings
3from collections.abc import Mapping, Sequence
4from enum import Enum, StrEnum
5from functools import _CacheInfo, lru_cache
6from importlib.util import find_spec
7from ipaddress import ip_address
8from typing import (
9 TYPE_CHECKING,
10 Any,
11 Literal,
12 NoReturn,
13 Self,
14 TypedDict,
15 TypeVar,
16 cast,
17 overload,
18)
19from urllib.parse import (
20 SplitResult,
21 quote,
22 scheme_chars,
23 unquote_to_bytes,
24 uses_relative,
25)
27import idna
28from multidict import MultiDict, MultiDictProxy, istr
29from propcache.api import under_cached_property as cached_property
31from ._parse import (
32 EMPTY_AUTHORITY,
33 EMPTY_FRAGMENT,
34 EMPTY_QUERY,
35 IP_FUTURE_RE,
36 SPECIAL_SCHEMES,
37 UNSAFE_URL_BYTES_TO_REMOVE,
38 WHATWG_C0_CONTROL_OR_SPACE,
39 ZONE_ID_RE,
40 SplitURLType,
41 _check_netloc,
42 check_zone_id,
43 has_dot_prefix,
44 make_netloc,
45 query_to_pairs,
46 split_netloc,
47 split_netloc_rfc,
48 split_url,
49 unsplit_result,
50 unsplit_result_empty,
51)
52from ._path import (
53 drive_letter,
54 is_drive_letter,
55 normalize_drive_letter,
56 normalize_file_path,
57 normalize_path,
58 normalize_path_segments,
59 remove_dot_segments,
60 starts_with_drive_letter,
61)
62from ._query import (
63 Query,
64 QueryVariable,
65 SimpleQuery,
66 get_str_query,
67 get_str_query_from_iterable,
68 get_str_query_from_sequence_iterable,
69)
70from ._quoters import (
71 FRAGMENT_QUOTER,
72 FRAGMENT_REQUOTER,
73 PATH_QUOTER,
74 PATH_REQUOTER,
75 PATH_SAFE_UNQUOTER,
76 QS_UNQUOTER,
77 QUERY_QUOTER,
78 QUERY_REQUOTER,
79 QUOTER,
80 REQUOTER,
81 UNQUOTER,
82 WHATWG_FRAGMENT_REQUOTER,
83 WHATWG_PATH_REQUOTER,
84 WHATWG_QUERY_REQUOTER,
85 ZONE_ID_UNQUOTER,
86 human_quote,
87)
89# Avoid Pydantic import if not used (increases yarl's import time by 3-7x).
90HAS_PYDANTIC = find_spec("pydantic_core") is not None
91if TYPE_CHECKING:
92 from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler
93 from pydantic.json_schema import JsonSchemaValue
94 from pydantic_core import CoreSchema
97DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21}
98USES_RELATIVE = frozenset(uses_relative)
99_SCHEME_CHARS = frozenset(scheme_chars)
101# Special schemes https://url.spec.whatwg.org/#special-scheme
102# are not allowed to have an empty host https://url.spec.whatwg.org/#url-representation
103SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp"))
106# reg-name: unreserved / pct-encoded / sub-delims
107# this pattern matches anything that is *not* in those classes. and is only used
108# on lower-cased ASCII values.
109NOT_REG_NAME = re.compile(
110 r"""
111 # any character not in the unreserved or sub-delims sets, plus %
112 # (validated with the additional check for pct-encoded sequences below)
113 [^a-z0-9\-._~!$&'()*+,;=%]
114 |
115 # % only allowed if it is part of a pct-encoded
116 # sequence of 2 hex digits.
117 %(?![0-9a-f]{2})
118 """,
119 re.VERBOSE,
120)
121# The characters that a WHATWG domain can have and an RFC 3986 reg-name
122# cannot: WHATWG only rejects the forbidden domain code points.
123_WHATWG_ONLY_HOST_RE = re.compile(r'["`{}]')
124# NOT_REG_NAME with those characters allowed.
125_NOT_WHATWG_DOMAIN_RE = re.compile(r"[^a-z0-9\-._~!$&'()*+,;=%\"`{}]|%(?![0-9a-f]{2})")
127# Invisible default-ignorable / format code points that must not appear in a
128# host (soft hyphen, zero-width space, word joiner, bidi controls, variation
129# selectors, ...). Depending on the code point IDNA either silently deletes it
130# (so ``e<ZWSP>vil.com`` encodes to ``evil.com``) or folds it into a different
131# punycode host; either way the parsed host differs from the string an
132# application validated. The set is the union of two authoritative sources,
133# matching the two encoders _idna_encode dispatches to:
134#
135# 1. Unicode Default_Ignorable_Code_Point (uts46=True path via the ``idna``
136# package). Ranges taken from the DerivedCoreProperties data file:
137# https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
138# (the E0000..E0FFF block is contiguous under this property).
139# 2. RFC 3454 (Stringprep) Table B.1 "commonly mapped to nothing", used by the
140# stdlib ``str.encode("idna")`` / IDNA2003 nameprep fallback:
141# https://www.rfc-editor.org/rfc/rfc3454#appendix-B.1
142# This is the source of U+1806, which is not Default_Ignorable.
143#
144# Coverage is pinned to the installed ``idna``/Unicode data by a sweep test
145# (test_default_ignorable_covers_idna_stripped in tests/test_url.py) that
146# brute-forces every code point through _idna_encode and fails if any point it
147# silently deletes is not matched here.
148_DEFAULT_IGNORABLE_RE = re.compile(
149 "["
150 "\u00ad" # SOFT HYPHEN
151 "\u034f" # COMBINING GRAPHEME JOINER
152 "\u061c" # ARABIC LETTER MARK
153 "\u115f-\u1160" # HANGUL CHOSEONG/JUNGSEONG FILLER
154 "\u17b4-\u17b5" # KHMER VOWEL INHERENT AQ/AA
155 "\u1806" # MONGOLIAN TODO SOFT HYPHEN (nameprep maps to nothing)
156 "\u180b-\u180f" # MONGOLIAN FVS ONE..FOUR and VOWEL SEPARATOR
157 "\u200b-\u200f" # ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK
158 "\u202a-\u202e" # bidi embedding/override controls
159 "\u2060-\u206f" # WORD JOINER..NOMINAL DIGIT SHAPES
160 "\u3164" # HANGUL FILLER
161 "\ufe00-\ufe0f" # VARIATION SELECTOR-1..16
162 "\ufeff" # ZERO WIDTH NO-BREAK SPACE (BOM)
163 "\uffa0" # HALFWIDTH HANGUL FILLER
164 "\ufff0-\ufff8" # reserved default-ignorables
165 "\U0001bca0-\U0001bca3" # SHORTHAND FORMAT controls
166 "\U0001d173-\U0001d17a" # MUSICAL SYMBOL begin/end controls
167 "\U000e0000-\U000e0fff" # tags and VARIATION SELECTOR SUPPLEMENT
168 "]"
169)
171# Zone IDs are OS-specific text strings with no format defined by the RFCs:
172# https://datatracker.ietf.org/doc/html/rfc4007#section-11.2
173# RFC 9844 §6.3 recommends rejecting characters inappropriate for the
174# environment; for yarl we reject ASCII control characters (CTL):
175# https://datatracker.ietf.org/doc/html/rfc9844#section-6-3
176_ZONE_ID_UNSAFE_RE = re.compile(r"[\x00-\x1f\x7f]")
177# A character of a zone identifier that RFC 6874 percent-encodes.
178_ZONE_ID_REQUOTE_RE = re.compile(r"%(?![0-9A-Fa-f]{2})|[^0-9A-Za-z._~%-]")
181def _quote_match(match: re.Match[str]) -> str:
182 return quote(match[0], safe="")
185_T = TypeVar("_T")
188class UndefinedType(Enum):
189 """Singleton type for use with not set sentinel values."""
191 _singleton = 0
194UNDEFINED = UndefinedType._singleton
197class Mode(StrEnum):
198 """URL standard a :class:`URL` follows."""
200 RFC = "rfc"
201 WHATWG = "whatwg"
204Mode.__module__ = "yarl"
205_WHATWG = Mode.WHATWG
206# Members hash and compare as their values, so this maps both the members and
207# the plain strings; much faster than calling ``Mode(value)``.
208_MODE_BY_VALUE: dict[str, Mode] = {c.value: c for c in Mode}
210ModeType = Mode | Literal["rfc", "whatwg"]
213def _to_mode(value: ModeType) -> Mode:
214 """Normalize a compatibility argument, rejecting unknown values."""
215 try:
216 mode = _MODE_BY_VALUE.get(value)
217 except TypeError: # unhashable, e.g. a list
218 mode = None
219 if mode is None:
220 raise ValueError(f"{value!r} is not a valid Mode")
221 return mode
224class CacheInfo(TypedDict):
225 """Host encoding cache."""
227 idna_encode: _CacheInfo
228 idna_decode: _CacheInfo
229 ip_address: _CacheInfo
230 host_validate: _CacheInfo
231 encode_host: _CacheInfo
234class _InternalURLCache(TypedDict, total=False):
235 _val: SplitURLType
236 _join_path: str
237 _origin: "URL"
238 absolute: bool
239 hash: int
240 scheme: str
241 raw_authority: str
242 authority: str
243 raw_user: str | None
244 user: str | None
245 raw_password: str | None
246 password: str | None
247 raw_host: str | None
248 host: str | None
249 host_subcomponent: str | None
250 host_port_subcomponent: str | None
251 port: int | None
252 explicit_port: int | None
253 raw_path: str
254 path: str
255 _parsed_query: list[tuple[str, str]]
256 query: "MultiDictProxy[str]"
257 raw_query_string: str
258 query_string: str
259 path_qs: str
260 raw_path_qs: str
261 raw_fragment: str
262 fragment: str
263 raw_parts: tuple[str, ...]
264 parts: tuple[str, ...]
265 parent: "URL"
266 raw_name: str
267 name: str
268 raw_suffix: str
269 suffix: str
270 raw_suffixes: tuple[str, ...]
271 suffixes: tuple[str, ...]
274def rewrite_module(obj: _T) -> _T:
275 obj.__module__ = "yarl"
276 return obj
279def _encode_relative_scheme_colon(path: str) -> str:
280 """Re-encode a scheme-shaped leading ``:`` in a relative path to ``%3A``."""
281 colon_pos = path.find(":")
282 if colon_pos <= 0:
283 return path
284 for c in path[:colon_pos]:
285 if c not in _SCHEME_CHARS:
286 return path
287 return path[:colon_pos] + "%3A" + path[colon_pos + 1 :]
290def _encode_relative_colon(path: str, mode: Mode) -> str:
291 """Re-encode a ``:`` in a relative path that would not parse back.
293 WHATWG mode only encodes one that would read as a scheme. RFC 3986
294 section 4.2 allows no ``:`` in the first segment at all, and RFC 3986
295 mode rejects one when it parses a string, so it encodes all of them.
296 """
297 if mode is _WHATWG:
298 return _encode_relative_scheme_colon(path)
299 first, slash, rest = path.partition("/")
300 return first.replace(":", "%3A") + slash + rest
303def _check_missing_host(scheme: str, mode: Mode) -> None:
304 """Reject an authority that has userinfo or a port but no host.
306 Schemes that need a host always reject it. RFC 3986 allows an empty
307 host otherwise, WHATWG rejects it for every scheme.
308 """
309 if scheme in SCHEME_REQUIRES_HOST:
310 raise ValueError(
311 f"Invalid URL: host is required for absolute urls with the {scheme} scheme"
312 )
313 if mode is Mode.WHATWG:
314 raise ValueError("Invalid URL: host is required with userinfo or a port")
317def _check_whatwg_file_authority(authority: str) -> None:
318 """Reject the userinfo and the port of a file URL in WHATWG mode.
320 The WHATWG file host state reads the authority as a host only, which
321 cannot have "@" or ":", so "file://user@host/" and "file://host:1/"
322 fail. RFC 3986 mode keeps both. A Windows drive letter is not a host
323 with an empty port: callers move it into the path first, see
324 _whatwg_file_drive().
325 """
326 if "@" in authority or ":" in authority.rpartition("]")[2]:
327 raise ValueError(
328 f"Invalid URL: a file URL cannot have userinfo or a port, got {authority!r}"
329 )
332# The characters the userinfo percent-encode set of the WHATWG URL Standard
333# leaves alone: the unreserved characters, the sub-delims other than ";" and
334# "=", and "%", which is kept as written, also when it starts no
335# percent-encoded octet.
336_WHATWG_USERINFO_UNSAFE_RE = re.compile(r"[^A-Za-z0-9\-._~!$&'()*+,%]+")
339def _quote_userinfo_match(match: "re.Match[str]") -> str:
340 return quote(match.group(), safe="", errors="ignore")
343def _whatwg_requote_userinfo(value: str) -> str:
344 """Encode a parsed username or password as WHATWG mode does.
346 The WHATWG URL Standard splits the userinfo at the last "@" and
347 percent-encodes the username and the password with the userinfo
348 percent-encode set, so "http://a b;@c@h" has the username "a%20b%3B%40c".
349 A "%" is kept as written, "http://a%b@h" stays as it is. RFC 3986 mode
350 accepts only a valid userinfo, which it requotes. See quote_userinfo()
351 in _parse.py for a value that is not encoded.
352 """
353 if value.isascii() and value.isalnum():
354 return value
355 return _WHATWG_USERINFO_UNSAFE_RE.sub(_quote_userinfo_match, value)
358def _empty_path(scheme: str, mode: Mode) -> str:
359 """Return the path of a URL whose path became empty.
361 A file URL has a path in WHATWG mode, "/" at least, so "file:///a" has
362 the parent "file:///". Every other URL keeps the empty path.
363 """
364 return "/" if scheme == "file" and mode is _WHATWG else ""
367def _whatwg_file_drive(netloc: str, path: str) -> str:
368 """Return the path of a file URL whose authority is a drive letter.
370 The WHATWG file host state reads a Windows drive letter in the authority
371 as the start of the path, so "file://C:/x" and "file://C|/x" are
372 "file:///C:/x", with an empty host.
373 """
374 return f"/{netloc[0]}:{path}"
377def _normalize_path(path: str, scheme: str, mode: Mode) -> str:
378 """Drop '.' and '..' from path, keeping the drive letter of a file URL.
380 WHATWG mode keeps it, see normalize_file_path().
381 """
382 if scheme == "file" and mode is _WHATWG:
383 return normalize_file_path(path)
384 return normalize_path(path)
387def _file_join_path(base: str, path: str) -> str:
388 """Resolve the path of a reference against the path of a file URL base.
390 The WHATWG file and file slash states, in WHATWG mode: a path that
391 starts with a drive letter replaces the path of the base, an absolute
392 path keeps the drive letter of the base, and the drive letter alone is
393 never dropped from the base, so "x" against "file:///C:" is
394 "file:///C:/x".
395 """
396 if path[0] == "/":
397 if not starts_with_drive_letter(path, 1) and (drive := drive_letter(base)):
398 path = f"/{drive}{path}"
399 elif starts_with_drive_letter(path):
400 path = f"/{path}"
401 else:
402 if base[:1] != "/":
403 # A rootless base, or an empty one, which only encoded=True
404 # leaves in a file URL.
405 base = f"/{base}"
406 if len(base) == 3 and drive_letter(base):
407 path = f"{base}/{path}"
408 else:
409 path = base[: base.rfind("/") + 1] + path
410 return normalize_file_path(path) if "." in path else path
413def _is_pipe_drive_authority(join_path: str) -> bool:
414 """Tell if a reference saved for join() has an authority like "C|".
416 Only a file URL reads it as a drive letter, it is no host for others.
417 """
418 return (
419 join_path[:2] == "//"
420 and join_path[3:4] == "|"
421 and starts_with_drive_letter(join_path, 2)
422 )
425def _rfc_host_error(host: str) -> str:
426 """Explain why RFC 3986 mode rejected a host that WHATWG mode accepts.
428 A domain can have '"', "`", "{" and "}", which an RFC 3986 reg-name
429 cannot, see _encode_host(). RFC 3987 converts an internationalized host
430 with IDNA, now IDNA2008 (RFC 5891), which disallows code points such as
431 emoji. WHATWG mode keeps the IDNA2003 fallback of _idna_encode(), as
432 UTS #46 accepts them.
433 """
434 if invalid := _WHATWG_ONLY_HOST_RE.search(host):
435 return (
436 f"Host {host!r} cannot contain {invalid.group()!r}"
437 f" (at position {invalid.start()})"
438 )
439 return f"Host {host!r} is not a valid IDNA2008 name"
442# The RFC 3986 mode check below is guarded with ``mode is not _WHATWG`` at
443# the call sites, so that the default mode does not pay for a call on the
444# URL.build() hot path (CodSpeed).
445def _check_rfc_authority(authority: str) -> None:
446 """Reject "@" in the userinfo in RFC 3986 mode.
448 RFC 3986 userinfo cannot contain "@", so "sc://a@b@c/" has no valid
449 parse; WHATWG splits at the last "@" and percent-encodes the others.
450 """
451 if authority.count("@") > 1:
452 raise ValueError(f"Invalid URL: userinfo cannot contain '@' in {authority!r}")
455def _char_ranges(*pairs: tuple[int, int]) -> str:
456 return "".join(f"{chr(first)}-{chr(last)}" for first, last in pairs)
459# The characters of an IRI outside ASCII, RFC 3987 section 2.2, less the
460# bidi formatting characters that section 4.1 forbids (U+200E, U+200F and
461# U+202A to U+202E).
462_UCSCHAR = _char_ranges(
463 (0xA0, 0x200D),
464 (0x2010, 0x2029),
465 (0x202F, 0xD7FF),
466 (0xF900, 0xFDCF),
467 (0xFDF0, 0xFFEF),
468 *((plane << 16, (plane << 16) + 0xFFFD) for plane in range(1, 14)),
469 (0xE1000, 0xEFFFD),
470)
471_IPRIVATE = _char_ranges((0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD))
472_UNRESERVED = r"A-Za-z0-9\-._~"
473_IUNRESERVED = _UNRESERVED + _UCSCHAR
474_SUB_DELIMS = "!$&'()*+,;="
475_RFC_SPLIT_RE = re.compile(
476 r"(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?", re.S
477)
478_RFC_SCHEME_RE = re.compile(r"[A-Za-z][A-Za-z0-9+\-.]*")
481def _rfc_re(chars: str) -> re.Pattern[str]:
482 return re.compile(f"(?:[{chars}]|%[0-9A-Fa-f]{{2}})*")
485# Several "@" are left to _check_rfc_authority(), which says so.
486_RFC_USERINFO_RE = _rfc_re(f"{_IUNRESERVED}{_SUB_DELIMS}:@")
487_RFC_REG_NAME_RE = _rfc_re(f"{_IUNRESERVED}{_SUB_DELIMS}")
488_RFC_PATH_RE = _rfc_re(f"{_IUNRESERVED}{_SUB_DELIMS}:@/")
489_RFC_QUERY_RE = _rfc_re(f"{_IUNRESERVED}{_SUB_DELIMS}:@/?{_IPRIVATE}")
490_RFC_FRAGMENT_RE = _rfc_re(f"{_IUNRESERVED}{_SUB_DELIMS}:@/?")
491_RFC_PORT_RE = re.compile("[0-9]*")
492# The characters of IPv6address and IPvFuture, and of a zone identifier
493# (RFC 6874); the parser checks the rest.
494_RFC_IP_LITERAL_RE = _rfc_re(f"{_UNRESERVED}{_SUB_DELIMS}:")
497def _check_rfc_component(
498 url_str: str, name: str, value: str | None, pattern: re.Pattern[str]
499) -> None:
500 if value is None:
501 return
502 end = cast(re.Match[str], pattern.match(value)).end()
503 if end == len(value):
504 return
505 char = value[end]
506 if char == "%" and pattern is not _RFC_PORT_RE:
507 what = "a '%' not followed by two hexadecimal digits"
508 else:
509 what = repr(char)
510 raise ValueError(
511 f"Invalid URL: RFC 3986 and RFC 3987 do not allow {what} in the {name}"
512 f" of {url_str!r}"
513 )
516def _check_rfc_url(url_str: str) -> None:
517 """Reject a URL string outside the grammar of RFC 3986 and RFC 3987.
519 RFC 3986 mode takes IRIs (RFC 3987): URL() percent-encodes their
520 non-ASCII characters and encodes a non-ASCII host with IDNA2008. Other
521 characters, such as a space, "<" or a "%" that does not start a
522 percent-encoded octet, make the string invalid, where WHATWG mode
523 percent-encodes or removes them. The characters of an IP-literal host are
524 checked here, its syntax when it is parsed.
525 """
526 if "\\" in url_str:
527 raise ValueError(f"Invalid URL: RFC 3986 does not allow '\\' in {url_str!r}")
528 # RFC 3986 Appendix B, with no preprocessing.
529 match = cast(re.Match[str], _RFC_SPLIT_RE.fullmatch(url_str))
530 scheme, authority, path, query, fragment = match.groups()
531 if scheme is not None and _RFC_SCHEME_RE.fullmatch(scheme) is None:
532 raise ValueError(
533 f"Invalid URL: {scheme!r} is not a valid scheme in {url_str!r}"
534 )
535 if authority is not None:
536 _check_rfc_authority(authority)
537 userinfo, _, hostport = authority.rpartition("@")
538 _check_rfc_component(url_str, "userinfo", userinfo, _RFC_USERINFO_RE)
539 if hostport.startswith("["):
540 literal, _, after = hostport[1:].partition("]")
541 _check_rfc_component(url_str, "host", literal, _RFC_IP_LITERAL_RE)
542 port = after.partition(":")[2]
543 else:
544 host, _, port = hostport.partition(":")
545 _check_rfc_component(url_str, "host", host, _RFC_REG_NAME_RE)
546 _check_rfc_component(url_str, "port", port, _RFC_PORT_RE)
547 elif scheme is None and ":" in path.partition("/")[0]:
548 # RFC 3986 section 4.2: "./" must come first to make it relative.
549 raise ValueError(
550 "Invalid URL: the first segment of a relative path cannot contain"
551 f" ':' in {url_str!r}"
552 )
553 _check_rfc_component(url_str, "path", path, _RFC_PATH_RE)
554 _check_rfc_component(url_str, "query", query, _RFC_QUERY_RE)
555 _check_rfc_component(url_str, "fragment", fragment, _RFC_FRAGMENT_RE)
558def _special_authority_url(
559 scheme: str, path: str, query: str, fragment: str, empty: int, encoded: bool
560) -> "URL":
561 """Parse "http:host/p" or "http:///host/p" in WHATWG mode.
563 The WHATWG URL Standard skips any slashes after the scheme of a special
564 URL and reads an authority, so "http:/example.com/" and
565 "http:///example.com/" are "http://example.com/". Against a base with the
566 same scheme, an input without "//" is the relative reference
567 "/example.com/" instead; its path is kept for join().
568 """
569 url_str = f"{scheme}://{path.lstrip('/')}"
570 if query or empty & EMPTY_QUERY:
571 url_str = f"{url_str}?{query}"
572 if fragment or empty & EMPTY_FRAGMENT:
573 url_str = f"{url_str}#{fragment}"
574 if encoded:
575 url = _pre_encoded_url(url_str, _WHATWG)
576 else:
577 url = _encode_url(url_str, _WHATWG)
578 path = WHATWG_PATH_REQUOTER(path)
579 if not empty & EMPTY_AUTHORITY:
580 url._cache["_join_path"] = path
581 return url
584def _is_special_authority_path(scheme: str, path: str, mode: Mode) -> bool:
585 """Tell if the path of a URL without a host is its authority in WHATWG mode.
587 Only a path with text after the leading slashes is, so "http:", "http:/"
588 and "http:///" stay as they are: against a base with the same scheme the
589 WHATWG parser reads the first two as references to the base, and alone
590 it rejects all of them.
591 """
592 return mode is _WHATWG and scheme in SCHEME_REQUIRES_HOST and path.lstrip("/") != ""
595def _backslashes(url_str: str) -> tuple[str, bool]:
596 """Handle "\\" in a URL string before it is parsed in WHATWG mode.
598 RFC 3986 has no "\\" in its grammar, so RFC 3986 mode rejects it (see
599 _check_rfc_url()). The WHATWG URL Standard reads it as "/" in the
600 authority and the path of a special URL, and so does WHATWG mode; other
601 schemes keep it, percent-encoded, as do the query and the fragment.
603 Return the string with "\\" read as "/" and whether it is a relative
604 reference, whose "\\" is read as "/" only when it is joined with a
605 special base: against another base, "\\/a" must not become the
606 network-path reference "//a".
607 """
608 # Find the scheme the way split_url() does.
609 url = url_str.strip(WHATWG_C0_CONTROL_OR_SPACE)
610 for b in UNSAFE_URL_BYTES_TO_REMOVE:
611 url = url.replace(b, "")
612 i = url.find(":")
613 relative = True
614 if i > 0 and all(c in _SCHEME_CHARS for c in url[:i]):
615 if url[:i].lower() not in SPECIAL_SCHEMES:
616 return url_str, False
617 relative = False
618 end = len(url)
619 for c in "?#":
620 if (pos := url.find(c)) >= 0 and pos < end:
621 end = pos
622 return url[:end].replace("\\", "/") + url[end:], relative
625def _encode_url(url_str: str, mode: Mode, backslashes: bool = True) -> "URL":
626 """Parse unencoded URL."""
627 cache: _InternalURLCache = {}
628 if backslashes and "\\" in url_str:
629 special_str, relative = _backslashes(url_str)
630 if not relative:
631 url_str = special_str
632 elif not url_str.lstrip(WHATWG_C0_CONTROL_OR_SPACE).startswith("//"):
633 # Keep "\\" as a character; join() reads it as "/" when the base
634 # is special.
635 url = _encode_url(url_str, mode, False)
636 url._cache["_join_path"] = special_str
637 return url
638 else:
639 # A network-path reference has an authority either way, and one
640 # with "\\" in it would be rejected.
641 url_str = special_str
642 host: str | None
643 scheme, netloc, path, query, fragment, empty = split_url(url_str)
644 if not netloc and _is_special_authority_path(scheme, path, mode):
645 return _special_authority_url(scheme, path, query, fragment, empty, False)
646 if scheme == "file":
647 if mode is _WHATWG:
648 if is_drive_letter(netloc):
649 path = _whatwg_file_drive(netloc, path)
650 netloc = ""
651 empty |= EMPTY_AUTHORITY
652 if "|" in path:
653 path = normalize_drive_letter(path)
654 elif (
655 not scheme
656 and mode is _WHATWG
657 and (
658 is_drive_letter(netloc)
659 or ("|" in path and normalize_drive_letter(path) is not path)
660 )
661 ):
662 # A drive letter means something else when the reference is joined
663 # with a file URL, see join(); it is kept for that.
664 cache["_join_path"] = (
665 f"//{netloc}{path}" if netloc or empty & EMPTY_AUTHORITY else path
666 )
667 if netloc[1:] == "|":
668 # Not a host, so the file URL is the only meaning it has.
669 path = _whatwg_file_drive(netloc, path)
670 netloc = ""
671 empty |= EMPTY_AUTHORITY
672 if not netloc: # netloc
673 if empty & EMPTY_AUTHORITY and scheme in SCHEME_REQUIRES_HOST:
674 # The WHATWG host parser fails on the empty host, and RFC 9110
675 # section 4.2.1 rejects it: "http://", "http:///" and "http://?q"
676 # are invalid. "http:" and "http:?q" have no authority, they are
677 # references to a base.
678 _check_missing_host(scheme, mode)
679 elif (
680 empty & EMPTY_AUTHORITY
681 and not scheme
682 and mode is _WHATWG
683 and ("%" in path or "[" in path)
684 and "_join_path" not in cache
685 ):
686 # Against an http base join() reads the path of "///h/p" as an
687 # authority, where "%38" and "[" must stay as written; the path
688 # quoter would decode the first and encode the second. "//" keeps
689 # the empty authority for a file base.
690 cache["_join_path"] = f"//{path}"
691 host = ""
692 else:
693 if ":" in netloc or "@" in netloc or "[" in netloc:
694 # Complex netloc
695 if mode is _WHATWG:
696 username, password, host, port = split_netloc(netloc)
697 else:
698 username, password, host, port = split_netloc_rfc(netloc)
699 else:
700 username = password = port = None
701 host = netloc
702 if host is None:
703 # The authority is not empty, so it has userinfo or a port.
704 _check_missing_host(scheme, mode)
705 host = ""
706 if port is None and username is None:
707 # Only an empty port, which RFC 3986 section 6.2.3 drops:
708 # "sc://:/" is "sc:///", the authority is kept.
709 empty |= EMPTY_AUTHORITY
710 # The parser historically encoded without validation, which let
711 # control characters (NUL/C0) and IDNA-normalized delimiters into
712 # the host, producing a ``str(url)`` that yarl cannot re-parse
713 # (#1829). Validate like the builder APIs do, but keep accepting an
714 # empty IPv6 zone identifier, which parsing has always allowed (#998).
715 if (
716 scheme not in SPECIAL_SCHEMES
717 and mode is _WHATWG
718 and _is_opaque_host(scheme, host)
719 ):
720 host = _encode_opaque_host(host)
721 else:
722 encoded, whatwg_host, not_rfc = _encode_host(
723 host, validate_host=True, reject_empty_zone=False
724 )
725 if not_rfc and mode is not _WHATWG:
726 raise ValueError(_rfc_host_error(host))
727 host = encoded
728 if (
729 whatwg_host is not host
730 and mode is _WHATWG
731 and (scheme in SPECIAL_SCHEMES or host[0] == "[")
732 ):
733 host = _whatwg_special_host(host, whatwg_host)
734 if mode is _WHATWG and scheme == "file":
735 _check_whatwg_file_authority(netloc)
736 if host == "localhost":
737 # The WHATWG file host state turns "localhost" into the empty
738 # host: "file://localhost/p" is "file:///p".
739 host = ""
740 empty |= EMPTY_AUTHORITY
741 # Remove brackets as host encoder adds back brackets for IPv6 addresses.
742 # An IPvFuture address keeps them, see split_netloc().
743 # An empty host (RFC mode, e.g. "sc://user@:8080") is None, as for
744 # built and pre-encoded URLs.
745 cache["raw_host"] = (
746 host[1:-1] if "[" in host and host[1] != "v" else host
747 ) or None
748 cache["explicit_port"] = port
749 if password is None and username is None:
750 # Fast path for URLs without user, password
751 netloc = host if port is None else f"{host}:{port}"
752 cache["raw_user"] = None
753 cache["raw_password"] = None
754 else:
755 if mode is _WHATWG:
756 # WHATWG writes ":" only before a password that is not
757 # empty: "http://a:@h" is "http://a@h", "http://:@h" is
758 # "http://h". RFC 3986 keeps the empty userinfo and an
759 # empty username is "", see split_netloc_rfc().
760 raw_user = _whatwg_requote_userinfo(username) if username else None
761 raw_password = _whatwg_requote_userinfo(password) if password else None
762 else:
763 raw_user = REQUOTER(username) if username else username
764 raw_password = REQUOTER(password) if password else password
765 netloc = make_netloc(raw_user, raw_password, host, port)
766 cache["raw_user"] = raw_user
767 cache["raw_password"] = raw_password
769 if path:
770 # WHATWG mode keeps a "%" that does not start a "%XX" escape, RFC
771 # mode encodes it as "%25"
772 path = (WHATWG_PATH_REQUOTER if mode is _WHATWG else PATH_REQUOTER)(path)
773 if "." in path and (netloc or (mode is _WHATWG and scheme == "file")):
774 # A file URL always has an authority in WHATWG mode, also an
775 # empty one, and its path is absolute: "file:///a/../b" and
776 # "file:a/../b" are "file:///b".
777 if not netloc and path[0] != "/":
778 # As a reference the path is relative to the path of a file
779 # base, so join() takes the path as it is written.
780 cache["_join_path"] = path
781 path = f"/{path}"
782 empty |= EMPTY_AUTHORITY
783 path = _normalize_path(path, scheme, mode)
784 elif not scheme and not netloc:
785 path = _encode_relative_scheme_colon(path)
786 elif mode is not _WHATWG and "." in path:
787 # RFC 3986 section 5.2.2 removes the dot segments of any URI
788 # with a scheme, also one without an authority ("file:.." is
789 # "file:").
790 path = remove_dot_segments(path)
791 if not netloc and has_dot_prefix(path) and not empty & EMPTY_AUTHORITY:
792 # Undo the "/." that str() puts in front of a path starting
793 # with "//" when there is no authority.
794 path = path[2:]
795 if query:
796 query = (WHATWG_QUERY_REQUOTER if mode is _WHATWG else QUERY_REQUOTER)(query)
797 if fragment:
798 fragment = (WHATWG_FRAGMENT_REQUOTER if mode is _WHATWG else FRAGMENT_REQUOTER)(
799 fragment
800 )
802 if not path and scheme == "file" and mode is _WHATWG:
803 # A file URL has a path in WHATWG mode, "/" at least: "file:" and
804 # "file://h" are "file:///" and "file://h/".
805 if not netloc and not empty & EMPTY_AUTHORITY:
806 # As a reference "file:" and "file:?q" keep the path of a file
807 # base, so join() takes the path as it is written.
808 cache["_join_path"] = ""
809 empty |= EMPTY_AUTHORITY
810 path = "/"
811 cache["scheme"] = scheme
812 cache["raw_path"] = "/" if not path and netloc else path
813 cache["raw_query_string"] = query
814 cache["raw_fragment"] = fragment
816 self = object.__new__(URL)
817 self._scheme = scheme
818 self._netloc = netloc
819 self._path = path
820 self._query = query
821 self._fragment = fragment
822 self._empty = empty
823 self._cache = cache
824 self._mode = mode
825 return self
828def _pre_encoded_url(url_str: str, mode: Mode) -> "URL":
829 """Parse pre-encoded URL."""
830 val = split_url(url_str)
831 scheme, netloc, path, query, fragment, empty = val
832 if not netloc and _is_special_authority_path(scheme, path, mode):
833 return _special_authority_url(scheme, path, query, fragment, empty, True)
834 self = object.__new__(URL)
835 (
836 self._scheme,
837 self._netloc,
838 self._path,
839 self._query,
840 self._fragment,
841 self._empty,
842 ) = val
843 if (
844 not self._netloc
845 and has_dot_prefix(self._path)
846 and not self._empty & EMPTY_AUTHORITY
847 ):
848 # Undo the "/." that str() puts in front of a path starting with
849 # "//" when there is no authority, as the unencoded parser does.
850 self._path = self._path[2:]
851 self._cache = {}
852 self._mode = mode
853 return self
856# One cache per compatibility mode keeps the single ``str`` argument, which
857# ``lru_cache`` uses as the key directly instead of building a tuple.
858@lru_cache
859def encode_url(url_str: str) -> "URL":
860 """Parse unencoded URL in WHATWG mode."""
861 return _encode_url(url_str, Mode.WHATWG)
864@lru_cache
865def encode_url_rfc(url_str: str) -> "URL":
866 """Parse unencoded URL in RFC 3986 mode."""
867 _check_rfc_url(url_str)
868 return _encode_url(url_str, Mode.RFC)
871@lru_cache
872def pre_encoded_url(url_str: str) -> "URL":
873 """Parse pre-encoded URL in WHATWG mode."""
874 return _pre_encoded_url(url_str, Mode.WHATWG)
877@lru_cache
878def pre_encoded_url_rfc(url_str: str) -> "URL":
879 """Parse pre-encoded URL in RFC 3986 mode."""
880 return _pre_encoded_url(url_str, Mode.RFC)
883@lru_cache
884def build_pre_encoded_url(
885 scheme: str,
886 authority: str,
887 user: str | None,
888 password: str | None,
889 host: str,
890 port: int | None,
891 path: str,
892 query_string: str,
893 fragment: str,
894 mode: Mode,
895) -> "URL":
896 """Build a pre-encoded URL from parts."""
897 self = object.__new__(URL)
898 self._scheme = scheme
899 if authority:
900 self._netloc = authority
901 elif host:
902 if port is not None:
903 port = None if port == DEFAULT_PORTS.get(scheme) else port
904 if user is None and password is None:
905 self._netloc = host if port is None else f"{host}:{port}"
906 else:
907 if not user and mode is _WHATWG:
908 # WHATWG has no empty userinfo, see make_netloc().
909 user = None
910 self._netloc = make_netloc(user, password, host, port)
911 else:
912 self._netloc = ""
913 if path and not scheme and not self._netloc and ":" in path:
914 path = _encode_relative_colon(path, mode)
915 self._path = path
916 self._query = query_string
917 self._fragment = fragment
918 self._empty = 0
919 self._cache = {}
920 self._mode = mode
921 return self
924def _kept_empty(keep_query: bool, keep_fragment: bool) -> int:
925 """Return the mask of empty components kept by a path replacement."""
926 return (
927 EMPTY_AUTHORITY
928 | (EMPTY_QUERY if keep_query else 0)
929 | (EMPTY_FRAGMENT if keep_fragment else 0)
930 )
933def from_parts_uncached(
934 scheme: str,
935 netloc: str,
936 path: str,
937 query: str,
938 fragment: str,
939 mode: Mode,
940 empty: int = 0,
941) -> "URL":
942 """Create a new URL from parts.
944 *empty* marks which empty components are present, see split_url(); it
945 must not have bits for components that are not empty.
946 """
947 self = object.__new__(URL)
948 self._scheme = scheme
949 self._netloc = netloc
950 if path and not scheme and not netloc and ":" in path:
951 path = _encode_relative_colon(path, mode)
952 self._path = path
953 self._query = query
954 self._fragment = fragment
955 self._empty = empty
956 self._cache = {}
957 self._mode = mode
958 return self
961from_parts = lru_cache(from_parts_uncached)
964@rewrite_module
965class URL:
966 # Don't derive from str
967 # follow pathlib.Path design
968 # probably URL will not suffer from pathlib problems:
969 # it's intended for libraries like aiohttp,
970 # not to be passed into standard library functions like os.open etc.
972 # URL grammar (RFC 3986)
973 # pct-encoded = "%" HEXDIG HEXDIG
974 # reserved = gen-delims / sub-delims
975 # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
976 # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
977 # / "*" / "+" / "," / ";" / "="
978 # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
979 # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
980 # hier-part = "//" authority path-abempty
981 # / path-absolute
982 # / path-rootless
983 # / path-empty
984 # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
985 # authority = [ userinfo "@" ] host [ ":" port ]
986 # userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
987 # host = IP-literal / IPv4address / reg-name
988 # IP-literal = "[" ( IPv6address / IPvFuture ) "]"
989 # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )
990 # IPv6address = 6( h16 ":" ) ls32
991 # / "::" 5( h16 ":" ) ls32
992 # / [ h16 ] "::" 4( h16 ":" ) ls32
993 # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
994 # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
995 # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
996 # / [ *4( h16 ":" ) h16 ] "::" ls32
997 # / [ *5( h16 ":" ) h16 ] "::" h16
998 # / [ *6( h16 ":" ) h16 ] "::"
999 # ls32 = ( h16 ":" h16 ) / IPv4address
1000 # ; least-significant 32 bits of address
1001 # h16 = 1*4HEXDIG
1002 # ; 16 bits of address represented in hexadecimal
1003 # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
1004 # dec-octet = DIGIT ; 0-9
1005 # / %x31-39 DIGIT ; 10-99
1006 # / "1" 2DIGIT ; 100-199
1007 # / "2" %x30-34 DIGIT ; 200-249
1008 # / "25" %x30-35 ; 250-255
1009 # reg-name = *( unreserved / pct-encoded / sub-delims )
1010 # port = *DIGIT
1011 # path = path-abempty ; begins with "/" or is empty
1012 # / path-absolute ; begins with "/" but not "//"
1013 # / path-noscheme ; begins with a non-colon segment
1014 # / path-rootless ; begins with a segment
1015 # / path-empty ; zero characters
1016 # path-abempty = *( "/" segment )
1017 # path-absolute = "/" [ segment-nz *( "/" segment ) ]
1018 # path-noscheme = segment-nz-nc *( "/" segment )
1019 # path-rootless = segment-nz *( "/" segment )
1020 # path-empty = 0<pchar>
1021 # segment = *pchar
1022 # segment-nz = 1*pchar
1023 # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
1024 # ; non-zero-length segment without any colon ":"
1025 # pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
1026 # query = *( pchar / "/" / "?" )
1027 # fragment = *( pchar / "/" / "?" )
1028 # URI-reference = URI / relative-ref
1029 # relative-ref = relative-part [ "?" query ] [ "#" fragment ]
1030 # relative-part = "//" authority path-abempty
1031 # / path-absolute
1032 # / path-noscheme
1033 # / path-empty
1034 # absolute-URI = scheme ":" hier-part [ "?" query ]
1035 __slots__ = (
1036 "_cache",
1037 "_scheme",
1038 "_netloc",
1039 "_path",
1040 "_query",
1041 "_fragment",
1042 "_empty",
1043 "_mode",
1044 )
1046 _cache: _InternalURLCache
1047 _scheme: str
1048 _netloc: str
1049 _path: str
1050 _query: str
1051 _fragment: str
1052 # EMPTY_AUTHORITY, EMPTY_QUERY and EMPTY_FRAGMENT bits for components
1053 # that are present but empty, e.g. the query of "http://h/?"
1054 _empty: int
1055 _mode: Mode
1057 def __new__(
1058 cls,
1059 val: "str | SplitResult | URL | UndefinedType" = UNDEFINED,
1060 *,
1061 encoded: bool = False,
1062 strict: bool | None = None,
1063 # Internally ``None`` stands for ``Mode.WHATWG``; it is not a
1064 # documented value. It is the default rather than the enum member
1065 # because ``mode is None`` compares against a constant, while
1066 # ``mode is Mode.WHATWG`` (or a module-level alias) needs a
1067 # global lookup; on the cached ``URL(str)`` hot path that lookup alone
1068 # was measured at about 3% of the whole call (CodSpeed). The
1069 # ``== "whatwg"`` test then matches both the string and the enum
1070 # member without calling ``_to_mode()``.
1071 mode: ModeType = None, # type: ignore[assignment]
1072 ) -> "URL":
1073 if strict is not None: # pragma: no cover
1074 warnings.warn("strict parameter is ignored")
1075 if type(val) is str:
1076 if mode is None or mode == "whatwg": # type: ignore[redundant-expr]
1077 return pre_encoded_url(val) if encoded else encode_url(val)
1078 _to_mode(mode) # reject unknown values; the rest is RFC
1079 return pre_encoded_url_rfc(val) if encoded else encode_url_rfc(val)
1080 mode = _WHATWG if mode is None else _to_mode(mode) # type: ignore[redundant-expr]
1081 if type(val) is cls:
1082 if val._mode is mode:
1083 return val
1084 return _moved_url(val, val._scheme, mode)
1085 if type(val) is SplitResult:
1086 if not encoded:
1087 raise ValueError("Cannot apply decoding to SplitResult")
1088 return from_parts(*val, mode)
1089 if isinstance(val, str):
1090 return URL(str(val), encoded=encoded, mode=mode)
1091 if val is UNDEFINED:
1092 # Special case for UNDEFINED since it might be unpickling and we do
1093 # not want to cache as the `__set_state__` call would mutate the URL
1094 # object in the `pre_encoded_url` or `encoded_url` caches.
1095 self = object.__new__(URL)
1096 self._scheme = self._netloc = self._path = self._query = self._fragment = ""
1097 self._empty = 0
1098 self._cache = {}
1099 self._mode = mode
1100 return self
1101 raise TypeError("Constructor parameter should be str")
1103 @classmethod
1104 def build(
1105 cls,
1106 *,
1107 scheme: str = "",
1108 authority: str = "",
1109 user: str | None = None,
1110 password: str | None = None,
1111 host: str = "",
1112 port: int | None = None,
1113 path: str = "",
1114 query: Query | None = None,
1115 query_string: str = "",
1116 fragment: str = "",
1117 encoded: bool = False,
1118 # Internally ``None`` stands for ``Mode.WHATWG``, as in ``URL()``.
1119 mode: ModeType = None, # type: ignore[assignment]
1120 ) -> "URL":
1121 """Creates and returns a new URL"""
1122 mode = _WHATWG if mode is None else _to_mode(mode) # type: ignore[redundant-expr]
1124 if authority and (user or password or host or port):
1125 raise ValueError(
1126 'Can\'t mix "authority" with "user", "password", "host" or "port".'
1127 )
1128 if port is not None and not isinstance(port, int):
1129 raise TypeError(f"The port is required to be int, got {type(port)!r}.")
1130 if port and not host:
1131 raise ValueError('Can\'t build URL with "port" but without "host".')
1132 if query and query_string:
1133 raise ValueError('Only one of "query" or "query_string" should be passed')
1134 if (
1135 scheme is None # type: ignore[redundant-expr]
1136 or authority is None # type: ignore[redundant-expr]
1137 or host is None # type: ignore[redundant-expr]
1138 or path is None # type: ignore[redundant-expr]
1139 or query_string is None # type: ignore[redundant-expr]
1140 or fragment is None
1141 ):
1142 raise TypeError(
1143 'NoneType is illegal for "scheme", "authority", "host", "path", '
1144 '"query_string", and "fragment" args, use empty string instead.'
1145 )
1147 if query:
1148 query_string = get_str_query(query) or ""
1150 if encoded:
1151 return build_pre_encoded_url(
1152 scheme,
1153 authority,
1154 user,
1155 password,
1156 host,
1157 port,
1158 path,
1159 query_string,
1160 fragment,
1161 mode,
1162 )
1164 if scheme == "file" and mode is _WHATWG:
1165 # As the parser does, see _whatwg_file_drive() and
1166 # normalize_drive_letter().
1167 if is_drive_letter(authority) and path[:1] in ("", "/"):
1168 path = _whatwg_file_drive(authority, path)
1169 authority = ""
1170 if "|" in path:
1171 path = normalize_drive_letter(path)
1173 self = object.__new__(URL)
1174 self._scheme = scheme
1175 self._empty = 0
1176 _host: str | None = None
1177 if authority:
1178 if mode is _WHATWG:
1179 user, password, _host, port = split_netloc(authority)
1180 else:
1181 _check_rfc_authority(authority)
1182 user, password, _host, port = split_netloc_rfc(authority)
1183 if not _host:
1184 _check_missing_host(scheme, mode)
1185 if (
1186 _host
1187 and scheme not in SPECIAL_SCHEMES
1188 and mode is _WHATWG
1189 and _is_opaque_host(scheme, _host)
1190 ):
1191 _host = _encode_opaque_host(_host)
1192 elif _host:
1193 if "%" in _host and ":" in _host and _host[0] != "[":
1194 check_zone_id(_host)
1195 encoded_host, whatwg_host, not_rfc = _encode_host(
1196 _host, validate_host=False
1197 )
1198 if not_rfc and mode is not _WHATWG:
1199 raise ValueError(_rfc_host_error(_host))
1200 _host = encoded_host
1201 if (
1202 whatwg_host is not _host
1203 and mode is _WHATWG
1204 and (scheme in SPECIAL_SCHEMES or _host[0] == "[")
1205 ):
1206 _host = _whatwg_special_host(_host, whatwg_host)
1207 else:
1208 _host = ""
1209 if port is None and user is None:
1210 # As the parser does, see _encode_url().
1211 self._empty = EMPTY_AUTHORITY
1212 elif (
1213 host
1214 and scheme not in SPECIAL_SCHEMES
1215 and mode is _WHATWG
1216 and _is_opaque_host(scheme, host)
1217 ):
1218 _host = _encode_opaque_host(host)
1219 elif host:
1220 _host, whatwg_host, not_rfc = _encode_host(host, validate_host=True)
1221 if not_rfc and mode is not _WHATWG:
1222 raise ValueError(_rfc_host_error(host))
1223 if (
1224 whatwg_host is not _host
1225 and mode is _WHATWG
1226 and (scheme in SPECIAL_SCHEMES or _host[0] == "[")
1227 ):
1228 _host = _whatwg_special_host(_host, whatwg_host)
1229 else:
1230 self._netloc = ""
1232 if _host is not None:
1233 if port is not None:
1234 port = None if port == DEFAULT_PORTS.get(scheme) else port
1235 if user is None and password is None:
1236 self._netloc = _host if port is None else f"{_host}:{port}"
1237 else:
1238 if not password and mode is _WHATWG:
1239 # As the parser does, see _encode_url().
1240 password = None
1241 if not user and mode is _WHATWG:
1242 # An empty user is None, see make_netloc().
1243 user = None
1244 self._netloc = make_netloc(
1245 user, password, _host, port, True, mode is _WHATWG
1246 )
1248 path = PATH_QUOTER(path) if path else path
1249 if path and (self._netloc or self._empty):
1250 if "." in path:
1251 path = _normalize_path(path, scheme, mode)
1252 if path[0] != "/":
1253 msg = (
1254 "Path in a URL with authority should "
1255 "start with a slash ('/') if set"
1256 )
1257 raise ValueError(msg)
1258 elif path and scheme and mode is not _WHATWG and "." in path:
1259 # Without an authority, as the parser does in RFC 3986 mode.
1260 path = remove_dot_segments(path)
1261 elif path and scheme == "file" and "." in path:
1262 # A file URL has an authority in WHATWG mode, also an empty one,
1263 # and its path is absolute.
1264 path = normalize_file_path(path if path[0] == "/" else f"/{path}")
1266 if path and not self._scheme and not self._netloc and ":" in path:
1267 path = _encode_relative_colon(path, mode)
1268 self._path = path
1269 if not query and query_string:
1270 query_string = QUERY_QUOTER(query_string)
1271 self._query = query_string
1272 self._fragment = FRAGMENT_QUOTER(fragment) if fragment else fragment
1273 self._cache = {}
1274 self._mode = mode
1275 if scheme == "file" and mode is _WHATWG:
1276 if self._netloc:
1277 # As the parser does, see _check_whatwg_file_authority(). The
1278 # path was checked and normalized as one with an authority
1279 # above.
1280 _check_whatwg_file_authority(authority or self._netloc)
1281 if _host == "localhost":
1282 self._netloc = ""
1283 self._empty = EMPTY_AUTHORITY
1284 if not path:
1285 # A file URL has an authority and a path in WHATWG mode, the
1286 # path is "/" at least.
1287 self._path = "/"
1288 if not self._netloc:
1289 self._empty = EMPTY_AUTHORITY
1290 return self
1292 def __init_subclass__(cls) -> NoReturn:
1293 raise TypeError(f"Inheriting a class {cls!r} from URL is forbidden")
1295 def __str__(self) -> str:
1296 if self._empty or not self._netloc:
1297 # Rare: empty components, or no authority, which a special
1298 # scheme in WHATWG mode still prints as "//".
1299 return self._str_with_empty()
1300 if (
1301 not self._path
1302 and (self._query or self._fragment)
1303 and (
1304 self._scheme in SPECIAL_SCHEMES
1305 or not self._scheme
1306 or self._scheme.lower() in SPECIAL_SCHEMES
1307 )
1308 ):
1309 # A special scheme has the path "/" at least, so "http://h?q" is
1310 # "http://h/?q". A non-special URL keeps its empty path, as
1311 # RFC 3986 and WHATWG write it: "sc://h?q". A relative reference
1312 # has no scheme to tell, and keeps the "/". URL.build() keeps
1313 # the scheme as given, hence lower().
1314 path = "/"
1315 else:
1316 path = self._path
1317 if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get(
1318 self._scheme
1319 ):
1320 # port normalization - using None for default ports to remove from rendering
1321 # https://datatracker.ietf.org/doc/html/rfc3986.html#section-6.2.3
1322 host = self.host_subcomponent
1323 netloc = make_netloc(self.raw_user, self.raw_password, host, None)
1324 else:
1325 netloc = self._netloc
1326 return unsplit_result(self._scheme, netloc, path, self._query, self._fragment)
1328 def _str_with_empty(self) -> str:
1329 """Render a URL with empty components or without an authority."""
1330 if (
1331 not self._path
1332 and self._netloc
1333 and (
1334 self._scheme in SPECIAL_SCHEMES
1335 or not self._scheme
1336 or self._scheme.lower() in SPECIAL_SCHEMES
1337 )
1338 ):
1339 # As in __str__(), "sc://h?" keeps its empty path.
1340 path = "/"
1341 else:
1342 path = self._path
1343 if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get(
1344 self._scheme
1345 ):
1346 host = self.host_subcomponent
1347 netloc = make_netloc(self.raw_user, self.raw_password, host, None)
1348 else:
1349 netloc = self._netloc
1350 empty = self._str_empty if not netloc else self._empty
1351 if not empty:
1352 return unsplit_result(
1353 self._scheme, netloc, path, self._query, self._fragment
1354 )
1355 return unsplit_result_empty(
1356 self._scheme, netloc, path, self._query, self._fragment, empty
1357 )
1359 def __repr__(self) -> str:
1360 if self._mode is Mode.WHATWG:
1361 return f"{self.__class__.__name__}('{str(self)}')"
1362 return f"{self.__class__.__name__}('{str(self)}', mode='{self._mode}')"
1364 def __bytes__(self) -> bytes:
1365 return str(self).encode("ascii")
1367 def __eq__(self, other: object) -> bool:
1368 if type(other) is not URL:
1369 return NotImplemented
1371 path1 = "/" if not self._path and self._netloc else self._path
1372 path2 = "/" if not other._path and other._netloc else other._path
1373 return (
1374 self._scheme == other._scheme
1375 and self._netloc == other._netloc
1376 and path1 == path2
1377 and self._query == other._query
1378 and self._fragment == other._fragment
1379 and (
1380 self._empty == other._empty
1381 if self._netloc
1382 else self._cmp_empty == other._cmp_empty
1383 )
1384 )
1386 def __hash__(self) -> int:
1387 if (ret := self._cache.get("hash")) is None:
1388 path = "/" if not self._path and self._netloc else self._path
1389 if (self._empty or not self._netloc) and (cmp_empty := self._cmp_empty):
1390 ret = hash(
1391 (
1392 self._scheme,
1393 self._netloc,
1394 path,
1395 self._query,
1396 self._fragment,
1397 cmp_empty,
1398 )
1399 )
1400 else:
1401 ret = hash(
1402 (self._scheme, self._netloc, path, self._query, self._fragment)
1403 )
1404 self._cache["hash"] = ret
1405 return ret
1407 def __le__(self, other: object) -> bool:
1408 if type(other) is not URL:
1409 return NotImplemented
1410 return (self._val, self._cmp_empty) <= (other._val, other._cmp_empty)
1412 def __lt__(self, other: object) -> bool:
1413 if type(other) is not URL:
1414 return NotImplemented
1415 return (self._val, self._cmp_empty) < (other._val, other._cmp_empty)
1417 def __ge__(self, other: object) -> bool:
1418 if type(other) is not URL:
1419 return NotImplemented
1420 return (self._val, self._cmp_empty) >= (other._val, other._cmp_empty)
1422 def __gt__(self, other: object) -> bool:
1423 if type(other) is not URL:
1424 return NotImplemented
1425 return (self._val, self._cmp_empty) > (other._val, other._cmp_empty)
1427 def __truediv__(self, name: str) -> "URL":
1428 if not isinstance(name, str):
1429 return NotImplemented
1430 return self._make_child((str(name),))
1432 def __mod__(self, query: Query) -> "URL":
1433 return self.update_query(query)
1435 def __bool__(self) -> bool:
1436 return bool(
1437 self._netloc or self._path or self._query or self._fragment or self._empty
1438 )
1440 def __getstate__(
1441 self,
1442 ) -> tuple[SplitURLType, str, int] | tuple[SplitURLType, str, int, str]:
1443 # Return a plain tuple rather than a ``SplitResult``. Constructing a
1444 # ``SplitResult`` via ``tuple.__new__`` skips its ``__init__`` and on
1445 # Python 3.15+ leaves ``_keep_empty`` unset, which breaks pickling: the
1446 # new ``SplitResult.__getstate__`` indexes a state that ends up as
1447 # ``None`` (gh-1632). ``__setstate__`` already unpacks both shapes, so
1448 # pickles produced by older yarl releases (which embed a real
1449 # ``SplitResult``) still load correctly. The compatibility mode goes
1450 # second and the mask of empty components third; older releases
1451 # ignore trailing items of the state. What join() reads for "http:g"
1452 # (see _special_authority_url()) or for a relative reference with
1453 # "\\" (see _backslashes()) goes last, when there is one.
1454 if (join_path := self._cache.get("_join_path")) is not None:
1455 return (self._val, self._mode.value, self._empty, join_path)
1456 return (self._val, self._mode.value, self._empty)
1458 def __setstate__(
1459 self,
1460 # ``(val,)`` or ``(val, mode)`` from older releases,
1461 # ``(val, mode, empty)``, ``(val, mode, empty, join_path)``, or the
1462 # legacy default style
1463 # ``(None, {"_val": val})``.
1464 state: tuple[Any, ...],
1465 ) -> None:
1466 mode = Mode.WHATWG
1467 empty = 0
1468 cache: _InternalURLCache = {}
1469 if state[0] is None and isinstance(state[1], dict):
1470 # default style pickle
1471 val = state[1]["_val"]
1472 else:
1473 val, *rest = state
1474 if rest:
1475 mode = Mode(rest[0])
1476 if rest[1:]:
1477 empty = rest[1]
1478 if rest[2:]:
1479 cache["_join_path"] = rest[2]
1480 self._scheme, self._netloc, self._path, self._query, self._fragment = val
1481 self._empty = empty
1482 self._cache = cache
1483 self._mode = mode
1485 def _keep_join_path(self, url: "URL") -> None:
1486 """Give url, a copy of self with another query or fragment, what
1487 join() reads instead of self's path: the path of "http:g" (see
1488 _special_authority_url()) or, for a relative reference without an
1489 authority, the reference with "\\" read as "/" (see _backslashes()).
1491 url must be a new URL, not one shared through the from_parts() cache.
1492 """
1493 url._cache["_join_path"] = self._cache["_join_path"]
1495 def _cache_netloc(self) -> None:
1496 """Cache the netloc parts of the URL."""
1497 c = self._cache
1498 if self._mode is _WHATWG:
1499 split_loc = split_netloc(self._netloc)
1500 else:
1501 # An empty username is "" in RFC 3986 mode, see split_netloc_rfc().
1502 split_loc = split_netloc_rfc(self._netloc)
1503 c["raw_user"], c["raw_password"], c["raw_host"], c["explicit_port"] = split_loc
1505 def is_absolute(self) -> bool:
1506 """A check for absolute URLs.
1508 Return True for absolute ones (having scheme or starting
1509 with //), False otherwise.
1511 Is is preferred to call the .absolute property instead
1512 as it is cached.
1513 """
1514 return self.absolute
1516 def is_default_port(self) -> bool:
1517 """A check for default port.
1519 Return True if port is default for specified scheme,
1520 e.g. 'http://python.org' or 'http://python.org:80', False
1521 otherwise.
1523 Return False for relative URLs.
1525 """
1526 if (explicit := self.explicit_port) is None:
1527 # If the explicit port is None, then the URL must be
1528 # using the default port unless its a relative URL
1529 # which does not have an implicit port / default port
1530 return self._netloc != ""
1531 return explicit == DEFAULT_PORTS.get(self._scheme)
1533 def origin(self) -> "URL":
1534 """Return an URL with scheme, host and port parts only.
1536 user, password, path, query and fragment are removed.
1538 """
1539 # TODO: add a keyword-only option for keeping user/pass maybe?
1540 return self._origin
1542 @property
1543 def _cmp_empty(self) -> int:
1544 """The mask of empty components that matters for comparisons.
1546 It follows what str() writes out: in WHATWG mode a file URL always
1547 prints "//", so "file:/p" and "file:///p" are the same URL there,
1548 while RFC mode prints and compares them differently.
1549 """
1550 if not self._netloc and self._mode is _WHATWG and self._scheme == "file":
1551 return self._empty | EMPTY_AUTHORITY
1552 return self._empty
1554 @property
1555 def _str_empty(self) -> int:
1556 """The mask of empty components that str() writes out.
1558 In WHATWG mode a file URL always has an authority, so "//" is
1559 written even when it is missing. Other special schemes need a host;
1560 without one the URL is a reference to a base, such as "http:?q".
1561 """
1562 if self._mode is _WHATWG and self._scheme == "file":
1563 return self._empty | EMPTY_AUTHORITY
1564 return self._empty
1566 @cached_property
1567 def _val(self) -> SplitURLType:
1568 return (self._scheme, self._netloc, self._path, self._query, self._fragment)
1570 @cached_property
1571 def _origin(self) -> "URL":
1572 """Return an URL with scheme, host and port parts only.
1574 user, password, path, query and fragment are removed.
1575 """
1576 if not (netloc := self._netloc):
1577 raise ValueError("URL should be absolute")
1578 if not (scheme := self._scheme):
1579 raise ValueError("URL should have scheme")
1580 if "@" in netloc:
1581 # The host is None for an authority without one ("user@:8080").
1582 encoded_host = self.host_subcomponent or ""
1583 netloc = make_netloc(None, None, encoded_host, self.explicit_port)
1584 elif not self._path and not self._query and not self._fragment:
1585 if not self._empty:
1586 return self
1587 return from_parts(scheme, netloc, "", "", "", self._mode)
1589 def relative(self) -> "URL":
1590 """Return a relative part of the URL.
1592 scheme, user, password, host and port are removed.
1594 """
1595 if not self._netloc:
1596 raise ValueError("URL should be absolute")
1597 return from_parts(
1598 "", "", self._path, self._query, self._fragment, self._mode, self._empty
1599 )
1601 @cached_property
1602 def absolute(self) -> bool:
1603 """A check for absolute URLs.
1605 Return True for absolute ones (having scheme or starting
1606 with //), False otherwise.
1608 """
1609 # `netloc`` is an empty string for relative URLs
1610 # Checking `netloc` is faster than checking `hostname`
1611 # because `hostname` is a property that does some extra work
1612 # to parse the host from the `netloc`
1613 return self._netloc != ""
1615 @cached_property
1616 def scheme(self) -> str:
1617 """Scheme for absolute URLs.
1619 Empty string for relative URLs or URLs starting with //
1621 """
1622 return self._scheme
1624 @cached_property
1625 def raw_authority(self) -> str:
1626 """Encoded authority part of URL.
1628 Empty string for relative URLs.
1630 """
1631 return self._netloc
1633 @cached_property
1634 def authority(self) -> str:
1635 """Decoded authority part of URL.
1637 Empty string for relative URLs.
1639 """
1640 if (host := self.host) is None and self._netloc:
1641 # An authority without a host, e.g. "user@:8080" in RFC mode.
1642 host = ""
1643 return make_netloc(self.user, self.password, host, self.port)
1645 @cached_property
1646 def raw_user(self) -> str | None:
1647 """Encoded user part of URL.
1649 None if user is missing.
1651 """
1652 # not .username
1653 self._cache_netloc()
1654 return self._cache["raw_user"]
1656 @cached_property
1657 def user(self) -> str | None:
1658 """Decoded user part of URL.
1660 None if user is missing.
1662 """
1663 if (raw_user := self.raw_user) is None:
1664 return None
1665 return UNQUOTER(raw_user)
1667 @cached_property
1668 def raw_password(self) -> str | None:
1669 """Encoded password part of URL.
1671 None if password is missing.
1673 """
1674 self._cache_netloc()
1675 return self._cache["raw_password"]
1677 @cached_property
1678 def password(self) -> str | None:
1679 """Decoded password part of URL.
1681 None if password is missing.
1683 """
1684 if (raw_password := self.raw_password) is None:
1685 return None
1686 return UNQUOTER(raw_password)
1688 @property
1689 def mode(self) -> Mode:
1690 """Standard the URL follows: RFC 3986 or the WHATWG URL Standard."""
1691 return self._mode
1693 @cached_property
1694 def raw_host(self) -> str | None:
1695 """Encoded host part of URL.
1697 None for relative URLs.
1699 When working with IPv6 addresses, use the `host_subcomponent` property instead
1700 as it will return the host subcomponent with brackets.
1701 """
1702 # Use host instead of hostname for sake of shortness
1703 # May add .hostname prop later
1704 self._cache_netloc()
1705 return self._cache["raw_host"]
1707 @cached_property
1708 def host(self) -> str | None:
1709 """Decoded host part of URL.
1711 None for relative URLs.
1713 For IPv6 hosts that carry an RFC 6874 zone identifier, the
1714 ``%25`` zone separator is decoded back to ``%`` and the zone
1715 identifier is percent-decoded; the encoded form is still
1716 available via :attr:`raw_host` and :attr:`host_subcomponent`.
1718 The opaque host of a non-special URL in WHATWG mode has its
1719 non-ASCII characters percent-decoded, the rest stays as it is.
1721 """
1722 if (raw := self.raw_host) is None:
1723 return None
1724 if (
1725 self._scheme not in SPECIAL_SCHEMES
1726 and self._mode is _WHATWG
1727 and _is_opaque_host(self._scheme, raw)
1728 ):
1729 return _decode_opaque_host(raw)
1730 if raw and (raw[-1].isdigit() or raw[-1] == "]") or ":" in raw:
1731 # IP addresses are never IDNA encoded. The RFC 6874 zone
1732 # separator "%25" of an IPv6 address decodes to "%"; the zone
1733 # identifier is decoded except for "%", the delimiters of an
1734 # authority and octets that are not UTF-8, so that passing
1735 # the result to with_host() gives the same URL back.
1736 if "%25" in raw:
1737 if ":" not in raw:
1738 return raw.replace("%25", "%")
1739 address, _, zone = raw.partition("%25")
1740 return f"{address}%{ZONE_ID_UNQUOTER(zone)}"
1741 return raw
1742 return _idna_decode(raw)
1744 @cached_property
1745 def host_subcomponent(self) -> str | None:
1746 """Return the host subcomponent part of URL.
1748 None for relative URLs.
1750 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
1752 `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
1754 Examples:
1755 - `http://example.com:8080` -> `example.com`
1756 - `http://example.com:80` -> `example.com`
1757 - `https://127.0.0.1:8443` -> `127.0.0.1`
1758 - `https://[::1]:8443` -> `[::1]`
1759 - `http://[::1]` -> `[::1]`
1761 """
1762 if (raw := self.raw_host) is None:
1763 return None
1764 # An IPvFuture address keeps its brackets in the raw host.
1765 return f"[{raw}]" if ":" in raw and raw[-1] != "]" else raw
1767 @cached_property
1768 def host_port_subcomponent(self) -> str | None:
1769 """Return the host and port subcomponent part of URL.
1771 Trailing dots are removed from the host part.
1773 This value is suitable for use in the Host header of an HTTP request.
1775 None for relative URLs.
1777 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
1778 `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
1779 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.3
1780 port = *DIGIT
1782 Examples:
1783 - `http://example.com:8080` -> `example.com:8080`
1784 - `http://example.com:80` -> `example.com`
1785 - `http://example.com.:80` -> `example.com`
1786 - `https://127.0.0.1:8443` -> `127.0.0.1:8443`
1787 - `https://[::1]:8443` -> `[::1]:8443`
1788 - `http://[::1]` -> `[::1]`
1790 """
1791 if (raw := self.raw_host) is None:
1792 return None
1793 if raw and raw[-1] == ".":
1794 # Remove all trailing dots from the netloc as while
1795 # they are valid FQDNs in DNS, TLS validation fails.
1796 # See https://github.com/aio-libs/aiohttp/issues/3636.
1797 # To avoid string manipulation we only call rstrip if
1798 # the last character is a dot.
1799 raw = raw.rstrip(".")
1800 if ":" in raw and raw[-1] != "]":
1801 raw = f"[{raw}]"
1802 port = self.explicit_port
1803 if port is None or port == DEFAULT_PORTS.get(self._scheme):
1804 return raw
1805 return f"{raw}:{port}"
1807 @cached_property
1808 def port(self) -> int | None:
1809 """Port part of URL, with scheme-based fallback.
1811 None for relative URLs or URLs without explicit port and
1812 scheme without default port substitution.
1814 """
1815 if (explicit_port := self.explicit_port) is not None:
1816 return explicit_port
1817 return DEFAULT_PORTS.get(self._scheme)
1819 @cached_property
1820 def explicit_port(self) -> int | None:
1821 """Port part of URL, without scheme-based fallback.
1823 None for relative URLs or URLs without explicit port.
1825 """
1826 self._cache_netloc()
1827 return self._cache["explicit_port"]
1829 @cached_property
1830 def raw_path(self) -> str:
1831 """Encoded path of URL.
1833 / for absolute URLs without path part.
1835 """
1836 return self._path if self._path or not self._netloc else "/"
1838 @cached_property
1839 def path(self) -> str:
1840 """Decoded path of URL.
1842 / for absolute URLs without path part.
1844 """
1845 return UNQUOTER(self._path) if self._path else "/" if self._netloc else ""
1847 @cached_property
1848 def path_safe(self) -> str:
1849 """Decoded path of URL.
1851 / for absolute URLs without path part.
1853 / (%2F) and % (%25) are not decoded
1855 """
1856 if self._path:
1857 return PATH_SAFE_UNQUOTER(self._path)
1858 return "/" if self._netloc else ""
1860 @cached_property
1861 def _parsed_query(self) -> list[tuple[str, str]]:
1862 """Parse query part of URL."""
1863 return query_to_pairs(self._query)
1865 @cached_property
1866 def query(self) -> "MultiDictProxy[str]":
1867 """A MultiDictProxy representing parsed query parameters in decoded
1868 representation.
1870 Empty value if URL has no query part.
1872 """
1873 return MultiDictProxy(MultiDict(self._parsed_query))
1875 @cached_property
1876 def raw_query_string(self) -> str:
1877 """Encoded query part of URL.
1879 Empty string if query is missing.
1881 """
1882 return self._query
1884 @cached_property
1885 def query_string(self) -> str:
1886 """Decoded query part of URL.
1888 Empty string if query is missing.
1890 """
1891 return QS_UNQUOTER(self._query) if self._query else ""
1893 @cached_property
1894 def path_qs(self) -> str:
1895 """Decoded path of URL with query."""
1896 if q := self.query_string:
1897 return f"{self.path}?{q}"
1898 if (empty := self._empty) and empty & EMPTY_QUERY:
1899 return f"{self.path}?"
1900 return self.path
1902 @cached_property
1903 def raw_path_qs(self) -> str:
1904 """Encoded path of URL with query."""
1905 if q := self._query:
1906 return f"{self._path}?{q}" if self._path or not self._netloc else f"/?{q}"
1907 if (empty := self._empty) and empty & EMPTY_QUERY:
1908 return f"{self._path}?" if self._path or not self._netloc else "/?"
1909 return self._path if self._path or not self._netloc else "/"
1911 @cached_property
1912 def raw_fragment(self) -> str:
1913 """Encoded fragment part of URL.
1915 Empty string if fragment is missing.
1917 """
1918 return self._fragment
1920 @cached_property
1921 def fragment(self) -> str:
1922 """Decoded fragment part of URL.
1924 Empty string if fragment is missing.
1926 """
1927 return UNQUOTER(self._fragment) if self._fragment else ""
1929 @cached_property
1930 def raw_parts(self) -> tuple[str, ...]:
1931 """A tuple containing encoded *path* parts.
1933 ('/',) for absolute URLs if *path* is missing.
1935 """
1936 path = self._path
1937 if self._netloc or self._empty & EMPTY_AUTHORITY:
1938 return ("/", *path[1:].split("/")) if path else ("/",)
1939 if path and path[0] == "/":
1940 return ("/", *path[1:].split("/"))
1941 return tuple(path.split("/"))
1943 @cached_property
1944 def parts(self) -> tuple[str, ...]:
1945 """A tuple containing decoded *path* parts.
1947 ('/',) for absolute URLs if *path* is missing.
1949 """
1950 return tuple(UNQUOTER(part) for part in self.raw_parts)
1952 @cached_property
1953 def parent(self) -> "URL":
1954 """A new URL with last part of path removed and cleaned up query and
1955 fragment.
1957 """
1958 path = self._path
1959 authority_empty = self._empty & EMPTY_AUTHORITY
1960 if not path or path == "/":
1961 if self._fragment or self._query or self._empty & ~EMPTY_AUTHORITY:
1962 return from_parts(
1963 self._scheme,
1964 self._netloc,
1965 path,
1966 "",
1967 "",
1968 self._mode,
1969 authority_empty,
1970 )
1971 return self
1972 parts = path.split("/")
1973 if not (path := "/".join(parts[:-1])) and self._scheme == "file":
1974 # A file URL has a path in WHATWG mode, "/" at least.
1975 path = _empty_path(self._scheme, self._mode)
1976 return from_parts(
1977 self._scheme,
1978 self._netloc,
1979 path,
1980 "",
1981 "",
1982 self._mode,
1983 authority_empty,
1984 )
1986 @cached_property
1987 def raw_name(self) -> str:
1988 """The last part of raw_parts."""
1989 parts = self.raw_parts
1990 if not self._netloc:
1991 return parts[-1]
1992 parts = parts[1:]
1993 return parts[-1] if parts else ""
1995 @cached_property
1996 def name(self) -> str:
1997 """The last part of parts."""
1998 return UNQUOTER(self.raw_name)
2000 @cached_property
2001 def raw_suffix(self) -> str:
2002 name = self.raw_name
2003 i = name.rfind(".")
2004 return name[i:] if 0 < i < len(name) - 1 else ""
2006 @cached_property
2007 def suffix(self) -> str:
2008 return UNQUOTER(self.raw_suffix)
2010 @cached_property
2011 def raw_suffixes(self) -> tuple[str, ...]:
2012 name = self.raw_name
2013 if name.endswith("."):
2014 return ()
2015 name = name.lstrip(".")
2016 return tuple("." + suffix for suffix in name.split(".")[1:])
2018 @cached_property
2019 def suffixes(self) -> tuple[str, ...]:
2020 return tuple(UNQUOTER(suffix) for suffix in self.raw_suffixes)
2022 def _make_child(self, paths: "Sequence[str]", encoded: bool = False) -> "URL":
2023 """
2024 add paths to self._path, accounting for absolute vs relative paths,
2025 keep existing, but do not create new, empty segments
2026 """
2027 parsed: list[str] = []
2028 needs_normalize: bool = False
2029 if (
2030 self._scheme == "file"
2031 and self._mode is _WHATWG
2032 and self._path in ("", "/")
2033 and paths
2034 and not encoded
2035 ):
2036 # The first segment of a file URL can be a drive letter, see
2037 # normalize_drive_letter().
2038 paths = [normalize_drive_letter(paths[0]), *paths[1:]]
2039 for idx, path in enumerate(reversed(paths)):
2040 # empty segment of last is not removed
2041 last = idx == 0
2042 if path and path[0] == "/":
2043 raise ValueError(
2044 f"Appending path {path!r} starting from slash is forbidden"
2045 )
2046 # We need to quote the path if it is not already encoded
2047 # This cannot be done at the end because the existing
2048 # path is already quoted and we do not want to double quote
2049 # the existing path.
2050 path = path if encoded else PATH_QUOTER(path)
2051 needs_normalize |= "." in path
2052 segments = path.split("/")
2053 segments.reverse()
2054 # remove trailing empty segment for all but the last path
2055 parsed += segments[1:] if not last and segments[0] == "" else segments
2057 if (path := self._path) and (old_segments := path.split("/")):
2058 # If the old path ends with a slash, the last segment is an empty string
2059 # and should be removed before adding the new path segments.
2060 old = old_segments[:-1] if old_segments[-1] == "" else old_segments
2061 old.reverse()
2062 parsed += old
2064 # If the netloc is present, inject a leading slash when adding a
2065 # path to an absolute URL where there was none before.
2066 netloc = self._netloc
2067 authority_empty = self._empty & EMPTY_AUTHORITY
2068 # A file URL has an authority in WHATWG mode, also an empty one.
2069 has_authority = (
2070 netloc
2071 or authority_empty
2072 or (self._scheme == "file" and self._mode is _WHATWG)
2073 )
2074 if has_authority and parsed and parsed[-1] != "":
2075 parsed.append("")
2077 parsed.reverse()
2078 if not has_authority or not needs_normalize:
2079 path = "/".join(parsed)
2080 if needs_normalize and self._scheme and self._mode is not _WHATWG:
2081 # Without an authority, as the parser does in RFC 3986 mode.
2082 path = remove_dot_segments(path)
2083 return from_parts(
2084 self._scheme,
2085 netloc,
2086 path,
2087 "",
2088 "",
2089 self._mode,
2090 authority_empty,
2091 )
2093 if self._scheme == "file" and self._mode is _WHATWG:
2094 # Keep the drive letter, as the parser does.
2095 path = normalize_file_path("/".join(parsed))
2096 else:
2097 path = "/".join(normalize_path_segments(parsed))
2098 # If normalizing the path segments removed the leading slash, add it back.
2099 if path and path[0] != "/":
2100 path = f"/{path}"
2101 return from_parts(
2102 self._scheme, netloc, path, "", "", self._mode, authority_empty
2103 )
2105 def with_scheme(self, scheme: str) -> "URL":
2106 """Return a new URL with scheme replaced."""
2107 # N.B. doesn't cleanup query/fragment
2108 if not isinstance(scheme, str):
2109 raise TypeError("Invalid scheme type")
2110 lower_scheme = scheme.lower()
2111 netloc = self._netloc
2112 if not netloc and lower_scheme in SCHEME_REQUIRES_HOST:
2113 msg = (
2114 "scheme replacement is not allowed for "
2115 f"relative URLs for the {lower_scheme} scheme"
2116 )
2117 raise ValueError(msg)
2118 if lower_scheme == "file" or (self._scheme not in SPECIAL_SCHEMES and netloc):
2119 # A special scheme in WHATWG mode had its host parsed already, but
2120 # only for a file URL is "localhost" dropped, and the path of a
2121 # file URL is not empty.
2122 return _moved_url(self, lower_scheme, self._mode)
2123 return from_parts(
2124 lower_scheme,
2125 netloc,
2126 self._path,
2127 self._query,
2128 self._fragment,
2129 self._mode,
2130 self._empty,
2131 )
2133 def with_user(self, user: str | None) -> "URL":
2134 """Return a new URL with user replaced.
2136 Autoencode user if needed.
2138 Clear user/password if user is None.
2140 """
2141 # N.B. doesn't cleanup query/fragment
2142 if user:
2143 if not isinstance(user, str):
2144 raise TypeError("Invalid user type")
2145 user = QUOTER(user)
2146 if (";" in user or "=" in user) and self._mode is _WHATWG:
2147 # Inlined quote_userinfo(), with_user() is a benchmarked path.
2148 user = user.replace(";", "%3B").replace("=", "%3D")
2149 password = self.raw_password
2150 elif user is None:
2151 password = None
2152 elif isinstance(user, str):
2153 # An empty user keeps the empty userinfo in RFC 3986 mode,
2154 # "http://@h"; WHATWG has none, see make_netloc().
2155 if self._mode is _WHATWG:
2156 user = None
2157 password = self.raw_password
2158 else:
2159 raise TypeError("Invalid user type")
2160 if not (netloc := self._netloc):
2161 raise ValueError("user replacement is not allowed for relative URLs")
2162 encoded_host = self.host_subcomponent or ""
2163 netloc = make_netloc(user, password, encoded_host, self.explicit_port)
2164 if self._scheme == "file" and self._mode is _WHATWG:
2165 _check_whatwg_file_authority(netloc)
2166 return from_parts(
2167 self._scheme,
2168 netloc,
2169 self._path,
2170 self._query,
2171 self._fragment,
2172 self._mode,
2173 self._empty,
2174 )
2176 def with_password(self, password: str | None) -> "URL":
2177 """Return a new URL with password replaced.
2179 Autoencode password if needed.
2181 Clear password if argument is None.
2183 """
2184 # N.B. doesn't cleanup query/fragment
2185 if password is None:
2186 pass
2187 elif isinstance(password, str):
2188 if password:
2189 password = QUOTER(password)
2190 if (";" in password or "=" in password) and self._mode is _WHATWG:
2191 # Inlined quote_userinfo(), as in with_user().
2192 password = password.replace(";", "%3B").replace("=", "%3D")
2193 elif self._mode is _WHATWG:
2194 # WHATWG has no empty password, see _encode_url().
2195 password = None
2196 else:
2197 raise TypeError("Invalid password type")
2198 if not (netloc := self._netloc):
2199 raise ValueError("password replacement is not allowed for relative URLs")
2200 encoded_host = self.host_subcomponent or ""
2201 port = self.explicit_port
2202 netloc = make_netloc(self.raw_user, password, encoded_host, port)
2203 if self._scheme == "file" and self._mode is _WHATWG:
2204 _check_whatwg_file_authority(netloc)
2205 return from_parts(
2206 self._scheme,
2207 netloc,
2208 self._path,
2209 self._query,
2210 self._fragment,
2211 self._mode,
2212 self._empty,
2213 )
2215 def with_host(self, host: str) -> "URL":
2216 """Return a new URL with host replaced.
2218 Autoencode host if needed.
2220 Changing host for relative URLs is not allowed, use .join()
2221 instead.
2223 """
2224 # N.B. doesn't cleanup query/fragment
2225 if not isinstance(host, str):
2226 raise TypeError("Invalid host type")
2227 if not (netloc := self._netloc):
2228 raise ValueError("host replacement is not allowed for relative URLs")
2229 if not host:
2230 raise ValueError("host removing is not allowed")
2231 if (
2232 self._scheme not in SPECIAL_SCHEMES
2233 and self._mode is _WHATWG
2234 and _is_opaque_host(self._scheme, host)
2235 ):
2236 encoded_host = _encode_opaque_host(host)
2237 else:
2238 encoded_host, whatwg_host, not_rfc = _encode_host(host, validate_host=True)
2239 if not_rfc and self._mode is not _WHATWG:
2240 raise ValueError(_rfc_host_error(host))
2241 if (
2242 whatwg_host is not encoded_host
2243 and self._mode is _WHATWG
2244 and (self._scheme in SPECIAL_SCHEMES or encoded_host[0] == "[")
2245 ):
2246 encoded_host = _whatwg_special_host(encoded_host, whatwg_host)
2247 port = self.explicit_port
2248 netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
2249 if self._scheme == "file" and self._mode is _WHATWG:
2250 # As the parser does, see _check_whatwg_file_authority().
2251 _check_whatwg_file_authority(netloc)
2252 if encoded_host == "localhost":
2253 return from_parts(
2254 self._scheme,
2255 "",
2256 self._path or "/",
2257 self._query,
2258 self._fragment,
2259 self._mode,
2260 self._empty | EMPTY_AUTHORITY,
2261 )
2262 return from_parts(
2263 self._scheme,
2264 netloc,
2265 self._path,
2266 self._query,
2267 self._fragment,
2268 self._mode,
2269 self._empty,
2270 )
2272 def with_port(self, port: int | None) -> "URL":
2273 """Return a new URL with port replaced.
2275 Clear port to default if None is passed.
2277 """
2278 # N.B. doesn't cleanup query/fragment
2279 if port is not None:
2280 if isinstance(port, bool) or not isinstance(port, int):
2281 raise TypeError(f"port should be int or None, got {type(port)}")
2282 if not (0 <= port <= 65535):
2283 raise ValueError(f"port must be between 0 and 65535, got {port}")
2284 if not (netloc := self._netloc):
2285 raise ValueError("port replacement is not allowed for relative URLs")
2286 encoded_host = self.host_subcomponent or ""
2287 netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
2288 if self._scheme == "file" and self._mode is _WHATWG:
2289 _check_whatwg_file_authority(netloc)
2290 return from_parts(
2291 self._scheme,
2292 netloc,
2293 self._path,
2294 self._query,
2295 self._fragment,
2296 self._mode,
2297 self._empty,
2298 )
2300 def with_path(
2301 self,
2302 path: str,
2303 *,
2304 encoded: bool = False,
2305 keep_query: bool = False,
2306 keep_fragment: bool = False,
2307 ) -> "URL":
2308 """Return a new URL with path replaced."""
2309 netloc = self._netloc
2310 if not encoded:
2311 if "|" in path and self._scheme == "file" and self._mode is _WHATWG:
2312 # As the parser does, see normalize_drive_letter().
2313 path = normalize_drive_letter(path)
2314 path = PATH_QUOTER(path)
2315 if netloc or (self._scheme == "file" and self._mode is _WHATWG):
2316 # A file URL has an authority in WHATWG mode, also an empty one.
2317 if "." in path:
2318 path = _normalize_path(path, self._scheme, self._mode)
2319 elif self._scheme and self._mode is not _WHATWG and "." in path:
2320 # Without an authority, as the parser does in RFC 3986 mode.
2321 path = remove_dot_segments(path)
2322 if not path:
2323 path = _empty_path(self._scheme, self._mode)
2324 elif path[0] != "/":
2325 path = f"/{path}"
2326 query = self._query if keep_query else ""
2327 fragment = self._fragment if keep_fragment else ""
2328 if empty := self._empty:
2329 empty &= _kept_empty(keep_query, keep_fragment)
2330 return from_parts(
2331 self._scheme, netloc, path, query, fragment, self._mode, empty
2332 )
2333 return from_parts(self._scheme, netloc, path, query, fragment, self._mode)
2335 @overload
2336 def with_query(self, query: Query) -> "URL": ...
2338 @overload
2339 def with_query(self, **kwargs: QueryVariable) -> "URL": ...
2341 def with_query(self, *args: Any, **kwargs: Any) -> "URL":
2342 """Return a new URL with query part replaced.
2344 Accepts any Mapping (e.g. dict, multidict.MultiDict instances)
2345 or str, autoencode the argument if needed.
2347 A sequence of (key, value) pairs is supported as well.
2349 It also can take an arbitrary number of keyword arguments.
2351 Clear query if None is passed.
2353 """
2354 # N.B. doesn't cleanup query/fragment
2355 query = get_str_query(*args, **kwargs) or ""
2356 url = from_parts_uncached(
2357 self._scheme,
2358 self._netloc,
2359 self._path,
2360 query,
2361 self._fragment,
2362 self._mode,
2363 self._empty and self._empty & ~EMPTY_QUERY,
2364 )
2365 if "_join_path" in self._cache:
2366 self._keep_join_path(url)
2367 return url
2369 @overload
2370 def extend_query(self, query: Query) -> "URL": ...
2372 @overload
2373 def extend_query(self, **kwargs: QueryVariable) -> "URL": ...
2375 def extend_query(self, *args: Any, **kwargs: Any) -> "URL":
2376 """Return a new URL with query part combined with the existing.
2378 This method will not remove existing query parameters.
2380 Example:
2381 >>> url = URL('http://example.com/?a=1&b=2')
2382 >>> url.extend_query(a=3, c=4)
2383 URL('http://example.com/?a=1&b=2&a=3&c=4')
2384 """
2385 if not (new_query := get_str_query(*args, **kwargs)):
2386 return self
2387 if query := self._query:
2388 # both strings are already encoded so we can use a simple
2389 # string join
2390 query += new_query if query[-1] == "&" else f"&{new_query}"
2391 else:
2392 query = new_query
2393 url = from_parts_uncached(
2394 self._scheme,
2395 self._netloc,
2396 self._path,
2397 query,
2398 self._fragment,
2399 self._mode,
2400 self._empty and self._empty & ~EMPTY_QUERY,
2401 )
2402 if "_join_path" in self._cache:
2403 self._keep_join_path(url)
2404 return url
2406 @overload
2407 def update_query(self, query: Query) -> "URL": ...
2409 @overload
2410 def update_query(self, **kwargs: QueryVariable) -> "URL": ...
2412 def update_query(self, *args: Any, **kwargs: Any) -> "URL":
2413 """Return a new URL with query part updated.
2415 This method will overwrite existing query parameters.
2417 Example:
2418 >>> url = URL('http://example.com/?a=1&b=2')
2419 >>> url.update_query(a=3, c=4)
2420 URL('http://example.com/?a=3&b=2&c=4')
2421 """
2422 in_query: (
2423 str
2424 | Mapping[str, QueryVariable]
2425 | Sequence[tuple[str | istr, SimpleQuery]]
2426 | None
2427 )
2428 if kwargs:
2429 if args:
2430 msg = "Either kwargs or single query parameter must be present"
2431 raise ValueError(msg)
2432 in_query = kwargs
2433 elif len(args) == 1:
2434 in_query = args[0]
2435 else:
2436 raise ValueError("Either kwargs or single query parameter must be present")
2438 if in_query is None:
2439 query = ""
2440 elif not in_query:
2441 query = self._query
2442 elif isinstance(in_query, Mapping):
2443 qm: MultiDict[QueryVariable] = MultiDict(self._parsed_query)
2444 qm.update(in_query)
2445 query = get_str_query_from_sequence_iterable(qm.items())
2446 elif isinstance(in_query, str):
2447 qstr: MultiDict[str] = MultiDict(self._parsed_query)
2448 qstr.update(query_to_pairs(in_query))
2449 query = get_str_query_from_iterable(qstr.items())
2450 elif isinstance(in_query, (bytes, bytearray, memoryview)):
2451 msg = "Invalid query type: bytes, bytearray and memoryview are forbidden"
2452 raise TypeError(msg)
2453 elif isinstance(in_query, Sequence):
2454 # We don't expect sequence values if we're given a list of pairs
2455 # already; only mappings like builtin `dict` which can't have the
2456 # same key pointing to multiple values are allowed to use
2457 # `_query_seq_pairs`.
2458 if TYPE_CHECKING:
2459 in_query = cast(Sequence[tuple[str | istr, SimpleQuery]], in_query)
2460 qs: MultiDict[SimpleQuery] = MultiDict(self._parsed_query)
2461 qs.update(in_query)
2462 query = get_str_query_from_iterable(qs.items())
2463 else:
2464 raise TypeError(
2465 "Invalid query type: only str, mapping or "
2466 "sequence of (key, value) pairs is allowed"
2467 )
2468 # An empty query stays when nothing is added, "None" removes it.
2469 if (empty := self._empty) and (query or in_query is None):
2470 empty &= ~EMPTY_QUERY
2471 url = from_parts_uncached(
2472 self._scheme,
2473 self._netloc,
2474 self._path,
2475 query,
2476 self._fragment,
2477 self._mode,
2478 empty,
2479 )
2480 if "_join_path" in self._cache:
2481 self._keep_join_path(url)
2482 return url
2484 def without_query_params(self, *query_params: str) -> "URL":
2485 """Remove some keys from query part and return new URL."""
2486 params_to_remove = set(query_params) & self.query.keys()
2487 if not params_to_remove:
2488 return self
2489 return self.with_query(
2490 tuple(
2491 (name, value)
2492 for name, value in self.query.items()
2493 if name not in params_to_remove
2494 )
2495 )
2497 def with_fragment(self, fragment: str | None) -> "URL":
2498 """Return a new URL with fragment replaced.
2500 Autoencode fragment if needed.
2502 Clear fragment to default if None is passed.
2504 """
2505 # N.B. doesn't cleanup query/fragment
2506 if fragment is None:
2507 raw_fragment = ""
2508 elif not isinstance(fragment, str):
2509 raise TypeError("Invalid fragment type")
2510 else:
2511 raw_fragment = FRAGMENT_QUOTER(fragment)
2512 if empty := self._empty:
2513 if self._fragment == raw_fragment and not empty & EMPTY_FRAGMENT:
2514 return self
2515 empty &= ~EMPTY_FRAGMENT
2516 elif self._fragment == raw_fragment:
2517 return self
2518 if "_join_path" in self._cache:
2519 # from_parts() may return a shared URL, which must not get it.
2520 url = from_parts_uncached(
2521 self._scheme,
2522 self._netloc,
2523 self._path,
2524 self._query,
2525 raw_fragment,
2526 self._mode,
2527 empty,
2528 )
2529 self._keep_join_path(url)
2530 return url
2531 return from_parts(
2532 self._scheme,
2533 self._netloc,
2534 self._path,
2535 self._query,
2536 raw_fragment,
2537 self._mode,
2538 empty,
2539 )
2541 def with_name(
2542 self,
2543 name: str,
2544 *,
2545 keep_query: bool = False,
2546 keep_fragment: bool = False,
2547 ) -> "URL":
2548 """Return a new URL with name (last part of path) replaced.
2550 Query and fragment parts are cleaned up.
2552 Name is encoded if needed.
2554 """
2555 # N.B. DOES cleanup query/fragment
2556 if not isinstance(name, str):
2557 raise TypeError("Invalid name type")
2558 if "/" in name:
2559 raise ValueError("Slash in name is not allowed")
2560 name = PATH_QUOTER(name)
2561 if name in (".", ".."):
2562 raise ValueError(". and .. values are forbidden")
2563 parts = list(self.raw_parts)
2564 netloc = self._netloc
2565 if netloc or self._empty & EMPTY_AUTHORITY:
2566 if len(parts) == 1:
2567 parts.append(name)
2568 else:
2569 parts[-1] = name
2570 parts[0] = "" # replace leading '/'
2571 else:
2572 parts[-1] = name
2573 if parts[0] == "/":
2574 parts[0] = "" # replace leading '/'
2576 query = self._query if keep_query else ""
2577 fragment = self._fragment if keep_fragment else ""
2578 if empty := self._empty:
2579 empty &= _kept_empty(keep_query, keep_fragment)
2580 return from_parts(
2581 self._scheme,
2582 netloc,
2583 "/".join(parts),
2584 query,
2585 fragment,
2586 self._mode,
2587 empty,
2588 )
2589 return from_parts(
2590 self._scheme, netloc, "/".join(parts), query, fragment, self._mode
2591 )
2593 def with_suffix(
2594 self,
2595 suffix: str,
2596 *,
2597 keep_query: bool = False,
2598 keep_fragment: bool = False,
2599 ) -> "URL":
2600 """Return a new URL with suffix (file extension of name) replaced.
2602 Query and fragment parts are cleaned up.
2604 suffix is encoded if needed.
2605 """
2606 if not isinstance(suffix, str):
2607 raise TypeError("Invalid suffix type")
2608 if suffix and not suffix[0] == "." or suffix == "." or "/" in suffix:
2609 raise ValueError(f"Invalid suffix {suffix!r}")
2610 name = self.raw_name
2611 if not name:
2612 raise ValueError(f"{self!r} has an empty name")
2613 old_suffix = self.raw_suffix
2614 suffix = PATH_QUOTER(suffix)
2615 name = name + suffix if not old_suffix else name[: -len(old_suffix)] + suffix
2616 if name in (".", ".."):
2617 raise ValueError(". and .. values are forbidden")
2618 parts = list(self.raw_parts)
2619 netloc = self._netloc
2620 if netloc or self._empty & EMPTY_AUTHORITY:
2621 if len(parts) == 1:
2622 parts.append(name)
2623 else:
2624 parts[-1] = name
2625 parts[0] = "" # replace leading '/'
2626 else:
2627 parts[-1] = name
2628 if parts[0] == "/":
2629 parts[0] = "" # replace leading '/'
2631 query = self._query if keep_query else ""
2632 fragment = self._fragment if keep_fragment else ""
2633 if empty := self._empty:
2634 empty &= _kept_empty(keep_query, keep_fragment)
2635 return from_parts(
2636 self._scheme,
2637 netloc,
2638 "/".join(parts),
2639 query,
2640 fragment,
2641 self._mode,
2642 empty,
2643 )
2644 return from_parts(
2645 self._scheme, netloc, "/".join(parts), query, fragment, self._mode
2646 )
2648 def join(self, url: "URL") -> "URL":
2649 """Join URLs
2651 Construct a full (“absolute”) URL by combining a “base URL”
2652 (self) with another URL (url).
2654 Informally, this uses components of the base URL, in
2655 particular the addressing scheme, the network location and
2656 (part of) the path, to provide missing components in the
2657 relative URL.
2659 """
2660 if type(url) is not URL:
2661 raise TypeError("url should be URL")
2663 scheme = url._scheme or self._scheme
2664 # A reference with a scheme is used as is (RFC 3986 section 5.2.2).
2665 # In WHATWG mode "http:g" is still resolved against an http base, the
2666 # backward-compatible behavior RFC 3986 section 5.4.2 permits.
2667 if scheme != self._scheme or (
2668 url._scheme and (scheme not in USES_RELATIVE or self._mode is not _WHATWG)
2669 ):
2670 if (
2671 not url._netloc
2672 and self._mode is _WHATWG
2673 and scheme != self._scheme
2674 and scheme in SCHEME_REQUIRES_HOST
2675 ):
2676 # "http:" and "http:?q" are references to an http base only;
2677 # against any other base WHATWG reads them as absolute URLs
2678 # without a host and fails.
2679 raise ValueError(
2680 f"Invalid URL: host is required for absolute urls"
2681 f" with the {scheme} scheme"
2682 )
2683 # The result follows the base URL's compatibility mode.
2684 return url if url._mode is self._mode else URL(url, mode=self._mode)
2686 if (
2687 not self._netloc
2688 and self._mode is _WHATWG
2689 and not url._scheme
2690 and self._scheme
2691 and self._scheme not in SPECIAL_SCHEMES
2692 and not self._empty & EMPTY_AUTHORITY
2693 and self._path[:1] != "/"
2694 and (
2695 url._netloc
2696 or url._path
2697 or url._query
2698 or url._empty & ~EMPTY_FRAGMENT
2699 or not (url._fragment or url._empty & EMPTY_FRAGMENT)
2700 )
2701 ):
2702 # WHATWG resolves only a fragment against a base with an opaque
2703 # path, such as "sc:sd" or "mailto:a@b"; RFC 3986 section 5.2.2
2704 # merges the paths ("i" against "sc:sd" is "sc:i").
2705 raise ValueError(
2706 f"Invalid URL: cannot resolve {str(url)!r} against"
2707 f" {str(self)!r}, which has an opaque path"
2708 )
2710 if "_join_path" in url._cache:
2711 join_path = url._cache["_join_path"]
2712 if url._netloc and url._scheme:
2713 # "http:g" was parsed in WHATWG mode as "http://g/", but
2714 # against an http base it is the relative reference "g".
2715 url = from_parts(
2716 scheme,
2717 "",
2718 join_path,
2719 url._query,
2720 url._fragment,
2721 url._mode,
2722 url._empty,
2723 )
2724 elif url._scheme:
2725 # "file:a/../b", a rootless file path with dot segments, or
2726 # "file:" and "file:?q" without a path, see _encode_url().
2727 url = from_parts(
2728 scheme,
2729 "",
2730 join_path,
2731 url._query,
2732 url._fragment,
2733 url._mode,
2734 url._empty & ~EMPTY_AUTHORITY,
2735 )
2736 elif scheme == "file" and self._mode is _WHATWG:
2737 # WHATWG reads a reference against a file URL in the file
2738 # state, as it reads one with the "file:" scheme; there "C|"
2739 # is the drive letter "C:" and "//C:" is no authority.
2740 # "#" keeps a space before the query, which URL() would strip.
2741 ref = encode_url(f"file:{join_path}#")
2742 url = from_parts(
2743 scheme,
2744 ref._netloc,
2745 ref._path,
2746 url._query,
2747 url._fragment,
2748 url._mode,
2749 ref._empty & EMPTY_AUTHORITY | url._empty & ~EMPTY_AUTHORITY,
2750 )
2751 elif _is_pipe_drive_authority(join_path):
2752 raise ValueError(
2753 f"Invalid URL: {join_path!r} has a drive letter for an"
2754 f" authority, which only a file URL accepts"
2755 )
2756 elif url._empty & EMPTY_AUTHORITY and scheme in SCHEME_REQUIRES_HOST:
2757 # "///h/p" with "%" or "[" in it, see _encode_url().
2758 return self._join_empty_host(url, scheme, join_path)
2759 elif scheme in SPECIAL_SCHEMES and self._mode is _WHATWG:
2760 # A relative reference with "\\" (see _backslashes()); the
2761 # query and the fragment may have changed since it was parsed.
2762 ref = encode_url(join_path)
2763 if not ref._netloc and ref._empty & EMPTY_AUTHORITY:
2764 # "/\\/h/p" is "///h/p", see _join_empty_host().
2765 return self._join_empty_host(
2766 url, scheme, ref._cache.get("_join_path", ref._path)
2767 )
2768 url = from_parts(
2769 ref._scheme,
2770 ref._netloc,
2771 ref._path,
2772 url._query,
2773 url._fragment,
2774 url._mode,
2775 ref._empty & EMPTY_AUTHORITY | url._empty,
2776 )
2778 if url._empty or self._empty:
2779 return self._join_with_empty(url, scheme)
2781 if join_netloc := url._netloc:
2782 join_path = url._path
2783 join_empty = 0
2784 if not url._scheme or url._mode is not _WHATWG:
2785 # A schemeless or RFC-mode reference had no WHATWG host check.
2786 join_netloc, join_path, join_empty = _moved_parts(
2787 url, scheme, self._mode, 0
2788 )
2789 return from_parts(
2790 scheme,
2791 join_netloc,
2792 join_path,
2793 url._query,
2794 url._fragment,
2795 self._mode,
2796 join_empty,
2797 )
2799 orig_path = self._path
2800 if join_path := url._path:
2801 if scheme == "file" and self._mode is _WHATWG:
2802 path = _file_join_path(orig_path, join_path)
2803 else:
2804 if join_path[0] == "/":
2805 path = join_path
2806 elif not orig_path:
2807 path = f"/{join_path}" if self._netloc else join_path
2808 elif orig_path[-1] == "/":
2809 path = f"{orig_path}{join_path}"
2810 else:
2811 # Merge on the encoded base path, dropping its last
2812 # segment, so percent-encoded delimiters in the base are
2813 # kept as is.
2814 path = orig_path[: orig_path.rfind("/") + 1] + join_path
2815 if "." in path:
2816 path = (
2817 normalize_path(path)
2818 if self._netloc or self._mode is _WHATWG or not scheme
2819 # RFC 3986 section 5.2.4, also for a path without a
2820 # leading "/": "sd/../i" is "/i".
2821 else remove_dot_segments(path)
2822 )
2823 else:
2824 path = orig_path
2826 return from_parts(
2827 scheme,
2828 self._netloc,
2829 path,
2830 url._query if join_path or url._query else self._query,
2831 url._fragment,
2832 self._mode,
2833 )
2835 def _join_with_empty(self, url: "URL", scheme: str) -> "URL":
2836 """join() for URLs with empty components; the same algorithm as
2837 join() takes for other URLs, plus the empty component rules."""
2838 join_empty = url._empty
2839 if url._netloc or join_empty & EMPTY_AUTHORITY:
2840 if not url._netloc and scheme in SCHEME_REQUIRES_HOST:
2841 # A path edited after parsing, such as by with_name(), has
2842 # lost the authority as written, see _encode_url(); its "["
2843 # stays "%5B", which the host parser rejects.
2844 return self._join_empty_host(url, scheme, url._path)
2845 join_netloc = url._netloc
2846 join_path = url._path
2847 if not url._scheme or url._mode is not _WHATWG:
2848 join_netloc, join_path, join_empty = _moved_parts(
2849 url, scheme, self._mode, join_empty
2850 )
2851 return from_parts(
2852 scheme,
2853 join_netloc,
2854 join_path,
2855 url._query,
2856 url._fragment,
2857 self._mode,
2858 join_empty,
2859 )
2861 orig_path = self._path
2862 has_authority = self._netloc or self._empty & EMPTY_AUTHORITY
2863 if join_path := url._path:
2864 if scheme == "file" and self._mode is _WHATWG:
2865 path = _file_join_path(orig_path, join_path)
2866 else:
2867 if join_path[0] == "/":
2868 path = join_path
2869 elif not orig_path:
2870 path = f"/{join_path}" if has_authority else join_path
2871 elif orig_path[-1] == "/":
2872 path = f"{orig_path}{join_path}"
2873 else:
2874 # Merge on the encoded base path, dropping its last
2875 # segment, so percent-encoded delimiters in the base are
2876 # kept as is.
2877 path = orig_path[: orig_path.rfind("/") + 1] + join_path
2878 if "." in path:
2879 path = (
2880 normalize_path(path)
2881 if has_authority or self._mode is _WHATWG or not scheme
2882 # RFC 3986 section 5.2.4, also for a path without a
2883 # leading "/": "sd/../i" is "/i".
2884 else remove_dot_segments(path)
2885 )
2886 else:
2887 path = orig_path
2889 # The query comes from the reference when it has a path or a query,
2890 # and the fragment always does, per RFC 3986 section 5.2.2.
2891 if join_path or url._query or join_empty & EMPTY_QUERY:
2892 query = url._query
2893 query_empty = join_empty & EMPTY_QUERY
2894 else:
2895 query = self._query
2896 query_empty = self._empty & EMPTY_QUERY
2897 return from_parts(
2898 scheme,
2899 self._netloc,
2900 path,
2901 query,
2902 url._fragment,
2903 self._mode,
2904 self._empty & EMPTY_AUTHORITY | query_empty | join_empty & EMPTY_FRAGMENT,
2905 )
2907 def _join_empty_host(self, url: "URL", scheme: str, path: str) -> "URL":
2908 """join() for a reference with an empty authority, such as "///a",
2909 against a base whose scheme needs a host.
2911 RFC 3986 section 5.2.2 takes the empty authority of the reference,
2912 which RFC 9110 section 4.2.1 rejects for http, so RFC 3986 mode
2913 fails as it does on "http:///a". The WHATWG parser reads such a
2914 reference in the special authority ignore slashes state, where the
2915 host follows the slashes: "///a/b" is "http://a/b" and "//" fails.
2916 path is the path of the reference, as written when the path quoter
2917 would have changed its authority part, see _encode_url().
2918 """
2919 if self._mode is not _WHATWG:
2920 raise ValueError(
2921 f"Invalid URL: joining {str(url)!r} gives an empty host,"
2922 f" which the {scheme} scheme does not allow"
2923 )
2924 # "#" keeps a space before the query, which URL() would strip.
2925 joined = URL(f"{scheme}://{path.lstrip('/')}#")
2926 return from_parts(
2927 scheme,
2928 joined._netloc,
2929 joined._path,
2930 url._query,
2931 url._fragment,
2932 self._mode,
2933 url._empty & ~EMPTY_AUTHORITY,
2934 )
2936 def joinpath(self, *other: str, encoded: bool = False) -> "URL":
2937 """Return a new URL with the elements in other appended to the path."""
2938 return self._make_child(other, encoded=encoded)
2940 def human_repr(self) -> str:
2941 """Return decoded human readable string for URL representation."""
2942 user = human_quote(self.user, "#/:?@[]\\")
2943 password = human_quote(self.password, "#/:?@[]\\")
2944 if (host := self.host) and ":" in host and host[-1] != "]":
2945 # An IPv6 zone identifier stays percent-encoded, which is the
2946 # only form the parser accepts.
2947 host = f"[{self.raw_host}]" if "%" in host else f"[{host}]"
2948 elif (
2949 self._scheme not in SPECIAL_SCHEMES
2950 and host
2951 and host is not (raw_host := self.raw_host)
2952 ):
2953 # A decoded opaque host that the parser would reject, such as
2954 # "a\uff0fb", whose NFKC form has a "/", stays percent-encoded.
2955 try:
2956 _check_netloc(host)
2957 except ValueError:
2958 host = raw_host
2959 # WHATWG mode reads "\\" in the path as "/", so it stays encoded.
2960 # An empty path stays empty where str() keeps it so, see __str__().
2961 if (
2962 self._path
2963 or self._scheme in SPECIAL_SCHEMES
2964 or not self._scheme
2965 or self._scheme.lower() in SPECIAL_SCHEMES
2966 ):
2967 path = human_quote(self.path, "#?\\")
2968 else:
2969 path = ""
2970 if TYPE_CHECKING:
2971 assert path is not None
2972 if not self._scheme and not self._netloc:
2973 path = _encode_relative_colon(path, self._mode)
2974 query_string = "&".join(
2975 "{}={}".format(human_quote(k, "#&+;="), human_quote(v, "#&+;="))
2976 for k, v in self.query.items()
2977 )
2978 fragment = human_quote(self.fragment, "")
2979 if TYPE_CHECKING:
2980 assert fragment is not None
2981 if host is None and self._netloc:
2982 host = "" # an authority without a host, e.g. "user@:8080"
2983 netloc = make_netloc(user, password, host, self.explicit_port)
2984 if empty := self._str_empty:
2985 return unsplit_result_empty(
2986 self._scheme, netloc, path, query_string, fragment, empty
2987 )
2988 return unsplit_result(self._scheme, netloc, path, query_string, fragment)
2990 if HAS_PYDANTIC:
2991 # Borrowed from https://docs.pydantic.dev/latest/concepts/types/#handling-third-party-types
2992 @classmethod
2993 def __get_pydantic_json_schema__(
2994 cls,
2995 core_schema: "CoreSchema",
2996 handler: "GetJsonSchemaHandler",
2997 ) -> "JsonSchemaValue":
2998 field_schema: dict[str, Any] = {}
2999 field_schema.update(type="string", format="uri")
3000 return field_schema
3002 @classmethod
3003 def __get_pydantic_core_schema__(
3004 cls,
3005 source_type: type[Self] | type[str],
3006 handler: "GetCoreSchemaHandler",
3007 ) -> "CoreSchema":
3008 # Lazy import: pulling in pydantic_core at module load time
3009 # increases yarl's import cost 3-7x for users who don't use
3010 # pydantic. Keep this import function-scoped.
3011 from pydantic_core import core_schema # noqa: PLC0415
3013 from_str_schema = core_schema.chain_schema(
3014 [
3015 core_schema.str_schema(),
3016 core_schema.no_info_plain_validator_function(URL),
3017 ]
3018 )
3020 return core_schema.json_or_python_schema(
3021 json_schema=from_str_schema,
3022 python_schema=core_schema.union_schema(
3023 [
3024 # check if it's an instance first before doing any further work
3025 core_schema.is_instance_schema(URL),
3026 from_str_schema,
3027 ]
3028 ),
3029 serialization=core_schema.plain_serializer_function_ser_schema(str),
3030 )
3033_DEFAULT_IDNA_SIZE = 256
3034_DEFAULT_ENCODE_SIZE = 512
3037@lru_cache(_DEFAULT_IDNA_SIZE)
3038def _idna_decode(raw: str) -> str:
3039 try:
3040 return idna.decode(raw.encode("ascii"))
3041 except UnicodeError: # e.g. '::1'
3042 if _WHATWG_ONLY_HOST_RE.search(raw):
3043 # IDNA2008 rejects the whole host for one label with "{", see
3044 # _idna_encode_labels().
3045 return ".".join(map(_idna_decode_label, raw.split(".")))
3046 return raw.encode("ascii").decode("idna")
3049def _idna_decode_label(label: str) -> str:
3050 """Decode a label of a host that has '"', "`", "{" or "}".
3052 A label with them is decoded from Punycode without the IDNA checks, as
3053 _idna_encode_labels() encodes it; IDNA2003 would map "faß{b}" to
3054 "fass{b}" and fail to round-trip.
3055 """
3056 if not _WHATWG_ONLY_HOST_RE.search(label):
3057 return _idna_decode(label)
3058 if label[:4] == "xn--":
3059 try:
3060 return label[4:].encode("ascii").decode("punycode")
3061 except UnicodeError:
3062 pass
3063 return label
3066@lru_cache(_DEFAULT_IDNA_SIZE)
3067def _idna_encode(host: str) -> tuple[str, bool]:
3068 """Encode a host with IDNA2008 and UTS #46, or else with IDNA2003.
3070 The flag tells if the IDNA2003 fallback was needed, which RFC 3986 mode
3071 rejects.
3072 """
3073 try:
3074 return idna.encode(host, uts46=True).decode("ascii"), False
3075 except UnicodeError:
3076 try:
3077 mapped = idna.uts46_remap(host, std3_rules=False, transitional=False)
3078 except UnicodeError:
3079 mapped = ""
3080 if _WHATWG_ONLY_HOST_RE.search(mapped):
3081 return _idna_encode_labels(mapped), True
3082 return host.encode("idna").decode("ascii"), True
3085def _idna_encode_labels(mapped: str) -> str:
3086 """Encode a host mapped with UTS #46 that has '"', "`", "{" or "}".
3088 IDNA2008 rejects a label with them, which UTS #46 accepts without
3089 UseSTD3ASCIIRules, as WHATWG uses it. Such a label is encoded with
3090 Punycode, the other labels one by one; IDNA2003, the fallback for the
3091 whole host, would turn "faß" into "fass" where WHATWG has "xn--fa-hia".
3092 """
3093 return ".".join(
3094 label
3095 if label.isascii()
3096 else _idna_encode(label)[0]
3097 if not _WHATWG_ONLY_HOST_RE.search(label)
3098 else _punycode_label(label)
3099 for label in mapped.split(".")
3100 )
3103def _punycode_label(label: str) -> str:
3104 """Encode a label with Punycode after the checks UTS #46 makes.
3106 The label must not start with a combining mark and must follow the
3107 bidi rule of RFC 5893, as WHATWG has CheckBidi set: "x{א}" fails.
3108 """
3109 idna.check_initial_combiner(label)
3110 idna.check_bidi(label)
3111 return "xn--" + label.encode("punycode").decode("ascii")
3114_HEX_DIGITS = frozenset("0123456789abcdefABCDEF")
3115# A host that ends in a number ends in one of these; most hosts do not.
3116_NUMERIC_HOST_TAIL = frozenset("0123456789abcdefxABCDEFX.")
3119def _moved_url(url: "URL", scheme: str, mode: Mode) -> "URL":
3120 """Return url moved under scheme and mode, see _moved_parts()."""
3121 netloc, path, empty = _moved_parts(url, scheme, mode, url._empty)
3122 if path and not url._path and not (url._netloc or url._empty & EMPTY_AUTHORITY):
3123 # "file:" and "file:?q" became "file:///" and "file:///?q" in WHATWG
3124 # mode, but as a reference they keep the path of a file base, as
3125 # the parsed ones do, see _encode_url(). The cache of a from_parts()
3126 # URL is shared, so this one gets its own.
3127 moved = from_parts_uncached(
3128 scheme, netloc, path, url._query, url._fragment, mode, empty
3129 )
3130 moved._cache["_join_path"] = ""
3131 return moved
3132 return from_parts(scheme, netloc, path, url._query, url._fragment, mode, empty)
3135def _moved_parts(
3136 url: "URL", scheme: str, mode: Mode, empty: int
3137) -> tuple[str, str, int]:
3138 """Return the netloc, path and empty mask of url moved under scheme and mode.
3140 with_scheme(), join() and a change of mode can move a host that was
3141 parsed under another scheme or mode into a special-scheme WHATWG URL,
3142 where the host is percent-decoded and lowercased and one that ends in a
3143 number is parsed as an IPv4 address. The opaque host of a non-special
3144 WHATWG URL is kept under another non-special scheme, and validated as a
3145 domain under a special one. An RFC 3986 mode URL moved into WHATWG mode
3146 also loses an empty password and an empty userinfo, see
3147 _drop_empty_password(). A file URL also loses "localhost" and must not
3148 have userinfo or a port, see _check_whatwg_file_authority(), and reads a
3149 drive letter in the authority as the start of the path.
3150 """
3151 netloc = url._netloc
3152 if mode is not _WHATWG:
3153 return netloc, url._path, empty
3154 if not netloc:
3155 if not url._path and scheme == "file":
3156 # A file URL has an authority and a path, "/" at least.
3157 return "", "/", empty | EMPTY_AUTHORITY
3158 return netloc, url._path, empty
3159 host = url.raw_host
3160 if (scheme in SPECIAL_SCHEMES or "[" in netloc) and host:
3161 # The WHATWG host is cached with the encoding of the host. An opaque
3162 # host was not checked as a domain, see _encode_opaque_host().
3163 encoded, whatwg_host, _ = _encode_host(
3164 host,
3165 url._mode is _WHATWG
3166 and url._scheme not in SPECIAL_SCHEMES
3167 and _is_opaque_host(url._scheme, host),
3168 )
3169 if whatwg_host is not encoded:
3170 host = _whatwg_special_host(encoded, whatwg_host)
3171 netloc = make_netloc(
3172 url.raw_user, url.raw_password, host, url.explicit_port
3173 )
3174 elif encoded != host and "[" not in netloc:
3175 # An opaque host is not lowercased, see _encode_opaque_host().
3176 host = encoded
3177 netloc = make_netloc(
3178 url.raw_user, url.raw_password, host, url.explicit_port
3179 )
3180 if url._mode is not _WHATWG and "@" in netloc:
3181 # The userinfo of RFC 3986 mode can have ";" and "=", which WHATWG
3182 # mode encodes, see _whatwg_requote_userinfo().
3183 userinfo, _, hostport = netloc.rpartition("@")
3184 user, colon, password = userinfo.partition(":")
3185 user = _whatwg_requote_userinfo(user)
3186 password = _whatwg_requote_userinfo(password)
3187 netloc = f"{user}{colon}{password}@{hostport}"
3188 if ":@" in netloc or netloc[0] == "@":
3189 netloc = _drop_empty_password(netloc)
3190 if scheme == "file":
3191 # The netloc as written: the host above is lowercased.
3192 if is_drive_letter(url._netloc):
3193 path = _whatwg_file_drive(url._netloc, url._path)
3194 if "." in path:
3195 path = normalize_file_path(path)
3196 return "", path, empty | EMPTY_AUTHORITY
3197 _check_whatwg_file_authority(netloc)
3198 if host == "localhost":
3199 return "", url._path or "/", empty | EMPTY_AUTHORITY
3200 return netloc, url._path or "/", empty
3201 return netloc, url._path, empty
3204def _drop_empty_password(netloc: str) -> str:
3205 """Drop an empty password from an authority, as WHATWG writes it.
3207 WHATWG writes ":" only before a password that is not empty, and "@" only
3208 after a username or a password that is not empty: "a:@h" is "a@h" and
3209 ":@h" and "@h" are "h".
3210 """
3211 userinfo, _, hostinfo = netloc.rpartition("@")
3212 user, _, password = userinfo.partition(":")
3213 if password:
3214 return netloc
3215 return f"{user}@{hostinfo}" if user else hostinfo
3218def _whatwg_special_host(host: str, whatwg_host: str | None) -> str:
3219 """Return the host WHATWG parses a special-scheme host as.
3221 The WHATWG URL Standard percent-decodes the host and parses a host whose
3222 last label is a number, like "0x7f.1" or "example.123", as an IPv4
3223 address. It fails when the decoded host is not a valid domain, a label
3224 is not a number or a number is out of range. RFC 3986 mode accepts all
3225 of them as a reg-name.
3226 """
3227 if whatwg_host is None:
3228 if host[0] == "[":
3229 raise ValueError(
3230 f"Host {host!r} is an IPvFuture address, which the WHATWG URL"
3231 " Standard does not have"
3232 )
3233 if "%" in host:
3234 raise ValueError(f"Host {host!r} is not a valid host once percent-decoded")
3235 raise ValueError(f"Host {host!r} ends in a number but is not an IPv4 address")
3236 return whatwg_host
3239# The forbidden host code points of the WHATWG URL Standard, which the
3240# opaque-host parser rejects.
3241_FORBIDDEN_HOST_RE = re.compile(r"[\x00\t\n\r #/:<>?@\[\\\]^|]")
3242# The C0 control percent-encode set, which the opaque-host parser encodes.
3243_C0_CONTROL_PERCENT_ENCODE_RE = re.compile(r"[^\x20-\x7e]+")
3244# The percent-encoded octets of a non-ASCII host, see _decode_opaque_host().
3245_NON_ASCII_ESCAPES_RE = re.compile(r"(?:%[89A-Fa-f][0-9A-Fa-f])+")
3248def _is_opaque_host(scheme: str, host: str) -> bool:
3249 """Tell if the WHATWG mode host of a URL whose scheme is not special is
3250 an opaque host.
3252 The WHATWG URL Standard parses the host of a non-special URL as an opaque
3253 host, unless it is an IPv6 address in brackets. A relative reference has
3254 no scheme yet, its host is encoded as a domain. The callers check that
3255 the scheme is not special first, which keeps the call off the hot path;
3256 URL.build() keeps the scheme as given, so "HTTP" is checked here.
3257 """
3258 return (
3259 scheme != ""
3260 and ":" not in host
3261 and host[:1] != "["
3262 and scheme.lower() not in SPECIAL_SCHEMES
3263 )
3266def _encode_opaque_host(host: str) -> str:
3267 """Encode a host with the WHATWG opaque-host parser.
3269 The host is neither lowercased nor encoded with IDNA: it is rejected if
3270 it has a forbidden host code point and its C0 controls and non-ASCII
3271 code points are percent-encoded, so "sc://Ñ" is "sc://%C3%91". A "%"
3272 that does not start a percent-encoded octet is kept. RFC 3986 mode and
3273 special schemes encode a domain instead, see _encode_host().
3274 """
3275 if invalid := _FORBIDDEN_HOST_RE.search(host):
3276 raise ValueError(
3277 f"Host {host!r} cannot contain {invalid.group()!r}"
3278 f" (at position {invalid.start()})"
3279 )
3280 if host.isascii() and host.isprintable():
3281 return host
3282 return _C0_CONTROL_PERCENT_ENCODE_RE.sub(
3283 lambda match: quote(match.group(), safe=""), host
3284 )
3287def _decode_opaque_host(raw: str) -> str:
3288 """Decode the non-ASCII code points of an opaque host.
3290 The percent-encoded ASCII characters stay encoded, as does an octet that
3291 is not UTF-8, so that passing the result to with_host() gives the same
3292 host back.
3293 """
3294 if "%" not in raw:
3295 return raw
3296 return _NON_ASCII_ESCAPES_RE.sub(lambda match: UNQUOTER(match.group()), raw)
3299# A number with more significant digits than this in its base is at least
3300# 2**32, out of range for any part of an IPv4 address. Checking the length
3301# first keeps int() away from huge labels, which Python refuses to convert.
3302_IPV4_MAX_DIGITS = {16: 8, 8: 11, 10: 10}
3303_IPV4_OUT_OF_RANGE = 2**32
3306def _ipv4_number(label: str) -> int | None:
3307 """Parse a WHATWG IPv4 number: decimal, 0x hex or 0 octal."""
3308 if label[:2] in ("0x", "0X"):
3309 digits, base = label[2:], 16
3310 if not all(c in _HEX_DIGITS for c in digits):
3311 return None
3312 elif label == "" or not label.isascii() or not label.isdigit():
3313 return None
3314 elif label[0] == "0" and len(label) > 1:
3315 digits, base = label[1:], 8
3316 if "8" in digits or "9" in digits:
3317 return None
3318 else:
3319 digits, base = label, 10
3320 digits = digits.lstrip("0")
3321 if len(digits) > _IPV4_MAX_DIGITS[base]:
3322 return _IPV4_OUT_OF_RANGE
3323 return int(digits, base) if digits else 0
3326def _whatwg_numeric_host(host: str) -> str | None:
3327 """Return host as WHATWG sees it for a special scheme.
3329 A host that does not end in a number is returned as is. One that does
3330 is run through the WHATWG IPv4 parser: the result is the dotted-quad
3331 serialization, or None when parsing fails. The host must not be
3332 percent-encoded; see _whatwg_decoded_host.
3333 """
3334 if host[-1] not in _NUMERIC_HOST_TAIL:
3335 return host
3336 labels = host.split(".")
3337 if labels[-1] == "" and len(labels) > 1:
3338 labels.pop()
3339 last = labels[-1]
3340 if not (last.isascii() and last.isdigit()) and _ipv4_number(last) is None:
3341 return host
3342 if len(labels) > 4:
3343 return None
3344 numbers = [_ipv4_number(label) for label in labels]
3345 *head, tail = numbers
3346 if tail is None or tail >= 256 ** (5 - len(numbers)):
3347 return None
3348 address = tail
3349 for i, number in enumerate(head):
3350 if number is None or number > 255:
3351 return None
3352 address += number << (8 * (3 - i))
3353 return ".".join(str(address >> shift & 255) for shift in (24, 16, 8, 0))
3356def _whatwg_decoded_host(host: str) -> str | None:
3357 """Return a percent-encoded host as WHATWG sees it for a special scheme.
3359 The WHATWG host parser percent-decodes the host, decodes the bytes as
3360 UTF-8 with U+FFFD for invalid sequences and runs domain to ASCII, so
3361 "%e2%98%83" is "xn--n3h" and "ho%00st" fails. The result goes on to
3362 the IPv4 parser like any other host. None means WHATWG fails, or gives
3363 a host that yarl rejects anyway, like one with a default-ignorable code
3364 point.
3365 """
3366 decoded = unquote_to_bytes(host).decode("utf-8", "replace")
3367 if decoded.isascii():
3368 decoded = decoded.lower()
3369 elif _DEFAULT_IGNORABLE_RE.search(decoded):
3370 return None
3371 else:
3372 try:
3373 decoded = _idna_encode(decoded)[0]
3374 except UnicodeError:
3375 return None
3376 if "%" in decoded or _NOT_WHATWG_DOMAIN_RE.search(decoded):
3377 return None
3378 return _whatwg_numeric_host(decoded)
3381@lru_cache(_DEFAULT_ENCODE_SIZE)
3382def _encode_host(
3383 host: str, validate_host: bool, *, reject_empty_zone: bool = True
3384) -> tuple[str, str | None, bool]:
3385 """Encode host part of URL.
3387 Next to the encoded host, return the host WHATWG sees for a special
3388 scheme, which is the encoded host itself unless the host is
3389 percent-encoded (see _whatwg_decoded_host), ends in a number (see
3390 _whatwg_numeric_host) or is an IPv6 address not written compressed, and
3391 a flag that tells if RFC 3986 mode rejects the host: IDNA2008 could not
3392 encode it, or it has a character that a WHATWG domain can have and an
3393 RFC 3986 reg-name cannot, like "{". Both are computed here so that the
3394 checks are cached with the encoding.
3395 """
3396 # If the host ends with a digit or contains a colon, its likely
3397 # an IP address.
3398 if host and (host[-1].isdigit() or ":" in host):
3399 # RFC 6874 spells the IPv6 zone separator as the percent-encoded
3400 # ``%25``; the builders also take a bare ``%`` followed by the
3401 # decoded zone (e.g. ``with_host("fe80::1%1")``).
3402 part = "%25" if host.startswith("%25", host.find("%")) else "%"
3403 raw_ip, sep, zone = host.partition(part)
3404 # If it looks like an IP, we check with _ip_compressed_version
3405 # and fall-through if its not an IP address. This is a performance
3406 # optimization to avoid parsing IP addresses as much as possible
3407 # because it is orders of magnitude slower than almost any other
3408 # operation this library does.
3409 # Might be an IP address, check it
3410 #
3411 # IP Addresses can look like:
3412 # https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
3413 # - 127.0.0.1 (last character is a digit)
3414 # - 2001:db8::ff00:42:8329 (contains a colon)
3415 # - 2001:db8::ff00:42:8329%eth0 (contains a colon)
3416 # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should
3417 # have been removed before it gets here)
3418 # Rare IP Address formats are not supported per:
3419 # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4
3420 #
3421 # IP parsing is slow, so its wrapped in an LRU
3422 try:
3423 ip = ip_address(raw_ip)
3424 except ValueError:
3425 pass
3426 else:
3427 if sep and validate_host:
3428 if ip.version == 6:
3429 if sep == "%" and not _ZONE_ID_UNSAFE_RE.search(zone):
3430 # A zone identifier given after a bare "%" is
3431 # decoded, as host returns it; RFC 6874 writes it
3432 # as "%25" followed by the percent-encoded zone.
3433 # Percent-encoded octets are kept as they are.
3434 zone = _ZONE_ID_REQUOTE_RE.sub(_quote_match, zone)
3435 sep = "%25"
3436 bad_zone = ZONE_ID_RE.fullmatch(zone) is None
3437 else:
3438 bad_zone = _ZONE_ID_UNSAFE_RE.search(zone) is not None
3439 if bad_zone or (reject_empty_zone and not zone):
3440 raise ValueError("Invalid characters in zone identifier")
3441 # These checks should not happen in the
3442 # LRU to keep the cache size small
3443 host = ip.compressed
3444 if ip.version == 6:
3445 # RFC 3986 keeps an IPv6 address as written, lowercased per
3446 # section 6.2.2.1; WHATWG serializes it compressed, which is
3447 # the WHATWG host here, see _whatwg_special_host().
3448 host = f"[{host}{sep}{zone}]" if sep else f"[{host}]"
3449 if (written := raw_ip.lower()) != ip.compressed:
3450 return f"[{written}{sep}{zone}]", host, False
3451 elif sep:
3452 # WHATWG has no zone identifiers: "%" after an IPv4 address
3453 # is percent-encoding in the host.
3454 host = f"{host}{sep}{zone}"
3455 return host, _whatwg_decoded_host(host), False
3456 return host, host, False
3458 # IDNA encoding is slow, skip it for ASCII-only strings
3459 if host.isascii():
3460 # Check for invalid characters explicitly; _idna_encode() does this
3461 # for non-ascii host names.
3462 host = host.lower()
3463 if host[:2] == "[v":
3464 # Only an IPvFuture address keeps its brackets up to here, see
3465 # split_netloc(). WHATWG has no IPvFuture, hence None.
3466 end = len(host) - 1
3467 if host[end] != "]" or IP_FUTURE_RE.fullmatch(host, 1, end) is None:
3468 raise ValueError(f"IPvFuture address is invalid: {host!r}")
3469 return host, None, False
3470 not_rfc = False
3471 if validate_host and (invalid := NOT_REG_NAME.search(host)):
3472 if invalid := _NOT_WHATWG_DOMAIN_RE.search(host, invalid.start()):
3473 value, pos, extra = invalid.group(), invalid.start(), ""
3474 if value == "@" or (value == ":" and "@" in host[pos:]):
3475 # this looks like an authority string
3476 extra = (
3477 ", if the value includes a username or password, "
3478 "use 'authority' instead of 'host'"
3479 )
3480 raise ValueError(
3481 f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}"
3482 ) from None
3483 # The WHATWG URL Standard keeps '"', "`", "{" and "}" in a
3484 # domain, RFC 3986 rejects them, see _rfc_host_error().
3485 not_rfc = True
3486 if "%" in host:
3487 return host, _whatwg_decoded_host(host), not_rfc
3488 return host, host and _whatwg_numeric_host(host), not_rfc
3490 # IDNA/UTS-46 mapping silently deletes default-ignorable code points, which
3491 # would turn e.g. ``e<ZWSP>vil.com`` into ``evil.com``, a different host
3492 # than the string the caller supplied. Reject them on every path (this runs
3493 # regardless of ``validate_host`` since the plain ``URL(str)`` constructor
3494 # encodes with ``validate_host=False``) so the parsed host cannot diverge
3495 # from the input, matching idna/httpx/urllib3.
3496 if invalid := _DEFAULT_IGNORABLE_RE.search(host):
3497 raise ValueError(
3498 f"Host {host!r} cannot contain {invalid.group()!r} "
3499 f"(at position {invalid.start()})"
3500 ) from None
3501 encoded, not_rfc = _idna_encode(host)
3502 # IDNA uses NFKC equivalence, so normalization can expand a non-ascii
3503 # character into an ASCII delimiter (e.g. the fullwidth solidus U+FF0F
3504 # becomes '/'). split_url's _check_netloc screens '/?#@:%' but not '[',
3505 # ']' or '\\', so a host like ``exa［mple`` (fullwidth '[') slips
3506 # through the parser and _idna_encode turns it into ``exa[mple``, a netloc
3507 # that str(url) then renders but yarl itself rejects on re-parse. Run the
3508 # check on every path (like the default-ignorable check above, which also
3509 # ignores validate_host) so the parsed host cannot diverge from what the
3510 # serialized URL means.
3511 if invalid := NOT_REG_NAME.search(encoded):
3512 if invalid := _NOT_WHATWG_DOMAIN_RE.search(encoded, invalid.start()):
3513 raise ValueError(
3514 f"Host {host!r} cannot contain {invalid.group()!r} "
3515 f"after IDNA normalization to {encoded!r}"
3516 ) from None
3517 not_rfc = True
3518 # The host can be percent-encoded next to the non-ASCII code points, or
3519 # IDNA can map a code point to "%", like the fullwidth percent sign
3520 # U+FF05, which WHATWG then rejects.
3521 if "%" in encoded:
3522 return encoded, _whatwg_decoded_host(host), not_rfc
3523 return encoded, _whatwg_numeric_host(encoded), not_rfc
3526@rewrite_module
3527def cache_clear() -> None:
3528 """Clear all LRU caches."""
3529 _idna_encode.cache_clear()
3530 _idna_decode.cache_clear()
3531 _encode_host.cache_clear()
3534@rewrite_module
3535def cache_info() -> CacheInfo:
3536 """Report cache statistics."""
3537 return {
3538 "idna_encode": _idna_encode.cache_info(),
3539 "idna_decode": _idna_decode.cache_info(),
3540 "ip_address": _encode_host.cache_info(),
3541 "host_validate": _encode_host.cache_info(),
3542 "encode_host": _encode_host.cache_info(),
3543 }
3546@rewrite_module
3547def cache_configure(
3548 *,
3549 idna_encode_size: int | None = _DEFAULT_IDNA_SIZE,
3550 idna_decode_size: int | None = _DEFAULT_IDNA_SIZE,
3551 ip_address_size: int | None | UndefinedType = UNDEFINED,
3552 host_validate_size: int | None | UndefinedType = UNDEFINED,
3553 encode_host_size: int | None | UndefinedType = UNDEFINED,
3554) -> None:
3555 """Configure LRU cache sizes."""
3556 global _idna_decode, _idna_encode, _encode_host
3557 # ip_address_size, host_validate_size are no longer
3558 # used, but are kept for backwards compatibility.
3559 if ip_address_size is not UNDEFINED or host_validate_size is not UNDEFINED:
3560 warnings.warn(
3561 "cache_configure() no longer accepts the "
3562 "ip_address_size or host_validate_size arguments, "
3563 "they are used to set the encode_host_size instead "
3564 "and will be removed in the future",
3565 DeprecationWarning,
3566 stacklevel=2,
3567 )
3569 if encode_host_size is not None:
3570 for size in (ip_address_size, host_validate_size):
3571 if size is None:
3572 encode_host_size = None
3573 elif encode_host_size is UNDEFINED:
3574 if size is not UNDEFINED:
3575 encode_host_size = size
3576 elif size is not UNDEFINED:
3577 if TYPE_CHECKING:
3578 assert isinstance(size, int)
3579 assert isinstance(encode_host_size, int)
3580 encode_host_size = max(size, encode_host_size)
3581 if encode_host_size is UNDEFINED:
3582 encode_host_size = _DEFAULT_ENCODE_SIZE
3584 _encode_host = lru_cache(encode_host_size)(_encode_host.__wrapped__)
3585 _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__)
3586 _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__)