Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/yarl/_url.py: 43%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import re
2import sys
3import warnings
4from collections.abc import Mapping, Sequence
5from enum import Enum
6from functools import _CacheInfo, lru_cache
7from importlib.util import find_spec
8from ipaddress import ip_address
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 NoReturn,
13 TypedDict,
14 TypeVar,
15 Union,
16 cast,
17 overload,
18)
19from urllib.parse import SplitResult, scheme_chars, uses_relative
21import idna
22from multidict import MultiDict, MultiDictProxy, istr
23from propcache.api import under_cached_property as cached_property
25from ._parse import (
26 USES_AUTHORITY,
27 SplitURLType,
28 make_netloc,
29 query_to_pairs,
30 split_netloc,
31 split_url,
32 unsplit_result,
33)
34from ._path import normalize_path, normalize_path_segments
35from ._query import (
36 Query,
37 QueryVariable,
38 SimpleQuery,
39 get_str_query,
40 get_str_query_from_iterable,
41 get_str_query_from_sequence_iterable,
42)
43from ._quoters import (
44 FRAGMENT_QUOTER,
45 FRAGMENT_REQUOTER,
46 PATH_QUOTER,
47 PATH_REQUOTER,
48 PATH_SAFE_UNQUOTER,
49 PATH_UNQUOTER,
50 QS_UNQUOTER,
51 QUERY_QUOTER,
52 QUERY_REQUOTER,
53 QUOTER,
54 REQUOTER,
55 UNQUOTER,
56 human_quote,
57)
59# Avoid Pydantic import if not used (increases yarl's import time by 3-7x).
60HAS_PYDANTIC = find_spec("pydantic_core") is not None
61if TYPE_CHECKING:
62 from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler
63 from pydantic.json_schema import JsonSchemaValue
64 from pydantic_core import CoreSchema
67DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21}
68USES_RELATIVE = frozenset(uses_relative)
69_SCHEME_CHARS = frozenset(scheme_chars)
71# Special schemes https://url.spec.whatwg.org/#special-scheme
72# are not allowed to have an empty host https://url.spec.whatwg.org/#url-representation
73SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp"))
76# reg-name: unreserved / pct-encoded / sub-delims
77# this pattern matches anything that is *not* in those classes. and is only used
78# on lower-cased ASCII values.
79NOT_REG_NAME = re.compile(
80 r"""
81 # any character not in the unreserved or sub-delims sets, plus %
82 # (validated with the additional check for pct-encoded sequences below)
83 [^a-z0-9\-._~!$&'()*+,;=%]
84 |
85 # % only allowed if it is part of a pct-encoded
86 # sequence of 2 hex digits.
87 %(?![0-9a-f]{2})
88 """,
89 re.VERBOSE,
90)
92# Invisible default-ignorable / format code points that must not appear in a
93# host (soft hyphen, zero-width space, word joiner, bidi controls, variation
94# selectors, ...). Depending on the code point IDNA either silently deletes it
95# (so ``e<ZWSP>vil.com`` encodes to ``evil.com``) or folds it into a different
96# punycode host; either way the parsed host differs from the string an
97# application validated. The set is the union of two authoritative sources,
98# matching the two encoders _idna_encode dispatches to:
99#
100# 1. Unicode Default_Ignorable_Code_Point (uts46=True path via the ``idna``
101# package). Ranges taken from the DerivedCoreProperties data file:
102# https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
103# (the E0000..E0FFF block is contiguous under this property).
104# 2. RFC 3454 (Stringprep) Table B.1 "commonly mapped to nothing", used by the
105# stdlib ``str.encode("idna")`` / IDNA2003 nameprep fallback:
106# https://www.rfc-editor.org/rfc/rfc3454#appendix-B.1
107# This is the source of U+1806, which is not Default_Ignorable.
108#
109# Coverage is pinned to the installed ``idna``/Unicode data by a sweep test
110# (test_default_ignorable_covers_idna_stripped in tests/test_url.py) that
111# brute-forces every code point through _idna_encode and fails if any point it
112# silently deletes is not matched here.
113_DEFAULT_IGNORABLE_RE = re.compile(
114 "["
115 "\u00ad" # SOFT HYPHEN
116 "\u034f" # COMBINING GRAPHEME JOINER
117 "\u061c" # ARABIC LETTER MARK
118 "\u115f-\u1160" # HANGUL CHOSEONG/JUNGSEONG FILLER
119 "\u17b4-\u17b5" # KHMER VOWEL INHERENT AQ/AA
120 "\u1806" # MONGOLIAN TODO SOFT HYPHEN (nameprep maps to nothing)
121 "\u180b-\u180f" # MONGOLIAN FVS ONE..FOUR and VOWEL SEPARATOR
122 "\u200b-\u200f" # ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK
123 "\u202a-\u202e" # bidi embedding/override controls
124 "\u2060-\u206f" # WORD JOINER..NOMINAL DIGIT SHAPES
125 "\u3164" # HANGUL FILLER
126 "\ufe00-\ufe0f" # VARIATION SELECTOR-1..16
127 "\ufeff" # ZERO WIDTH NO-BREAK SPACE (BOM)
128 "\uffa0" # HALFWIDTH HANGUL FILLER
129 "\ufff0-\ufff8" # reserved default-ignorables
130 "\U0001bca0-\U0001bca3" # SHORTHAND FORMAT controls
131 "\U0001d173-\U0001d17a" # MUSICAL SYMBOL begin/end controls
132 "\U000e0000-\U000e0fff" # tags and VARIATION SELECTOR SUPPLEMENT
133 "]"
134)
136# Zone IDs are OS-specific text strings with no format defined by the RFCs:
137# https://datatracker.ietf.org/doc/html/rfc4007#section-11.2
138# RFC 9844 §6.3 recommends rejecting characters inappropriate for the
139# environment; for yarl we reject ASCII control characters (CTL):
140# https://datatracker.ietf.org/doc/html/rfc9844#section-6-3
141_ZONE_ID_UNSAFE_RE = re.compile(r"[\x00-\x1f\x7f]")
143_T = TypeVar("_T")
145if sys.version_info >= (3, 11):
146 from typing import Self
147else:
148 Self = Any
151class UndefinedType(Enum):
152 """Singleton type for use with not set sentinel values."""
154 _singleton = 0
157UNDEFINED = UndefinedType._singleton
160class CacheInfo(TypedDict):
161 """Host encoding cache."""
163 idna_encode: _CacheInfo
164 idna_decode: _CacheInfo
165 ip_address: _CacheInfo
166 host_validate: _CacheInfo
167 encode_host: _CacheInfo
170class _InternalURLCache(TypedDict, total=False):
171 _val: SplitURLType
172 _origin: "URL"
173 absolute: bool
174 hash: int
175 scheme: str
176 raw_authority: str
177 authority: str
178 raw_user: str | None
179 user: str | None
180 raw_password: str | None
181 password: str | None
182 raw_host: str | None
183 host: str | None
184 host_subcomponent: str | None
185 host_port_subcomponent: str | None
186 port: int | None
187 explicit_port: int | None
188 raw_path: str
189 path: str
190 _parsed_query: list[tuple[str, str]]
191 query: "MultiDictProxy[str]"
192 raw_query_string: str
193 query_string: str
194 path_qs: str
195 raw_path_qs: str
196 raw_fragment: str
197 fragment: str
198 raw_parts: tuple[str, ...]
199 parts: tuple[str, ...]
200 parent: "URL"
201 raw_name: str
202 name: str
203 raw_suffix: str
204 suffix: str
205 raw_suffixes: tuple[str, ...]
206 suffixes: tuple[str, ...]
209def rewrite_module(obj: _T) -> _T:
210 obj.__module__ = "yarl"
211 return obj
214def _encode_relative_scheme_colon(path: str) -> str:
215 """Re-encode a scheme-shaped leading ``:`` in a relative path to ``%3A``."""
216 colon_pos = path.find(":")
217 if colon_pos <= 0:
218 return path
219 for c in path[:colon_pos]:
220 if c not in _SCHEME_CHARS:
221 return path
222 return path[:colon_pos] + "%3A" + path[colon_pos + 1 :]
225@lru_cache
226def encode_url(url_str: str) -> "URL":
227 """Parse unencoded URL."""
228 cache: _InternalURLCache = {}
229 host: str | None
230 scheme, netloc, path, query, fragment = split_url(url_str)
231 if not netloc: # netloc
232 host = ""
233 else:
234 if ":" in netloc or "@" in netloc or "[" in netloc:
235 # Complex netloc
236 username, password, host, port = split_netloc(netloc)
237 else:
238 username = password = port = None
239 host = netloc
240 if host is None:
241 if scheme in SCHEME_REQUIRES_HOST:
242 msg = (
243 "Invalid URL: host is required for "
244 f"absolute urls with the {scheme} scheme"
245 )
246 raise ValueError(msg)
247 else:
248 host = ""
249 host = _encode_host(host, validate_host=False)
250 # Remove brackets as host encoder adds back brackets for IPv6 addresses
251 cache["raw_host"] = host[1:-1] if "[" in host else host
252 cache["explicit_port"] = port
253 if password is None and username is None:
254 # Fast path for URLs without user, password
255 netloc = host if port is None else f"{host}:{port}"
256 cache["raw_user"] = None
257 cache["raw_password"] = None
258 else:
259 raw_user = REQUOTER(username) if username else username
260 raw_password = REQUOTER(password) if password else password
261 netloc = make_netloc(raw_user, raw_password, host, port)
262 cache["raw_user"] = raw_user
263 cache["raw_password"] = raw_password
265 if path:
266 path = PATH_REQUOTER(path)
267 if netloc and "." in path:
268 path = normalize_path(path)
269 elif not scheme and not netloc:
270 path = _encode_relative_scheme_colon(path)
271 if query:
272 query = QUERY_REQUOTER(query)
273 if fragment:
274 fragment = FRAGMENT_REQUOTER(fragment)
276 cache["scheme"] = scheme
277 cache["raw_path"] = "/" if not path and netloc else path
278 cache["raw_query_string"] = query
279 cache["raw_fragment"] = fragment
281 self = object.__new__(URL)
282 self._scheme = scheme
283 self._netloc = netloc
284 self._path = path
285 self._query = query
286 self._fragment = fragment
287 self._cache = cache
288 return self
291@lru_cache
292def pre_encoded_url(url_str: str) -> "URL":
293 """Parse pre-encoded URL."""
294 self = object.__new__(URL)
295 val = split_url(url_str)
296 self._scheme, self._netloc, self._path, self._query, self._fragment = val
297 self._cache = {}
298 return self
301@lru_cache
302def build_pre_encoded_url(
303 scheme: str,
304 authority: str,
305 user: str | None,
306 password: str | None,
307 host: str,
308 port: int | None,
309 path: str,
310 query_string: str,
311 fragment: str,
312) -> "URL":
313 """Build a pre-encoded URL from parts."""
314 self = object.__new__(URL)
315 self._scheme = scheme
316 if authority:
317 self._netloc = authority
318 elif host:
319 if port is not None:
320 port = None if port == DEFAULT_PORTS.get(scheme) else port
321 if user is None and password is None:
322 self._netloc = host if port is None else f"{host}:{port}"
323 else:
324 self._netloc = make_netloc(user, password, host, port)
325 else:
326 self._netloc = ""
327 if path and not scheme and not self._netloc and ":" in path:
328 path = _encode_relative_scheme_colon(path)
329 self._path = path
330 self._query = query_string
331 self._fragment = fragment
332 self._cache = {}
333 return self
336def from_parts_uncached(
337 scheme: str, netloc: str, path: str, query: str, fragment: str
338) -> "URL":
339 """Create a new URL from parts."""
340 self = object.__new__(URL)
341 self._scheme = scheme
342 self._netloc = netloc
343 if path and not scheme and not netloc and ":" in path:
344 path = _encode_relative_scheme_colon(path)
345 self._path = path
346 self._query = query
347 self._fragment = fragment
348 self._cache = {}
349 return self
352from_parts = lru_cache(from_parts_uncached)
355@rewrite_module
356class URL:
357 # Don't derive from str
358 # follow pathlib.Path design
359 # probably URL will not suffer from pathlib problems:
360 # it's intended for libraries like aiohttp,
361 # not to be passed into standard library functions like os.open etc.
363 # URL grammar (RFC 3986)
364 # pct-encoded = "%" HEXDIG HEXDIG
365 # reserved = gen-delims / sub-delims
366 # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
367 # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
368 # / "*" / "+" / "," / ";" / "="
369 # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
370 # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
371 # hier-part = "//" authority path-abempty
372 # / path-absolute
373 # / path-rootless
374 # / path-empty
375 # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
376 # authority = [ userinfo "@" ] host [ ":" port ]
377 # userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
378 # host = IP-literal / IPv4address / reg-name
379 # IP-literal = "[" ( IPv6address / IPvFuture ) "]"
380 # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )
381 # IPv6address = 6( h16 ":" ) ls32
382 # / "::" 5( h16 ":" ) ls32
383 # / [ h16 ] "::" 4( h16 ":" ) ls32
384 # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
385 # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
386 # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
387 # / [ *4( h16 ":" ) h16 ] "::" ls32
388 # / [ *5( h16 ":" ) h16 ] "::" h16
389 # / [ *6( h16 ":" ) h16 ] "::"
390 # ls32 = ( h16 ":" h16 ) / IPv4address
391 # ; least-significant 32 bits of address
392 # h16 = 1*4HEXDIG
393 # ; 16 bits of address represented in hexadecimal
394 # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
395 # dec-octet = DIGIT ; 0-9
396 # / %x31-39 DIGIT ; 10-99
397 # / "1" 2DIGIT ; 100-199
398 # / "2" %x30-34 DIGIT ; 200-249
399 # / "25" %x30-35 ; 250-255
400 # reg-name = *( unreserved / pct-encoded / sub-delims )
401 # port = *DIGIT
402 # path = path-abempty ; begins with "/" or is empty
403 # / path-absolute ; begins with "/" but not "//"
404 # / path-noscheme ; begins with a non-colon segment
405 # / path-rootless ; begins with a segment
406 # / path-empty ; zero characters
407 # path-abempty = *( "/" segment )
408 # path-absolute = "/" [ segment-nz *( "/" segment ) ]
409 # path-noscheme = segment-nz-nc *( "/" segment )
410 # path-rootless = segment-nz *( "/" segment )
411 # path-empty = 0<pchar>
412 # segment = *pchar
413 # segment-nz = 1*pchar
414 # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
415 # ; non-zero-length segment without any colon ":"
416 # pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
417 # query = *( pchar / "/" / "?" )
418 # fragment = *( pchar / "/" / "?" )
419 # URI-reference = URI / relative-ref
420 # relative-ref = relative-part [ "?" query ] [ "#" fragment ]
421 # relative-part = "//" authority path-abempty
422 # / path-absolute
423 # / path-noscheme
424 # / path-empty
425 # absolute-URI = scheme ":" hier-part [ "?" query ]
426 __slots__ = ("_cache", "_scheme", "_netloc", "_path", "_query", "_fragment")
428 _cache: _InternalURLCache
429 _scheme: str
430 _netloc: str
431 _path: str
432 _query: str
433 _fragment: str
435 def __new__(
436 cls,
437 val: Union[str, SplitResult, "URL", UndefinedType] = UNDEFINED,
438 *,
439 encoded: bool = False,
440 strict: bool | None = None,
441 ) -> "URL":
442 if strict is not None: # pragma: no cover
443 warnings.warn("strict parameter is ignored")
444 if type(val) is str:
445 return pre_encoded_url(val) if encoded else encode_url(val)
446 if type(val) is cls:
447 return val
448 if type(val) is SplitResult:
449 if not encoded:
450 raise ValueError("Cannot apply decoding to SplitResult")
451 return from_parts(*val)
452 if isinstance(val, str):
453 return pre_encoded_url(str(val)) if encoded else encode_url(str(val))
454 if val is UNDEFINED:
455 # Special case for UNDEFINED since it might be unpickling and we do
456 # not want to cache as the `__set_state__` call would mutate the URL
457 # object in the `pre_encoded_url` or `encoded_url` caches.
458 self = object.__new__(URL)
459 self._scheme = self._netloc = self._path = self._query = self._fragment = ""
460 self._cache = {}
461 return self
462 raise TypeError("Constructor parameter should be str")
464 @classmethod
465 def build(
466 cls,
467 *,
468 scheme: str = "",
469 authority: str = "",
470 user: str | None = None,
471 password: str | None = None,
472 host: str = "",
473 port: int | None = None,
474 path: str = "",
475 query: Query | None = None,
476 query_string: str = "",
477 fragment: str = "",
478 encoded: bool = False,
479 ) -> "URL":
480 """Creates and returns a new URL"""
482 if authority and (user or password or host or port):
483 raise ValueError(
484 'Can\'t mix "authority" with "user", "password", "host" or "port".'
485 )
486 if port is not None and not isinstance(port, int):
487 raise TypeError(f"The port is required to be int, got {type(port)!r}.")
488 if port and not host:
489 raise ValueError('Can\'t build URL with "port" but without "host".')
490 if query and query_string:
491 raise ValueError('Only one of "query" or "query_string" should be passed')
492 if (
493 scheme is None # type: ignore[redundant-expr]
494 or authority is None # type: ignore[redundant-expr]
495 or host is None # type: ignore[redundant-expr]
496 or path is None # type: ignore[redundant-expr]
497 or query_string is None # type: ignore[redundant-expr]
498 or fragment is None
499 ):
500 raise TypeError(
501 'NoneType is illegal for "scheme", "authority", "host", "path", '
502 '"query_string", and "fragment" args, use empty string instead.'
503 )
505 if query:
506 query_string = get_str_query(query) or ""
508 if encoded:
509 return build_pre_encoded_url(
510 scheme,
511 authority,
512 user,
513 password,
514 host,
515 port,
516 path,
517 query_string,
518 fragment,
519 )
521 self = object.__new__(URL)
522 self._scheme = scheme
523 _host: str | None = None
524 if authority:
525 user, password, _host, port = split_netloc(authority)
526 _host = _encode_host(_host, validate_host=False) if _host else ""
527 elif host:
528 _host = _encode_host(host, validate_host=True)
529 else:
530 self._netloc = ""
532 if _host is not None:
533 if port is not None:
534 port = None if port == DEFAULT_PORTS.get(scheme) else port
535 if user is None and password is None:
536 self._netloc = _host if port is None else f"{_host}:{port}"
537 else:
538 self._netloc = make_netloc(user, password, _host, port, True)
540 path = PATH_QUOTER(path) if path else path
541 if path and self._netloc:
542 if "." in path:
543 path = normalize_path(path)
544 if path[0] != "/":
545 msg = (
546 "Path in a URL with authority should "
547 "start with a slash ('/') if set"
548 )
549 raise ValueError(msg)
551 if path and not self._scheme and not self._netloc and ":" in path:
552 path = _encode_relative_scheme_colon(path)
553 self._path = path
554 if not query and query_string:
555 query_string = QUERY_QUOTER(query_string)
556 self._query = query_string
557 self._fragment = FRAGMENT_QUOTER(fragment) if fragment else fragment
558 self._cache = {}
559 return self
561 def __init_subclass__(cls) -> NoReturn:
562 raise TypeError(f"Inheriting a class {cls!r} from URL is forbidden")
564 def __str__(self) -> str:
565 if not self._path and self._netloc and (self._query or self._fragment):
566 path = "/"
567 else:
568 path = self._path
569 if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get(
570 self._scheme
571 ):
572 # port normalization - using None for default ports to remove from rendering
573 # https://datatracker.ietf.org/doc/html/rfc3986.html#section-6.2.3
574 host = self.host_subcomponent
575 netloc = make_netloc(self.raw_user, self.raw_password, host, None)
576 else:
577 netloc = self._netloc
578 return unsplit_result(self._scheme, netloc, path, self._query, self._fragment)
580 def __repr__(self) -> str:
581 return f"{self.__class__.__name__}('{str(self)}')"
583 def __bytes__(self) -> bytes:
584 return str(self).encode("ascii")
586 def __eq__(self, other: object) -> bool:
587 if type(other) is not URL:
588 return NotImplemented
590 path1 = "/" if not self._path and self._netloc else self._path
591 path2 = "/" if not other._path and other._netloc else other._path
592 return (
593 self._scheme == other._scheme
594 and self._netloc == other._netloc
595 and path1 == path2
596 and self._query == other._query
597 and self._fragment == other._fragment
598 )
600 def __hash__(self) -> int:
601 if (ret := self._cache.get("hash")) is None:
602 path = "/" if not self._path and self._netloc else self._path
603 ret = self._cache["hash"] = hash(
604 (self._scheme, self._netloc, path, self._query, self._fragment)
605 )
606 return ret
608 def __le__(self, other: object) -> bool:
609 if type(other) is not URL:
610 return NotImplemented
611 return self._val <= other._val
613 def __lt__(self, other: object) -> bool:
614 if type(other) is not URL:
615 return NotImplemented
616 return self._val < other._val
618 def __ge__(self, other: object) -> bool:
619 if type(other) is not URL:
620 return NotImplemented
621 return self._val >= other._val
623 def __gt__(self, other: object) -> bool:
624 if type(other) is not URL:
625 return NotImplemented
626 return self._val > other._val
628 def __truediv__(self, name: str) -> "URL":
629 if not isinstance(name, str):
630 return NotImplemented
631 return self._make_child((str(name),))
633 def __mod__(self, query: Query) -> "URL":
634 return self.update_query(query)
636 def __bool__(self) -> bool:
637 return bool(self._netloc or self._path or self._query or self._fragment)
639 def __getstate__(self) -> tuple[SplitURLType]:
640 # Return a plain tuple rather than a ``SplitResult``. Constructing a
641 # ``SplitResult`` via ``tuple.__new__`` skips its ``__init__`` and on
642 # Python 3.15+ leaves ``_keep_empty`` unset, which breaks pickling: the
643 # new ``SplitResult.__getstate__`` indexes a state that ends up as
644 # ``None`` (gh-1632). ``__setstate__`` already unpacks both shapes, so
645 # pickles produced by older yarl releases (which embed a real
646 # ``SplitResult``) still load correctly.
647 return (self._val,)
649 def __setstate__(
650 self, state: tuple[SplitURLType] | tuple[None, _InternalURLCache]
651 ) -> None:
652 if state[0] is None and isinstance(state[1], dict):
653 # default style pickle
654 val = state[1]["_val"]
655 else:
656 unused: list[object]
657 val, *unused = state
658 self._scheme, self._netloc, self._path, self._query, self._fragment = val
659 self._cache = {}
661 def _cache_netloc(self) -> None:
662 """Cache the netloc parts of the URL."""
663 c = self._cache
664 split_loc = split_netloc(self._netloc)
665 c["raw_user"], c["raw_password"], c["raw_host"], c["explicit_port"] = split_loc
667 def is_absolute(self) -> bool:
668 """A check for absolute URLs.
670 Return True for absolute ones (having scheme or starting
671 with //), False otherwise.
673 Is is preferred to call the .absolute property instead
674 as it is cached.
675 """
676 return self.absolute
678 def is_default_port(self) -> bool:
679 """A check for default port.
681 Return True if port is default for specified scheme,
682 e.g. 'http://python.org' or 'http://python.org:80', False
683 otherwise.
685 Return False for relative URLs.
687 """
688 if (explicit := self.explicit_port) is None:
689 # If the explicit port is None, then the URL must be
690 # using the default port unless its a relative URL
691 # which does not have an implicit port / default port
692 return self._netloc != ""
693 return explicit == DEFAULT_PORTS.get(self._scheme)
695 def origin(self) -> "URL":
696 """Return an URL with scheme, host and port parts only.
698 user, password, path, query and fragment are removed.
700 """
701 # TODO: add a keyword-only option for keeping user/pass maybe?
702 return self._origin
704 @cached_property
705 def _val(self) -> SplitURLType:
706 return (self._scheme, self._netloc, self._path, self._query, self._fragment)
708 @cached_property
709 def _origin(self) -> "URL":
710 """Return an URL with scheme, host and port parts only.
712 user, password, path, query and fragment are removed.
713 """
714 if not (netloc := self._netloc):
715 raise ValueError("URL should be absolute")
716 if not (scheme := self._scheme):
717 raise ValueError("URL should have scheme")
718 if "@" in netloc:
719 encoded_host = self.host_subcomponent
720 netloc = make_netloc(None, None, encoded_host, self.explicit_port)
721 elif not self._path and not self._query and not self._fragment:
722 return self
723 return from_parts(scheme, netloc, "", "", "")
725 def relative(self) -> "URL":
726 """Return a relative part of the URL.
728 scheme, user, password, host and port are removed.
730 """
731 if not self._netloc:
732 raise ValueError("URL should be absolute")
733 return from_parts("", "", self._path, self._query, self._fragment)
735 @cached_property
736 def absolute(self) -> bool:
737 """A check for absolute URLs.
739 Return True for absolute ones (having scheme or starting
740 with //), False otherwise.
742 """
743 # `netloc`` is an empty string for relative URLs
744 # Checking `netloc` is faster than checking `hostname`
745 # because `hostname` is a property that does some extra work
746 # to parse the host from the `netloc`
747 return self._netloc != ""
749 @cached_property
750 def scheme(self) -> str:
751 """Scheme for absolute URLs.
753 Empty string for relative URLs or URLs starting with //
755 """
756 return self._scheme
758 @cached_property
759 def raw_authority(self) -> str:
760 """Encoded authority part of URL.
762 Empty string for relative URLs.
764 """
765 return self._netloc
767 @cached_property
768 def authority(self) -> str:
769 """Decoded authority part of URL.
771 Empty string for relative URLs.
773 """
774 return make_netloc(self.user, self.password, self.host, self.port)
776 @cached_property
777 def raw_user(self) -> str | None:
778 """Encoded user part of URL.
780 None if user is missing.
782 """
783 # not .username
784 self._cache_netloc()
785 return self._cache["raw_user"]
787 @cached_property
788 def user(self) -> str | None:
789 """Decoded user part of URL.
791 None if user is missing.
793 """
794 if (raw_user := self.raw_user) is None:
795 return None
796 return UNQUOTER(raw_user)
798 @cached_property
799 def raw_password(self) -> str | None:
800 """Encoded password part of URL.
802 None if password is missing.
804 """
805 self._cache_netloc()
806 return self._cache["raw_password"]
808 @cached_property
809 def password(self) -> str | None:
810 """Decoded password part of URL.
812 None if password is missing.
814 """
815 if (raw_password := self.raw_password) is None:
816 return None
817 return UNQUOTER(raw_password)
819 @cached_property
820 def raw_host(self) -> str | None:
821 """Encoded host part of URL.
823 None for relative URLs.
825 When working with IPv6 addresses, use the `host_subcomponent` property instead
826 as it will return the host subcomponent with brackets.
827 """
828 # Use host instead of hostname for sake of shortness
829 # May add .hostname prop later
830 self._cache_netloc()
831 return self._cache["raw_host"]
833 @cached_property
834 def host(self) -> str | None:
835 """Decoded host part of URL.
837 None for relative URLs.
839 For IPv6 hosts that carry an RFC 6874 zone identifier, the
840 ``%25`` zone separator is decoded back to ``%``; the encoded
841 form is still available via :attr:`raw_host` and
842 :attr:`host_subcomponent`.
844 """
845 if (raw := self.raw_host) is None:
846 return None
847 if raw and raw[-1].isdigit() or ":" in raw:
848 # IP addresses are never IDNA encoded. The replace decodes
849 # every %25 in the raw host, i.e. the RFC 6874 zone
850 # separator and any %25 that percent-encodes a literal %
851 # inside the zone identifier.
852 if "%25" in raw:
853 return raw.replace("%25", "%")
854 return raw
855 return _idna_decode(raw)
857 @cached_property
858 def host_subcomponent(self) -> str | None:
859 """Return the host subcomponent part of URL.
861 None for relative URLs.
863 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
865 `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
867 Examples:
868 - `http://example.com:8080` -> `example.com`
869 - `http://example.com:80` -> `example.com`
870 - `https://127.0.0.1:8443` -> `127.0.0.1`
871 - `https://[::1]:8443` -> `[::1]`
872 - `http://[::1]` -> `[::1]`
874 """
875 if (raw := self.raw_host) is None:
876 return None
877 return f"[{raw}]" if ":" in raw else raw
879 @cached_property
880 def host_port_subcomponent(self) -> str | None:
881 """Return the host and port subcomponent part of URL.
883 Trailing dots are removed from the host part.
885 This value is suitable for use in the Host header of an HTTP request.
887 None for relative URLs.
889 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
890 `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
891 https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.3
892 port = *DIGIT
894 Examples:
895 - `http://example.com:8080` -> `example.com:8080`
896 - `http://example.com:80` -> `example.com`
897 - `http://example.com.:80` -> `example.com`
898 - `https://127.0.0.1:8443` -> `127.0.0.1:8443`
899 - `https://[::1]:8443` -> `[::1]:8443`
900 - `http://[::1]` -> `[::1]`
902 """
903 if (raw := self.raw_host) is None:
904 return None
905 if raw[-1] == ".":
906 # Remove all trailing dots from the netloc as while
907 # they are valid FQDNs in DNS, TLS validation fails.
908 # See https://github.com/aio-libs/aiohttp/issues/3636.
909 # To avoid string manipulation we only call rstrip if
910 # the last character is a dot.
911 raw = raw.rstrip(".")
912 port = self.explicit_port
913 if port is None or port == DEFAULT_PORTS.get(self._scheme):
914 return f"[{raw}]" if ":" in raw else raw
915 return f"[{raw}]:{port}" if ":" in raw else f"{raw}:{port}"
917 @cached_property
918 def port(self) -> int | None:
919 """Port part of URL, with scheme-based fallback.
921 None for relative URLs or URLs without explicit port and
922 scheme without default port substitution.
924 """
925 if (explicit_port := self.explicit_port) is not None:
926 return explicit_port
927 return DEFAULT_PORTS.get(self._scheme)
929 @cached_property
930 def explicit_port(self) -> int | None:
931 """Port part of URL, without scheme-based fallback.
933 None for relative URLs or URLs without explicit port.
935 """
936 self._cache_netloc()
937 return self._cache["explicit_port"]
939 @cached_property
940 def raw_path(self) -> str:
941 """Encoded path of URL.
943 / for absolute URLs without path part.
945 """
946 return self._path if self._path or not self._netloc else "/"
948 @cached_property
949 def path(self) -> str:
950 """Decoded path of URL.
952 / for absolute URLs without path part.
954 """
955 return PATH_UNQUOTER(self._path) if self._path else "/" if self._netloc else ""
957 @cached_property
958 def path_safe(self) -> str:
959 """Decoded path of URL.
961 / for absolute URLs without path part.
963 / (%2F) and % (%25) are not decoded
965 """
966 if self._path:
967 return PATH_SAFE_UNQUOTER(self._path)
968 return "/" if self._netloc else ""
970 @cached_property
971 def _parsed_query(self) -> list[tuple[str, str]]:
972 """Parse query part of URL."""
973 return query_to_pairs(self._query)
975 @cached_property
976 def query(self) -> "MultiDictProxy[str]":
977 """A MultiDictProxy representing parsed query parameters in decoded
978 representation.
980 Empty value if URL has no query part.
982 """
983 return MultiDictProxy(MultiDict(self._parsed_query))
985 @cached_property
986 def raw_query_string(self) -> str:
987 """Encoded query part of URL.
989 Empty string if query is missing.
991 """
992 return self._query
994 @cached_property
995 def query_string(self) -> str:
996 """Decoded query part of URL.
998 Empty string if query is missing.
1000 """
1001 return QS_UNQUOTER(self._query) if self._query else ""
1003 @cached_property
1004 def path_qs(self) -> str:
1005 """Decoded path of URL with query."""
1006 return self.path if not (q := self.query_string) else f"{self.path}?{q}"
1008 @cached_property
1009 def raw_path_qs(self) -> str:
1010 """Encoded path of URL with query."""
1011 if q := self._query:
1012 return f"{self._path}?{q}" if self._path or not self._netloc else f"/?{q}"
1013 return self._path if self._path or not self._netloc else "/"
1015 @cached_property
1016 def raw_fragment(self) -> str:
1017 """Encoded fragment part of URL.
1019 Empty string if fragment is missing.
1021 """
1022 return self._fragment
1024 @cached_property
1025 def fragment(self) -> str:
1026 """Decoded fragment part of URL.
1028 Empty string if fragment is missing.
1030 """
1031 return UNQUOTER(self._fragment) if self._fragment else ""
1033 @cached_property
1034 def raw_parts(self) -> tuple[str, ...]:
1035 """A tuple containing encoded *path* parts.
1037 ('/',) for absolute URLs if *path* is missing.
1039 """
1040 path = self._path
1041 if self._netloc:
1042 return ("/", *path[1:].split("/")) if path else ("/",)
1043 if path and path[0] == "/":
1044 return ("/", *path[1:].split("/"))
1045 return tuple(path.split("/"))
1047 @cached_property
1048 def parts(self) -> tuple[str, ...]:
1049 """A tuple containing decoded *path* parts.
1051 ('/',) for absolute URLs if *path* is missing.
1053 """
1054 return tuple(UNQUOTER(part) for part in self.raw_parts)
1056 @cached_property
1057 def parent(self) -> "URL":
1058 """A new URL with last part of path removed and cleaned up query and
1059 fragment.
1061 """
1062 path = self._path
1063 if not path or path == "/":
1064 if self._fragment or self._query:
1065 return from_parts(self._scheme, self._netloc, path, "", "")
1066 return self
1067 parts = path.split("/")
1068 return from_parts(self._scheme, self._netloc, "/".join(parts[:-1]), "", "")
1070 @cached_property
1071 def raw_name(self) -> str:
1072 """The last part of raw_parts."""
1073 parts = self.raw_parts
1074 if not self._netloc:
1075 return parts[-1]
1076 parts = parts[1:]
1077 return parts[-1] if parts else ""
1079 @cached_property
1080 def name(self) -> str:
1081 """The last part of parts."""
1082 return UNQUOTER(self.raw_name)
1084 @cached_property
1085 def raw_suffix(self) -> str:
1086 name = self.raw_name
1087 i = name.rfind(".")
1088 return name[i:] if 0 < i < len(name) - 1 else ""
1090 @cached_property
1091 def suffix(self) -> str:
1092 return UNQUOTER(self.raw_suffix)
1094 @cached_property
1095 def raw_suffixes(self) -> tuple[str, ...]:
1096 name = self.raw_name
1097 if name.endswith("."):
1098 return ()
1099 name = name.lstrip(".")
1100 return tuple("." + suffix for suffix in name.split(".")[1:])
1102 @cached_property
1103 def suffixes(self) -> tuple[str, ...]:
1104 return tuple(UNQUOTER(suffix) for suffix in self.raw_suffixes)
1106 def _make_child(self, paths: "Sequence[str]", encoded: bool = False) -> "URL":
1107 """
1108 add paths to self._path, accounting for absolute vs relative paths,
1109 keep existing, but do not create new, empty segments
1110 """
1111 parsed: list[str] = []
1112 needs_normalize: bool = False
1113 for idx, path in enumerate(reversed(paths)):
1114 # empty segment of last is not removed
1115 last = idx == 0
1116 if path and path[0] == "/":
1117 raise ValueError(
1118 f"Appending path {path!r} starting from slash is forbidden"
1119 )
1120 # We need to quote the path if it is not already encoded
1121 # This cannot be done at the end because the existing
1122 # path is already quoted and we do not want to double quote
1123 # the existing path.
1124 path = path if encoded else PATH_QUOTER(path)
1125 needs_normalize |= "." in path
1126 segments = path.split("/")
1127 segments.reverse()
1128 # remove trailing empty segment for all but the last path
1129 parsed += segments[1:] if not last and segments[0] == "" else segments
1131 if (path := self._path) and (old_segments := path.split("/")):
1132 # If the old path ends with a slash, the last segment is an empty string
1133 # and should be removed before adding the new path segments.
1134 old = old_segments[:-1] if old_segments[-1] == "" else old_segments
1135 old.reverse()
1136 parsed += old
1138 # If the netloc is present, inject a leading slash when adding a
1139 # path to an absolute URL where there was none before.
1140 if (netloc := self._netloc) and parsed and parsed[-1] != "":
1141 parsed.append("")
1143 parsed.reverse()
1144 if not netloc or not needs_normalize:
1145 return from_parts(self._scheme, netloc, "/".join(parsed), "", "")
1147 path = "/".join(normalize_path_segments(parsed))
1148 # If normalizing the path segments removed the leading slash, add it back.
1149 if path and path[0] != "/":
1150 path = f"/{path}"
1151 return from_parts(self._scheme, netloc, path, "", "")
1153 def with_scheme(self, scheme: str) -> "URL":
1154 """Return a new URL with scheme replaced."""
1155 # N.B. doesn't cleanup query/fragment
1156 if not isinstance(scheme, str):
1157 raise TypeError("Invalid scheme type")
1158 lower_scheme = scheme.lower()
1159 netloc = self._netloc
1160 if not netloc and lower_scheme in SCHEME_REQUIRES_HOST:
1161 msg = (
1162 "scheme replacement is not allowed for "
1163 f"relative URLs for the {lower_scheme} scheme"
1164 )
1165 raise ValueError(msg)
1166 return from_parts(lower_scheme, netloc, self._path, self._query, self._fragment)
1168 def with_user(self, user: str | None) -> "URL":
1169 """Return a new URL with user replaced.
1171 Autoencode user if needed.
1173 Clear user/password if user is None.
1175 """
1176 # N.B. doesn't cleanup query/fragment
1177 if user is None:
1178 password = None
1179 elif isinstance(user, str):
1180 user = QUOTER(user)
1181 password = self.raw_password
1182 else:
1183 raise TypeError("Invalid user type")
1184 if not (netloc := self._netloc):
1185 raise ValueError("user replacement is not allowed for relative URLs")
1186 encoded_host = self.host_subcomponent or ""
1187 netloc = make_netloc(user, password, encoded_host, self.explicit_port)
1188 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
1190 def with_password(self, password: str | None) -> "URL":
1191 """Return a new URL with password replaced.
1193 Autoencode password if needed.
1195 Clear password if argument is None.
1197 """
1198 # N.B. doesn't cleanup query/fragment
1199 if password is None:
1200 pass
1201 elif isinstance(password, str):
1202 password = QUOTER(password)
1203 else:
1204 raise TypeError("Invalid password type")
1205 if not (netloc := self._netloc):
1206 raise ValueError("password replacement is not allowed for relative URLs")
1207 encoded_host = self.host_subcomponent or ""
1208 port = self.explicit_port
1209 netloc = make_netloc(self.raw_user, password, encoded_host, port)
1210 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
1212 def with_host(self, host: str) -> "URL":
1213 """Return a new URL with host replaced.
1215 Autoencode host if needed.
1217 Changing host for relative URLs is not allowed, use .join()
1218 instead.
1220 """
1221 # N.B. doesn't cleanup query/fragment
1222 if not isinstance(host, str):
1223 raise TypeError("Invalid host type")
1224 if not (netloc := self._netloc):
1225 raise ValueError("host replacement is not allowed for relative URLs")
1226 if not host:
1227 raise ValueError("host removing is not allowed")
1228 encoded_host = _encode_host(host, validate_host=True) if host else ""
1229 port = self.explicit_port
1230 netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
1231 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
1233 def with_port(self, port: int | None) -> "URL":
1234 """Return a new URL with port replaced.
1236 Clear port to default if None is passed.
1238 """
1239 # N.B. doesn't cleanup query/fragment
1240 if port is not None:
1241 if isinstance(port, bool) or not isinstance(port, int):
1242 raise TypeError(f"port should be int or None, got {type(port)}")
1243 if not (0 <= port <= 65535):
1244 raise ValueError(f"port must be between 0 and 65535, got {port}")
1245 if not (netloc := self._netloc):
1246 raise ValueError("port replacement is not allowed for relative URLs")
1247 encoded_host = self.host_subcomponent or ""
1248 netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
1249 return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
1251 def with_path(
1252 self,
1253 path: str,
1254 *,
1255 encoded: bool = False,
1256 keep_query: bool = False,
1257 keep_fragment: bool = False,
1258 ) -> "URL":
1259 """Return a new URL with path replaced."""
1260 netloc = self._netloc
1261 if not encoded:
1262 path = PATH_QUOTER(path)
1263 if netloc:
1264 path = normalize_path(path) if "." in path else path
1265 if path and path[0] != "/":
1266 path = f"/{path}"
1267 query = self._query if keep_query else ""
1268 fragment = self._fragment if keep_fragment else ""
1269 return from_parts(self._scheme, netloc, path, query, fragment)
1271 @overload
1272 def with_query(self, query: Query) -> "URL": ...
1274 @overload
1275 def with_query(self, **kwargs: QueryVariable) -> "URL": ...
1277 def with_query(self, *args: Any, **kwargs: Any) -> "URL":
1278 """Return a new URL with query part replaced.
1280 Accepts any Mapping (e.g. dict, multidict.MultiDict instances)
1281 or str, autoencode the argument if needed.
1283 A sequence of (key, value) pairs is supported as well.
1285 It also can take an arbitrary number of keyword arguments.
1287 Clear query if None is passed.
1289 """
1290 # N.B. doesn't cleanup query/fragment
1291 query = get_str_query(*args, **kwargs) or ""
1292 return from_parts_uncached(
1293 self._scheme, self._netloc, self._path, query, self._fragment
1294 )
1296 @overload
1297 def extend_query(self, query: Query) -> "URL": ...
1299 @overload
1300 def extend_query(self, **kwargs: QueryVariable) -> "URL": ...
1302 def extend_query(self, *args: Any, **kwargs: Any) -> "URL":
1303 """Return a new URL with query part combined with the existing.
1305 This method will not remove existing query parameters.
1307 Example:
1308 >>> url = URL('http://example.com/?a=1&b=2')
1309 >>> url.extend_query(a=3, c=4)
1310 URL('http://example.com/?a=1&b=2&a=3&c=4')
1311 """
1312 if not (new_query := get_str_query(*args, **kwargs)):
1313 return self
1314 if query := self._query:
1315 # both strings are already encoded so we can use a simple
1316 # string join
1317 query += new_query if query[-1] == "&" else f"&{new_query}"
1318 else:
1319 query = new_query
1320 return from_parts_uncached(
1321 self._scheme, self._netloc, self._path, query, self._fragment
1322 )
1324 @overload
1325 def update_query(self, query: Query) -> "URL": ...
1327 @overload
1328 def update_query(self, **kwargs: QueryVariable) -> "URL": ...
1330 def update_query(self, *args: Any, **kwargs: Any) -> "URL":
1331 """Return a new URL with query part updated.
1333 This method will overwrite existing query parameters.
1335 Example:
1336 >>> url = URL('http://example.com/?a=1&b=2')
1337 >>> url.update_query(a=3, c=4)
1338 URL('http://example.com/?a=3&b=2&c=4')
1339 """
1340 in_query: (
1341 str
1342 | Mapping[str, QueryVariable]
1343 | Sequence[tuple[str | istr, SimpleQuery]]
1344 | None
1345 )
1346 if kwargs:
1347 if args:
1348 msg = "Either kwargs or single query parameter must be present"
1349 raise ValueError(msg)
1350 in_query = kwargs
1351 elif len(args) == 1:
1352 in_query = args[0]
1353 else:
1354 raise ValueError("Either kwargs or single query parameter must be present")
1356 if in_query is None:
1357 query = ""
1358 elif not in_query:
1359 query = self._query
1360 elif isinstance(in_query, Mapping):
1361 qm: MultiDict[QueryVariable] = MultiDict(self._parsed_query)
1362 qm.update(in_query)
1363 query = get_str_query_from_sequence_iterable(qm.items())
1364 elif isinstance(in_query, str):
1365 qstr: MultiDict[str] = MultiDict(self._parsed_query)
1366 qstr.update(query_to_pairs(in_query))
1367 query = get_str_query_from_iterable(qstr.items())
1368 elif isinstance(in_query, (bytes, bytearray, memoryview)):
1369 msg = "Invalid query type: bytes, bytearray and memoryview are forbidden"
1370 raise TypeError(msg)
1371 elif isinstance(in_query, Sequence):
1372 # We don't expect sequence values if we're given a list of pairs
1373 # already; only mappings like builtin `dict` which can't have the
1374 # same key pointing to multiple values are allowed to use
1375 # `_query_seq_pairs`.
1376 if TYPE_CHECKING:
1377 in_query = cast(
1378 Sequence[tuple[Union[str, istr], SimpleQuery]], in_query
1379 )
1380 qs: MultiDict[SimpleQuery] = MultiDict(self._parsed_query)
1381 qs.update(in_query)
1382 query = get_str_query_from_iterable(qs.items())
1383 else:
1384 raise TypeError(
1385 "Invalid query type: only str, mapping or "
1386 "sequence of (key, value) pairs is allowed"
1387 )
1388 return from_parts_uncached(
1389 self._scheme, self._netloc, self._path, query, self._fragment
1390 )
1392 def without_query_params(self, *query_params: str) -> "URL":
1393 """Remove some keys from query part and return new URL."""
1394 params_to_remove = set(query_params) & self.query.keys()
1395 if not params_to_remove:
1396 return self
1397 return self.with_query(
1398 tuple(
1399 (name, value)
1400 for name, value in self.query.items()
1401 if name not in params_to_remove
1402 )
1403 )
1405 def with_fragment(self, fragment: str | None) -> "URL":
1406 """Return a new URL with fragment replaced.
1408 Autoencode fragment if needed.
1410 Clear fragment to default if None is passed.
1412 """
1413 # N.B. doesn't cleanup query/fragment
1414 if fragment is None:
1415 raw_fragment = ""
1416 elif not isinstance(fragment, str):
1417 raise TypeError("Invalid fragment type")
1418 else:
1419 raw_fragment = FRAGMENT_QUOTER(fragment)
1420 if self._fragment == raw_fragment:
1421 return self
1422 return from_parts(
1423 self._scheme, self._netloc, self._path, self._query, raw_fragment
1424 )
1426 def with_name(
1427 self,
1428 name: str,
1429 *,
1430 keep_query: bool = False,
1431 keep_fragment: bool = False,
1432 ) -> "URL":
1433 """Return a new URL with name (last part of path) replaced.
1435 Query and fragment parts are cleaned up.
1437 Name is encoded if needed.
1439 """
1440 # N.B. DOES cleanup query/fragment
1441 if not isinstance(name, str):
1442 raise TypeError("Invalid name type")
1443 if "/" in name:
1444 raise ValueError("Slash in name is not allowed")
1445 name = PATH_QUOTER(name)
1446 if name in (".", ".."):
1447 raise ValueError(". and .. values are forbidden")
1448 parts = list(self.raw_parts)
1449 if netloc := self._netloc:
1450 if len(parts) == 1:
1451 parts.append(name)
1452 else:
1453 parts[-1] = name
1454 parts[0] = "" # replace leading '/'
1455 else:
1456 parts[-1] = name
1457 if parts[0] == "/":
1458 parts[0] = "" # replace leading '/'
1460 query = self._query if keep_query else ""
1461 fragment = self._fragment if keep_fragment else ""
1462 return from_parts(self._scheme, netloc, "/".join(parts), query, fragment)
1464 def with_suffix(
1465 self,
1466 suffix: str,
1467 *,
1468 keep_query: bool = False,
1469 keep_fragment: bool = False,
1470 ) -> "URL":
1471 """Return a new URL with suffix (file extension of name) replaced.
1473 Query and fragment parts are cleaned up.
1475 suffix is encoded if needed.
1476 """
1477 if not isinstance(suffix, str):
1478 raise TypeError("Invalid suffix type")
1479 if suffix and not suffix[0] == "." or suffix == "." or "/" in suffix:
1480 raise ValueError(f"Invalid suffix {suffix!r}")
1481 name = self.raw_name
1482 if not name:
1483 raise ValueError(f"{self!r} has an empty name")
1484 old_suffix = self.raw_suffix
1485 suffix = PATH_QUOTER(suffix)
1486 name = name + suffix if not old_suffix else name[: -len(old_suffix)] + suffix
1487 if name in (".", ".."):
1488 raise ValueError(". and .. values are forbidden")
1489 parts = list(self.raw_parts)
1490 if netloc := self._netloc:
1491 if len(parts) == 1:
1492 parts.append(name)
1493 else:
1494 parts[-1] = name
1495 parts[0] = "" # replace leading '/'
1496 else:
1497 parts[-1] = name
1498 if parts[0] == "/":
1499 parts[0] = "" # replace leading '/'
1501 query = self._query if keep_query else ""
1502 fragment = self._fragment if keep_fragment else ""
1503 return from_parts(self._scheme, netloc, "/".join(parts), query, fragment)
1505 def join(self, url: "URL") -> "URL":
1506 """Join URLs
1508 Construct a full (“absolute”) URL by combining a “base URL”
1509 (self) with another URL (url).
1511 Informally, this uses components of the base URL, in
1512 particular the addressing scheme, the network location and
1513 (part of) the path, to provide missing components in the
1514 relative URL.
1516 """
1517 if type(url) is not URL:
1518 raise TypeError("url should be URL")
1520 scheme = url._scheme or self._scheme
1521 if scheme != self._scheme or scheme not in USES_RELATIVE:
1522 return url
1524 # scheme is in uses_authority as uses_authority is a superset of uses_relative
1525 if (join_netloc := url._netloc) and scheme in USES_AUTHORITY:
1526 return from_parts(scheme, join_netloc, url._path, url._query, url._fragment)
1528 orig_path = self._path
1529 if join_path := url._path:
1530 if join_path[0] == "/":
1531 path = join_path
1532 elif not orig_path:
1533 path = f"/{join_path}"
1534 elif orig_path[-1] == "/":
1535 path = f"{orig_path}{join_path}"
1536 else:
1537 # …
1538 # and relativizing ".."
1539 # parts[0] is / for absolute urls,
1540 # this join will add a double slash there
1541 path = "/".join([*self.parts[:-1], ""]) + join_path
1542 # which has to be removed
1543 if orig_path[0] == "/":
1544 path = path[1:]
1545 path = normalize_path(path) if "." in path else path
1546 else:
1547 path = orig_path
1549 return from_parts(
1550 scheme,
1551 self._netloc,
1552 path,
1553 url._query if join_path or url._query else self._query,
1554 url._fragment if join_path or url._fragment else self._fragment,
1555 )
1557 def joinpath(self, *other: str, encoded: bool = False) -> "URL":
1558 """Return a new URL with the elements in other appended to the path."""
1559 return self._make_child(other, encoded=encoded)
1561 def human_repr(self) -> str:
1562 """Return decoded human readable string for URL representation."""
1563 user = human_quote(self.user, "#/:?@[]\\")
1564 password = human_quote(self.password, "#/:?@[]\\")
1565 if (host := self.host) and ":" in host:
1566 host = f"[{host}]"
1567 path = human_quote(self.path, "#?")
1568 if TYPE_CHECKING:
1569 assert path is not None
1570 if not self._scheme and not self._netloc:
1571 path = _encode_relative_scheme_colon(path)
1572 query_string = "&".join(
1573 "{}={}".format(human_quote(k, "#&+;="), human_quote(v, "#&+;="))
1574 for k, v in self.query.items()
1575 )
1576 fragment = human_quote(self.fragment, "")
1577 if TYPE_CHECKING:
1578 assert fragment is not None
1579 netloc = make_netloc(user, password, host, self.explicit_port)
1580 return unsplit_result(self._scheme, netloc, path, query_string, fragment)
1582 if HAS_PYDANTIC:
1583 # Borrowed from https://docs.pydantic.dev/latest/concepts/types/#handling-third-party-types
1584 @classmethod
1585 def __get_pydantic_json_schema__(
1586 cls,
1587 core_schema: "CoreSchema",
1588 handler: "GetJsonSchemaHandler",
1589 ) -> "JsonSchemaValue":
1590 field_schema: dict[str, Any] = {}
1591 field_schema.update(type="string", format="uri")
1592 return field_schema
1594 @classmethod
1595 def __get_pydantic_core_schema__(
1596 cls,
1597 source_type: type[Self] | type[str],
1598 handler: "GetCoreSchemaHandler",
1599 ) -> "CoreSchema":
1600 # Lazy import: pulling in pydantic_core at module load time
1601 # increases yarl's import cost 3-7x for users who don't use
1602 # pydantic. Keep this import function-scoped.
1603 from pydantic_core import core_schema # noqa: PLC0415
1605 from_str_schema = core_schema.chain_schema(
1606 [
1607 core_schema.str_schema(),
1608 core_schema.no_info_plain_validator_function(URL),
1609 ]
1610 )
1612 return core_schema.json_or_python_schema(
1613 json_schema=from_str_schema,
1614 python_schema=core_schema.union_schema(
1615 [
1616 # check if it's an instance first before doing any further work
1617 core_schema.is_instance_schema(URL),
1618 from_str_schema,
1619 ]
1620 ),
1621 serialization=core_schema.plain_serializer_function_ser_schema(str),
1622 )
1625_DEFAULT_IDNA_SIZE = 256
1626_DEFAULT_ENCODE_SIZE = 512
1629@lru_cache(_DEFAULT_IDNA_SIZE)
1630def _idna_decode(raw: str) -> str:
1631 try:
1632 return idna.decode(raw.encode("ascii"))
1633 except UnicodeError: # e.g. '::1'
1634 return raw.encode("ascii").decode("idna")
1637@lru_cache(_DEFAULT_IDNA_SIZE)
1638def _idna_encode(host: str) -> str:
1639 try:
1640 return idna.encode(host, uts46=True).decode("ascii")
1641 except UnicodeError:
1642 return host.encode("idna").decode("ascii")
1645@lru_cache(_DEFAULT_ENCODE_SIZE)
1646def _encode_host(host: str, validate_host: bool) -> str:
1647 """Encode host part of URL."""
1648 # If the host ends with a digit or contains a colon, its likely
1649 # an IP address.
1650 if host and (host[-1].isdigit() or ":" in host):
1651 # RFC 6874 spells the IPv6 zone separator as the percent-encoded
1652 # ``%25``; bare ``%`` is still accepted so that hosts constructed
1653 # programmatically (e.g. ``with_host("fe80::1%1")``) keep working.
1654 part = "%25" if "%25" in host else "%"
1655 raw_ip, sep, zone = host.partition(part)
1656 # If it looks like an IP, we check with _ip_compressed_version
1657 # and fall-through if its not an IP address. This is a performance
1658 # optimization to avoid parsing IP addresses as much as possible
1659 # because it is orders of magnitude slower than almost any other
1660 # operation this library does.
1661 # Might be an IP address, check it
1662 #
1663 # IP Addresses can look like:
1664 # https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
1665 # - 127.0.0.1 (last character is a digit)
1666 # - 2001:db8::ff00:42:8329 (contains a colon)
1667 # - 2001:db8::ff00:42:8329%eth0 (contains a colon)
1668 # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should
1669 # have been removed before it gets here)
1670 # Rare IP Address formats are not supported per:
1671 # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4
1672 #
1673 # IP parsing is slow, so its wrapped in an LRU
1674 try:
1675 ip = ip_address(raw_ip)
1676 except ValueError:
1677 pass
1678 else:
1679 if sep and validate_host and (not zone or _ZONE_ID_UNSAFE_RE.search(zone)):
1680 raise ValueError("Invalid characters in zone identifier")
1681 # These checks should not happen in the
1682 # LRU to keep the cache size small
1683 host = ip.compressed
1684 if ip.version == 6:
1685 return f"[{host}{sep}{zone}]" if sep else f"[{host}]"
1686 return f"{host}{sep}{zone}" if sep else host
1688 # IDNA encoding is slow, skip it for ASCII-only strings
1689 if host.isascii():
1690 # Check for invalid characters explicitly; _idna_encode() does this
1691 # for non-ascii host names.
1692 host = host.lower()
1693 if validate_host and (invalid := NOT_REG_NAME.search(host)):
1694 value, pos, extra = invalid.group(), invalid.start(), ""
1695 if value == "@" or (value == ":" and "@" in host[pos:]):
1696 # this looks like an authority string
1697 extra = (
1698 ", if the value includes a username or password, "
1699 "use 'authority' instead of 'host'"
1700 )
1701 raise ValueError(
1702 f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}"
1703 ) from None
1704 return host
1706 # IDNA/UTS-46 mapping silently deletes default-ignorable code points, which
1707 # would turn e.g. ``e<ZWSP>vil.com`` into ``evil.com``, a different host
1708 # than the string the caller supplied. Reject them on every path (this runs
1709 # regardless of ``validate_host`` since the plain ``URL(str)`` constructor
1710 # encodes with ``validate_host=False``) so the parsed host cannot diverge
1711 # from the input, matching idna/httpx/urllib3.
1712 if invalid := _DEFAULT_IGNORABLE_RE.search(host):
1713 raise ValueError(
1714 f"Host {host!r} cannot contain {invalid.group()!r} "
1715 f"(at position {invalid.start()})"
1716 ) from None
1717 encoded = _idna_encode(host)
1718 # IDNA uses NFKC equivalence, so normalization can expand a non-ascii
1719 # character into an ASCII delimiter (e.g. the fullwidth solidus U+FF0F
1720 # becomes '/'). The ascii branch above rejects such delimiters directly;
1721 # apply the same check to the IDNA output so the builder APIs agree with
1722 # the parser's _check_netloc.
1723 if validate_host and (invalid := NOT_REG_NAME.search(encoded)):
1724 raise ValueError(
1725 f"Host {host!r} cannot contain {invalid.group()!r} "
1726 f"after IDNA normalization to {encoded!r}"
1727 ) from None
1728 return encoded
1731@rewrite_module
1732def cache_clear() -> None:
1733 """Clear all LRU caches."""
1734 _idna_encode.cache_clear()
1735 _idna_decode.cache_clear()
1736 _encode_host.cache_clear()
1739@rewrite_module
1740def cache_info() -> CacheInfo:
1741 """Report cache statistics."""
1742 return {
1743 "idna_encode": _idna_encode.cache_info(),
1744 "idna_decode": _idna_decode.cache_info(),
1745 "ip_address": _encode_host.cache_info(),
1746 "host_validate": _encode_host.cache_info(),
1747 "encode_host": _encode_host.cache_info(),
1748 }
1751@rewrite_module
1752def cache_configure(
1753 *,
1754 idna_encode_size: int | None = _DEFAULT_IDNA_SIZE,
1755 idna_decode_size: int | None = _DEFAULT_IDNA_SIZE,
1756 ip_address_size: int | None | UndefinedType = UNDEFINED,
1757 host_validate_size: int | None | UndefinedType = UNDEFINED,
1758 encode_host_size: int | None | UndefinedType = UNDEFINED,
1759) -> None:
1760 """Configure LRU cache sizes."""
1761 global _idna_decode, _idna_encode, _encode_host
1762 # ip_address_size, host_validate_size are no longer
1763 # used, but are kept for backwards compatibility.
1764 if ip_address_size is not UNDEFINED or host_validate_size is not UNDEFINED:
1765 warnings.warn(
1766 "cache_configure() no longer accepts the "
1767 "ip_address_size or host_validate_size arguments, "
1768 "they are used to set the encode_host_size instead "
1769 "and will be removed in the future",
1770 DeprecationWarning,
1771 stacklevel=2,
1772 )
1774 if encode_host_size is not None:
1775 for size in (ip_address_size, host_validate_size):
1776 if size is None:
1777 encode_host_size = None
1778 elif encode_host_size is UNDEFINED:
1779 if size is not UNDEFINED:
1780 encode_host_size = size
1781 elif size is not UNDEFINED:
1782 if TYPE_CHECKING:
1783 assert isinstance(size, int)
1784 assert isinstance(encode_host_size, int)
1785 encode_host_size = max(size, encode_host_size)
1786 if encode_host_size is UNDEFINED:
1787 encode_host_size = _DEFAULT_ENCODE_SIZE
1789 _encode_host = lru_cache(encode_host_size)(_encode_host.__wrapped__)
1790 _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__)
1791 _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__)