Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2URL cleaning, normalization, and page-level link extraction for :mod:`turbohtml.extract`.
4The normalization core follows the `WHATWG URL standard <https://url.spec.whatwg.org/>`__ wherever it defines the
5behavior: input stripping (basic URL parser, spec 4.4), scheme lowercasing (scheme state), domain-to-ASCII for special
6URL hosts and percent-encoding for opaque hosts (host parsing, spec 3.5), default-port removal (port state), and
7dot-segment resolution including the
8``%2e`` forms (path state), the empty-path-to-``/`` serialization of special URLs (URL serializing, spec 4.5), and
9the path/query/fragment percent-encode sets (percent-encoded bytes, spec 1.3). On top of that sits the crawl-oriented
10canonicalization ``courlan`` and ``w3lib`` users expect -- query-parameter sorting, tracking-parameter removal, an
11optional strict parameter allowlist, and a URL-based language filter -- each documented where it goes beyond the spec.
12The pipeline runs in C (``_url_clean``, ``_url_normalize`` and ``Document._extract_links``); this module holds the
13option record and the two vocabularies it hands over.
14"""
16from __future__ import annotations
18from dataclasses import dataclass
19from typing import Final
21from turbohtml._html import _url_clean, _url_normalize, parse
23__all__ = [
24 "UrlCleaning",
25 "clean_url",
26 "extract_links",
27 "normalize_url",
28]
30_ISO_639_1: Final[frozenset[str]] = frozenset([
31 *("aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az", "ba", "be", "bg", "bh", "bi", "bm"),
32 *("bn", "bo", "br", "bs", "ca", "ce", "ch", "co", "cr", "cs", "cu", "cv", "cy", "da", "de", "dv", "dz", "ee"),
33 *("el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr", "fy", "ga", "gd", "gl", "gn", "gu"),
34 *("gv", "ha", "he", "hi", "ho", "hr", "ht", "hu", "hy", "hz", "ia", "id", "ie", "ig", "ii", "ik", "io", "is"),
35 *("it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", "kk", "kl", "km", "kn", "ko", "kr", "ks", "ku", "kv", "kw"),
36 *("ky", "la", "lb", "lg", "li", "ln", "lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms"),
37 *("mt", "my", "na", "nb", "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv", "ny", "oc", "oj", "om", "or", "os"),
38 *("pa", "pi", "pl", "ps", "pt", "qu", "rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk"),
39 *("sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "ti", "tk", "tl"),
40 *("tn", "to", "tr", "ts", "tt", "tw", "ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", "wa", "wo", "xh", "yi"),
41 *("yo", "za", "zh", "zu"),
42])
43"""The ISO 639-1 two-letter language codes, gating which URL segments count as language markers."""
45_CONTENT_PARAMS: Final[frozenset[str]] = frozenset({
46 "aid",
47 "article_id",
48 "artnr",
49 "id",
50 "itemid",
51 "objectid",
52 "p",
53 "page",
54 "page_id",
55 "pagenum",
56 "pid",
57 "post",
58 "postid",
59 "product_id",
60})
61"""The content-identifying query parameters strict mode keeps; everything else is presumed decorative."""
63_LANGUAGE_PARAMS: Final[frozenset[str]] = frozenset({"lang", "language"})
66@dataclass(frozen=True, slots=True)
67class UrlCleaning:
68 """
69 Options shared by :func:`clean_url`, :func:`normalize_url`, and :func:`extract_links`.
71 :param strict: keep only the content-identifying query parameters (page, id, post, ... plus the language
72 parameters) instead of merely dropping known trackers, and drop the fragment. The default keeps every
73 parameter that is not a known tracker.
74 :param trailing_slash: keep a path's trailing slash. ``False`` trims it from any path but the root ``/`` when no
75 query string follows, folding ``/dir/`` and ``/dir`` into one form.
76 :param strip_fragment: always drop the fragment. The default keeps it (scrubbed of tracker parameters), since the
77 fragment can address content (``#page2``, text fragments).
78 :param language: an ISO 639-1 code; :func:`clean_url` and :func:`extract_links` then reject URLs whose language
79 markers (a leading path segment such as ``/de/``, a ``lang``/``language`` query parameter, or an anchor's
80 ``hreflang``) point at another language. :func:`normalize_url` ignores this field.
81 :param query_allow: when set, keep only these query parameters (matched case-insensitively against the decoded
82 name), the ``w3lib.url.url_query_cleaner`` keep-list; a listed parameter survives even when it is a known
83 tracker. Mutually exclusive with ``strict``, which is itself an allowlist.
84 :param query_deny: always drop these query parameters (matched the same way), the ``url_query_cleaner``
85 ``remove=True`` mode; the tracker or ``strict`` filtering still applies to the rest.
86 """
88 strict: bool = False
89 trailing_slash: bool = True
90 strip_fragment: bool = False
91 language: str | None = None
92 query_allow: frozenset[str] | None = None
93 query_deny: frozenset[str] = frozenset()
95 def __post_init__(self) -> None:
96 """Reject a non-ISO-639-1 language and the contradiction of two query allowlists at once."""
97 if self.language is not None and self.language not in _ISO_639_1:
98 msg = f"language must be an ISO 639-1 code, got {self.language!r}"
99 raise ValueError(msg)
100 if self.strict and self.query_allow is not None:
101 msg = "strict and query_allow are mutually exclusive, each is a query-parameter allowlist"
102 raise ValueError(msg)
104 @classmethod
105 def w3lib(cls) -> UrlCleaning:
106 """Return ``w3lib.url.canonicalize_url``'s mode: fragments dropped, every non-tracker parameter kept."""
107 return cls(strip_fragment=True)
110_DEFAULT: Final = UrlCleaning()
113def clean_url(url: str, options: UrlCleaning | None = None, /) -> str | None:
114 """
115 Scrub a URL scraped from markup and normalize it, or return ``None`` when nothing usable remains.
117 The scrub recovers from HTML transport damage: it strips the surrounding whitespace and control characters the
118 WHATWG basic URL parser removes (spec 4.4 steps 1-2, extended to embedded spaces since those only appear in
119 scraped junk), unwraps ``<![CDATA[...]]>``, truncates at a stray ``<``/``>``/``"`` delimiter, and undoes a
120 leftover ``&`` escape. The survivor must be an ``http``/``https`` URL with a plausible host and pass the
121 ``language`` filter, then it is returned through :func:`normalize_url`.
123 :param url: the URL as found in the wild.
124 :param options: the cleaning options; defaults to :class:`UrlCleaning` (drop trackers, keep slash and fragment).
125 :returns: the cleaned, normalized URL, or ``None`` for anything that is not a fetchable web URL.
126 :raises TypeError: if ``url`` is not a ``str``.
127 """
128 active = options or _DEFAULT
129 return _url_clean(url, *_knobs(active), active.language, _ISO_639_1)
132def normalize_url(url: str, options: UrlCleaning | None = None, /) -> str:
133 """
134 Return the canonical form of a URL, so that two spellings of the same resource compare equal.
136 The spec-defined part lowercases the scheme. Special URL hosts are lowercased and converted to ASCII (browsers
137 serialize ``http://münchen.de`` as ``http://xn--mnchen-3ya.de``). Non-special hosts preserve ASCII case and
138 percent-encode Unicode (``git://münchen.de`` becomes ``git://m%C3%BCnchen.de``). Normalization also drops a default
139 port (port state), resolves ``.``/``..``/``%2e`` path segments (path state), percent-encodes what
140 the path/query/fragment percent-encode sets require -- with uppercase hex digits, leaving existing escapes alone
141 -- and serializes an empty special-URL path as ``/`` (URL serializing). Beyond the spec, query parameters are
142 sorted and known tracking parameters dropped (strict mode instead keeps only the content-identifying allowlist),
143 and a fragment shaped like a query string is scrubbed the same way. Unlike ``courlan``, repeated slashes are kept
144 (the spec preserves them) and punycode is the output form, not the input form.
145 For special schemes, a host after the scheme is recognized even when the input omits the authority slashes;
146 a hostless ``file:`` path is serialized with an empty authority.
148 A Unicode host longer than 16384 code points, or holding a code point UTS #46 disallows in a domain (a C1
149 control, a noncharacter), keeps its lowercased Unicode form instead of punycode. The cap bounds the domain-to-ASCII
150 step, whose combining-mark reorder and punycode encoder are quadratic in the host length. The WHATWG standard
151 leaves overlong labels undefined (`whatwg/url#824 <https://github.com/whatwg/url/issues/824>`_), and DNS caps a
152 domain at 253 octets, far below the cap.
154 :param url: an absolute or relative URL; a relative one keeps its shape, only its components are normalized.
155 :param options: the cleaning options; defaults to :class:`UrlCleaning` (drop trackers, keep slash and fragment).
156 :returns: the normalized URL.
157 :raises TypeError: if ``url`` is not a ``str``.
158 :raises ValueError: if the URL cannot be split into components (e.g. an unclosed IPv6 bracket) or carries a
159 forbidden or empty domain after host decoding or IDNA mapping, or a character that cannot be percent-encoded
160 (a lone surrogate).
161 """
162 return _url_normalize(url, *_knobs(options or _DEFAULT))
165def extract_links(
166 html: str,
167 base_url: str | None = None,
168 options: UrlCleaning | None = None,
169 /,
170 *,
171 external_only: bool = False,
172) -> set[str]:
173 """
174 Collect the cleaned page links of an HTML document, the ``courlan.extract_links`` counterpart.
176 The document is parsed with the WHATWG tree builder, so links are read from the real DOM rather than regex
177 matches. An anchor (``<a>``/``<area>``) contributes its ``href`` unless its ``rel`` carries ``nofollow`` or, with
178 a ``language`` filter active, its ``hreflang`` names another language (``x-default`` passes). Each candidate is
179 resolved against the document base URL (a ``<base href>`` wins over ``base_url``, per HTML spec 4.2.3), cleaned
180 through :func:`clean_url`, and deduplicated across trivial variants (the ``http``/``https`` twin and the
181 trailing-slash twin), the first occurrence in document order winning.
183 :param html: the page markup.
184 :param base_url: the URL the page was fetched from; relative links resolve against it, and it anchors the
185 external/internal split. Without it relative links are dropped, since they cannot be made absolute.
186 :param options: the cleaning options; defaults to :class:`UrlCleaning` (drop trackers, keep slash and fragment).
187 :param external_only: keep only links leaving ``base_url``'s site, where the site boundary is the registrable
188 domain (the public suffix plus one label, ``spam.example.co.uk`` and ``example.co.uk`` counting as one site);
189 the eTLD+1 is read from the shipped IANA and Public Suffix List tables, so sibling subdomains stay internal.
190 :returns: the surviving absolute URLs.
191 :raises ValueError: if ``external_only`` is set without a ``base_url`` to define what external means.
192 """
193 active = options or _DEFAULT
194 return parse(html)._extract_links( # ruff:ignore[private-member-access] # the Document type's private C entry point
195 base_url, external_only, *_knobs(active), active.language, _ISO_639_1
196 )
199def _knobs(
200 options: UrlCleaning,
201) -> tuple[bool, bool, bool, frozenset[str] | None, frozenset[str], frozenset[str], frozenset[str]]:
202 """Return the option fields and vocabularies every C entry point takes, in its argument order."""
203 return (
204 options.strict,
205 options.trailing_slash,
206 options.strip_fragment,
207 options.query_allow,
208 options.query_deny,
209 _CONTENT_PARAMS,
210 _LANGUAGE_PARAMS,
211 )