Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2turbohtml.extract: pull content and data out of HTML.
4Home for the extraction result types and (as the campaign lands) the content/date/url helpers that replace
5``trafilatura``/``readability-lxml``/``newspaper3k``/``goose3``/``justext``/``htmldate``/``courlan``/``extruct``.
6The extraction itself runs through the node methods (``node.article()``, ``node.links()``, ``node.tables()``,
7``node.structured_data()``); the records they return are re-exported here for discoverability and also stay
8importable from the package root.
10:func:`boilerplate` adds the per-paragraph view those methods do not give: it segments a page into paragraph units
11and marks each one good or boilerplate, the call shape ``justext`` and ``boilerpy3`` expose. The classification is a
12thin layer over the same C main-content scoring :meth:`turbohtml.Node.main_content` runs, refined per paragraph by
13the length and link-density thresholds a frozen :class:`Extraction` config carries.
15The URL helpers live here directly: :func:`clean_url`, :func:`normalize_url`, and :func:`extract_links`, configured
16by a frozen :class:`UrlCleaning`, replace ``courlan`` and the ``w3lib.url`` canonicalization surface.
18:func:`microdata` gives the ``microdata`` library's ``get_items(html)`` entry point over the same C walk
19:meth:`turbohtml.Document.microdata` runs, returning the :class:`MicrodataItem` records whose ``.get``/``.get_all``/
20``.json`` accessors mirror that library's ``Item``. :func:`opengraph` gives the ``opengraph`` library's
21``OpenGraph(html=...)`` shape over :meth:`turbohtml.Document.opengraph`: an :class:`OpenGraph` mapping of the
22``og:``-stripped keys with an :meth:`~OpenGraph.is_valid` check.
24:func:`dates` recovers a page's publication or modification date from its ``<meta>`` tags, JSON-LD, ``<time>``
25elements, and URL, the standalone entry point ``htmldate`` exposes; a frozen :class:`DateExtraction` carries its
26knobs and it returns a :class:`PublicationDate` naming the signal the date came from.
28:func:`feed` normalizes an RSS 2.0, Atom 1.0, or RDF/RSS-1.0 document into one :class:`Feed` of :class:`Entry`
29records, the ``feedparser.parse`` entry point over :meth:`turbohtml.Document.feed`.
30"""
32from __future__ import annotations
34from dataclasses import dataclass
35from itertools import starmap
36from typing import Final, NamedTuple, cast
38from turbohtml._html import Element, HTMLParseError, _boilerplate, parse, parse_xml
40from ._article import Article
41from ._dates import DateExtraction, PublicationDate, dates
42from ._feed import Entry, Feed
43from ._links import Link
44from ._structured_data import MicrodataItem, OpenGraph, RdfaItem, StructuredData
45from ._urls import UrlCleaning, clean_url, extract_links, normalize_url
47__all__ = [
48 "Article",
49 "DateExtraction",
50 "Element",
51 "Entry",
52 "Extraction",
53 "Feed",
54 "Link",
55 "MicrodataItem",
56 "OpenGraph",
57 "Paragraph",
58 "PublicationDate",
59 "RdfaItem",
60 "StructuredData",
61 "UrlCleaning",
62 "boilerplate",
63 "clean_url",
64 "dates",
65 "extract_links",
66 "feed",
67 "microdata",
68 "normalize_url",
69 "opengraph",
70]
73class Paragraph(NamedTuple):
74 """
75 One paragraph unit of a page and its classification, the record :func:`boilerplate` returns.
77 ``text`` is the unit's whitespace-normalized visible text (never empty; a blank unit is not reported).
78 ``is_boilerplate`` is ``True`` for navigation, footers, sidebars, and any in-content unit that fails the
79 :class:`Extraction` thresholds; the ``False`` entries concatenate to the article. ``is_heading`` marks
80 ``<h1>``-``<h6>`` units whatever their classification.
81 """
83 text: str
84 is_boilerplate: bool
85 is_heading: bool
88@dataclass(frozen=True, slots=True)
89class Extraction:
90 """
91 Options for :func:`boilerplate`: the per-paragraph thresholds refining the main-content scoring.
93 :param min_length: a non-heading paragraph inside the content body shorter than this many normalized characters
94 is still boilerplate. The default 25 is the floor the C scoring itself uses for a paragraph to contribute.
95 :param max_link_density: a paragraph whose text sits inside links beyond this fraction is boilerplate wherever
96 it lives; 0.5 tolerates prose with citations while dropping link lists that survive inside the content body.
97 :param keep_headings: with ``True`` a heading inside the content body is content regardless of ``min_length``
98 (headings are short by nature); with ``False`` headings face the same length threshold as prose, justext's
99 ``no_headings`` mode.
100 """
102 min_length: int = 25
103 max_link_density: float = 0.5
104 keep_headings: bool = True
106 def __post_init__(self) -> None:
107 """Reject a negative length floor and a link-density bound outside the unit interval."""
108 if self.min_length < 0:
109 msg = f"min_length must be non-negative, got {self.min_length}"
110 raise ValueError(msg)
111 if not 0.0 <= self.max_link_density <= 1.0:
112 msg = f"max_link_density must be within [0.0, 1.0], got {self.max_link_density}"
113 raise ValueError(msg)
115 @classmethod
116 def justext(cls) -> Extraction:
117 """Justext's defaults: its 70-character ``length_low`` floor and 0.2 ``max_link_density``."""
118 return cls(min_length=70, max_link_density=0.2)
121_DEFAULT: Final = Extraction()
124def microdata(html: str, base_url: str | None = None, /) -> list[MicrodataItem]:
125 """
126 Extract a page's top-level HTML Microdata items, the successor to ``microdata.get_items(html)``.
128 Each returned :class:`~turbohtml.MicrodataItem` carries the ``itemtype`` as
129 :attr:`~turbohtml.MicrodataItem.type`, the ``itemid`` as :attr:`~turbohtml.MicrodataItem.id`, and the ``itemprop``
130 values under :attr:`~turbohtml.MicrodataItem.properties`, with :meth:`~turbohtml.MicrodataItem.get`,
131 :meth:`~turbohtml.MicrodataItem.get_all`, and :meth:`~turbohtml.MicrodataItem.json` matching the library's ``Item``.
132 Shorthand for :meth:`turbohtml.Document.microdata` when a page string is all you hold.
134 :param html: the page markup.
135 :param base_url: when given, the URL each relative URL-valued property (an ``a``/``area``/``link`` href, a media
136 ``src``, an ``object`` data) is resolved against, the ``microdata`` library's ``base_url``; a ``<base href>``
137 refines it. ``None`` (the default) returns every value verbatim.
138 :returns: one :class:`~turbohtml.MicrodataItem` per top-level ``itemscope`` element, in document order.
139 :raises ValueError: if ``base_url`` is given and is not a valid absolute URL.
140 """
141 return parse(html).microdata(base_url)
144def opengraph(html: str, base_url: str | None = None, /) -> OpenGraph:
145 """
146 Extract a page's Open Graph metadata, the successor to ``opengraph.OpenGraph(html=...)``.
148 Reads the ``og:`` ``<meta>`` tags :meth:`turbohtml.Document.opengraph` gathers, with the ``twitter:`` tags dropped
149 and the ``og:`` prefix stripped from each key so ``og["title"]`` reads the ``og:title`` tag. Shorthand for
150 :meth:`turbohtml.Document.opengraph` when a page string is all you hold.
152 :param html: the page markup.
153 :param base_url: when given, the URL each relative URL-valued property (``og:url``, ``og:image``, ``og:video``,
154 ...) is resolved against, extruct's and metadata_parser's absolutization; a ``<base href>`` refines it.
155 ``None`` (the default) returns every value verbatim.
156 :returns: an :class:`OpenGraph` mapping of the prefix-stripped ``og:`` properties, empty when the page has none.
157 :raises ValueError: if ``base_url`` is given and is not a valid absolute URL.
158 """
159 return parse(html).opengraph(base_url)
162def feed(xml: str, /) -> Feed | None:
163 """
164 Parse an RSS 2.0, Atom 1.0, or RDF/RSS-1.0 document into a :class:`Feed`, the ``feedparser.parse`` successor.
166 Detects the format from the root element and maps each format's spelling of a field onto one shape: the feed's
167 ``title``, ``link``, ``description``, and ``updated``, and its ``entries``, each :class:`Entry` carrying ``title``,
168 ``link``, ``id``, ``updated``/``published``, ``summary``/``content``, and ``author``. Timestamps are returned
169 verbatim, not parsed. A well-formed feed parses under :func:`turbohtml.parse_xml`, so CDATA sections and nested
170 elements such as a channel ``<image>`` keep their XML meaning. An Atom XML root must declare the
171 ``http://www.w3.org/2005/Atom`` namespace; an RSS 2.0 XML root must declare ``version="2.0"``. A feed that is not
172 well-formed XML falls back to the HTML tree builder's recovery. Shorthand for
173 :meth:`turbohtml.Document.feed` on either tree.
175 :param xml: the feed document.
176 :returns: the parsed :class:`Feed`, or ``None`` when ``xml`` has no ``<rss>``, ``<feed>``, or ``<rdf:RDF>`` root.
177 """
178 try:
179 document = parse_xml(xml)
180 except HTMLParseError:
181 document = parse(xml)
182 return document.feed()
185def boilerplate(html: str, options: Extraction | None = None, /) -> list[Paragraph]:
186 """
187 Segment a page into paragraph units and classify each as content or boilerplate, in document order.
189 The successor to ``justext.justext`` and ``boilerpy3``'s per-block ``is_content``: the C main-content scoring
190 picks the content body exactly as :meth:`turbohtml.Node.main_content` does, every unit outside it is
191 boilerplate, and a unit inside it must still clear the ``options`` length and link-density thresholds. A page
192 with no scoring content body (a stub, pure navigation) classifies every unit as boilerplate.
194 :param html: the page markup.
195 :param options: the classification thresholds; defaults to :class:`Extraction`.
196 :returns: one :class:`Paragraph` per non-blank unit.
197 """
198 thresholds = options or _DEFAULT
199 document = parse(html)
200 rows = _boilerplate(
201 # a parsed document always has an <html> root, though the type admits None
202 cast("Element", document.root),
203 document.main_content(),
204 thresholds.min_length,
205 thresholds.max_link_density,
206 thresholds.keep_headings,
207 )
208 return list(starmap(Paragraph, rows))