Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/turbohtml/extract/_structured_data.py: 87%

1""" 

2The typed result classes and JSON-LD parsing facade behind :meth:`turbohtml.Document.structured_data`. 

3 

4The tree walk that locates structured data and the OpenGraph/Microdata extraction all live in the C core 

5(``turbohtml._html``), exposed as the :meth:`~turbohtml.Document.structured_data`, :meth:`~turbohtml.Document.json_ld`, 

6:meth:`~turbohtml.Document.opengraph`, and :meth:`~turbohtml.Document.microdata` methods. The C pass assembles the plain 

7dict/list/str pieces and then hands them to the :class:`MicrodataItem`, :class:`StructuredData`, and :class:`OpenGraph` 

8record classes defined here, so the typed, read-only result shapes live in Python while every walk stays in C. The other 

9piece of real Python logic is parsing the gathered ``<script type="application/ld+json">`` texts with the standard 

10library :mod:`json`. Importing this module registers the parser and every record class with the core. 

11""" 

12 

13from __future__ import annotations 

14 

15import json 

16from collections.abc import Mapping 

17from dataclasses import dataclass 

18from typing import TYPE_CHECKING, Final, NoReturn, TypeAlias 

19 

20from turbohtml._html import _microdata_as_dict, _register_structured_data 

21 

22if TYPE_CHECKING: 

23 from collections.abc import Iterator 

24 

25JSONValue: TypeAlias = bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] | None 

26"""A decoded JSON value: a scalar, ``None``, or a list/dict nesting more of the same.""" 

27 

28 

29@dataclass(frozen=True, slots=True) 

30class MicrodataItem: 

31 """ 

32 One HTML Microdata item: an element carrying ``itemscope`` and the properties beneath it. 

33 

34 :param type: the ``itemtype`` attribute verbatim, or ``None`` when the element has none. 

35 :param id: the ``itemid`` attribute verbatim, or ``None`` when the element has none. 

36 :param properties: each ``itemprop`` name mapped to its values in document order, a value being the property's 

37 text/URL as a ``str`` or a nested :class:`MicrodataItem` for a property that is itself an ``itemscope``. 

38 """ 

39 

40 type: str | None 

41 """the ``itemtype`` attribute verbatim, or ``None`` when the element has none.""" 

42 id: str | None 

43 """the ``itemid`` attribute verbatim, or ``None`` when the element has none.""" 

44 properties: dict[str, list[str | MicrodataItem]] 

45 """each ``itemprop`` name mapped to its values in document order.""" 

46 

47 def get(self, name: str) -> str | MicrodataItem | None: 

48 """Return the property's first value, or ``None`` when the item lacks it (``microdata.Item.get``).""" 

49 values = self.properties.get(name) 

50 return values[0] if values else None 

51 

52 def get_all(self, name: str) -> list[str | MicrodataItem]: 

53 """Return the property's values in document order, an empty list when absent (``microdata.Item.get_all``).""" 

54 return self.properties.get(name, []) 

55 

56 def json(self) -> str: 

57 """Serialize as ``microdata.Item.json`` does: a two-space-indented JSON object of the tree below the item.""" 

58 return json.dumps(_microdata_as_dict(self), indent=2) 

59 

60 

61@dataclass(frozen=True, slots=True) 

62class RdfaItem: 

63 """ 

64 One RDFa resource: an element carrying ``typeof`` and the ``property`` values beneath it. 

65 

66 Mirrors :class:`MicrodataItem` with the RDFa vocabulary context added. ``property`` keys and the ``type`` IRIs are 

67 expanded against the in-scope ``@vocab`` and ``@prefix`` mappings (the RDFa 1.1 initial context seeds the common 

68 prefixes, so ``schema:``, ``dc:``, ``foaf:`` resolve without a page-level declaration). Prefix matching ignores 

69 case; a token whose prefix is undeclared, or a bare term with no ``@vocab`` in scope, is kept verbatim. 

70 

71 :param vocab: the in-scope ``@vocab`` IRI, resolved against ``base_url`` when supplied, or ``None`` when no 

72 vocabulary is set. 

73 :param type: the expanded ``@typeof`` IRIs in document order, an empty list for a valueless ``typeof``. 

74 :param resource: the resource subject (``@about``/``@resource``/``@href``/``@src``), or ``None`` for a blank node. 

75 A present empty reference resolves to the document base when ``base_url`` is supplied. 

76 :param properties: each expanded ``property`` IRI mapped to its values in document order, a value being a literal or 

77 IRI ``str`` or a nested :class:`RdfaItem` for a property that is itself a ``typeof``. 

78 """ 

79 

80 vocab: str | None 

81 """the in-scope ``@vocab`` IRI, or ``None`` when no vocabulary is set.""" 

82 type: list[str] 

83 """the expanded ``@typeof`` IRIs in document order, an empty list for a valueless ``typeof``.""" 

84 resource: str | None 

85 """the resource subject (``@about``/``@resource``/``@href``/``@src``), or ``None`` for a blank node.""" 

86 properties: dict[str, list[str | RdfaItem]] 

87 """each expanded ``property`` IRI mapped to its values in document order.""" 

88 

89 def get(self, name: str) -> str | RdfaItem | None: 

90 """Return the property's first value, or ``None`` when the item lacks it.""" 

91 values = self.properties.get(name) 

92 return values[0] if values else None 

93 

94 def get_all(self, name: str) -> list[str | RdfaItem]: 

95 """Return the property's values in document order, an empty list when absent.""" 

96 return self.properties.get(name, []) 

97 

98 

99@dataclass(frozen=True, slots=True) 

100class StructuredData: 

101 """ 

102 Every machine-readable metadata format a document embeds, the combined result of one walk. 

103 

104 :param json_ld: each ``<script type="application/ld+json">`` block decoded with :mod:`json`. 

105 :param microdata: every top-level :class:`MicrodataItem`. 

106 :param opengraph: each ``og:``/``twitter:`` ``<meta>`` key mapped to its content. 

107 :param microformats: reserved for a later phase, an empty list for now. 

108 :param rdfa: every top-level :class:`RdfaItem`. 

109 :param dublin_core: each ``dc.*``/``dcterms.*`` ``<meta>`` name (lower-cased) mapped to its content. 

110 """ 

111 

112 json_ld: list[JSONValue] 

113 """each ``<script type="application/ld+json">`` block decoded with :mod:`json`.""" 

114 microdata: list[MicrodataItem] 

115 """every top-level :class:`MicrodataItem`.""" 

116 opengraph: dict[str, str] 

117 """each ``og:``/``twitter:`` ``<meta>`` key mapped to its content.""" 

118 microformats: list[JSONValue] 

119 """reserved for a later phase, an empty list for now.""" 

120 rdfa: list[RdfaItem] 

121 """every top-level :class:`RdfaItem`.""" 

122 dublin_core: dict[str, str] 

123 """each ``dc.*``/``dcterms.*`` ``<meta>`` name (lower-cased) mapped to its content.""" 

124 

125 

126_OG_REQUIRED: Final = ("title", "type", "image", "url") 

127"""The Open Graph protocol's four required properties; the check behind :meth:`OpenGraph.is_valid`.""" 

128 

129 

130class OpenGraph(Mapping[str, str]): 

131 """ 

132 A page's Open Graph metadata as a read-only mapping, the record :meth:`turbohtml.Document.opengraph` returns. 

133 

134 Keys are the ``og:`` property names with that prefix stripped (``og:title`` reads as ``og["title"]``), matching the 

135 ``opengraph`` library's ``OpenGraph`` dict. The mapping supports the full read surface -- ``og["title"]``, 

136 ``"title" in og``, ``og.get("title")``, iteration, and equality against a plain ``dict`` -- and adds 

137 :meth:`is_valid`. 

138 """ 

139 

140 __slots__ = ("_properties",) 

141 

142 def __init__(self, properties: dict[str, str], /) -> None: 

143 """Wrap the already prefix-stripped ``og:`` property mapping the C walk builds.""" 

144 self._properties = properties 

145 

146 def __getitem__(self, key: str) -> str: 

147 """Return the value of the ``og:<key>`` property, raising :exc:`KeyError` when the page lacks it.""" 

148 return self._properties[key] 

149 

150 def __iter__(self) -> Iterator[str]: 

151 """Iterate the prefix-stripped property names in document order.""" 

152 return iter(self._properties) 

153 

154 def __len__(self) -> int: 

155 """Return the number of ``og:`` properties the page carries.""" 

156 return len(self._properties) 

157 

158 def __repr__(self) -> str: 

159 """Render as ``OpenGraph({...})`` around the underlying property mapping.""" 

160 return f"OpenGraph({self._properties!r})" 

161 

162 def is_valid(self) -> bool: 

163 """ 

164 Return whether the page carries the four properties the Open Graph protocol requires, each non-empty. 

165 

166 Mirrors the ``opengraph`` library's ``is_valid``: every one of ``og:title``, ``og:type``, ``og:image``, and 

167 ``og:url`` is present with a non-empty value. The ``opengraph_py3`` fork also demands ``og:description``; the 

168 `Open Graph protocol <https://ogp.me/>`_ does not, so it is not required here. 

169 """ 

170 return all(self._properties.get(name) for name in _OG_REQUIRED) 

171 

172 

173def _reject_constant(name: str) -> NoReturn: 

174 # json.loads accepts NaN, Infinity, and -Infinity, which RFC 8259 JSON does not 

175 msg = f"{name} is not a JSON value" 

176 raise ValueError(msg) 

177 

178 

179def _decode_json(text: str) -> JSONValue: 

180 return json.loads(text, parse_constant=_reject_constant) 

181 

182 

183_register_structured_data(_decode_json, MicrodataItem, RdfaItem, StructuredData, OpenGraph)