Coverage for /pythoncovmergedfiles/medio/medio/src/turbohtml/tools/fuzz/content_targets.py: 99%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Fixed envelopes give content consumers independent expected results per input."""
3from __future__ import annotations
5import html
6import json
7import unicodedata
8from functools import partial
9from typing import TYPE_CHECKING, Final, cast
11import turbohtml
12from turbohtml import clean, detect, extract, parse_fragment
13from turbohtml.migration import bleach, markupsafe, stdlib
15from .atheris_header import BYTEWISE, Header, feed_chunks
16from .atheris_registry import Target
17from .idna_nfc_oracles import idna_nfc_check
18from .round_trip_oracles import (
19 clean_url_check,
20 css_semantics_check,
21 encoding_decode_check,
22 encoding_stream_check,
23 fixpoint_check,
24 idna_host_check,
25 normalize_url_check,
26 url_reparse_check,
27)
29if TYPE_CHECKING:
30 from collections.abc import Callable
32__all__ = [
33 "content_targets",
34 "detector_check",
35 "minifier_check",
36 "minifier_observation",
37 "sanitizer_check",
38 "sanitizer_observation",
39 "stdlib_check",
40 "stdlib_observation",
41 "url_check",
42]
45def content_targets() -> tuple[Target, ...]:
46 """Keep each qualified content export attached to an executable contract."""
47 return (
48 Target(
49 "content-sanitize",
50 _sanitize,
51 tuple(
52 f"turbohtml.clean.{name}"
53 for name in (
54 "DEFAULT_ATTRIBUTES",
55 "DEFAULT_CSS_PROPERTIES",
56 "DEFAULT_SCHEMES",
57 "DEFAULT_TAGS",
58 "OnDisallowed",
59 "Policy",
60 "Removed",
61 "Sanitizer",
62 "Transform",
63 "sanitize",
64 "sanitize_node",
65 "sanitize_report",
66 "sanitize_report_node",
67 "collapse_whitespace_node",
68 "strip_comments_node",
69 "transform_node",
70 )
71 ),
72 ),
73 Target(
74 "content-links",
75 _links,
76 tuple(
77 f"turbohtml.clean.{name}"
78 for name in (
79 "DEFAULT_CALLBACKS",
80 "DEFAULT_PHONE_LABELS",
81 "Callback",
82 "LinkCandidate",
83 "LinkDetector",
84 "LinkSpan",
85 "Linker",
86 "Linkify",
87 "PhoneFormat",
88 "PhoneGrouping",
89 "PhoneNumber",
90 "PhoneNumbers",
91 "PhoneType",
92 "linkify",
93 "linkify_node",
94 "nofollow",
95 "target_blank",
96 )
97 ),
98 ),
99 Target(
100 "content-minify",
101 _minify,
102 tuple(
103 f"turbohtml.clean.{name}"
104 for name in (
105 "CSSMinify",
106 "JSMinify",
107 "Minify",
108 "minify",
109 "minify_css_inline",
110 )
111 ),
112 ),
113 Target(
114 "content-detect",
115 _detect,
116 tuple(
117 f"turbohtml.detect.{name}"
118 for name in (
119 "Detection",
120 "EncodingDetector",
121 "EncodingMatch",
122 "LanguageDetection",
123 "LanguageMatch",
124 "NormalizationForm",
125 "detect",
126 "detect_all",
127 "detect_language",
128 "is_normalized",
129 "normalize",
130 )
131 ),
132 chunked=_detect_chunks,
133 ),
134 Target(
135 "content-extract",
136 _extract,
137 (
138 *(
139 f"turbohtml.extract.{name}"
140 for name in (
141 "Article",
142 "DateExtraction",
143 "Element",
144 "Entry",
145 "Extraction",
146 "Feed",
147 "Link",
148 "MicrodataItem",
149 "OpenGraph",
150 "Paragraph",
151 "PublicationDate",
152 "RdfaItem",
153 "StructuredData",
154 "UrlCleaning",
155 "boilerplate",
156 "clean_url",
157 "dates",
158 "extract_links",
159 "feed",
160 "microdata",
161 "normalize_url",
162 "opengraph",
163 )
164 ),
165 *(
166 f"turbohtml.{name}"
167 for name in (
168 "Article",
169 "Entry",
170 "Feed",
171 "Link",
172 "MicrodataItem",
173 "OpenGraph",
174 "RdfaItem",
175 "StructuredData",
176 )
177 ),
178 ),
179 ),
180 Target(
181 "content-migration",
182 _migration,
183 (
184 *(
185 f"turbohtml.migration.bleach.{name}"
186 for name in ("ALLOWED_ATTRIBUTES", "ALLOWED_PROTOCOLS", "ALLOWED_TAGS", "attribute_policy", "clean")
187 ),
188 *(
189 f"turbohtml.migration.markupsafe.{name}"
190 for name in ("EscapeFormatter", "Markup", "escape", "escape_silent", "soft_str")
191 ),
192 ),
193 ),
194 Target(
195 "content-stdlib",
196 _stdlib,
197 ("turbohtml.migration.stdlib.HTMLParser",),
198 (UnicodeDecodeError,),
199 _stdlib_chunks,
200 ),
201 )
204def _sanitize(data: bytes) -> None:
205 sanitizer_check(data)
206 text: Final = _text(data)
207 _equal(clean.Policy.strict().tags, frozenset())
208 _equal(clean.Policy.basic().tags, clean.DEFAULT_TAGS)
209 _equal("h1" in clean.Policy.relaxed().tags, expected=True)
210 markup: Final = f'<b title="{text}" onclick="ignored">{text}</b><!--comment--><unknown>{text}</unknown>'
211 options: Final = clean.Policy(
212 tags=clean.DEFAULT_TAGS,
213 attributes=clean.DEFAULT_ATTRIBUTES,
214 url_schemes=clean.DEFAULT_SCHEMES,
215 css_properties=clean.DEFAULT_CSS_PROPERTIES,
216 on_disallowed_tag=clean.OnDisallowed.STRIP,
217 transform_tags={"unknown": clean.Transform("b")},
218 )
219 expected: Final = f"<b>{text}</b><b>{text}</b>"
220 _equal(sanitizer_observation(data), (expected, [clean.Removed("b", "title"), clean.Removed("b", "onclick")]))
221 _equal(clean.sanitize(markup, options), expected)
222 sanitizer: Final = clean.Sanitizer(options)
223 _equal(sanitizer.sanitize(markup), expected)
224 rendered, removed = clean.sanitize_report(markup, options)
225 _equal(rendered, expected)
226 _equal(removed, [clean.Removed("b", "title"), clean.Removed("b", "onclick")])
227 _equal(sanitizer.sanitize_report(markup), (rendered, removed))
228 _equal(clean.sanitize_node(parse_fragment(markup), options).inner_html, expected)
229 node, report = clean.sanitize_report_node(parse_fragment(markup), options)
230 _equal((node.inner_html, report), (expected, removed))
231 _equal(sanitizer.sanitize_node(parse_fragment(markup)).inner_html, expected)
232 node, report = sanitizer.sanitize_report_node(parse_fragment(markup))
233 _equal((node.inner_html, report), (expected, removed))
234 node = parse_fragment(f"<p>{text} x<!--comment--></p>")
235 _equal(clean.transform_node(node, clean.strip_comments_node, clean.collapse_whitespace_node).text, f"{text} x")
238def _links(data: bytes) -> None:
239 text: Final = _text(data)
240 url: Final = f"https://example.com/{text}"
241 candidate: Final = clean.LinkCandidate(url, text)
242 callback: Final[clean.Callback] = clean.target_blank
243 _equal(callback(clean.nofollow(candidate)).attrs, {"rel": "nofollow", "target": "_blank"})
244 options: Final = clean.Linkify(callbacks=(*clean.DEFAULT_CALLBACKS, callback))
245 linked: Final = f'<a href="{url}" rel="nofollow" target="_blank">{url}</a>'
246 _equal(clean.linkify(url, options), linked)
247 _equal(clean.Linker(options).linkify(url), linked)
248 _equal(clean.linkify_node(parse_fragment(url), options).inner_html, linked)
249 _equal(clean.Linker(options).linkify_node(parse_fragment(url)).inner_html, linked)
250 _equal(clean.LinkDetector().find(url), [clean.LinkSpan(0, len(url), url, url, is_email=False)])
251 phones: Final = clean.PhoneNumbers(
252 regions=("US",),
253 grouping=clean.PhoneGrouping.EXACT,
254 types=(clean.PhoneType.FIXED_LINE_OR_MOBILE,),
255 ignore_numbers_after=clean.DEFAULT_PHONE_LABELS,
256 )
257 number: Final = clean.PhoneNumber.parse("650-253-0000", regions=("US",))
258 _equal(number, clean.PhoneNumber(1, "6502530000", None, "US", clean.PhoneType.FIXED_LINE_OR_MOBILE))
259 _equal(number.format(clean.PhoneFormat.E164), "+16502530000")
260 spans: Final = clean.LinkDetector(phones=phones).find(f"{text} 650-253-0000")
261 _equal([span.phone for span in spans], [number])
264def _minify(data: bytes) -> None:
265 minifier_check(data)
266 text: Final = _text(data)
267 _equal(
268 minifier_observation(data),
269 (
270 f"<html><head></head><body><p>{text}</p></body></html>",
271 f".{text}{{color:red}}",
272 "color:red",
273 f'const value="{text}"',
274 ),
275 )
278def _detect(data: bytes) -> None:
279 text: Final = _text(data)
280 _equal(encoding_stream_check(f"utf-8-bom\n{text}"), None)
281 _equal(encoding_decode_check(f"utf-8-bom\n{text}"), None)
282 encoded: Final = b"\xef\xbb\xbf" + text.encode()
283 options: Final = detect.Detection()
284 match: Final = detect.detect(encoded, options)
285 _equal(match, detect.EncodingMatch("UTF-8-SIG", 1.0, None, bom=True, codec="whatwg-utf-8-sig"))
286 _equal(detect.detect_all(encoded, options)[0], match)
287 stream: Final = detect.EncodingDetector(options)
288 stream.feed(b"")
289 for chunk in encoded:
290 stream.feed(bytes((chunk,)))
291 _equal(stream.close(), match)
292 language: Final = detect.detect_language(str(len(data)), detect.LanguageDetection(allowed=frozenset({"eng"})))
293 _equal(language, detect.LanguageMatch(None, 0.0, None))
294 for form in cast("tuple[detect.NormalizationForm, ...]", ("NFC", "NFD", "NFKC", "NFKD")):
295 normalized: Final = detect.normalize(form, f"{text}e\u0301")
296 _equal(normalized, unicodedata.normalize(form, f"{text}e\u0301"))
297 _equal(detect.is_normalized(form, normalized), expected=True)
300def _detect_chunks(data: bytes, header: Header) -> None:
301 _detect(data)
302 detector_check(data, header)
305def _extract(data: bytes) -> None:
306 url_check(data)
307 text: Final = _text(data)
308 url: Final = f"https://example.com/{text}"
309 markup: Final = (
310 f'<title>{text}</title><meta property="og:title" content="{text}">'
311 f'<meta property="og:type" content="article"><meta property="og:image" content="{url}">'
312 f'<meta property="og:url" content="{url}"><meta name="date" content="2020-01-02">'
313 f'<article><p>{text}</p></article><a href="{url}">{text}</a>'
314 f'<div itemscope itemtype="https://schema.org/Thing"><span itemprop="name">{text}</span></div>'
315 f'<div vocab="https://schema.org/" typeof="Thing"><span property="name">{text}</span></div>'
316 )
317 document: Final = turbohtml.parse(markup)
318 article: Final = document.article()
319 _equal(isinstance(article, (extract.Article, turbohtml.Article)), expected=True)
320 _equal(article.title, text)
321 links: Final = document.links()
322 _equal(isinstance(links[0], (extract.Link, turbohtml.Link)), expected=True)
323 _equal(links[0].url, url)
324 _equal(extract.extract_links(markup), {url})
325 _equal(extract.normalize_url(url, extract.UrlCleaning()), url)
326 _equal(extract.clean_url(url, extract.UrlCleaning()), url)
327 paragraphs: Final = extract.boilerplate(f"<article><p>{text}</p></article>", extract.Extraction(min_length=0))
328 _equal(isinstance(paragraphs[0], extract.Paragraph), expected=True)
329 _equal(paragraphs[0].text, text)
330 _equal(isinstance(document.select_one("article"), extract.Element), expected=True)
331 items: Final = extract.microdata(markup)
332 _equal(isinstance(items[0], (extract.MicrodataItem, turbohtml.MicrodataItem)), expected=True)
333 _equal(items[0].get_all("name"), [text])
334 _equal(json.loads(items[0].json())["properties"]["name"], [text])
335 graph: Final = extract.opengraph(markup)
336 _equal(isinstance(graph, (extract.OpenGraph, turbohtml.OpenGraph)), expected=True)
337 _equal((graph["title"], graph.is_valid()), (text, True))
338 structured: Final = document.structured_data()
339 _equal(isinstance(structured, (extract.StructuredData, turbohtml.StructuredData)), expected=True)
340 _equal(isinstance(structured.rdfa[0], (extract.RdfaItem, turbohtml.RdfaItem)), expected=True)
341 _equal(structured.rdfa[0].get("https://schema.org/name"), text)
342 publication: Final = cast("extract.PublicationDate", extract.dates(markup, extract.DateExtraction()))
343 _equal(isinstance(publication, extract.PublicationDate), expected=True)
344 _equal(str(publication.date), "2020-01-02")
345 feed: Final = cast(
346 "extract.Feed",
347 extract.feed(
348 f"<rss version='2.0'><channel><title>{text}</title><item><title>{text}</title></item></channel></rss>"
349 ),
350 )
351 _equal(isinstance(feed, (extract.Feed, turbohtml.Feed)), expected=True)
352 _equal((feed.title, feed.entries[0].title), (text, text))
353 _equal(isinstance(feed.entries[0], (extract.Entry, turbohtml.Entry)), expected=True)
356def _migration(data: bytes) -> None:
357 text: Final = _text(data)
358 _equal(
359 bleach.clean(
360 f"<b>{text}</b><!--comment-->",
361 tags=bleach.ALLOWED_TAGS,
362 attributes=bleach.ALLOWED_ATTRIBUTES,
363 protocols=bleach.ALLOWED_PROTOCOLS,
364 ),
365 f"<b>{text}</b>",
366 )
367 names, predicate = bleach.attribute_policy({"b": ("title",)})
368 _equal((names, predicate), ({"b": frozenset({"title"})}, None))
369 raw: Final = f"<{text}>&\"'"
370 expected: Final = html.escape(raw).replace("'", "'").replace(""", """)
371 escaped: Final = markupsafe.escape(raw)
372 _equal((isinstance(escaped, markupsafe.Markup), str(escaped)), (True, expected))
373 _equal(str(markupsafe.escape_silent(raw)), expected)
374 _equal(str(markupsafe.escape_silent(None)), "")
375 _equal(markupsafe.soft_str(escaped), escaped)
376 _equal(markupsafe.Markup(f"<b>{text}</b>").striptags(), text)
377 _equal(markupsafe.EscapeFormatter(markupsafe.escape).vformat("{}", (raw,), {}), expected)
380def _stdlib(data: bytes) -> None:
381 text: Final = _text(data)
382 parser: Final = _TextParser()
383 markup: Final = f"<b>{text}&</b>"
384 parser.feed("")
385 for char in markup:
386 parser.feed(char)
387 parser.close()
388 _equal("".join(parser.text), f"{text}&")
389 _equal(stdlib_observation(data), f"{text}&")
390 parser.reset()
391 _equal(parser.getpos(), (1, 0))
394def _stdlib_chunks(data: bytes, header: Header) -> None:
395 _stdlib(data)
396 stdlib_check(data, header)
399def detector_check(data: bytes, header: Header = BYTEWISE) -> None:
400 """Require the fuzz bytes fed in chunks, empty ones included, to reach the answer one ``detect`` call gives."""
401 options: Final = detect.Detection()
402 stream: Final = detect.EncodingDetector(options)
403 feed_chunks(stream.feed, data, header.max_chunk)
404 _equal(stream.close(), detect.detect(data, options))
407def stdlib_check(data: bytes, header: Header = BYTEWISE) -> None:
408 """Require the decoded fuzz text fed in chunks to reach the handlers as one ``feed`` call does."""
409 source: Final = data.decode("utf-8")
410 chunked: Final = _TextParser()
411 feed_chunks(chunked.feed, source, header.max_chunk)
412 chunked.close()
413 whole: Final = _TextParser()
414 whole.feed(source)
415 whole.close()
416 _equal("".join(chunked.text), "".join(whole.text))
419def minifier_check(
420 data: bytes,
421 css: Callable[[str], str] = clean.minify_css,
422 js: Callable[[str], str] | None = None,
423) -> None:
424 """Stable minifier output must also preserve the stylesheet meaning."""
425 text: Final = _text(data)
426 sheet: Final = f".{text} {{ color: #ff0000; margin: 0px 0px }}"
427 _equal(fixpoint_check(sheet, css, numeric=True), None)
428 _equal(css_semantics_check(f'<style>{sheet}</style><p class="{text}">x</p>', css), None)
429 _equal(
430 fixpoint_check(
431 f'const value = "{text}";',
432 js or partial(clean.minify_js, options=clean.JSMinify(fold=False, mangle=False)),
433 numeric=False,
434 ),
435 None,
436 )
439def url_check(data: bytes, normalize: Callable[[str], str] = extract.normalize_url) -> None:
440 """Known-valid Unicode families distinguish stable wrong hosts from valid normalization."""
441 url: Final = f"https://example.com/{_text(data)}"
442 _equal(url_reparse_check(url, normalize), None)
443 affix: Final = data[:4].hex()
444 _equal(idna_host_check(f"acute-decomposed\n{affix}", normalize), None)
445 _equal(idna_nfc_check(f"reorder:{affix}", normalize), None)
446 _equal(normalize_url_check(url, normalize), None)
447 _equal(clean_url_check(url), None)
450def sanitizer_observation(data: bytes) -> tuple[str, list[clean.Removed]]:
451 """Distinguish stripping from a no-op sanitizer with an explicit envelope."""
452 text: Final = _text(data)
453 return clean.sanitize_report(
454 f'<b title="{text}" onclick="ignored">{text}</b><!--comment--><unknown>{text}</unknown>',
455 clean.Policy(on_disallowed_tag=clean.OnDisallowed.STRIP, transform_tags={"unknown": clean.Transform("b")}),
456 )
459def sanitizer_check(
460 data: bytes,
461 render: Callable[[bytes], tuple[str, list[clean.Removed]]] = sanitizer_observation,
462) -> None:
463 """Reject wrong renderer results before treating an input as a valid oracle case."""
464 text: Final = _text(data)
465 _equal(render(data), (f"<b>{text}</b><b>{text}</b>", [clean.Removed("b", "title"), clean.Removed("b", "onclick")]))
468def minifier_observation(data: bytes) -> tuple[str, str, str, str]:
469 """Keep printer results available for independent behavioral controls."""
470 text: Final = _text(data)
471 return (
472 clean.minify(f"<p><!--comment-->{text}</p>", clean.Minify(omit_optional_tags=False)),
473 clean.minify_css(f".{text} {{ color: red; }}", clean.CSSMinify()),
474 clean.minify_css_inline("color: red;", clean.CSSMinify()),
475 clean.minify_js(f'const value = "{text}";', clean.JSMinify()),
476 )
479def stdlib_observation(data: bytes) -> str:
480 """Chunked callbacks must retain decoded text rather than just complete parsing."""
481 parser: Final = _TextParser()
482 for char in f"<b>{_text(data)}&</b>":
483 parser.feed(char)
484 parser.close()
485 return "".join(parser.text)
488def _text(data: bytes) -> str:
489 return "value" + data[:32].hex()
492def _equal(actual: object, expected: object) -> None:
493 if actual != expected:
494 msg = f"Content API mismatch: {actual!r} != {expected!r}"
495 raise AssertionError(msg)
498class _TextParser(stdlib.HTMLParser):
499 def __init__(self) -> None:
500 super().__init__()
501 self.text: Final[list[str]] = []
503 def handle_data(self, data: str) -> None:
504 self.text.append(data)