Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/turbohtml/extract/_dates.py: 92%

1""" 

2Publication-date extraction for :mod:`turbohtml.extract`, the ``htmldate.find_date`` counterpart. 

3 

4:func:`dates` reads the same signals ``htmldate`` reads -- ``<meta>`` publication/modification tags, JSON-LD 

5``datePublished``/``dateModified``, ``<time>`` elements, a date pattern in the canonical URL, and (as a last resort) 

6visible text -- but off the parsed DOM and the :meth:`~turbohtml.Document.structured_data` engine rather than a 

7second parse. Each signal is a stage tried in htmldate's priority order; the first stage that yields a bounded date 

8wins, and within a stage the :class:`DateExtraction` ``original`` flag routes a publication date against a 

9modification date. The stages and the date parsing run in C (``Document._dates``); this module holds the options, 

10the result record, and the window and output formatting the :mod:`datetime` module owns. The date parsing needs no 

11``dateparser``: ISO 8601, the common numeric spellings, an 8-digit stamp, and a compact multilingual month vocabulary. 

12""" 

13 

14from __future__ import annotations 

15 

16from dataclasses import dataclass 

17from datetime import UTC, date, datetime 

18from typing import Final, Literal, NamedTuple 

19 

20from turbohtml._html import parse 

21 

22__all__ = [ 

23 "DateExtraction", 

24 "PublicationDate", 

25 "dates", 

26] 

27 

28Signal = Literal["url", "meta", "json-ld", "time", "text"] 

29"""Which engine a :class:`PublicationDate` came from, in htmldate's stage-priority order.""" 

30 

31_EARLIEST: Final = date(1995, 1, 1) 

32"""The default lower bound, htmldate's ``MIN_DATE``: the web's date metadata does not predate it.""" 

33 

34 

35class PublicationDate(NamedTuple): 

36 """ 

37 A date :func:`dates` recovered and the signal it read it from. 

38 

39 ``date`` is the formatted string (``output_format``, ISO ``YYYY-MM-DD`` by default). ``signal`` names the engine: 

40 ``"meta"`` a publication/modification ``<meta>`` tag, ``"json-ld"`` a JSON-LD ``datePublished``/``dateModified``, 

41 ``"time"`` a ``<time>`` element, ``"url"`` a date pattern in the canonical URL, ``"text"`` visible page text. 

42 """ 

43 

44 date: str 

45 signal: Signal 

46 

47 

48@dataclass(frozen=True, slots=True) 

49class DateExtraction: 

50 """ 

51 Options for :func:`dates`, mirroring ``htmldate.find_date``'s knobs. 

52 

53 :param original: prefer the first-published date over the last-modified one, ``htmldate``'s ``original_date``. 

54 The default (``False``) prefers the modification date, the most recent the page reports. 

55 :param output_format: an :meth:`~datetime.date.strftime` format for the returned string; the default is 

56 ISO ``%Y-%m-%d``. 

57 :param min_date: the earliest acceptable date; a candidate before it is skipped. Defaults to 1995-01-01, the floor 

58 ``htmldate`` uses. 

59 :param max_date: the latest acceptable date; a candidate after it is skipped. Defaults to today, so a stray future 

60 stamp never wins. 

61 :param extensive_search: scan visible page text when no metadata carries a date, ``htmldate``'s 

62 ``extensive_search``. With ``False`` only the structured signals (meta, JSON-LD, time, URL) are read. 

63 """ 

64 

65 original: bool = False 

66 output_format: str = "%Y-%m-%d" 

67 min_date: date | None = None 

68 max_date: date | None = None 

69 extensive_search: bool = True 

70 

71 def __post_init__(self) -> None: 

72 """Reject a min/max window whose bounds cross, the one contradiction the fields can express.""" 

73 if self.min_date is not None and self.max_date is not None and self.min_date > self.max_date: 

74 msg = f"min_date {self.min_date.isoformat()} is after max_date {self.max_date.isoformat()}" 

75 raise ValueError(msg) 

76 

77 

78_DEFAULT: Final = DateExtraction() 

79 

80 

81def dates(html: str, options: DateExtraction | None = None, /) -> PublicationDate | None: 

82 """ 

83 Find a document's publication (or, by default, modification) date, the ``htmldate.find_date`` counterpart. 

84 

85 The signals are tried in htmldate's order -- a date in the canonical URL, then publication/modification 

86 ``<meta>`` tags, then JSON-LD, then ``<time>`` elements, then (with ``extensive_search``) visible text -- and the 

87 first that yields a date inside the ``[min_date, max_date]`` window wins. Within a stage the ``original`` flag 

88 picks a publication date over a modification date, or the reverse. 

89 

90 :param html: the page markup. 

91 :param options: the extraction knobs; defaults to :class:`DateExtraction` (modification date, ISO output, the 

92 1995-to-today window, text search on). 

93 :returns: the date and the signal it came from, or ``None`` when no bounded date is found. 

94 """ 

95 active = options or _DEFAULT 

96 today = _today() 

97 earliest = active.min_date or _EARLIEST 

98 latest = active.max_date or today 

99 found = parse(html)._dates( # ruff:ignore[private-member-access] # the Document type's private C entry point 

100 1 if active.original else 2, 

101 today.year, 

102 earliest.year, 

103 earliest.month, 

104 earliest.day, 

105 latest.year, 

106 latest.month, 

107 latest.day, 

108 active.extensive_search, 

109 ) 

110 if found is None: 

111 return None 

112 year, month, day, signal = found 

113 return PublicationDate(date(year, month, day).strftime(active.output_format), signal) 

114 

115 

116def _today() -> date: 

117 """Today's date in UTC, the default upper bound so a stray future stamp never wins.""" 

118 return datetime.now(UTC).date()