Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# encoding: utf-8
2"""Use the HTMLParser library to parse HTML files that aren't too bad."""
3from __future__ import annotations
5# Use of this source code is governed by the MIT license.
6__license__ = "MIT"
8__all__ = [
9 "HTMLParserTreeBuilder",
10]
12from html.parser import HTMLParser
14from typing import (
15 Any,
16 Callable,
17 cast,
18 Dict,
19 Iterable,
20 List,
21 Optional,
22 TYPE_CHECKING,
23 Tuple,
24 Type,
25 Union,
26)
28from bs4.element import (
29 AttributeDict,
30 CData,
31 Comment,
32 Declaration,
33 Doctype,
34 ProcessingInstruction,
35)
36from bs4.dammit import EntitySubstitution, UnicodeDammit
38from bs4.builder import (
39 DetectsXMLParsedAsHTML,
40 HTML,
41 HTMLTreeBuilder,
42 STRICT,
43)
45from bs4.exceptions import ParserRejectedMarkup
47if TYPE_CHECKING:
48 from bs4 import BeautifulSoup
49 from bs4.element import NavigableString
50 from bs4._typing import (
51 _Encoding,
52 _Encodings,
53 _RawMarkup,
54 )
56HTMLPARSER = "html.parser"
58_DuplicateAttributeHandler = Callable[[Dict[str, str], str, str], None]
61class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
62 #: Constant to handle duplicate attributes by ignoring later values
63 #: and keeping the earlier ones.
64 REPLACE: str = "replace"
66 #: Constant to handle duplicate attributes by replacing earlier values
67 #: with later ones.
68 IGNORE: str = "ignore"
70 """A subclass of the Python standard library's HTMLParser class, which
71 listens for HTMLParser events and translates them into calls
72 to Beautiful Soup's tree construction API.
74 :param on_duplicate_attribute: A strategy for what to do if a
75 tag includes the same attribute more than once. Accepted
76 values are: REPLACE (replace earlier values with later
77 ones, the default), IGNORE (keep the earliest value
78 encountered), or a callable. A callable must take three
79 arguments: the dictionary of attributes already processed,
80 the name of the duplicate attribute, and the most recent value
81 encountered.
82 """
84 def __init__(
85 self,
86 soup: BeautifulSoup,
87 *args: Any,
88 on_duplicate_attribute: Union[str, _DuplicateAttributeHandler] = REPLACE,
89 **kwargs: Any,
90 ):
91 self.soup = soup
92 self.on_duplicate_attribute = on_duplicate_attribute
93 self.attribute_dict_class = soup.builder.attribute_dict_class
94 HTMLParser.__init__(self, *args, **kwargs)
96 # Keep a list of empty-element tags that were encountered
97 # without an explicit closing tag. If we encounter a closing tag
98 # of this type, we'll associate it with one of those entries.
99 #
100 # This isn't a stack because we don't care about the
101 # order. It's a list of closing tags we've already handled and
102 # will ignore, assuming they ever show up.
103 self.already_closed_empty_element = []
105 self._initialize_xml_detector()
107 on_duplicate_attribute: Union[str, _DuplicateAttributeHandler]
108 already_closed_empty_element: List[str]
109 soup: BeautifulSoup
111 def error(self, message: str) -> None:
112 # NOTE: This method is required so long as Python 3.9 is
113 # supported. The corresponding code is removed from HTMLParser
114 # in 3.5, but not removed from ParserBase until 3.10.
115 # https://github.com/python/cpython/issues/76025
116 #
117 # The original implementation turned the error into a warning,
118 # but in every case I discovered, this made HTMLParser
119 # immediately crash with an error message that was less
120 # helpful than the warning. The new implementation makes it
121 # more clear that html.parser just can't parse this
122 # markup. The 3.10 implementation does the same, though it
123 # raises AssertionError rather than calling a method. (We
124 # catch this error and wrap it in a ParserRejectedMarkup.)
125 raise ParserRejectedMarkup(message)
127 def handle_startendtag(
128 self, name: str, attrs: List[Tuple[str, Optional[str]]]
129 ) -> None:
130 """Handle an incoming empty-element tag.
132 html.parser only calls this method when the markup looks like
133 <tag/>.
134 """
135 # `handle_empty_element` tells handle_starttag not to close the tag
136 # just because its name matches a known empty-element tag. We
137 # know that this is an empty-element tag, and we want to call
138 # handle_endtag ourselves.
139 self.handle_starttag(name, attrs, handle_empty_element=False)
140 self.handle_endtag(name)
142 def handle_starttag(
143 self,
144 name: str,
145 attrs: List[Tuple[str, Optional[str]]],
146 handle_empty_element: bool = True,
147 ) -> None:
148 """Handle an opening tag, e.g. '<tag>'
150 :param handle_empty_element: True if this tag is known to be
151 an empty-element tag (i.e. there is not expected to be any
152 closing tag).
153 """
154 # TODO: handle namespaces here?
155 attr_dict: AttributeDict = self.attribute_dict_class()
156 for key, value in attrs:
157 # Change None attribute values to the empty string
158 # for consistency with the other tree builders.
159 if value is None:
160 value = ""
161 if key in attr_dict:
162 # A single attribute shows up multiple times in this
163 # tag. How to handle it depends on the
164 # on_duplicate_attribute setting.
165 on_dupe = self.on_duplicate_attribute
166 if on_dupe == self.IGNORE:
167 pass
168 elif on_dupe in (None, self.REPLACE):
169 attr_dict[key] = value
170 else:
171 on_dupe = cast(_DuplicateAttributeHandler, on_dupe)
172 on_dupe(attr_dict, key, value)
173 else:
174 attr_dict[key] = value
175 # print("START", name)
176 sourceline: Optional[int]
177 sourcepos: Optional[int]
178 if self.soup.builder.store_line_numbers:
179 sourceline, sourcepos = self.getpos()
180 else:
181 sourceline = sourcepos = None
182 tag = self.soup.handle_starttag(
183 name, None, None, attr_dict, sourceline=sourceline, sourcepos=sourcepos
184 )
185 if tag and tag.is_empty_element and handle_empty_element:
186 # Unlike other parsers, html.parser doesn't send separate end tag
187 # events for empty-element tags. (It's handled in
188 # handle_startendtag, but only if the original markup looked like
189 # <tag/>.)
190 #
191 # So we need to call handle_endtag() ourselves. Since we
192 # know the start event is identical to the end event, we
193 # don't want handle_endtag() to cross off any previous end
194 # events for tags of this name.
195 self.handle_endtag(name, check_already_closed=False)
197 # But we might encounter an explicit closing tag for this tag
198 # later on. If so, we want to ignore it.
199 self.already_closed_empty_element.append(name)
201 if self._root_tag_name is None:
202 self._root_tag_encountered(name)
204 def handle_endtag(self, name: str, check_already_closed: bool = True) -> None:
205 """Handle a closing tag, e.g. '</tag>'
207 :param name: A tag name.
208 :param check_already_closed: True if this tag is expected to
209 be the closing portion of an empty-element tag,
210 e.g. '<tag></tag>'.
211 """
212 # print("END", name)
213 if check_already_closed and name in self.already_closed_empty_element:
214 # This is a redundant end tag for an empty-element tag.
215 # We've already called handle_endtag() for it, so just
216 # check it off the list.
217 # print("ALREADY CLOSED", name)
218 self.already_closed_empty_element.remove(name)
219 else:
220 self.soup.handle_endtag(name)
222 def handle_data(self, data: str) -> None:
223 """Handle some textual data that shows up between tags."""
224 self.soup.handle_data(data)
226 def handle_charref(self, name: str) -> None:
227 """Handle a numeric character reference by converting it to the
228 corresponding Unicode character and treating it as textual
229 data.
231 :param name: Character number, possibly in hexadecimal.
232 """
233 # TODO: This was originally a workaround for a bug in
234 # HTMLParser. (http://bugs.python.org/issue13633) The bug has
235 # been fixed, but removing this code still makes some
236 # Beautiful Soup tests fail. This needs investigation.
237 if name.startswith("x"):
238 real_name = int(name.lstrip("x"), 16)
239 elif name.startswith("X"):
240 real_name = int(name.lstrip("X"), 16)
241 else:
242 real_name = int(name)
244 data = None
245 if real_name < 256:
246 # HTML numeric entities are supposed to reference Unicode
247 # code points, but sometimes they reference code points in
248 # some other encoding (ahem, Windows-1252). E.g. “
249 # instead of É for LEFT DOUBLE QUOTATION MARK. This
250 # code tries to detect this situation and compensate.
251 for encoding in (self.soup.original_encoding, "windows-1252"):
252 if not encoding:
253 continue
254 try:
255 data = bytearray([real_name]).decode(encoding)
256 except UnicodeDecodeError:
257 pass
258 if not data:
259 try:
260 data = chr(real_name)
261 except (ValueError, OverflowError):
262 pass
263 data = data or "\N{REPLACEMENT CHARACTER}"
264 self.handle_data(data)
266 def handle_entityref(self, name: str) -> None:
267 """Handle a named entity reference by converting it to the
268 corresponding Unicode character(s) and treating it as textual
269 data.
271 :param name: Name of the entity reference.
272 """
273 character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
274 if character is not None:
275 data = character
276 else:
277 # If this were XML, it would be ambiguous whether "&foo"
278 # was an character entity reference with a missing
279 # semicolon or the literal string "&foo". Since this is
280 # HTML, we have a complete list of all character entity references,
281 # and this one wasn't found, so assume it's the literal string "&foo".
282 data = "&%s" % name
283 self.handle_data(data)
285 def handle_comment(self, data: str) -> None:
286 """Handle an HTML comment.
288 :param data: The text of the comment.
289 """
290 self.soup.endData()
291 self.soup.handle_data(data)
292 self.soup.endData(Comment)
294 def handle_decl(self, data: str) -> None:
295 """Handle a DOCTYPE declaration.
297 :param data: The text of the declaration.
298 """
299 self.soup.endData()
300 data = data[len("DOCTYPE ") :]
301 self.soup.handle_data(data)
302 self.soup.endData(Doctype)
304 def unknown_decl(self, data: str) -> None:
305 """Handle a declaration of unknown type -- probably a CDATA block.
307 :param data: The text of the declaration.
308 """
309 cls: Type[NavigableString]
310 if data.upper().startswith("CDATA["):
311 cls = CData
312 data = data[len("CDATA[") :]
313 else:
314 cls = Declaration
315 self.soup.endData()
316 self.soup.handle_data(data)
317 self.soup.endData(cls)
319 def handle_pi(self, data: str) -> None:
320 """Handle a processing instruction.
322 :param data: The text of the instruction.
323 """
324 self.soup.endData()
325 self.soup.handle_data(data)
326 self._document_might_be_xml(data)
327 self.soup.endData(ProcessingInstruction)
330class HTMLParserTreeBuilder(HTMLTreeBuilder):
331 """A Beautiful soup `bs4.builder.TreeBuilder` that uses the
332 :py:class:`html.parser.HTMLParser` parser, found in the Python
333 standard library.
335 """
337 is_xml: bool = False
338 picklable: bool = True
339 NAME: str = HTMLPARSER
340 features: Iterable[str] = [NAME, HTML, STRICT]
341 parser_args: Tuple[Iterable[Any], Dict[str, Any]]
343 #: The html.parser knows which line number and position in the
344 #: original file is the source of an element.
345 TRACKS_LINE_NUMBERS: bool = True
347 def __init__(
348 self,
349 parser_args: Optional[Iterable[Any]] = None,
350 parser_kwargs: Optional[Dict[str, Any]] = None,
351 **kwargs: Any,
352 ):
353 """Constructor.
355 :param parser_args: Positional arguments to pass into
356 the BeautifulSoupHTMLParser constructor, once it's
357 invoked.
358 :param parser_kwargs: Keyword arguments to pass into
359 the BeautifulSoupHTMLParser constructor, once it's
360 invoked.
361 :param kwargs: Keyword arguments for the superclass constructor.
362 """
363 # Some keyword arguments will be pulled out of kwargs and placed
364 # into parser_kwargs.
365 extra_parser_kwargs = dict()
366 for arg in ("on_duplicate_attribute",):
367 if arg in kwargs:
368 value = kwargs.pop(arg)
369 extra_parser_kwargs[arg] = value
370 super(HTMLParserTreeBuilder, self).__init__(**kwargs)
371 parser_args = parser_args or []
372 parser_kwargs = parser_kwargs or {}
373 parser_kwargs.update(extra_parser_kwargs)
374 parser_kwargs["convert_charrefs"] = False
375 self.parser_args = (parser_args, parser_kwargs)
377 def prepare_markup(
378 self,
379 markup: _RawMarkup,
380 user_specified_encoding: Optional[_Encoding] = None,
381 document_declared_encoding: Optional[_Encoding] = None,
382 exclude_encodings: Optional[_Encodings] = None,
383 ) -> Iterable[Tuple[str, Optional[_Encoding], Optional[_Encoding], bool]]:
384 """Run any preliminary steps necessary to make incoming markup
385 acceptable to the parser.
387 :param markup: Some markup -- probably a bytestring.
388 :param user_specified_encoding: The user asked to try this encoding.
389 :param document_declared_encoding: The markup itself claims to be
390 in this encoding.
391 :param exclude_encodings: The user asked _not_ to try any of
392 these encodings.
394 :yield: A series of 4-tuples: (markup, encoding, declared encoding,
395 has undergone character replacement)
397 Each 4-tuple represents a strategy for parsing the document.
398 This TreeBuilder uses Unicode, Dammit to convert the markup
399 into Unicode, so the ``markup`` element of the tuple will
400 always be a string.
401 """
402 if isinstance(markup, str):
403 # Parse Unicode as-is.
404 yield (markup, None, None, False)
405 return
407 # Ask UnicodeDammit to sniff the most likely encoding.
409 known_definite_encodings: List[_Encoding] = []
410 if user_specified_encoding:
411 # This was provided by the end-user; treat it as a known
412 # definite encoding per the algorithm laid out in the
413 # HTML5 spec. (See the EncodingDetector class for
414 # details.)
415 known_definite_encodings.append(user_specified_encoding)
417 user_encodings: List[_Encoding] = []
418 if document_declared_encoding:
419 # This was found in the document; treat it as a slightly
420 # lower-priority user encoding.
421 user_encodings.append(document_declared_encoding)
423 dammit = UnicodeDammit(
424 markup,
425 known_definite_encodings=known_definite_encodings,
426 user_encodings=user_encodings,
427 is_html=True,
428 exclude_encodings=exclude_encodings,
429 )
431 if dammit.unicode_markup is None:
432 # In every case I've seen, Unicode, Dammit is able to
433 # convert the markup into Unicode, even if it needs to use
434 # REPLACEMENT CHARACTER. But there is a code path that
435 # could result in unicode_markup being None, and
436 # HTMLParser can only parse Unicode, so here we handle
437 # that code path.
438 raise ParserRejectedMarkup(
439 "Could not convert input to Unicode, and html.parser will not accept bytestrings."
440 )
441 else:
442 yield (
443 dammit.unicode_markup,
444 dammit.original_encoding,
445 dammit.declared_html_encoding,
446 dammit.contains_replacement_characters,
447 )
449 def feed(self, markup: _RawMarkup) -> None:
450 args, kwargs = self.parser_args
452 # HTMLParser.feed will only handle str, but
453 # BeautifulSoup.markup is allowed to be _RawMarkup, because
454 # it's set by the yield value of
455 # TreeBuilder.prepare_markup. Fortunately,
456 # HTMLParserTreeBuilder.prepare_markup always yields a str
457 # (UnicodeDammit.unicode_markup).
458 assert isinstance(markup, str)
460 # We know BeautifulSoup calls TreeBuilder.initialize_soup
461 # before calling feed(), so we can assume self.soup
462 # is set.
463 assert self.soup is not None
464 parser = BeautifulSoupHTMLParser(self.soup, *args, **kwargs)
466 try:
467 parser.feed(markup)
468 parser.close()
469 except AssertionError as e:
470 # html.parser raises AssertionError in rare cases to
471 # indicate a fatal problem with the markup, especially
472 # when there's an error in the doctype declaration.
473 raise ParserRejectedMarkup(e)
474 parser.already_closed_empty_element = []