Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# encoding: utf-8
2"""Use the HTMLParser library to parse HTML files that aren't too bad."""
3from __future__ import annotations
5# Use of this source code is governed by the MIT license.
6__license__ = "MIT"
8__all__ = [
9 "HTMLParserTreeBuilder",
10]
12from html.parser import HTMLParser
13import re
15from typing import (
16 Any,
17 Callable,
18 cast,
19 Dict,
20 Iterable,
21 List,
22 Optional,
23 TYPE_CHECKING,
24 Tuple,
25 Type,
26 Union,
27)
29from bs4.element import (
30 AttributeDict,
31 CData,
32 Comment,
33 Declaration,
34 Doctype,
35 ProcessingInstruction,
36)
37from bs4.dammit import EntitySubstitution, UnicodeDammit
39from bs4.builder import (
40 DetectsXMLParsedAsHTML,
41 HTML,
42 HTMLTreeBuilder,
43 STRICT,
44)
46from bs4.exceptions import ParserRejectedMarkup
48if TYPE_CHECKING:
49 from bs4 import BeautifulSoup
50 from bs4.element import NavigableString
51 from bs4._typing import (
52 _Encoding,
53 _Encodings,
54 _RawMarkup,
55 )
57HTMLPARSER = "html.parser"
59_DuplicateAttributeHandler = Callable[[Dict[str, str], str, str], None]
62class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
63 #: Constant to handle duplicate attributes by ignoring later values
64 #: and keeping the earlier ones.
65 REPLACE: str = "replace"
67 #: Constant to handle duplicate attributes by replacing earlier values
68 #: with later ones.
69 IGNORE: str = "ignore"
71 """A subclass of the Python standard library's HTMLParser class, which
72 listens for HTMLParser events and translates them into calls
73 to Beautiful Soup's tree construction API.
75 :param on_duplicate_attribute: A strategy for what to do if a
76 tag includes the same attribute more than once. Accepted
77 values are: REPLACE (replace earlier values with later
78 ones, the default), IGNORE (keep the earliest value
79 encountered), or a callable. A callable must take three
80 arguments: the dictionary of attributes already processed,
81 the name of the duplicate attribute, and the most recent value
82 encountered.
83 """
85 def __init__(
86 self,
87 soup: BeautifulSoup,
88 *args: Any,
89 on_duplicate_attribute: Union[str, _DuplicateAttributeHandler] = REPLACE,
90 **kwargs: Any,
91 ):
92 self.soup = soup
93 self.on_duplicate_attribute = on_duplicate_attribute
94 self.attribute_dict_class = soup.builder.attribute_dict_class
95 HTMLParser.__init__(self, *args, **kwargs)
97 # Keep a list of empty-element tags that were encountered
98 # without an explicit closing tag. If we encounter a closing tag
99 # of this type, we'll associate it with one of those entries.
100 #
101 # This isn't a stack because we don't care about the
102 # order. It's a list of closing tags we've already handled and
103 # will ignore, assuming they ever show up.
104 self.already_closed_empty_element = []
106 self._initialize_xml_detector()
108 on_duplicate_attribute: Union[str, _DuplicateAttributeHandler]
109 already_closed_empty_element: List[str]
110 soup: BeautifulSoup
112 def error(self, message: str) -> None:
113 # NOTE: This method is required so long as Python 3.9 is
114 # supported. The corresponding code is removed from HTMLParser
115 # in 3.5, but not removed from ParserBase until 3.10.
116 # https://github.com/python/cpython/issues/76025
117 #
118 # The original implementation turned the error into a warning,
119 # but in every case I discovered, this made HTMLParser
120 # immediately crash with an error message that was less
121 # helpful than the warning. The new implementation makes it
122 # more clear that html.parser just can't parse this
123 # markup. The 3.10 implementation does the same, though it
124 # raises AssertionError rather than calling a method. (We
125 # catch this error and wrap it in a ParserRejectedMarkup.)
126 raise ParserRejectedMarkup(message)
128 def handle_startendtag(
129 self, tag: str, attrs: List[Tuple[str, Optional[str]]]
130 ) -> None:
131 """Handle an incoming empty-element tag.
133 html.parser only calls this method when the markup looks like
134 <tag/>.
135 """
136 # `handle_empty_element` tells handle_starttag not to close the tag
137 # just because its name matches a known empty-element tag. We
138 # know that this is an empty-element tag, and we want to call
139 # handle_endtag ourselves.
140 self.handle_starttag(tag, attrs, handle_empty_element=False)
142 # Similarly, we set `check_already_closed` when calling
143 # handle_endtag. Since we know the start event is identical to
144 # the end event, we don't want handle_endtag() to cross off
145 # any previous end events for tags of this name.
146 self.handle_endtag(tag, check_already_closed=False)
148 def handle_starttag(
149 self,
150 tag: str,
151 attrs: List[Tuple[str, Optional[str]]],
152 handle_empty_element: bool = True,
153 ) -> None:
154 """Handle an opening tag, e.g. '<tag>'
156 :param handle_empty_element: True if this tag is known to be
157 an empty-element tag (i.e. there is not expected to be any
158 closing tag).
159 """
160 # TODO: handle namespaces here?
161 attr_dict: AttributeDict = self.attribute_dict_class()
162 for key, value in attrs:
163 # Change None attribute values to the empty string
164 # for consistency with the other tree builders.
165 if value is None:
166 value = ""
167 if key in attr_dict:
168 # A single attribute shows up multiple times in this
169 # tag. How to handle it depends on the
170 # on_duplicate_attribute setting.
171 on_dupe = self.on_duplicate_attribute
172 if on_dupe == self.IGNORE:
173 pass
174 elif on_dupe in (None, self.REPLACE):
175 attr_dict[key] = value
176 else:
177 on_dupe = cast(_DuplicateAttributeHandler, on_dupe)
178 on_dupe(attr_dict, key, value)
179 else:
180 attr_dict[key] = value
181 # print("START", tag)
182 sourceline: Optional[int]
183 sourcepos: Optional[int]
184 if self.soup.builder.store_line_numbers:
185 sourceline, sourcepos = self.getpos()
186 else:
187 sourceline = sourcepos = None
188 tagObj = self.soup.handle_starttag(
189 tag, None, None, attr_dict, sourceline=sourceline, sourcepos=sourcepos
190 )
191 if tagObj is not None and tagObj.is_empty_element and handle_empty_element:
192 # Unlike other parsers, html.parser doesn't send separate end tag
193 # events for empty-element tags. (It's handled in
194 # handle_startendtag, but only if the original markup looked like
195 # <tag/>.)
196 #
197 # So we need to call handle_endtag() ourselves. Since we
198 # know the start event is identical to the end event, we
199 # don't want handle_endtag() to cross off any previous end
200 # events for tags of this name.
201 self.handle_endtag(tag, check_already_closed=False)
203 # But we might encounter an explicit closing tag for this tag
204 # later on. If so, we want to ignore it.
205 self.already_closed_empty_element.append(tag)
207 if self._root_tag_name is None:
208 self._root_tag_encountered(tag)
210 def handle_endtag(self, tag: str, check_already_closed: bool = True) -> None:
211 """Handle a closing tag, e.g. '</tag>'
213 :param tag: A tag name.
214 :param check_already_closed: True if this tag is expected to
215 be the closing portion of an empty-element tag,
216 e.g. '<tag></tag>'.
217 """
218 # print("END", tag)
219 if check_already_closed and tag in self.already_closed_empty_element:
220 # This is a redundant end tag for an empty-element tag.
221 # We've already called handle_endtag() for it, so just
222 # check it off the list.
223 # print("ALREADY CLOSED", tag)
224 self.already_closed_empty_element.remove(tag)
225 else:
226 self.soup.handle_endtag(tag)
228 def handle_data(self, data: str) -> None:
229 """Handle some textual data that shows up between tags."""
230 self.soup.handle_data(data)
232 _DECIMAL_REFERENCE_WITH_FOLLOWING_DATA = re.compile("^([0-9]+)(.*)")
233 _HEX_REFERENCE_WITH_FOLLOWING_DATA = re.compile("^([0-9a-f]+)(.*)")
235 @classmethod
236 def _dereference_numeric_character_reference(cls, name:str) -> Tuple[str, bool, str]:
237 """Convert a numeric character reference into an actual character.
239 :param name: The number of the character reference, as
240 obtained by html.parser
242 :return: A 3-tuple (dereferenced, replacement_added,
243 extra_data). `dereferenced` is the dereferenced character
244 reference, or the empty string if there was no
245 reference. `replacement_added` is True if the reference
246 could only be dereferenced by replacing content with U+FFFD
247 REPLACEMENT CHARACTER. `extra_data` is a portion of data
248 following the character reference, which was deemed to be
249 normal data and not part of the reference at all.
250 """
251 dereferenced:str = ""
252 replacement_added:bool = False
253 extra_data:str = ""
255 base:int = 10
256 reg = cls._DECIMAL_REFERENCE_WITH_FOLLOWING_DATA
257 if name.startswith("x") or name.startswith("X"):
258 # Hex reference
259 name = name[1:]
260 base = 16
261 reg = cls._HEX_REFERENCE_WITH_FOLLOWING_DATA
263 real_name:Optional[int] = None
264 try:
265 real_name = int(name, base)
266 except ValueError:
267 # This is either bad data that starts with what looks like
268 # a numeric character reference, or a real numeric
269 # reference that wasn't terminated by a semicolon.
270 #
271 # The fix to https://bugs.python.org/issue13633 made it
272 # our responsibility to handle the extra data.
273 #
274 # To preserve the old behavior, we extract the numeric
275 # portion of the incoming "reference" and treat that as a
276 # numeric reference. All subsequent data will be processed
277 # as string data.
278 match = reg.search(name)
279 if match is not None:
280 real_name = int(match.groups()[0], base)
281 extra_data = match.groups()[1]
283 if real_name is None:
284 dereferenced = ""
285 extra_data = name
286 else:
287 dereferenced, replacement_added = UnicodeDammit.numeric_character_reference(real_name)
288 return dereferenced, replacement_added, extra_data
290 def handle_charref(self, name: str) -> None:
291 """Handle a numeric character reference by converting it to the
292 corresponding Unicode character and treating it as textual
293 data.
295 :param name: Character number, possibly in hexadecimal.
296 """
297 dereferenced, replacement_added, extra_data = self._dereference_numeric_character_reference(name)
298 if replacement_added:
299 self.soup.contains_replacement_characters = True
300 if dereferenced is not None:
301 self.handle_data(dereferenced)
302 if extra_data is not None:
303 self.handle_data(extra_data)
305 def handle_entityref(self, name: str) -> None:
306 """Handle a named entity reference by converting it to the
307 corresponding Unicode character(s) and treating it as textual
308 data.
310 :param name: Name of the entity reference.
311 """
312 character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
313 if character is not None:
314 data = character
315 else:
316 # If this were XML, it would be ambiguous whether "&foo"
317 # was an character entity reference with a missing
318 # semicolon or the literal string "&foo". Since this is
319 # HTML, we have a complete list of all character entity references,
320 # and this one wasn't found, so assume it's the literal string "&foo".
321 data = "&%s" % name
322 self.handle_data(data)
324 def handle_comment(self, data: str) -> None:
325 """Handle an HTML comment.
327 :param data: The text of the comment.
328 """
329 self.soup.endData()
330 self.soup.handle_data(data)
331 self.soup.endData(Comment)
333 def handle_decl(self, decl: str) -> None:
334 """Handle a DOCTYPE declaration.
336 :param data: The text of the declaration.
337 """
338 self.soup.endData()
339 decl = decl[len("DOCTYPE ") :]
340 self.soup.handle_data(decl)
341 self.soup.endData(Doctype)
343 def unknown_decl(self, data: str) -> None:
344 """Handle a declaration of unknown type -- probably a CDATA block.
346 :param data: The text of the declaration.
347 """
348 cls: Type[NavigableString]
349 if data.upper().startswith("CDATA["):
350 cls = CData
351 data = data[len("CDATA[") :]
352 else:
353 cls = Declaration
354 self.soup.endData()
355 self.soup.handle_data(data)
356 self.soup.endData(cls)
358 def handle_pi(self, data: str) -> None:
359 """Handle a processing instruction.
361 :param data: The text of the instruction.
362 """
363 self.soup.endData()
364 self.soup.handle_data(data)
365 self._document_might_be_xml(data)
366 self.soup.endData(ProcessingInstruction)
369class HTMLParserTreeBuilder(HTMLTreeBuilder):
370 """A Beautiful soup `bs4.builder.TreeBuilder` that uses the
371 :py:class:`html.parser.HTMLParser` parser, found in the Python
372 standard library.
374 """
376 is_xml: bool = False
377 picklable: bool = True
378 NAME: str = HTMLPARSER
379 features: Iterable[str] = [NAME, HTML, STRICT]
380 parser_args: Tuple[Iterable[Any], Dict[str, Any]]
382 #: The html.parser knows which line number and position in the
383 #: original file is the source of an element.
384 TRACKS_LINE_NUMBERS: bool = True
386 def __init__(
387 self,
388 parser_args: Optional[Iterable[Any]] = None,
389 parser_kwargs: Optional[Dict[str, Any]] = None,
390 **kwargs: Any,
391 ):
392 """Constructor.
394 :param parser_args: Positional arguments to pass into
395 the BeautifulSoupHTMLParser constructor, once it's
396 invoked.
397 :param parser_kwargs: Keyword arguments to pass into
398 the BeautifulSoupHTMLParser constructor, once it's
399 invoked.
400 :param kwargs: Keyword arguments for the superclass constructor.
401 """
402 # Some keyword arguments will be pulled out of kwargs and placed
403 # into parser_kwargs.
404 extra_parser_kwargs = dict()
405 for arg in ("on_duplicate_attribute",):
406 if arg in kwargs:
407 value = kwargs.pop(arg)
408 extra_parser_kwargs[arg] = value
409 super(HTMLParserTreeBuilder, self).__init__(**kwargs)
410 parser_args = parser_args or []
411 parser_kwargs = parser_kwargs or {}
412 parser_kwargs.update(extra_parser_kwargs)
413 parser_kwargs["convert_charrefs"] = False
414 self.parser_args = (parser_args, parser_kwargs)
416 def prepare_markup(
417 self,
418 markup: _RawMarkup,
419 user_specified_encoding: Optional[_Encoding] = None,
420 document_declared_encoding: Optional[_Encoding] = None,
421 exclude_encodings: Optional[_Encodings] = None,
422 ) -> Iterable[Tuple[str, Optional[_Encoding], Optional[_Encoding], bool]]:
423 """Run any preliminary steps necessary to make incoming markup
424 acceptable to the parser.
426 :param markup: Some markup -- probably a bytestring.
427 :param user_specified_encoding: The user asked to try this encoding.
428 :param document_declared_encoding: The markup itself claims to be
429 in this encoding.
430 :param exclude_encodings: The user asked _not_ to try any of
431 these encodings.
433 :yield: A series of 4-tuples: (markup, encoding, declared encoding,
434 has undergone character replacement)
436 Each 4-tuple represents a strategy for parsing the document.
437 This TreeBuilder uses Unicode, Dammit to convert the markup
438 into Unicode, so the ``markup`` element of the tuple will
439 always be a string.
440 """
441 if isinstance(markup, str):
442 # Parse Unicode as-is.
443 yield (markup, None, None, False)
444 return
446 # Ask UnicodeDammit to sniff the most likely encoding.
448 known_definite_encodings: List[_Encoding] = []
449 if user_specified_encoding:
450 # This was provided by the end-user; treat it as a known
451 # definite encoding per the algorithm laid out in the
452 # HTML5 spec. (See the EncodingDetector class for
453 # details.)
454 known_definite_encodings.append(user_specified_encoding)
456 user_encodings: List[_Encoding] = []
457 if document_declared_encoding:
458 # This was found in the document; treat it as a slightly
459 # lower-priority user encoding.
460 user_encodings.append(document_declared_encoding)
462 dammit = UnicodeDammit(
463 markup,
464 known_definite_encodings=known_definite_encodings,
465 user_encodings=user_encodings,
466 is_html=True,
467 exclude_encodings=exclude_encodings,
468 )
470 if dammit.unicode_markup is None:
471 # In every case I've seen, Unicode, Dammit is able to
472 # convert the markup into Unicode, even if it needs to use
473 # REPLACEMENT CHARACTER. But there is a code path that
474 # could result in unicode_markup being None, and
475 # HTMLParser can only parse Unicode, so here we handle
476 # that code path.
477 raise ParserRejectedMarkup(
478 "Could not convert input to Unicode, and html.parser will not accept bytestrings."
479 )
480 else:
481 yield (
482 dammit.unicode_markup,
483 dammit.original_encoding,
484 dammit.declared_html_encoding,
485 dammit.contains_replacement_characters,
486 )
488 def feed(self, markup: _RawMarkup, _parser_class:type[BeautifulSoupHTMLParser] =BeautifulSoupHTMLParser) -> None:
489 """
490 :param markup: The markup to feed into the parser.
491 :param _parser_class: An HTMLParser subclass to use. This is only intended for use in unit tests.
492 """
493 args, kwargs = self.parser_args
495 # HTMLParser.feed will only handle str, but
496 # BeautifulSoup.markup is allowed to be _RawMarkup, because
497 # it's set by the yield value of
498 # TreeBuilder.prepare_markup. Fortunately,
499 # HTMLParserTreeBuilder.prepare_markup always yields a str
500 # (UnicodeDammit.unicode_markup).
501 assert isinstance(markup, str)
503 # We know BeautifulSoup calls TreeBuilder.initialize_soup
504 # before calling feed(), so we can assume self.soup
505 # is set.
506 assert self.soup is not None
507 parser = _parser_class(self.soup, *args, **kwargs)
509 try:
510 parser.feed(markup)
511 parser.close()
512 except AssertionError as e:
513 # html.parser raises AssertionError in rare cases to
514 # indicate a fatal problem with the markup, especially
515 # when there's an error in the doctype declaration.
516 raise ParserRejectedMarkup(e)
517 parser.already_closed_empty_element = []