Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/html.py: 21%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2:mod:`pandas.io.html` is a module containing functionality for dealing with
3HTML IO.
5"""
7from __future__ import annotations
9from collections import abc
10import numbers
11import re
12from re import Pattern
13from typing import (
14 TYPE_CHECKING,
15 Literal,
16 cast,
17)
19from pandas._libs import lib
20from pandas.compat._optional import import_optional_dependency
21from pandas.errors import (
22 AbstractMethodError,
23 EmptyDataError,
24)
25from pandas.util._decorators import set_module
26from pandas.util._validators import check_dtype_backend
28from pandas.core.dtypes.common import is_list_like
30from pandas import isna
31from pandas.core.indexes.base import Index
32from pandas.core.indexes.multi import MultiIndex
33from pandas.core.series import Series
35from pandas.io.common import (
36 get_handle,
37 is_url,
38 stringify_path,
39 validate_header_arg,
40)
41from pandas.io.formats.printing import pprint_thing
42from pandas.io.parsers import TextParser
44if TYPE_CHECKING:
45 from collections.abc import (
46 Iterable,
47 Sequence,
48 )
50 from pandas._typing import (
51 BaseBuffer,
52 DtypeBackend,
53 FilePath,
54 HTMLFlavors,
55 ReadBuffer,
56 StorageOptions,
57 )
59 from pandas import DataFrame
61#############
62# READ HTML #
63#############
64_RE_WHITESPACE = re.compile(r"[\r\n]+|\s{2,}")
67def _remove_whitespace(s: str, regex: Pattern = _RE_WHITESPACE) -> str:
68 """
69 Replace extra whitespace inside of a string with a single space.
71 Parameters
72 ----------
73 s : str or unicode
74 The string from which to remove extra whitespace.
75 regex : re.Pattern
76 The regular expression to use to remove extra whitespace.
78 Returns
79 -------
80 subd : str or unicode
81 `s` with all extra whitespace replaced with a single space.
82 """
83 return regex.sub(" ", s.strip())
86def _get_skiprows(skiprows: int | Sequence[int] | slice | None) -> int | Sequence[int]:
87 """
88 Get an iterator given an integer, slice or container.
90 Parameters
91 ----------
92 skiprows : int, slice, container
93 The iterator to use to skip rows; can also be a slice.
95 Raises
96 ------
97 TypeError
98 * If `skiprows` is not a slice, integer, or Container
100 Returns
101 -------
102 it : iterable
103 A proper iterator to use to skip rows of a DataFrame.
104 """
105 if isinstance(skiprows, slice):
106 start, step = skiprows.start or 0, skiprows.step or 1
107 return list(range(start, skiprows.stop, step))
108 elif isinstance(skiprows, numbers.Integral) or is_list_like(skiprows):
109 return cast("int | Sequence[int]", skiprows)
110 elif skiprows is None:
111 return 0
112 raise TypeError(f"{type(skiprows).__name__} is not a valid type for skipping rows")
115def _read(
116 obj: FilePath | BaseBuffer,
117 encoding: str | None,
118 storage_options: StorageOptions | None,
119) -> str | bytes:
120 """
121 Try to read from a url, file or string.
123 Parameters
124 ----------
125 obj : str, unicode, path object, or file-like object
127 Returns
128 -------
129 raw_text : str
130 """
131 with get_handle(
132 obj, "r", encoding=encoding, storage_options=storage_options
133 ) as handles:
134 return handles.handle.read()
137class _HtmlFrameParser:
138 """
139 Base class for parsers that parse HTML into DataFrames.
141 Parameters
142 ----------
143 io : str or file-like
144 This can be either a string path, a valid URL using the HTTP,
145 FTP, or FILE protocols or a file-like object.
147 match : str or regex
148 The text to match in the document.
150 attrs : dict
151 List of HTML <table> element attributes to match.
153 encoding : str
154 Encoding to be used by parser
156 displayed_only : bool
157 Whether or not items with "display:none" should be ignored
159 extract_links : {None, "all", "header", "body", "footer"}
160 Table elements in the specified section(s) with <a> tags will have their
161 href extracted.
163 Attributes
164 ----------
165 io : str or file-like
166 raw HTML, URL, or file-like object
168 match : regex
169 The text to match in the raw HTML
171 attrs : dict-like
172 A dictionary of valid table attributes to use to search for table
173 elements.
175 encoding : str
176 Encoding to be used by parser
178 displayed_only : bool
179 Whether or not items with "display:none" should be ignored
181 extract_links : {None, "all", "header", "body", "footer"}
182 Table elements in the specified section(s) with <a> tags will have their
183 href extracted.
185 Notes
186 -----
187 To subclass this class effectively you must override the following methods:
188 * :func:`_build_doc`
189 * :func:`_attr_getter`
190 * :func:`_href_getter`
191 * :func:`_text_getter`
192 * :func:`_parse_td`
193 * :func:`_parse_thead_tr`
194 * :func:`_parse_tbody_tr`
195 * :func:`_parse_tfoot_tr`
196 * :func:`_parse_tables`
197 * :func:`_equals_tag`
198 See each method's respective documentation for details on their
199 functionality.
200 """
202 def __init__(
203 self,
204 io: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
205 match: str | Pattern,
206 attrs: dict[str, str] | None,
207 encoding: str,
208 displayed_only: bool,
209 extract_links: Literal["header", "footer", "body", "all"] | None,
210 storage_options: StorageOptions = None,
211 ) -> None:
212 self.io = io
213 self.match = match
214 self.attrs = attrs
215 self.encoding = encoding
216 self.displayed_only = displayed_only
217 self.extract_links = extract_links
218 self.storage_options = storage_options
220 def parse_tables(self):
221 """
222 Parse and return all tables from the DOM.
224 Returns
225 -------
226 list of parsed (header, body, footer) tuples from tables.
227 """
228 tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
229 return (self._parse_thead_tbody_tfoot(table) for table in tables)
231 def _attr_getter(self, obj, attr):
232 """
233 Return the attribute value of an individual DOM node.
235 Parameters
236 ----------
237 obj : node-like
238 A DOM node.
240 attr : str or unicode
241 The attribute, such as "colspan"
243 Returns
244 -------
245 str or unicode
246 The attribute value.
247 """
248 # Both lxml and BeautifulSoup have the same implementation:
249 return obj.get(attr)
251 def _href_getter(self, obj) -> str | None:
252 """
253 Return an href if the DOM node contains a child <a> or None.
255 Parameters
256 ----------
257 obj : node-like
258 A DOM node.
260 Returns
261 -------
262 href : str or unicode
263 The href from the <a> child of the DOM node.
264 """
265 raise AbstractMethodError(self)
267 def _text_getter(self, obj):
268 """
269 Return the text of an individual DOM node.
271 Parameters
272 ----------
273 obj : node-like
274 A DOM node.
276 Returns
277 -------
278 text : str or unicode
279 The text from an individual DOM node.
280 """
281 raise AbstractMethodError(self)
283 def _parse_td(self, obj):
284 """
285 Return the td elements from a row element.
287 Parameters
288 ----------
289 obj : node-like
290 A DOM <tr> node.
292 Returns
293 -------
294 list of node-like
295 These are the elements of each row, i.e., the columns.
296 """
297 raise AbstractMethodError(self)
299 def _parse_thead_tr(self, table):
300 """
301 Return the list of thead row elements from the parsed table element.
303 Parameters
304 ----------
305 table : a table element that contains zero or more thead elements.
307 Returns
308 -------
309 list of node-like
310 These are the <tr> row elements of a table.
311 """
312 raise AbstractMethodError(self)
314 def _parse_tbody_tr(self, table):
315 """
316 Return the list of tbody row elements from the parsed table element.
318 HTML5 table bodies consist of either 0 or more <tbody> elements (which
319 only contain <tr> elements) or 0 or more <tr> elements. This method
320 checks for both structures.
322 Parameters
323 ----------
324 table : a table element that contains row elements.
326 Returns
327 -------
328 list of node-like
329 These are the <tr> row elements of a table.
330 """
331 raise AbstractMethodError(self)
333 def _parse_tfoot_tr(self, table):
334 """
335 Return the list of tfoot row elements from the parsed table element.
337 Parameters
338 ----------
339 table : a table element that contains row elements.
341 Returns
342 -------
343 list of node-like
344 These are the <tr> row elements of a table.
345 """
346 raise AbstractMethodError(self)
348 def _parse_tables(self, document, match, attrs):
349 """
350 Return all tables from the parsed DOM.
352 Parameters
353 ----------
354 document : the DOM from which to parse the table element.
356 match : str or regular expression
357 The text to search for in the DOM tree.
359 attrs : dict
360 A dictionary of table attributes that can be used to disambiguate
361 multiple tables on a page.
363 Raises
364 ------
365 ValueError : `match` does not match any text in the document.
367 Returns
368 -------
369 list of node-like
370 HTML <table> elements to be parsed into raw data.
371 """
372 raise AbstractMethodError(self)
374 def _equals_tag(self, obj, tag) -> bool:
375 """
376 Return whether an individual DOM node matches a tag
378 Parameters
379 ----------
380 obj : node-like
381 A DOM node.
383 tag : str
384 Tag name to be checked for equality.
386 Returns
387 -------
388 boolean
389 Whether `obj`'s tag name is `tag`
390 """
391 raise AbstractMethodError(self)
393 def _build_doc(self):
394 """
395 Return a tree-like object that can be used to iterate over the DOM.
397 Returns
398 -------
399 node-like
400 The DOM from which to parse the table element.
401 """
402 raise AbstractMethodError(self)
404 def _parse_thead_tbody_tfoot(self, table_html):
405 """
406 Given a table, return parsed header, body, and foot.
408 Parameters
409 ----------
410 table_html : node-like
412 Returns
413 -------
414 tuple of (header, body, footer), each a list of list-of-text rows.
416 Notes
417 -----
418 Header and body are lists-of-lists. Top level list is a list of
419 rows. Each row is a list of str text.
421 Logic: Use <thead>, <tbody>, <tfoot> elements to identify
422 header, body, and footer, otherwise:
423 - Put all rows into body
424 - Move rows from top of body to header only if
425 all elements inside row are <th>
426 - Move rows from bottom of body to footer only if
427 all elements inside row are <th>
428 """
429 header_rows = self._parse_thead_tr(table_html)
430 body_rows = self._parse_tbody_tr(table_html)
431 footer_rows = self._parse_tfoot_tr(table_html)
433 def row_is_all_th(row):
434 return all(self._equals_tag(t, "th") for t in self._parse_td(row))
436 if not header_rows:
437 # The table has no <thead>. Move the top all-<th> rows from
438 # body_rows to header_rows. (This is a common case because many
439 # tables in the wild have no <thead> or <tfoot>
440 while body_rows and row_is_all_th(body_rows[0]):
441 header_rows.append(body_rows.pop(0))
443 header, rem = self._expand_colspan_rowspan(header_rows, section="header")
444 body, rem = self._expand_colspan_rowspan(
445 body_rows,
446 section="body",
447 remainder=rem,
448 overflow=len(footer_rows) > 0,
449 )
450 footer, _ = self._expand_colspan_rowspan(
451 footer_rows, section="footer", remainder=rem, overflow=False
452 )
454 return header, body, footer
456 def _expand_colspan_rowspan(
457 self,
458 rows,
459 section: Literal["header", "footer", "body"],
460 remainder: list[tuple[int, str | tuple, int]] | None = None,
461 overflow: bool = True,
462 ) -> tuple[list[list], list[tuple[int, str | tuple, int]]]:
463 """
464 Given a list of <tr>s, return a list of text rows.
466 Parameters
467 ----------
468 rows : list of node-like
469 List of <tr>s
470 section : the section that the rows belong to (header, body or footer).
471 remainder: list[tuple[int, str | tuple, int]] | None
472 Any remainder from the expansion of previous section
473 overflow: bool
474 If true, return any partial rows as 'remainder'. If not, use up any
475 partial rows. True by default.
477 Returns
478 -------
479 list of list
480 Each returned row is a list of str text, or tuple (text, link)
481 if extract_links is not None.
482 remainder
483 Remaining partial rows if any. If overflow is False, an empty list
484 is returned.
486 Notes
487 -----
488 Any cell with ``rowspan`` or ``colspan`` will have its contents copied
489 to subsequent cells.
490 """
491 all_texts = [] # list of rows, each a list of str
492 text: str | tuple
493 remainder = remainder if remainder is not None else []
495 for tr in rows:
496 texts = [] # the output for this row
497 next_remainder = []
499 index = 0
500 tds = self._parse_td(tr)
501 for td in tds:
502 # Append texts from previous rows with rowspan>1 that come
503 # before this <td>
504 while remainder and remainder[0][0] <= index:
505 prev_i, prev_text, prev_rowspan = remainder.pop(0)
506 texts.append(prev_text)
507 if prev_rowspan > 1:
508 next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
509 index += 1
511 # Append the text from this <td>, colspan times
512 text = _remove_whitespace(self._text_getter(td))
513 if self.extract_links in ("all", section):
514 href = self._href_getter(td)
515 text = (text, href)
516 rowspan = int(self._attr_getter(td, "rowspan") or 1)
517 colspan = int(self._attr_getter(td, "colspan") or 1)
519 for _ in range(colspan):
520 texts.append(text)
521 if rowspan > 1:
522 next_remainder.append((index, text, rowspan - 1))
523 index += 1
525 # Append texts from previous rows at the final position
526 for prev_i, prev_text, prev_rowspan in remainder:
527 texts.append(prev_text)
528 if prev_rowspan > 1:
529 next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
531 all_texts.append(texts)
532 remainder = next_remainder
534 if not overflow:
535 # Append rows that only appear because the previous row had non-1
536 # rowspan
537 while remainder:
538 next_remainder = []
539 texts = []
540 for prev_i, prev_text, prev_rowspan in remainder:
541 texts.append(prev_text)
542 if prev_rowspan > 1:
543 next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
544 all_texts.append(texts)
545 remainder = next_remainder
547 return all_texts, remainder
549 def _handle_hidden_tables(self, tbl_list, attr_name: str):
550 """
551 Return list of tables, potentially removing hidden elements
553 Parameters
554 ----------
555 tbl_list : list of node-like
556 Type of list elements will vary depending upon parser used
557 attr_name : str
558 Name of the accessor for retrieving HTML attributes
560 Returns
561 -------
562 list of node-like
563 Return type matches `tbl_list`
564 """
565 if not self.displayed_only:
566 return tbl_list
568 return [
569 x
570 for x in tbl_list
571 if "display:none"
572 not in getattr(x, attr_name).get("style", "").replace(" ", "")
573 ]
576class _BeautifulSoupHtml5LibFrameParser(_HtmlFrameParser):
577 """
578 HTML to DataFrame parser that uses BeautifulSoup under the hood.
580 See Also
581 --------
582 pandas.io.html._HtmlFrameParser
583 pandas.io.html._LxmlFrameParser
585 Notes
586 -----
587 Documentation strings for this class are in the base class
588 :class:`pandas.io.html._HtmlFrameParser`.
589 """
591 def _parse_tables(self, document, match, attrs):
592 element_name = "table"
593 tables = document.find_all(element_name, attrs=attrs)
594 if not tables:
595 raise ValueError("No tables found")
597 result = []
598 unique_tables = set()
599 tables = self._handle_hidden_tables(tables, "attrs")
601 for table in tables:
602 if self.displayed_only:
603 for elem in table.find_all("style"):
604 elem.decompose()
606 for elem in table.find_all(style=re.compile(r"display:\s*none")):
607 elem.decompose()
609 if table not in unique_tables and table.find(string=match) is not None:
610 result.append(table)
611 unique_tables.add(table)
612 if not result:
613 raise ValueError(f"No tables found matching pattern {match.pattern!r}")
614 return result
616 def _href_getter(self, obj) -> str | None:
617 a = obj.find("a", href=True)
618 return None if not a else a["href"]
620 def _text_getter(self, obj):
621 return obj.text
623 def _equals_tag(self, obj, tag) -> bool:
624 return obj.name == tag
626 def _parse_td(self, row):
627 return row.find_all(("td", "th"), recursive=False)
629 def _parse_thead_tr(self, table):
630 return table.select(":scope thead tr")
632 def _parse_tbody_tr(self, table):
633 from_tbody = table.select(":scope tbody tr")
634 from_root = table.find_all("tr", recursive=False)
635 # HTML spec: at most one of these lists has content
636 return from_tbody + from_root
638 def _parse_tfoot_tr(self, table):
639 return table.select(":scope tfoot tr")
641 def _setup_build_doc(self):
642 raw_text = _read(self.io, self.encoding, self.storage_options)
643 if not raw_text:
644 raise ValueError(f"No text parsed from document: {self.io}")
645 return raw_text
647 def _build_doc(self):
648 from bs4 import BeautifulSoup
650 bdoc = self._setup_build_doc()
651 if isinstance(bdoc, bytes) and self.encoding is not None:
652 udoc = bdoc.decode(self.encoding)
653 from_encoding = None
654 else:
655 udoc = bdoc
656 from_encoding = self.encoding
658 soup = BeautifulSoup(udoc, features="html5lib", from_encoding=from_encoding)
660 for br in soup.find_all("br"):
661 br.replace_with("\n" + br.text)
663 return soup
666def _build_xpath_expr(attrs) -> str:
667 """
668 Build an xpath expression to simulate bs4's ability to pass in kwargs to
669 search for attributes when using the lxml parser.
671 Parameters
672 ----------
673 attrs : dict
674 A dict of HTML attributes. These are NOT checked for validity.
676 Returns
677 -------
678 expr : unicode
679 An XPath expression that checks for the given HTML attributes.
680 """
681 # give class attribute as class_ because class is a python keyword
682 if "class_" in attrs:
683 attrs["class"] = attrs.pop("class_")
685 s = " and ".join([f"@{k}={v!r}" for k, v in attrs.items()])
686 return f"[{s}]"
689_re_namespace = {"re": "http://exslt.org/regular-expressions"}
692class _LxmlFrameParser(_HtmlFrameParser):
693 """
694 HTML to DataFrame parser that uses lxml under the hood.
696 Warning
697 -------
698 This parser can only handle HTTP, FTP, and FILE urls.
700 See Also
701 --------
702 _HtmlFrameParser
703 _BeautifulSoupLxmlFrameParser
705 Notes
706 -----
707 Documentation strings for this class are in the base class
708 :class:`_HtmlFrameParser`.
709 """
711 def _href_getter(self, obj) -> str | None:
712 href = obj.xpath(".//a/@href")
713 return None if not href else href[0]
715 def _text_getter(self, obj):
716 return obj.text_content()
718 def _parse_td(self, row):
719 # Look for direct children only: the "row" element here may be a
720 # <thead> or <tfoot> (see _parse_thead_tr).
721 return row.xpath("./td|./th")
723 def _parse_tables(self, document, match, kwargs):
724 pattern = match.pattern
726 # 1. check all descendants for the given pattern and only search tables
727 # GH 49929
728 xpath_expr = f"//table[.//text()[re:test(., {pattern!r})]]"
730 # if any table attributes were given build an xpath expression to
731 # search for them
732 if kwargs:
733 xpath_expr += _build_xpath_expr(kwargs)
735 tables = document.xpath(xpath_expr, namespaces=_re_namespace)
737 tables = self._handle_hidden_tables(tables, "attrib")
738 if self.displayed_only:
739 for table in tables:
740 # lxml utilizes XPATH 1.0 which does not have regex
741 # support. As a result, we find all elements with a style
742 # attribute and iterate them to check for display:none
743 for elem in table.xpath(".//style"):
744 elem.drop_tree()
745 for elem in table.xpath(".//*[@style]"):
746 if "display:none" in elem.attrib.get("style", "").replace(" ", ""):
747 elem.drop_tree()
748 if not tables:
749 raise ValueError(f"No tables found matching regex {pattern!r}")
750 return tables
752 def _equals_tag(self, obj, tag) -> bool:
753 return obj.tag == tag
755 def _raise_if_unreadable(self) -> None:
756 # lxml does not report an unreadable local file consistently -- some
757 # builds raise an errno-less OSError, others parse it as an empty
758 # document -- so reopen the path to let the real error escape. GH#29125
759 if isinstance(self.io, (str, bytes)) and not is_url(self.io):
760 with open(self.io, "rb"):
761 pass
763 def _build_doc(self):
764 """
765 Raises
766 ------
767 ValueError
768 * If a URL that lxml cannot parse is passed.
770 Exception
771 * Any other ``Exception`` thrown. For example, trying to parse a
772 URL that is syntactically correct on a machine with no internet
773 connection will fail.
775 See Also
776 --------
777 pandas.io.html._HtmlFrameParser._build_doc
778 """
779 from lxml.etree import XMLSyntaxError
780 from lxml.html import (
781 HTMLParser,
782 parse,
783 )
785 parser = HTMLParser(recover=True, encoding=self.encoding)
787 if is_url(self.io):
788 with get_handle(self.io, "r", storage_options=self.storage_options) as f:
789 r = parse(f.handle, parser=parser)
790 else:
791 # try to parse the input in the simplest way
792 try:
793 r = parse(self.io, parser=parser)
794 except OSError:
795 self._raise_if_unreadable()
796 raise
797 try:
798 r = r.getroot()
799 except AttributeError:
800 pass
801 else:
802 if not hasattr(r, "text_content"):
803 self._raise_if_unreadable()
804 raise XMLSyntaxError("no text parsed from document", 0, 0, 0)
806 for br in r.xpath("*//br"):
807 br.tail = "\n" + (br.tail or "")
809 return r
811 def _parse_thead_tr(self, table):
812 rows = []
814 for thead in table.xpath(".//thead"):
815 rows.extend(thead.xpath("./tr"))
817 # HACK: lxml does not clean up the clearly-erroneous
818 # <thead><th>foo</th><th>bar</th></thead>. (Missing <tr>). Add
819 # the <thead> and _pretend_ it's a <tr>; _parse_td() will find its
820 # children as though it's a <tr>.
821 #
822 # Better solution would be to use html5lib.
823 elements_at_root = thead.xpath("./td|./th")
824 if elements_at_root:
825 rows.append(thead)
827 return rows
829 def _parse_tbody_tr(self, table):
830 from_tbody = table.xpath(".//tbody//tr")
831 from_root = table.xpath("./tr")
832 # HTML spec: at most one of these lists has content
833 return from_tbody + from_root
835 def _parse_tfoot_tr(self, table):
836 return table.xpath(".//tfoot//tr")
839def _expand_elements(body) -> None:
840 data = [len(elem) for elem in body]
841 lens = Series(data)
842 lens_max = lens.max()
843 not_max = lens[lens != lens_max]
845 empty = [""]
846 for ind, length in not_max.items():
847 body[ind] += empty * (lens_max - length)
850def _data_to_frame(**kwargs):
851 head, body, foot = kwargs.pop("data")
852 header = kwargs.pop("header")
853 kwargs["skiprows"] = _get_skiprows(kwargs["skiprows"])
854 if head:
855 body = head + body
857 # Infer header when there is a <thead> or top <th>-only rows
858 if header is None:
859 if len(head) == 1:
860 header = 0
861 else:
862 # ignore all-empty-text rows
863 header = [i for i, row in enumerate(head) if any(text for text in row)]
865 if foot:
866 body += foot
868 # fill out elements of body that are "ragged"
869 _expand_elements(body)
870 with TextParser(body, header=header, **kwargs) as tp:
871 return tp.read()
874_valid_parsers = {
875 "lxml": _LxmlFrameParser,
876 None: _LxmlFrameParser,
877 "html5lib": _BeautifulSoupHtml5LibFrameParser,
878 "bs4": _BeautifulSoupHtml5LibFrameParser,
879}
882def _parser_dispatch(flavor: HTMLFlavors | None) -> type[_HtmlFrameParser]:
883 """
884 Choose the parser based on the input flavor.
886 Parameters
887 ----------
888 flavor : {"lxml", "html5lib", "bs4"} or None
889 The type of parser to use. This must be a valid backend.
891 Returns
892 -------
893 cls : _HtmlFrameParser subclass
894 The parser class based on the requested input flavor.
896 Raises
897 ------
898 ValueError
899 * If `flavor` is not a valid backend.
900 ImportError
901 * If you do not have the requested `flavor`
902 """
903 valid_parsers = list(_valid_parsers.keys())
904 if flavor not in valid_parsers:
905 raise ValueError(
906 f"{flavor!r} is not a valid flavor, valid flavors are {valid_parsers}"
907 )
909 if flavor in ("bs4", "html5lib"):
910 import_optional_dependency("html5lib")
911 import_optional_dependency("bs4")
912 else:
913 import_optional_dependency("lxml.etree")
914 return _valid_parsers[flavor]
917def _print_as_set(s) -> str:
918 arg = ", ".join([pprint_thing(el) for el in s])
919 return f"{{{arg}}}"
922def _validate_flavor(flavor):
923 if flavor is None:
924 flavor = "lxml", "bs4"
925 elif isinstance(flavor, str):
926 flavor = (flavor,)
927 elif isinstance(flavor, abc.Iterable):
928 if not all(isinstance(flav, str) for flav in flavor):
929 raise TypeError(
930 f"Object of type {type(flavor).__name__!r} "
931 f"is not an iterable of strings"
932 )
933 else:
934 msg = repr(flavor) if isinstance(flavor, str) else str(flavor)
935 msg += " is not a valid flavor"
936 raise ValueError(msg)
938 flavor = tuple(flavor)
939 valid_flavors = set(_valid_parsers)
940 flavor_set = set(flavor)
942 if not flavor_set & valid_flavors:
943 raise ValueError(
944 f"{_print_as_set(flavor_set)} is not a valid set of flavors, valid "
945 f"flavors are {_print_as_set(valid_flavors)}"
946 )
947 return flavor
950def _parse(
951 flavor,
952 io,
953 match,
954 attrs,
955 encoding,
956 displayed_only,
957 extract_links,
958 storage_options,
959 **kwargs,
960):
961 flavor = _validate_flavor(flavor)
962 compiled_match = re.compile(match) # you can pass a compiled regex here
964 retained = None
965 for flav in flavor:
966 parser = _parser_dispatch(flav)
967 p = parser(
968 io,
969 compiled_match,
970 attrs,
971 encoding,
972 displayed_only,
973 extract_links,
974 storage_options,
975 )
977 try:
978 tables = p.parse_tables()
979 except ValueError as caught:
980 # if `io` is an io-like object, check if it's seekable
981 # and try to rewind it before trying the next parser
982 if hasattr(io, "seekable") and io.seekable():
983 io.seek(0)
984 elif hasattr(io, "seekable") and not io.seekable():
985 # if we couldn't rewind it, let the user know
986 raise ValueError(
987 f"The flavor {flav} failed to parse your input. "
988 "Since you passed a non-rewindable file "
989 "object, we can't rewind it to try "
990 "another parser. Try read_html() with a different flavor."
991 ) from caught
993 retained = caught
994 else:
995 break
996 else:
997 assert retained is not None # for mypy
998 raise retained
1000 ret = []
1001 for table in tables:
1002 try:
1003 df = _data_to_frame(data=table, **kwargs)
1004 # Cast MultiIndex header to an Index of tuples when extracting header
1005 # links and replace nan with None (therefore can't use mi.to_flat_index()).
1006 # This maintains consistency of selection (e.g. df.columns.str[1])
1007 if extract_links in ("all", "header") and isinstance(
1008 df.columns, MultiIndex
1009 ):
1010 df.columns = Index(
1011 ((col[0], None if isna(col[1]) else col[1]) for col in df.columns),
1012 tupleize_cols=False,
1013 )
1015 ret.append(df)
1016 except EmptyDataError: # empty table
1017 continue
1018 return ret
1021@set_module("pandas")
1022def read_html(
1023 io: FilePath | ReadBuffer[str],
1024 *,
1025 match: str | Pattern = ".+",
1026 flavor: HTMLFlavors | Sequence[HTMLFlavors] | None = None,
1027 header: int | Sequence[int] | None = None,
1028 index_col: int | Sequence[int] | None = None,
1029 skiprows: int | Sequence[int] | slice | None = None,
1030 attrs: dict[str, str] | None = None,
1031 parse_dates: bool = False,
1032 thousands: str | None = ",",
1033 encoding: str | None = None,
1034 decimal: str = ".",
1035 converters: dict | None = None,
1036 na_values: Iterable[object] | None = None,
1037 keep_default_na: bool = True,
1038 displayed_only: bool = True,
1039 extract_links: Literal["header", "footer", "body", "all"] | None = None,
1040 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
1041 storage_options: StorageOptions = None,
1042) -> list[DataFrame]:
1043 r"""
1044 Read HTML tables into a ``list`` of ``DataFrame`` objects.
1046 This function requires one of the following libraries:
1047 `lxml <https://lxml.de/>`_, `html5lib <https://github.com/html5lib/html5lib-python>`_,
1048 or `beautifulsoup4 <https://www.crummy.com/software/BeautifulSoup/>`_.
1050 This function searches for ``<table>`` elements within an HTML document
1051 and parses their rows and columns into DataFrames. It can read from a URL,
1052 a file path, or a raw HTML string, and supports filtering tables by
1053 matching text content via a regular expression.
1055 Parameters
1056 ----------
1057 io : str, path object, or file-like object
1058 String path, path object (implementing ``os.PathLike[str]``), or file-like
1059 object implementing a string ``read()`` function.
1060 The string can represent a URL. Note that
1061 lxml only accepts the http, ftp and file url protocols. If you have a
1062 URL that starts with ``'https'`` you might try removing the ``'s'``.
1064 match : str or compiled regular expression, optional
1065 The set of tables containing text matching this regex or string will be
1066 returned. Unless the HTML is extremely simple you will probably need to
1067 pass a non-empty string here. Defaults to '.+' (match any non-empty
1068 string). The default value will return all tables contained on a page.
1069 This value is converted to a regular expression so that there is
1070 consistent behavior between Beautiful Soup and lxml.
1072 flavor : {"lxml", "html5lib", "bs4"} or list-like, optional
1073 The parsing engine (or list of parsing engines) to use. 'bs4' and
1074 'html5lib' are synonymous with each other, they are both there for
1075 backwards compatibility. The default of ``None`` tries to use ``lxml``
1076 to parse and if that fails it falls back on ``bs4`` + ``html5lib``.
1078 header : int or list-like, optional
1079 The row (or list of rows for a :class:`~pandas.MultiIndex`) to use to
1080 make the columns headers.
1082 index_col : int or list-like, optional
1083 The column (or list of columns) to use to create the index.
1085 skiprows : int, list-like or slice, optional
1086 Number of rows to skip after parsing the header. 0-based. If a
1087 sequence of integers or a slice is given, will skip the rows indexed by
1088 that sequence. Note that a single element sequence means 'skip the nth
1089 row' whereas an integer means 'skip n rows'.
1091 attrs : dict, optional
1092 This is a dictionary of attributes that you can pass to use to identify
1093 the table in the HTML. These are not checked for validity before being
1094 passed to lxml or Beautiful Soup. However, these attributes must be
1095 valid HTML table attributes to work correctly. For example, ::
1097 attrs = {"id": "table"}
1099 is a valid attribute dictionary because the 'id' HTML tag attribute is
1100 a valid HTML attribute for *any* HTML tag as per `this document
1101 <https://html.spec.whatwg.org/multipage/dom.html#global-attributes>`__. ::
1103 attrs = {"asdf": "table"}
1105 is *not* a valid attribute dictionary because 'asdf' is not a valid
1106 HTML attribute even if it is a valid XML attribute. Valid HTML 4.01
1107 table attributes can be found `here
1108 <http://www.w3.org/TR/REC-html40/struct/tables.html#h-11.2>`__. A
1109 working draft of the HTML 5 spec can be found `here
1110 <https://html.spec.whatwg.org/multipage/tables.html>`__. It contains the
1111 latest information on table attributes for the modern web.
1113 parse_dates : bool, optional
1114 See :func:`~read_csv` for more details.
1116 thousands : str, optional
1117 Separator to use to parse thousands. Defaults to ``','``.
1119 encoding : str, optional
1120 The encoding used to decode the web page. Defaults to ``None``.``None``
1121 preserves the previous encoding behavior, which depends on the
1122 underlying parser library (e.g., the parser library will try to use
1123 the encoding provided by the document).
1125 decimal : str, default '.'
1126 Character to recognize as decimal point (e.g. use ',' for European
1127 data).
1129 converters : dict, default None
1130 Dict of functions for converting values in certain columns. Keys can
1131 either be integers or column labels, values are functions that take one
1132 input argument, the cell (not column) content, and return the
1133 transformed content.
1135 na_values : iterable, default None
1136 Custom NA values.
1138 keep_default_na : bool, default True
1139 If na_values are specified and keep_default_na is False the default NaN
1140 values are overridden, otherwise they're appended to.
1142 displayed_only : bool, default True
1143 Whether elements with "display: none" should be parsed.
1145 extract_links : {None, "all", "header", "body", "footer"}
1146 Table elements in the specified section(s) with <a> tags will have their
1147 href extracted.
1149 dtype_backend : {'numpy_nullable', 'pyarrow'}
1150 Back-end data type applied to the resultant :class:`DataFrame`
1151 (still experimental). If not specified, the default behavior
1152 is to not use nullable data types. If specified, the behavior
1153 is as follows:
1155 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
1156 * ``"pyarrow"``: returns pyarrow-backed nullable
1157 :class:`ArrowDtype` :class:`DataFrame`
1159 .. versionadded:: 2.0
1161 storage_options : dict, optional
1162 Extra options that make sense for a particular storage connection, e.g.
1163 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
1164 are forwarded to ``urllib.request.Request`` as header options. For other
1165 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
1166 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
1167 details, and for more examples on storage options refer `here
1168 <https://pandas.pydata.org/docs/user_guide/io.html?
1169 highlight=storage_options#reading-writing-remote-files>`_.
1171 .. versionadded:: 2.1.0
1173 Returns
1174 -------
1175 dfs
1176 A list of DataFrames.
1178 See Also
1179 --------
1180 read_csv : Read a comma-separated values (csv) file into DataFrame.
1182 Notes
1183 -----
1184 Before using this function you should read the :ref:`gotchas about the
1185 HTML parsing libraries <io.html.gotchas>`.
1187 Expect to do some cleanup after you call this function. For example, you
1188 might need to manually assign column names if the column names are
1189 converted to NaN when you pass the `header=0` argument. We try to assume as
1190 little as possible about the structure of the table and push the
1191 idiosyncrasies of the HTML contained in the table to the user.
1193 This function searches for ``<table>`` elements and only for ``<tr>``
1194 and ``<th>`` rows and ``<td>`` elements within each ``<tr>`` or ``<th>``
1195 element in the table. ``<td>`` stands for "table data". This function
1196 attempts to properly handle ``colspan`` and ``rowspan`` attributes.
1197 If the function has a ``<thead>`` argument, it is used to construct
1198 the header, otherwise the function attempts to find the header within
1199 the body (by putting rows with only ``<th>`` elements into the header).
1201 Similar to :func:`~read_csv` the `header` argument is applied
1202 **after** `skiprows` is applied.
1204 This function will *always* return a list of :class:`DataFrame` *or*
1205 it will fail, i.e., it will *not* return an empty list, save for some
1206 rare cases.
1207 It might return an empty list in case of inputs with single row and
1208 ``<td>`` containing only whitespaces.
1210 Examples
1211 --------
1212 See the :ref:`read_html documentation in the IO section of the docs
1213 <io.read_html>` for some examples of reading in HTML tables.
1214 """
1215 # Type check here. We don't want to parse only to fail because of an
1216 # invalid value of an integer skiprows.
1217 if isinstance(skiprows, numbers.Integral) and skiprows < 0:
1218 raise ValueError(
1219 "cannot skip rows starting from the end of the "
1220 "data (you passed a negative value)"
1221 )
1222 if extract_links not in [None, "header", "footer", "body", "all"]:
1223 raise ValueError(
1224 "`extract_links` must be one of "
1225 '{None, "header", "footer", "body", "all"}, got '
1226 f'"{extract_links}"'
1227 )
1229 validate_header_arg(header)
1230 check_dtype_backend(dtype_backend)
1232 io = stringify_path(io)
1234 return _parse(
1235 flavor=flavor,
1236 io=io,
1237 match=match,
1238 header=header,
1239 index_col=index_col,
1240 skiprows=skiprows,
1241 parse_dates=parse_dates,
1242 thousands=thousands,
1243 attrs=attrs,
1244 encoding=encoding,
1245 decimal=decimal,
1246 converters=converters,
1247 na_values=na_values,
1248 keep_default_na=keep_default_na,
1249 displayed_only=displayed_only,
1250 extract_links=extract_links,
1251 dtype_backend=dtype_backend,
1252 storage_options=storage_options,
1253 )