Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/html.py: 21%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2:mod:`pandas.io.html` is a module containing functionality for dealing with
3HTML IO.
5"""
7from __future__ import annotations
9from collections import abc
10import errno
11import numbers
12import os
13import re
14from re import Pattern
15from typing import (
16 TYPE_CHECKING,
17 Literal,
18 cast,
19)
21from pandas._libs import lib
22from pandas.compat._optional import import_optional_dependency
23from pandas.errors import (
24 AbstractMethodError,
25 EmptyDataError,
26)
27from pandas.util._decorators import set_module
28from pandas.util._validators import check_dtype_backend
30from pandas.core.dtypes.common import is_list_like
32from pandas import isna
33from pandas.core.indexes.base import Index
34from pandas.core.indexes.multi import MultiIndex
35from pandas.core.series import Series
37from pandas.io.common import (
38 get_handle,
39 is_url,
40 stringify_path,
41 validate_header_arg,
42)
43from pandas.io.formats.printing import pprint_thing
44from pandas.io.parsers import TextParser
46if TYPE_CHECKING:
47 from collections.abc import (
48 Iterable,
49 Sequence,
50 )
52 from pandas._typing import (
53 BaseBuffer,
54 DtypeBackend,
55 FilePath,
56 HTMLFlavors,
57 ReadBuffer,
58 StorageOptions,
59 )
61 from pandas import DataFrame
63#############
64# READ HTML #
65#############
66_RE_WHITESPACE = re.compile(r"[\r\n]+|\s{2,}")
69def _remove_whitespace(s: str, regex: Pattern = _RE_WHITESPACE) -> str:
70 """
71 Replace extra whitespace inside of a string with a single space.
73 Parameters
74 ----------
75 s : str or unicode
76 The string from which to remove extra whitespace.
77 regex : re.Pattern
78 The regular expression to use to remove extra whitespace.
80 Returns
81 -------
82 subd : str or unicode
83 `s` with all extra whitespace replaced with a single space.
84 """
85 return regex.sub(" ", s.strip())
88def _get_skiprows(skiprows: int | Sequence[int] | slice | None) -> int | Sequence[int]:
89 """
90 Get an iterator given an integer, slice or container.
92 Parameters
93 ----------
94 skiprows : int, slice, container
95 The iterator to use to skip rows; can also be a slice.
97 Raises
98 ------
99 TypeError
100 * If `skiprows` is not a slice, integer, or Container
102 Returns
103 -------
104 it : iterable
105 A proper iterator to use to skip rows of a DataFrame.
106 """
107 if isinstance(skiprows, slice):
108 start, step = skiprows.start or 0, skiprows.step or 1
109 return list(range(start, skiprows.stop, step))
110 elif isinstance(skiprows, numbers.Integral) or is_list_like(skiprows):
111 return cast("int | Sequence[int]", skiprows)
112 elif skiprows is None:
113 return 0
114 raise TypeError(f"{type(skiprows).__name__} is not a valid type for skipping rows")
117def _read(
118 obj: FilePath | BaseBuffer,
119 encoding: str | None,
120 storage_options: StorageOptions | None,
121) -> str | bytes:
122 """
123 Try to read from a url, file or string.
125 Parameters
126 ----------
127 obj : str, unicode, path object, or file-like object
129 Returns
130 -------
131 raw_text : str
132 """
133 try:
134 with get_handle(
135 obj, "r", encoding=encoding, storage_options=storage_options
136 ) as handles:
137 return handles.handle.read()
138 except OSError as err:
139 if not is_url(obj):
140 raise FileNotFoundError(
141 f"[Errno {errno.ENOENT}] {os.strerror(errno.ENOENT)}: {obj}"
142 ) from err
143 raise
146class _HtmlFrameParser:
147 """
148 Base class for parsers that parse HTML into DataFrames.
150 Parameters
151 ----------
152 io : str or file-like
153 This can be either a string path, a valid URL using the HTTP,
154 FTP, or FILE protocols or a file-like object.
156 match : str or regex
157 The text to match in the document.
159 attrs : dict
160 List of HTML <table> element attributes to match.
162 encoding : str
163 Encoding to be used by parser
165 displayed_only : bool
166 Whether or not items with "display:none" should be ignored
168 extract_links : {None, "all", "header", "body", "footer"}
169 Table elements in the specified section(s) with <a> tags will have their
170 href extracted.
172 Attributes
173 ----------
174 io : str or file-like
175 raw HTML, URL, or file-like object
177 match : regex
178 The text to match in the raw HTML
180 attrs : dict-like
181 A dictionary of valid table attributes to use to search for table
182 elements.
184 encoding : str
185 Encoding to be used by parser
187 displayed_only : bool
188 Whether or not items with "display:none" should be ignored
190 extract_links : {None, "all", "header", "body", "footer"}
191 Table elements in the specified section(s) with <a> tags will have their
192 href extracted.
194 Notes
195 -----
196 To subclass this class effectively you must override the following methods:
197 * :func:`_build_doc`
198 * :func:`_attr_getter`
199 * :func:`_href_getter`
200 * :func:`_text_getter`
201 * :func:`_parse_td`
202 * :func:`_parse_thead_tr`
203 * :func:`_parse_tbody_tr`
204 * :func:`_parse_tfoot_tr`
205 * :func:`_parse_tables`
206 * :func:`_equals_tag`
207 See each method's respective documentation for details on their
208 functionality.
209 """
211 def __init__(
212 self,
213 io: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
214 match: str | Pattern,
215 attrs: dict[str, str] | None,
216 encoding: str,
217 displayed_only: bool,
218 extract_links: Literal["header", "footer", "body", "all"] | None,
219 storage_options: StorageOptions = None,
220 ) -> None:
221 self.io = io
222 self.match = match
223 self.attrs = attrs
224 self.encoding = encoding
225 self.displayed_only = displayed_only
226 self.extract_links = extract_links
227 self.storage_options = storage_options
229 def parse_tables(self):
230 """
231 Parse and return all tables from the DOM.
233 Returns
234 -------
235 list of parsed (header, body, footer) tuples from tables.
236 """
237 tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
238 return (self._parse_thead_tbody_tfoot(table) for table in tables)
240 def _attr_getter(self, obj, attr):
241 """
242 Return the attribute value of an individual DOM node.
244 Parameters
245 ----------
246 obj : node-like
247 A DOM node.
249 attr : str or unicode
250 The attribute, such as "colspan"
252 Returns
253 -------
254 str or unicode
255 The attribute value.
256 """
257 # Both lxml and BeautifulSoup have the same implementation:
258 return obj.get(attr)
260 def _href_getter(self, obj) -> str | None:
261 """
262 Return an href if the DOM node contains a child <a> or None.
264 Parameters
265 ----------
266 obj : node-like
267 A DOM node.
269 Returns
270 -------
271 href : str or unicode
272 The href from the <a> child of the DOM node.
273 """
274 raise AbstractMethodError(self)
276 def _text_getter(self, obj):
277 """
278 Return the text of an individual DOM node.
280 Parameters
281 ----------
282 obj : node-like
283 A DOM node.
285 Returns
286 -------
287 text : str or unicode
288 The text from an individual DOM node.
289 """
290 raise AbstractMethodError(self)
292 def _parse_td(self, obj):
293 """
294 Return the td elements from a row element.
296 Parameters
297 ----------
298 obj : node-like
299 A DOM <tr> node.
301 Returns
302 -------
303 list of node-like
304 These are the elements of each row, i.e., the columns.
305 """
306 raise AbstractMethodError(self)
308 def _parse_thead_tr(self, table):
309 """
310 Return the list of thead row elements from the parsed table element.
312 Parameters
313 ----------
314 table : a table element that contains zero or more thead elements.
316 Returns
317 -------
318 list of node-like
319 These are the <tr> row elements of a table.
320 """
321 raise AbstractMethodError(self)
323 def _parse_tbody_tr(self, table):
324 """
325 Return the list of tbody row elements from the parsed table element.
327 HTML5 table bodies consist of either 0 or more <tbody> elements (which
328 only contain <tr> elements) or 0 or more <tr> elements. This method
329 checks for both structures.
331 Parameters
332 ----------
333 table : a table element that contains row elements.
335 Returns
336 -------
337 list of node-like
338 These are the <tr> row elements of a table.
339 """
340 raise AbstractMethodError(self)
342 def _parse_tfoot_tr(self, table):
343 """
344 Return the list of tfoot row elements from the parsed table element.
346 Parameters
347 ----------
348 table : a table element that contains row elements.
350 Returns
351 -------
352 list of node-like
353 These are the <tr> row elements of a table.
354 """
355 raise AbstractMethodError(self)
357 def _parse_tables(self, document, match, attrs):
358 """
359 Return all tables from the parsed DOM.
361 Parameters
362 ----------
363 document : the DOM from which to parse the table element.
365 match : str or regular expression
366 The text to search for in the DOM tree.
368 attrs : dict
369 A dictionary of table attributes that can be used to disambiguate
370 multiple tables on a page.
372 Raises
373 ------
374 ValueError : `match` does not match any text in the document.
376 Returns
377 -------
378 list of node-like
379 HTML <table> elements to be parsed into raw data.
380 """
381 raise AbstractMethodError(self)
383 def _equals_tag(self, obj, tag) -> bool:
384 """
385 Return whether an individual DOM node matches a tag
387 Parameters
388 ----------
389 obj : node-like
390 A DOM node.
392 tag : str
393 Tag name to be checked for equality.
395 Returns
396 -------
397 boolean
398 Whether `obj`'s tag name is `tag`
399 """
400 raise AbstractMethodError(self)
402 def _build_doc(self):
403 """
404 Return a tree-like object that can be used to iterate over the DOM.
406 Returns
407 -------
408 node-like
409 The DOM from which to parse the table element.
410 """
411 raise AbstractMethodError(self)
413 def _parse_thead_tbody_tfoot(self, table_html):
414 """
415 Given a table, return parsed header, body, and foot.
417 Parameters
418 ----------
419 table_html : node-like
421 Returns
422 -------
423 tuple of (header, body, footer), each a list of list-of-text rows.
425 Notes
426 -----
427 Header and body are lists-of-lists. Top level list is a list of
428 rows. Each row is a list of str text.
430 Logic: Use <thead>, <tbody>, <tfoot> elements to identify
431 header, body, and footer, otherwise:
432 - Put all rows into body
433 - Move rows from top of body to header only if
434 all elements inside row are <th>
435 - Move rows from bottom of body to footer only if
436 all elements inside row are <th>
437 """
438 header_rows = self._parse_thead_tr(table_html)
439 body_rows = self._parse_tbody_tr(table_html)
440 footer_rows = self._parse_tfoot_tr(table_html)
442 def row_is_all_th(row):
443 return all(self._equals_tag(t, "th") for t in self._parse_td(row))
445 if not header_rows:
446 # The table has no <thead>. Move the top all-<th> rows from
447 # body_rows to header_rows. (This is a common case because many
448 # tables in the wild have no <thead> or <tfoot>
449 while body_rows and row_is_all_th(body_rows[0]):
450 header_rows.append(body_rows.pop(0))
452 header, rem = self._expand_colspan_rowspan(header_rows, section="header")
453 body, rem = self._expand_colspan_rowspan(
454 body_rows,
455 section="body",
456 remainder=rem,
457 overflow=len(footer_rows) > 0,
458 )
459 footer, _ = self._expand_colspan_rowspan(
460 footer_rows, section="footer", remainder=rem, overflow=False
461 )
463 return header, body, footer
465 def _expand_colspan_rowspan(
466 self,
467 rows,
468 section: Literal["header", "footer", "body"],
469 remainder: list[tuple[int, str | tuple, int]] | None = None,
470 overflow: bool = True,
471 ) -> tuple[list[list], list[tuple[int, str | tuple, int]]]:
472 """
473 Given a list of <tr>s, return a list of text rows.
475 Parameters
476 ----------
477 rows : list of node-like
478 List of <tr>s
479 section : the section that the rows belong to (header, body or footer).
480 remainder: list[tuple[int, str | tuple, int]] | None
481 Any remainder from the expansion of previous section
482 overflow: bool
483 If true, return any partial rows as 'remainder'. If not, use up any
484 partial rows. True by default.
486 Returns
487 -------
488 list of list
489 Each returned row is a list of str text, or tuple (text, link)
490 if extract_links is not None.
491 remainder
492 Remaining partial rows if any. If overflow is False, an empty list
493 is returned.
495 Notes
496 -----
497 Any cell with ``rowspan`` or ``colspan`` will have its contents copied
498 to subsequent cells.
499 """
500 all_texts = [] # list of rows, each a list of str
501 text: str | tuple
502 remainder = remainder if remainder is not None else []
504 for tr in rows:
505 texts = [] # the output for this row
506 next_remainder = []
508 index = 0
509 tds = self._parse_td(tr)
510 for td in tds:
511 # Append texts from previous rows with rowspan>1 that come
512 # before this <td>
513 while remainder and remainder[0][0] <= index:
514 prev_i, prev_text, prev_rowspan = remainder.pop(0)
515 texts.append(prev_text)
516 if prev_rowspan > 1:
517 next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
518 index += 1
520 # Append the text from this <td>, colspan times
521 text = _remove_whitespace(self._text_getter(td))
522 if self.extract_links in ("all", section):
523 href = self._href_getter(td)
524 text = (text, href)
525 rowspan = int(self._attr_getter(td, "rowspan") or 1)
526 colspan = int(self._attr_getter(td, "colspan") or 1)
528 for _ in range(colspan):
529 texts.append(text)
530 if rowspan > 1:
531 next_remainder.append((index, text, rowspan - 1))
532 index += 1
534 # Append texts from previous rows at the final position
535 for prev_i, prev_text, prev_rowspan in remainder:
536 texts.append(prev_text)
537 if prev_rowspan > 1:
538 next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
540 all_texts.append(texts)
541 remainder = next_remainder
543 if not overflow:
544 # Append rows that only appear because the previous row had non-1
545 # rowspan
546 while remainder:
547 next_remainder = []
548 texts = []
549 for prev_i, prev_text, prev_rowspan in remainder:
550 texts.append(prev_text)
551 if prev_rowspan > 1:
552 next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
553 all_texts.append(texts)
554 remainder = next_remainder
556 return all_texts, remainder
558 def _handle_hidden_tables(self, tbl_list, attr_name: str):
559 """
560 Return list of tables, potentially removing hidden elements
562 Parameters
563 ----------
564 tbl_list : list of node-like
565 Type of list elements will vary depending upon parser used
566 attr_name : str
567 Name of the accessor for retrieving HTML attributes
569 Returns
570 -------
571 list of node-like
572 Return type matches `tbl_list`
573 """
574 if not self.displayed_only:
575 return tbl_list
577 return [
578 x
579 for x in tbl_list
580 if "display:none"
581 not in getattr(x, attr_name).get("style", "").replace(" ", "")
582 ]
585class _BeautifulSoupHtml5LibFrameParser(_HtmlFrameParser):
586 """
587 HTML to DataFrame parser that uses BeautifulSoup under the hood.
589 See Also
590 --------
591 pandas.io.html._HtmlFrameParser
592 pandas.io.html._LxmlFrameParser
594 Notes
595 -----
596 Documentation strings for this class are in the base class
597 :class:`pandas.io.html._HtmlFrameParser`.
598 """
600 def _parse_tables(self, document, match, attrs):
601 element_name = "table"
602 tables = document.find_all(element_name, attrs=attrs)
603 if not tables:
604 raise ValueError("No tables found")
606 result = []
607 unique_tables = set()
608 tables = self._handle_hidden_tables(tables, "attrs")
610 for table in tables:
611 if self.displayed_only:
612 for elem in table.find_all("style"):
613 elem.decompose()
615 for elem in table.find_all(style=re.compile(r"display:\s*none")):
616 elem.decompose()
618 if table not in unique_tables and table.find(string=match) is not None:
619 result.append(table)
620 unique_tables.add(table)
621 if not result:
622 raise ValueError(f"No tables found matching pattern {match.pattern!r}")
623 return result
625 def _href_getter(self, obj) -> str | None:
626 a = obj.find("a", href=True)
627 return None if not a else a["href"]
629 def _text_getter(self, obj):
630 return obj.text
632 def _equals_tag(self, obj, tag) -> bool:
633 return obj.name == tag
635 def _parse_td(self, row):
636 return row.find_all(("td", "th"), recursive=False)
638 def _parse_thead_tr(self, table):
639 return table.select("thead tr")
641 def _parse_tbody_tr(self, table):
642 from_tbody = table.select("tbody tr")
643 from_root = table.find_all("tr", recursive=False)
644 # HTML spec: at most one of these lists has content
645 return from_tbody + from_root
647 def _parse_tfoot_tr(self, table):
648 return table.select("tfoot tr")
650 def _setup_build_doc(self):
651 raw_text = _read(self.io, self.encoding, self.storage_options)
652 if not raw_text:
653 raise ValueError(f"No text parsed from document: {self.io}")
654 return raw_text
656 def _build_doc(self):
657 from bs4 import BeautifulSoup
659 bdoc = self._setup_build_doc()
660 if isinstance(bdoc, bytes) and self.encoding is not None:
661 udoc = bdoc.decode(self.encoding)
662 from_encoding = None
663 else:
664 udoc = bdoc
665 from_encoding = self.encoding
667 soup = BeautifulSoup(udoc, features="html5lib", from_encoding=from_encoding)
669 for br in soup.find_all("br"):
670 br.replace_with("\n" + br.text)
672 return soup
675def _build_xpath_expr(attrs) -> str:
676 """
677 Build an xpath expression to simulate bs4's ability to pass in kwargs to
678 search for attributes when using the lxml parser.
680 Parameters
681 ----------
682 attrs : dict
683 A dict of HTML attributes. These are NOT checked for validity.
685 Returns
686 -------
687 expr : unicode
688 An XPath expression that checks for the given HTML attributes.
689 """
690 # give class attribute as class_ because class is a python keyword
691 if "class_" in attrs:
692 attrs["class"] = attrs.pop("class_")
694 s = " and ".join([f"@{k}={v!r}" for k, v in attrs.items()])
695 return f"[{s}]"
698_re_namespace = {"re": "http://exslt.org/regular-expressions"}
701class _LxmlFrameParser(_HtmlFrameParser):
702 """
703 HTML to DataFrame parser that uses lxml under the hood.
705 Warning
706 -------
707 This parser can only handle HTTP, FTP, and FILE urls.
709 See Also
710 --------
711 _HtmlFrameParser
712 _BeautifulSoupLxmlFrameParser
714 Notes
715 -----
716 Documentation strings for this class are in the base class
717 :class:`_HtmlFrameParser`.
718 """
720 def _href_getter(self, obj) -> str | None:
721 href = obj.xpath(".//a/@href")
722 return None if not href else href[0]
724 def _text_getter(self, obj):
725 return obj.text_content()
727 def _parse_td(self, row):
728 # Look for direct children only: the "row" element here may be a
729 # <thead> or <tfoot> (see _parse_thead_tr).
730 return row.xpath("./td|./th")
732 def _parse_tables(self, document, match, kwargs):
733 pattern = match.pattern
735 # 1. check all descendants for the given pattern and only search tables
736 # GH 49929
737 xpath_expr = f"//table[.//text()[re:test(., {pattern!r})]]"
739 # if any table attributes were given build an xpath expression to
740 # search for them
741 if kwargs:
742 xpath_expr += _build_xpath_expr(kwargs)
744 tables = document.xpath(xpath_expr, namespaces=_re_namespace)
746 tables = self._handle_hidden_tables(tables, "attrib")
747 if self.displayed_only:
748 for table in tables:
749 # lxml utilizes XPATH 1.0 which does not have regex
750 # support. As a result, we find all elements with a style
751 # attribute and iterate them to check for display:none
752 for elem in table.xpath(".//style"):
753 elem.drop_tree()
754 for elem in table.xpath(".//*[@style]"):
755 if "display:none" in elem.attrib.get("style", "").replace(" ", ""):
756 elem.drop_tree()
757 if not tables:
758 raise ValueError(f"No tables found matching regex {pattern!r}")
759 return tables
761 def _equals_tag(self, obj, tag) -> bool:
762 return obj.tag == tag
764 def _build_doc(self):
765 """
766 Raises
767 ------
768 ValueError
769 * If a URL that lxml cannot parse is passed.
771 Exception
772 * Any other ``Exception`` thrown. For example, trying to parse a
773 URL that is syntactically correct on a machine with no internet
774 connection will fail.
776 See Also
777 --------
778 pandas.io.html._HtmlFrameParser._build_doc
779 """
780 from lxml.etree import XMLSyntaxError
781 from lxml.html import (
782 HTMLParser,
783 parse,
784 )
786 parser = HTMLParser(recover=True, encoding=self.encoding)
788 if is_url(self.io):
789 with get_handle(self.io, "r", storage_options=self.storage_options) as f:
790 r = parse(f.handle, parser=parser)
791 else:
792 # try to parse the input in the simplest way
793 try:
794 r = parse(self.io, parser=parser)
795 except OSError as err:
796 raise FileNotFoundError(
797 f"[Errno {errno.ENOENT}] {os.strerror(errno.ENOENT)}: {self.io}"
798 ) from err
799 try:
800 r = r.getroot()
801 except AttributeError:
802 pass
803 else:
804 if not hasattr(r, "text_content"):
805 raise XMLSyntaxError("no text parsed from document", 0, 0, 0)
807 for br in r.xpath("*//br"):
808 br.tail = "\n" + (br.tail or "")
810 return r
812 def _parse_thead_tr(self, table):
813 rows = []
815 for thead in table.xpath(".//thead"):
816 rows.extend(thead.xpath("./tr"))
818 # HACK: lxml does not clean up the clearly-erroneous
819 # <thead><th>foo</th><th>bar</th></thead>. (Missing <tr>). Add
820 # the <thead> and _pretend_ it's a <tr>; _parse_td() will find its
821 # children as though it's a <tr>.
822 #
823 # Better solution would be to use html5lib.
824 elements_at_root = thead.xpath("./td|./th")
825 if elements_at_root:
826 rows.append(thead)
828 return rows
830 def _parse_tbody_tr(self, table):
831 from_tbody = table.xpath(".//tbody//tr")
832 from_root = table.xpath("./tr")
833 # HTML spec: at most one of these lists has content
834 return from_tbody + from_root
836 def _parse_tfoot_tr(self, table):
837 return table.xpath(".//tfoot//tr")
840def _expand_elements(body) -> None:
841 data = [len(elem) for elem in body]
842 lens = Series(data)
843 lens_max = lens.max()
844 not_max = lens[lens != lens_max]
846 empty = [""]
847 for ind, length in not_max.items():
848 body[ind] += empty * (lens_max - length)
851def _data_to_frame(**kwargs):
852 head, body, foot = kwargs.pop("data")
853 header = kwargs.pop("header")
854 kwargs["skiprows"] = _get_skiprows(kwargs["skiprows"])
855 if head:
856 body = head + body
858 # Infer header when there is a <thead> or top <th>-only rows
859 if header is None:
860 if len(head) == 1:
861 header = 0
862 else:
863 # ignore all-empty-text rows
864 header = [i for i, row in enumerate(head) if any(text for text in row)]
866 if foot:
867 body += foot
869 # fill out elements of body that are "ragged"
870 _expand_elements(body)
871 with TextParser(body, header=header, **kwargs) as tp:
872 return tp.read()
875_valid_parsers = {
876 "lxml": _LxmlFrameParser,
877 None: _LxmlFrameParser,
878 "html5lib": _BeautifulSoupHtml5LibFrameParser,
879 "bs4": _BeautifulSoupHtml5LibFrameParser,
880}
883def _parser_dispatch(flavor: HTMLFlavors | None) -> type[_HtmlFrameParser]:
884 """
885 Choose the parser based on the input flavor.
887 Parameters
888 ----------
889 flavor : {"lxml", "html5lib", "bs4"} or None
890 The type of parser to use. This must be a valid backend.
892 Returns
893 -------
894 cls : _HtmlFrameParser subclass
895 The parser class based on the requested input flavor.
897 Raises
898 ------
899 ValueError
900 * If `flavor` is not a valid backend.
901 ImportError
902 * If you do not have the requested `flavor`
903 """
904 valid_parsers = list(_valid_parsers.keys())
905 if flavor not in valid_parsers:
906 raise ValueError(
907 f"{flavor!r} is not a valid flavor, valid flavors are {valid_parsers}"
908 )
910 if flavor in ("bs4", "html5lib"):
911 import_optional_dependency("html5lib")
912 import_optional_dependency("bs4")
913 else:
914 import_optional_dependency("lxml.etree")
915 return _valid_parsers[flavor]
918def _print_as_set(s) -> str:
919 arg = ", ".join([pprint_thing(el) for el in s])
920 return f"{{{arg}}}"
923def _validate_flavor(flavor):
924 if flavor is None:
925 flavor = "lxml", "bs4"
926 elif isinstance(flavor, str):
927 flavor = (flavor,)
928 elif isinstance(flavor, abc.Iterable):
929 if not all(isinstance(flav, str) for flav in flavor):
930 raise TypeError(
931 f"Object of type {type(flavor).__name__!r} "
932 f"is not an iterable of strings"
933 )
934 else:
935 msg = repr(flavor) if isinstance(flavor, str) else str(flavor)
936 msg += " is not a valid flavor"
937 raise ValueError(msg)
939 flavor = tuple(flavor)
940 valid_flavors = set(_valid_parsers)
941 flavor_set = set(flavor)
943 if not flavor_set & valid_flavors:
944 raise ValueError(
945 f"{_print_as_set(flavor_set)} is not a valid set of flavors, valid "
946 f"flavors are {_print_as_set(valid_flavors)}"
947 )
948 return flavor
951def _parse(
952 flavor,
953 io,
954 match,
955 attrs,
956 encoding,
957 displayed_only,
958 extract_links,
959 storage_options,
960 **kwargs,
961):
962 flavor = _validate_flavor(flavor)
963 compiled_match = re.compile(match) # you can pass a compiled regex here
965 retained = None
966 for flav in flavor:
967 parser = _parser_dispatch(flav)
968 p = parser(
969 io,
970 compiled_match,
971 attrs,
972 encoding,
973 displayed_only,
974 extract_links,
975 storage_options,
976 )
978 try:
979 tables = p.parse_tables()
980 except ValueError as caught:
981 # if `io` is an io-like object, check if it's seekable
982 # and try to rewind it before trying the next parser
983 if hasattr(io, "seekable") and io.seekable():
984 io.seek(0)
985 elif hasattr(io, "seekable") and not io.seekable():
986 # if we couldn't rewind it, let the user know
987 raise ValueError(
988 f"The flavor {flav} failed to parse your input. "
989 "Since you passed a non-rewindable file "
990 "object, we can't rewind it to try "
991 "another parser. Try read_html() with a different flavor."
992 ) from caught
994 retained = caught
995 else:
996 break
997 else:
998 assert retained is not None # for mypy
999 raise retained
1001 ret = []
1002 for table in tables:
1003 try:
1004 df = _data_to_frame(data=table, **kwargs)
1005 # Cast MultiIndex header to an Index of tuples when extracting header
1006 # links and replace nan with None (therefore can't use mi.to_flat_index()).
1007 # This maintains consistency of selection (e.g. df.columns.str[1])
1008 if extract_links in ("all", "header") and isinstance(
1009 df.columns, MultiIndex
1010 ):
1011 df.columns = Index(
1012 ((col[0], None if isna(col[1]) else col[1]) for col in df.columns),
1013 tupleize_cols=False,
1014 )
1016 ret.append(df)
1017 except EmptyDataError: # empty table
1018 continue
1019 return ret
1022@set_module("pandas")
1023def read_html(
1024 io: FilePath | ReadBuffer[str],
1025 *,
1026 match: str | Pattern = ".+",
1027 flavor: HTMLFlavors | Sequence[HTMLFlavors] | None = None,
1028 header: int | Sequence[int] | None = None,
1029 index_col: int | Sequence[int] | None = None,
1030 skiprows: int | Sequence[int] | slice | None = None,
1031 attrs: dict[str, str] | None = None,
1032 parse_dates: bool = False,
1033 thousands: str | None = ",",
1034 encoding: str | None = None,
1035 decimal: str = ".",
1036 converters: dict | None = None,
1037 na_values: Iterable[object] | None = None,
1038 keep_default_na: bool = True,
1039 displayed_only: bool = True,
1040 extract_links: Literal["header", "footer", "body", "all"] | None = None,
1041 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
1042 storage_options: StorageOptions = None,
1043) -> list[DataFrame]:
1044 r"""
1045 Read HTML tables into a ``list`` of ``DataFrame`` objects.
1047 Parameters
1048 ----------
1049 io : str, path object, or file-like object
1050 String path, path object (implementing ``os.PathLike[str]``), or file-like
1051 object implementing a string ``read()`` function.
1052 The string can represent a URL. Note that
1053 lxml only accepts the http, ftp and file url protocols. If you have a
1054 URL that starts with ``'https'`` you might try removing the ``'s'``.
1056 match : str or compiled regular expression, optional
1057 The set of tables containing text matching this regex or string will be
1058 returned. Unless the HTML is extremely simple you will probably need to
1059 pass a non-empty string here. Defaults to '.+' (match any non-empty
1060 string). The default value will return all tables contained on a page.
1061 This value is converted to a regular expression so that there is
1062 consistent behavior between Beautiful Soup and lxml.
1064 flavor : {"lxml", "html5lib", "bs4"} or list-like, optional
1065 The parsing engine (or list of parsing engines) to use. 'bs4' and
1066 'html5lib' are synonymous with each other, they are both there for
1067 backwards compatibility. The default of ``None`` tries to use ``lxml``
1068 to parse and if that fails it falls back on ``bs4`` + ``html5lib``.
1070 header : int or list-like, optional
1071 The row (or list of rows for a :class:`~pandas.MultiIndex`) to use to
1072 make the columns headers.
1074 index_col : int or list-like, optional
1075 The column (or list of columns) to use to create the index.
1077 skiprows : int, list-like or slice, optional
1078 Number of rows to skip after parsing the column integer. 0-based. If a
1079 sequence of integers or a slice is given, will skip the rows indexed by
1080 that sequence. Note that a single element sequence means 'skip the nth
1081 row' whereas an integer means 'skip n rows'.
1083 attrs : dict, optional
1084 This is a dictionary of attributes that you can pass to use to identify
1085 the table in the HTML. These are not checked for validity before being
1086 passed to lxml or Beautiful Soup. However, these attributes must be
1087 valid HTML table attributes to work correctly. For example, ::
1089 attrs = {"id": "table"}
1091 is a valid attribute dictionary because the 'id' HTML tag attribute is
1092 a valid HTML attribute for *any* HTML tag as per `this document
1093 <https://html.spec.whatwg.org/multipage/dom.html#global-attributes>`__. ::
1095 attrs = {"asdf": "table"}
1097 is *not* a valid attribute dictionary because 'asdf' is not a valid
1098 HTML attribute even if it is a valid XML attribute. Valid HTML 4.01
1099 table attributes can be found `here
1100 <http://www.w3.org/TR/REC-html40/struct/tables.html#h-11.2>`__. A
1101 working draft of the HTML 5 spec can be found `here
1102 <https://html.spec.whatwg.org/multipage/tables.html>`__. It contains the
1103 latest information on table attributes for the modern web.
1105 parse_dates : bool, optional
1106 See :func:`~read_csv` for more details.
1108 thousands : str, optional
1109 Separator to use to parse thousands. Defaults to ``','``.
1111 encoding : str, optional
1112 The encoding used to decode the web page. Defaults to ``None``.``None``
1113 preserves the previous encoding behavior, which depends on the
1114 underlying parser library (e.g., the parser library will try to use
1115 the encoding provided by the document).
1117 decimal : str, default '.'
1118 Character to recognize as decimal point (e.g. use ',' for European
1119 data).
1121 converters : dict, default None
1122 Dict of functions for converting values in certain columns. Keys can
1123 either be integers or column labels, values are functions that take one
1124 input argument, the cell (not column) content, and return the
1125 transformed content.
1127 na_values : iterable, default None
1128 Custom NA values.
1130 keep_default_na : bool, default True
1131 If na_values are specified and keep_default_na is False the default NaN
1132 values are overridden, otherwise they're appended to.
1134 displayed_only : bool, default True
1135 Whether elements with "display: none" should be parsed.
1137 extract_links : {None, "all", "header", "body", "footer"}
1138 Table elements in the specified section(s) with <a> tags will have their
1139 href extracted.
1141 dtype_backend : {'numpy_nullable', 'pyarrow'}
1142 Back-end data type applied to the resultant :class:`DataFrame`
1143 (still experimental). If not specified, the default behavior
1144 is to not use nullable data types. If specified, the behavior
1145 is as follows:
1147 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
1148 * ``"pyarrow"``: returns pyarrow-backed nullable
1149 :class:`ArrowDtype` :class:`DataFrame`
1151 .. versionadded:: 2.0
1153 storage_options : dict, optional
1154 Extra options that make sense for a particular storage connection, e.g.
1155 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
1156 are forwarded to ``urllib.request.Request`` as header options. For other
1157 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
1158 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
1159 details, and for more examples on storage options refer `here
1160 <https://pandas.pydata.org/docs/user_guide/io.html?
1161 highlight=storage_options#reading-writing-remote-files>`_.
1163 .. versionadded:: 2.1.0
1165 Returns
1166 -------
1167 dfs
1168 A list of DataFrames.
1170 See Also
1171 --------
1172 read_csv : Read a comma-separated values (csv) file into DataFrame.
1174 Notes
1175 -----
1176 Before using this function you should read the :ref:`gotchas about the
1177 HTML parsing libraries <io.html.gotchas>`.
1179 Expect to do some cleanup after you call this function. For example, you
1180 might need to manually assign column names if the column names are
1181 converted to NaN when you pass the `header=0` argument. We try to assume as
1182 little as possible about the structure of the table and push the
1183 idiosyncrasies of the HTML contained in the table to the user.
1185 This function searches for ``<table>`` elements and only for ``<tr>``
1186 and ``<th>`` rows and ``<td>`` elements within each ``<tr>`` or ``<th>``
1187 element in the table. ``<td>`` stands for "table data". This function
1188 attempts to properly handle ``colspan`` and ``rowspan`` attributes.
1189 If the function has a ``<thead>`` argument, it is used to construct
1190 the header, otherwise the function attempts to find the header within
1191 the body (by putting rows with only ``<th>`` elements into the header).
1193 Similar to :func:`~read_csv` the `header` argument is applied
1194 **after** `skiprows` is applied.
1196 This function will *always* return a list of :class:`DataFrame` *or*
1197 it will fail, i.e., it will *not* return an empty list, save for some
1198 rare cases.
1199 It might return an empty list in case of inputs with single row and
1200 ``<td>`` containing only whitespaces.
1202 Examples
1203 --------
1204 See the :ref:`read_html documentation in the IO section of the docs
1205 <io.read_html>` for some examples of reading in HTML tables.
1206 """
1207 # Type check here. We don't want to parse only to fail because of an
1208 # invalid value of an integer skiprows.
1209 if isinstance(skiprows, numbers.Integral) and skiprows < 0:
1210 raise ValueError(
1211 "cannot skip rows starting from the end of the "
1212 "data (you passed a negative value)"
1213 )
1214 if extract_links not in [None, "header", "footer", "body", "all"]:
1215 raise ValueError(
1216 "`extract_links` must be one of "
1217 '{None, "header", "footer", "body", "all"}, got '
1218 f'"{extract_links}"'
1219 )
1221 validate_header_arg(header)
1222 check_dtype_backend(dtype_backend)
1224 io = stringify_path(io)
1226 return _parse(
1227 flavor=flavor,
1228 io=io,
1229 match=match,
1230 header=header,
1231 index_col=index_col,
1232 skiprows=skiprows,
1233 parse_dates=parse_dates,
1234 thousands=thousands,
1235 attrs=attrs,
1236 encoding=encoding,
1237 decimal=decimal,
1238 converters=converters,
1239 na_values=na_values,
1240 keep_default_na=keep_default_na,
1241 displayed_only=displayed_only,
1242 extract_links=extract_links,
1243 dtype_backend=dtype_backend,
1244 storage_options=storage_options,
1245 )