Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/xml.py: 16%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2:mod:``pandas.io.xml`` is a module for reading XML.
3"""
5from __future__ import annotations
7import io
8from os import PathLike
9from typing import (
10 TYPE_CHECKING,
11 Any,
12)
14from pandas._libs import lib
15from pandas.compat._optional import import_optional_dependency
16from pandas.errors import (
17 AbstractMethodError,
18 ParserError,
19)
20from pandas.util._decorators import set_module
21from pandas.util._validators import check_dtype_backend
23from pandas.core.dtypes.common import is_list_like
25from pandas.io.common import (
26 get_handle,
27 infer_compression,
28 is_fsspec_url,
29 is_url,
30 stringify_path,
31)
32from pandas.io.parsers import TextParser
34if TYPE_CHECKING:
35 from collections.abc import (
36 Callable,
37 Sequence,
38 )
39 from xml.etree.ElementTree import Element
41 from lxml import etree
43 from pandas._typing import (
44 CompressionOptions,
45 ConvertersArg,
46 DtypeArg,
47 DtypeBackend,
48 FilePath,
49 ParseDatesArg,
50 ReadBuffer,
51 StorageOptions,
52 XMLParsers,
53 )
55 from pandas import DataFrame
58class _XMLFrameParser:
59 """
60 Internal subclass to parse XML into DataFrames.
62 Parameters
63 ----------
64 path_or_buffer : a valid JSON ``str``, path object or file-like object
65 Any valid string path is acceptable. The string could be a URL. Valid
66 URL schemes include http, ftp, s3, and file.
68 xpath : str or regex
69 The ``XPath`` expression to parse required set of nodes for
70 migration to :class:`~pandas.DataFrame`. ``etree`` supports limited ``XPath``.
72 namespaces : dict
73 The namespaces defined in XML document (``xmlns:namespace='URI'``)
74 as dicts with key being namespace and value the URI.
76 elems_only : bool
77 Parse only the child elements at the specified ``xpath``.
79 attrs_only : bool
80 Parse only the attributes at the specified ``xpath``.
82 names : list
83 Column names for :class:`~pandas.DataFrame` of parsed XML data.
85 dtype : dict
86 Data type for data or columns. E.g. {'a': np.float64,
87 'b': np.int32, 'c': 'Int64'}
89 converters : dict, optional
90 Dict of functions for converting values in certain columns. Keys can
91 either be integers or column labels.
93 parse_dates : bool or list of int or names or list of lists or dict
94 Converts either index or select columns to datetimes
96 encoding : str
97 Encoding of xml object or document.
99 stylesheet : str or file-like
100 URL, file, file-like object, or a raw string containing XSLT,
101 ``etree`` does not support XSLT but retained for consistency.
103 iterparse : dict, optional
104 Dict with row element as key and list of descendant elements
105 and/or attributes as value to be retrieved in iterparsing of
106 XML document.
108 compression : str or dict, default 'infer'
109 For on-the-fly decompression of on-disk data. If 'infer' and
110 'path_or_buffer' is path-like, then detect compression from the
111 following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar',
112 '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression).
113 If using 'zip' or 'tar', the ZIP file must contain only one data
114 file to be read in. Set to ``None`` for no decompression.
115 Can also be a dict with key ``'method'`` set to one of
116 {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``}
117 and other key-value pairs are forwarded to ``zipfile.ZipFile``,
118 ``gzip.GzipFile``, ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``,
119 ``lzma.LZMAFile`` or ``tarfile.TarFile``, respectively.
120 As an example, the following could be passed for Zstandard
121 decompression using a custom compression dictionary:
122 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
124 storage_options : dict, optional
125 Extra options that make sense for a particular storage connection,
126 e.g. host, port, username, password, etc. For HTTP(S) URLs the
127 key-value pairs are forwarded to ``urllib.request.Request`` as header
128 options. For other URLs (e.g. starting with "s3://", and "gcs://")
129 the key-value pairs are forwarded to ``fsspec.open``. Please see
130 ``fsspec`` and ``urllib`` for more details, and for more examples on
131 storage options refer `here <https://pandas.pydata.org/docs/
132 user_guide/io.html?highlight=storage_options#reading-writing-remote-
133 files>`_.
135 See also
136 --------
137 pandas.io.xml._EtreeFrameParser
138 pandas.io.xml._LxmlFrameParser
140 Notes
141 -----
142 To subclass this class effectively you must override the following methods:`
143 * :func:`parse_data`
144 * :func:`_parse_nodes`
145 * :func:`_iterparse_nodes`
146 * :func:`_parse_doc`
147 * :func:`_validate_names`
148 * :func:`_validate_path`
151 See each method's respective documentation for details on their
152 functionality.
153 """
155 def __init__(
156 self,
157 path_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
158 xpath: str,
159 namespaces: dict[str, str] | None,
160 elems_only: bool,
161 attrs_only: bool,
162 names: Sequence[str] | None,
163 dtype: DtypeArg | None,
164 converters: ConvertersArg | None,
165 parse_dates: ParseDatesArg | None,
166 encoding: str | None,
167 stylesheet: FilePath | ReadBuffer[bytes] | ReadBuffer[str] | None,
168 iterparse: dict[str, list[str]] | None,
169 compression: CompressionOptions,
170 storage_options: StorageOptions,
171 ) -> None:
172 self.path_or_buffer = path_or_buffer
173 self.xpath = xpath
174 self.namespaces = namespaces
175 self.elems_only = elems_only
176 self.attrs_only = attrs_only
177 self.names = names
178 self.dtype = dtype
179 self.converters = converters
180 self.parse_dates = parse_dates
181 self.encoding = encoding
182 self.stylesheet = stylesheet
183 self.iterparse = iterparse
184 self.compression: CompressionOptions = compression
185 self.storage_options = storage_options
187 def parse_data(self) -> list[dict[str, str | None]]:
188 """
189 Parse xml data.
191 This method will call the other internal methods to
192 validate ``xpath``, names, parse and return specific nodes.
193 """
195 raise AbstractMethodError(self)
197 def _parse_nodes(self, elems: list[Any]) -> list[dict[str, str | None]]:
198 """
199 Parse xml nodes.
201 This method will parse the children and attributes of elements
202 in ``xpath``, conditionally for only elements, only attributes
203 or both while optionally renaming node names.
205 Raises
206 ------
207 ValueError
208 * If only elements and only attributes are specified.
210 Notes
211 -----
212 Namespace URIs will be removed from return node values. Also,
213 elements with missing children or attributes compared to siblings
214 will have optional keys filled with None values.
215 """
217 dicts: list[dict[str, str | None]]
219 if self.elems_only and self.attrs_only:
220 raise ValueError("Either element or attributes can be parsed not both.")
221 if self.elems_only:
222 if self.names:
223 dicts = [
224 {
225 **(
226 {el.tag: el.text}
227 if el.text and not el.text.isspace()
228 else {}
229 ),
230 **{
231 nm: ch.text if ch.text else None
232 for nm, ch in zip(self.names, el.findall("*"), strict=True)
233 },
234 }
235 for el in elems
236 ]
237 else:
238 dicts = [
239 {ch.tag: ch.text if ch.text else None for ch in el.findall("*")}
240 for el in elems
241 ]
243 elif self.attrs_only:
244 dicts = [
245 {k: v if v else None for k, v in el.attrib.items()} for el in elems
246 ]
248 elif self.names:
249 dicts = [
250 {
251 **el.attrib,
252 **({el.tag: el.text} if el.text and not el.text.isspace() else {}),
253 **{
254 nm: ch.text if ch.text else None
255 for nm, ch in zip(self.names, el.findall("*"), strict=False)
256 },
257 }
258 for el in elems
259 ]
261 else:
262 dicts = [
263 {
264 **el.attrib,
265 **({el.tag: el.text} if el.text and not el.text.isspace() else {}),
266 **{ch.tag: ch.text if ch.text else None for ch in el.findall("*")},
267 }
268 for el in elems
269 ]
271 dicts = [
272 {k.split("}")[1] if "}" in k else k: v for k, v in d.items()} for d in dicts
273 ]
275 keys = list(dict.fromkeys([k for d in dicts for k in d.keys()]))
276 dicts = [{k: d[k] if k in d.keys() else None for k in keys} for d in dicts]
278 if self.names:
279 dicts = [dict(zip(self.names, d.values(), strict=True)) for d in dicts]
281 return dicts
283 def _iterparse_nodes(self, iterparse: Callable) -> list[dict[str, str | None]]:
284 """
285 Iterparse xml nodes.
287 This method will read in local disk, decompressed XML files for elements
288 and underlying descendants using iterparse, a method to iterate through
289 an XML tree without holding entire XML tree in memory.
291 Raises
292 ------
293 TypeError
294 * If ``iterparse`` is not a dict or its dict value is not list-like.
295 ParserError
296 * If ``path_or_buffer`` is not a physical file on disk or file-like object.
297 * If no data is returned from selected items in ``iterparse``.
299 Notes
300 -----
301 Namespace URIs will be removed from return node values. Also,
302 elements with missing children or attributes in submitted list
303 will have optional keys filled with None values.
304 """
306 dicts: list[dict[str, str | None]] = []
307 row: dict[str, str | None] | None = None
309 if not isinstance(self.iterparse, dict):
310 raise TypeError(
311 f"{type(self.iterparse).__name__} is not a valid type for iterparse"
312 )
314 row_node = next(iter(self.iterparse.keys())) if self.iterparse else ""
315 if not is_list_like(self.iterparse[row_node]):
316 raise TypeError(
317 f"{type(self.iterparse[row_node])} is not a valid type "
318 "for value in iterparse"
319 )
321 if (not hasattr(self.path_or_buffer, "read")) and (
322 not isinstance(self.path_or_buffer, (str, PathLike))
323 or is_url(self.path_or_buffer)
324 or is_fsspec_url(self.path_or_buffer)
325 or (
326 isinstance(self.path_or_buffer, str)
327 and self.path_or_buffer.startswith(("<?xml", "<"))
328 )
329 or infer_compression(self.path_or_buffer, "infer") is not None
330 ):
331 raise ParserError(
332 "iterparse is designed for large XML files that are fully extracted on "
333 "local disk and not as compressed files or online sources."
334 )
336 iterparse_repeats = len(self.iterparse[row_node]) != len(
337 set(self.iterparse[row_node])
338 )
340 for event, elem in iterparse(self.path_or_buffer, events=("start", "end")):
341 curr_elem = elem.tag.split("}")[1] if "}" in elem.tag else elem.tag
343 if event == "start":
344 if curr_elem == row_node:
345 row = {}
347 if row is not None:
348 if self.names and iterparse_repeats:
349 for col, nm in zip(
350 self.iterparse[row_node], self.names, strict=True
351 ):
352 if curr_elem == col:
353 elem_val = elem.text if elem.text else None
354 if elem_val not in row.values() and nm not in row:
355 row[nm] = elem_val
357 if col in elem.attrib:
358 if elem.attrib[col] not in row.values() and nm not in row:
359 row[nm] = elem.attrib[col]
360 else:
361 for col in self.iterparse[row_node]:
362 if curr_elem == col:
363 row[col] = elem.text if elem.text else None
364 if col in elem.attrib:
365 row[col] = elem.attrib[col]
367 if event == "end":
368 if curr_elem == row_node and row is not None:
369 dicts.append(row)
370 row = None
372 elem.clear()
373 if hasattr(elem, "getprevious"):
374 while (
375 elem.getprevious() is not None and elem.getparent() is not None
376 ):
377 del elem.getparent()[0]
379 if dicts == []:
380 raise ParserError("No result from selected items in iterparse.")
382 keys = list(dict.fromkeys([k for d in dicts for k in d.keys()]))
383 dicts = [{k: d[k] if k in d.keys() else None for k in keys} for d in dicts]
385 if self.names:
386 dicts = [dict(zip(self.names, d.values(), strict=True)) for d in dicts]
388 return dicts
390 def _validate_path(self) -> list[Any]:
391 """
392 Validate ``xpath``.
394 This method checks for syntax, evaluation, or empty nodes return.
396 Raises
397 ------
398 SyntaxError
399 * If xpah is not supported or issues with namespaces.
401 ValueError
402 * If xpah does not return any nodes.
403 """
405 raise AbstractMethodError(self)
407 def _validate_names(self) -> None:
408 """
409 Validate names.
411 This method will check if names is a list-like and aligns
412 with length of parse nodes.
414 Raises
415 ------
416 ValueError
417 * If value is not a list and less then length of nodes.
418 """
419 raise AbstractMethodError(self)
421 def _parse_doc(
422 self, raw_doc: FilePath | ReadBuffer[bytes] | ReadBuffer[str]
423 ) -> Element | etree._Element:
424 """
425 Build tree from path_or_buffer.
427 This method will parse XML object into tree
428 either from string/bytes or file location.
429 """
430 raise AbstractMethodError(self)
433class _EtreeFrameParser(_XMLFrameParser):
434 """
435 Internal class to parse XML into DataFrames with the Python
436 standard library XML module: `xml.etree.ElementTree`.
437 """
439 def parse_data(self) -> list[dict[str, str | None]]:
440 from xml.etree.ElementTree import iterparse
442 if self.stylesheet is not None:
443 raise ValueError(
444 "To use stylesheet, you need lxml installed and selected as parser."
445 )
447 if self.iterparse is None:
448 self.xml_doc = self._parse_doc(self.path_or_buffer)
449 elems = self._validate_path()
451 self._validate_names()
453 xml_dicts: list[dict[str, str | None]] = (
454 self._parse_nodes(elems)
455 if self.iterparse is None
456 else self._iterparse_nodes(iterparse)
457 )
459 return xml_dicts
461 def _validate_path(self) -> list[Any]:
462 """
463 Notes
464 -----
465 ``etree`` supports limited ``XPath``. If user attempts a more complex
466 expression syntax error will raise.
467 """
469 msg = (
470 "xpath does not return any nodes or attributes. "
471 "Be sure to specify in `xpath` the parent nodes of "
472 "children and attributes to parse. "
473 "If document uses namespaces denoted with "
474 "xmlns, be sure to define namespaces and "
475 "use them in xpath."
476 )
477 try:
478 elems = self.xml_doc.findall(self.xpath, namespaces=self.namespaces)
479 children = [ch for el in elems for ch in el.findall("*")]
480 attrs = {k: v for el in elems for k, v in el.attrib.items()}
482 if elems is None:
483 raise ValueError(msg)
485 if elems is not None:
486 if self.elems_only and children == []:
487 raise ValueError(msg)
488 if self.attrs_only and attrs == {}:
489 raise ValueError(msg)
490 if children == [] and attrs == {}:
491 raise ValueError(msg)
493 except (KeyError, SyntaxError) as err:
494 raise SyntaxError(
495 "You have used an incorrect or unsupported XPath "
496 "expression for etree library or you used an "
497 "undeclared namespace prefix."
498 ) from err
500 return elems
502 def _validate_names(self) -> None:
503 children: list[Any]
505 if self.names:
506 if self.iterparse:
507 children = self.iterparse[next(iter(self.iterparse))]
508 else:
509 parent = self.xml_doc.find(self.xpath, namespaces=self.namespaces)
510 children = parent.findall("*") if parent is not None else []
512 if is_list_like(self.names):
513 if len(self.names) < len(children):
514 raise ValueError(
515 "names does not match length of child elements in xpath."
516 )
517 else:
518 raise TypeError(
519 f"{type(self.names).__name__} is not a valid type for names"
520 )
522 def _parse_doc(
523 self, raw_doc: FilePath | ReadBuffer[bytes] | ReadBuffer[str]
524 ) -> Element:
525 from xml.etree.ElementTree import (
526 XMLParser,
527 parse,
528 )
530 handle_data = get_data_from_filepath(
531 filepath_or_buffer=raw_doc,
532 encoding=self.encoding,
533 compression=self.compression,
534 storage_options=self.storage_options,
535 )
537 with handle_data as xml_data:
538 curr_parser = XMLParser(encoding=self.encoding)
539 document = parse(xml_data, parser=curr_parser)
541 return document.getroot()
544class _LxmlFrameParser(_XMLFrameParser):
545 """
546 Internal class to parse XML into :class:`~pandas.DataFrame` with third-party
547 full-featured XML library, ``lxml``, that supports
548 ``XPath`` 1.0 and XSLT 1.0.
549 """
551 def parse_data(self) -> list[dict[str, str | None]]:
552 """
553 Parse xml data.
555 This method will call the other internal methods to
556 validate ``xpath``, names, optionally parse and run XSLT,
557 and parse original or transformed XML and return specific nodes.
558 """
559 from lxml.etree import iterparse
561 if self.iterparse is None:
562 self.xml_doc = self._parse_doc(self.path_or_buffer)
564 if self.stylesheet:
565 self.xsl_doc = self._parse_doc(self.stylesheet)
566 self.xml_doc = self._transform_doc()
568 elems = self._validate_path()
570 self._validate_names()
572 xml_dicts: list[dict[str, str | None]] = (
573 self._parse_nodes(elems)
574 if self.iterparse is None
575 else self._iterparse_nodes(iterparse)
576 )
578 return xml_dicts
580 def _validate_path(self) -> list[Any]:
581 msg = (
582 "xpath does not return any nodes or attributes. "
583 "Be sure to specify in `xpath` the parent nodes of "
584 "children and attributes to parse. "
585 "If document uses namespaces denoted with "
586 "xmlns, be sure to define namespaces and "
587 "use them in xpath."
588 )
590 elems = self.xml_doc.xpath(self.xpath, namespaces=self.namespaces)
591 children = [ch for el in elems for ch in el.xpath("*")]
592 attrs = {k: v for el in elems for k, v in el.attrib.items()}
594 if elems == []:
595 raise ValueError(msg)
597 if elems != []:
598 if self.elems_only and children == []:
599 raise ValueError(msg)
600 if self.attrs_only and attrs == {}:
601 raise ValueError(msg)
602 if children == [] and attrs == {}:
603 raise ValueError(msg)
605 return elems
607 def _validate_names(self) -> None:
608 children: list[Any]
610 if self.names:
611 if self.iterparse:
612 children = self.iterparse[next(iter(self.iterparse))]
613 else:
614 children = self.xml_doc.xpath(
615 self.xpath + "[1]/*", namespaces=self.namespaces
616 )
618 if is_list_like(self.names):
619 if len(self.names) < len(children):
620 raise ValueError(
621 "names does not match length of child elements in xpath."
622 )
623 else:
624 raise TypeError(
625 f"{type(self.names).__name__} is not a valid type for names"
626 )
628 def _parse_doc(
629 self, raw_doc: FilePath | ReadBuffer[bytes] | ReadBuffer[str]
630 ) -> etree._Element:
631 from lxml.etree import (
632 XMLParser,
633 fromstring,
634 parse,
635 )
637 handle_data = get_data_from_filepath(
638 filepath_or_buffer=raw_doc,
639 encoding=self.encoding,
640 compression=self.compression,
641 storage_options=self.storage_options,
642 )
644 with handle_data as xml_data:
645 curr_parser = XMLParser(encoding=self.encoding)
647 if isinstance(xml_data, io.StringIO):
648 if self.encoding is None:
649 raise TypeError(
650 "Can not pass encoding None when input is StringIO."
651 )
653 document = fromstring(
654 xml_data.getvalue().encode(self.encoding), parser=curr_parser
655 )
656 else:
657 document = parse(xml_data, parser=curr_parser)
659 return document
661 def _transform_doc(self) -> etree._XSLTResultTree:
662 """
663 Transform original tree using stylesheet.
665 This method will transform original xml using XSLT script into
666 am ideally flatter xml document for easier parsing and migration
667 to Data Frame.
668 """
669 from lxml.etree import XSLT
671 transformer = XSLT(self.xsl_doc)
672 new_doc = transformer(self.xml_doc)
674 return new_doc
677def get_data_from_filepath(
678 filepath_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
679 encoding: str | None,
680 compression: CompressionOptions,
681 storage_options: StorageOptions,
682):
683 """
684 Extract raw XML data.
686 The method accepts two input types:
687 1. filepath (string-like)
688 2. file-like object (e.g. open file object, StringIO)
689 """
690 filepath_or_buffer = stringify_path(filepath_or_buffer)
691 with get_handle(
692 filepath_or_buffer,
693 "r",
694 encoding=encoding,
695 compression=compression,
696 storage_options=storage_options,
697 ) as handle_obj:
698 return (
699 preprocess_data(handle_obj.handle.read())
700 if hasattr(handle_obj.handle, "read")
701 else handle_obj.handle
702 )
705def preprocess_data(
706 data: str | bytes | io.StringIO | io.BytesIO,
707) -> io.StringIO | io.BytesIO:
708 """
709 Convert extracted raw data.
711 This method will return underlying data of extracted XML content.
712 The data either has a `read` attribute (e.g. a file object or a
713 StringIO/BytesIO) or is a string or bytes that is an XML document.
714 """
716 if isinstance(data, str):
717 data = io.StringIO(data)
719 elif isinstance(data, bytes):
720 data = io.BytesIO(data)
722 return data
725def _data_to_frame(data: list[dict[str, str | None]], **kwargs) -> DataFrame:
726 """
727 Convert parsed data to Data Frame.
729 This method will bind xml dictionary data of keys and values
730 into named columns of Data Frame using the built-in TextParser
731 class that build Data Frame and infers specific dtypes.
732 """
734 tags = next(iter(data))
735 nodes = [list(d.values()) for d in data]
737 try:
738 with TextParser(nodes, names=tags, **kwargs) as tp:
739 return tp.read()
740 except ParserError as err:
741 raise ParserError(
742 "XML document may be too complex for import. "
743 "Try to flatten document and use distinct "
744 "element and attribute names."
745 ) from err
748def _parse(
749 path_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
750 xpath: str,
751 namespaces: dict[str, str] | None,
752 elems_only: bool,
753 attrs_only: bool,
754 names: Sequence[str] | None,
755 dtype: DtypeArg | None,
756 converters: ConvertersArg | None,
757 parse_dates: ParseDatesArg | None,
758 encoding: str | None,
759 parser: XMLParsers,
760 stylesheet: FilePath | ReadBuffer[bytes] | ReadBuffer[str] | None,
761 iterparse: dict[str, list[str]] | None,
762 compression: CompressionOptions,
763 storage_options: StorageOptions,
764 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
765 **kwargs,
766) -> DataFrame:
767 """
768 Call internal parsers.
770 This method will conditionally call internal parsers:
771 LxmlFrameParser and/or EtreeParser.
773 Raises
774 ------
775 ImportError
776 * If lxml is not installed if selected as parser.
778 ValueError
779 * If parser is not lxml or etree.
780 """
782 p: _EtreeFrameParser | _LxmlFrameParser
784 if parser == "lxml":
785 lxml = import_optional_dependency("lxml.etree", errors="ignore")
787 if lxml is not None:
788 p = _LxmlFrameParser(
789 path_or_buffer,
790 xpath,
791 namespaces,
792 elems_only,
793 attrs_only,
794 names,
795 dtype,
796 converters,
797 parse_dates,
798 encoding,
799 stylesheet,
800 iterparse,
801 compression,
802 storage_options,
803 )
804 else:
805 raise ImportError("lxml not found, please install or use the etree parser.")
807 elif parser == "etree":
808 p = _EtreeFrameParser(
809 path_or_buffer,
810 xpath,
811 namespaces,
812 elems_only,
813 attrs_only,
814 names,
815 dtype,
816 converters,
817 parse_dates,
818 encoding,
819 stylesheet,
820 iterparse,
821 compression,
822 storage_options,
823 )
824 else:
825 raise ValueError("Values for parser can only be lxml or etree.")
827 data_dicts = p.parse_data()
829 return _data_to_frame(
830 data=data_dicts,
831 dtype=dtype,
832 converters=converters,
833 parse_dates=parse_dates,
834 dtype_backend=dtype_backend,
835 **kwargs,
836 )
839@set_module("pandas")
840def read_xml(
841 path_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
842 *,
843 xpath: str = "./*",
844 namespaces: dict[str, str] | None = None,
845 elems_only: bool = False,
846 attrs_only: bool = False,
847 names: Sequence[str] | None = None,
848 dtype: DtypeArg | None = None,
849 converters: ConvertersArg | None = None,
850 parse_dates: ParseDatesArg | None = None,
851 # encoding can not be None for lxml and StringIO input
852 encoding: str | None = "utf-8",
853 parser: XMLParsers = "lxml",
854 stylesheet: FilePath | ReadBuffer[bytes] | ReadBuffer[str] | None = None,
855 iterparse: dict[str, list[str]] | None = None,
856 compression: CompressionOptions = "infer",
857 storage_options: StorageOptions | None = None,
858 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
859) -> DataFrame:
860 r"""
861 Read XML document into a :class:`~pandas.DataFrame` object.
863 Parameters
864 ----------
865 path_or_buffer : str, path object, or file-like object
866 String path, path object (implementing ``os.PathLike[str]``), or file-like
867 object implementing a ``read()`` function. The string can be a path.
868 The string can further be a URL. Valid URL schemes
869 include http, ftp, s3, and file.
871 xpath : str, optional, default './\*'
872 The ``XPath`` to parse required set of nodes for migration to
873 :class:`~pandas.DataFrame`.``XPath`` should return a collection of elements
874 and not a single element. Note: The ``etree`` parser supports limited ``XPath``
875 expressions. For more complex ``XPath``, use ``lxml`` which requires
876 installation.
878 namespaces : dict, optional
879 The namespaces defined in XML document as dicts with key being
880 namespace prefix and value the URI. There is no need to include all
881 namespaces in XML, only the ones used in ``xpath`` expression.
882 Note: if XML document uses default namespace denoted as
883 `xmlns='<URI>'` without a prefix, you must assign any temporary
884 namespace prefix such as 'doc' to the URI in order to parse
885 underlying nodes and/or attributes.
887 elems_only : bool, optional, default False
888 Parse only the child elements at the specified ``xpath``. By default,
889 all child elements and non-empty text nodes are returned.
891 attrs_only : bool, optional, default False
892 Parse only the attributes at the specified ``xpath``.
893 By default, all attributes are returned.
895 names : list-like, optional
896 Column names for DataFrame of parsed XML data. Use this parameter to
897 rename original element names and distinguish same named elements and
898 attributes.
900 dtype : Type name or dict of column -> type, optional
901 Data type for data or columns. E.g. {'a': np.float64, 'b': np.int32,
902 'c': 'Int64'}
903 Use `str` or `object` together with suitable `na_values` settings
904 to preserve and not interpret dtype.
905 If converters are specified, they will be applied INSTEAD
906 of dtype conversion.
908 converters : dict, optional
909 Dict of functions for converting values in certain columns. Keys can either
910 be integers or column labels.
912 parse_dates : bool or list of int or names or list of lists or dict, default False
913 Identifiers to parse index or columns to datetime. The behavior is as follows:
915 * boolean. If True -> try parsing the index.
916 * list of int or names. e.g. If [1, 2, 3] -> try parsing columns 1, 2, 3
917 each as a separate date column.
918 * list of lists. e.g. If [[1, 3]] -> combine columns 1 and 3 and parse as
919 a single date column.
920 * dict, e.g. {'foo' : [1, 3]} -> parse columns 1, 3 as date and call
921 result 'foo'
923 encoding : str, optional, default 'utf-8'
924 Encoding of XML document.
926 parser : {'lxml','etree'}, default 'lxml'
927 Parser module to use for retrieval of data. Only 'lxml' and
928 'etree' are supported. With 'lxml' more complex ``XPath`` searches
929 and ability to use XSLT stylesheet are supported.
931 stylesheet : str, path object or file-like object
932 A URL, file-like object, or a string path containing an XSLT script.
933 This stylesheet should flatten complex, deeply nested XML documents
934 for easier parsing. To use this feature you must have ``lxml`` module
935 installed and specify 'lxml' as ``parser``. The ``xpath`` must
936 reference nodes of transformed XML document generated after XSLT
937 transformation and not the original XML document. Only XSLT 1.0
938 scripts and not later versions is currently supported.
940 iterparse : dict, optional
941 The nodes or attributes to retrieve in iterparsing of XML document
942 as a dict with key being the name of repeating element and value being
943 list of elements or attribute names that are descendants of the repeated
944 element. Note: If this option is used, it will replace ``xpath`` parsing
945 and unlike ``xpath``, descendants do not need to relate to each other but can
946 exist any where in document under the repeating element. This memory-
947 efficient method should be used for very large XML files (500MB, 1GB, or 5GB+).
948 For example, ``{"row_element": ["child_elem", "attr", "grandchild_elem"]}``.
950 compression : str or dict, default 'infer'
951 For on-the-fly decompression of on-disk data. If 'infer' and
952 'path_or_buffer' is path-like, then detect compression from the
953 following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar',
954 '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression).
955 If using 'zip' or 'tar', the ZIP file must contain only one data
956 file to be read in. Set to ``None`` for no decompression.
957 Can also be a dict with key ``'method'`` set to one of
958 {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``}
959 and other key-value pairs are forwarded to ``zipfile.ZipFile``,
960 ``gzip.GzipFile``, ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``,
961 ``lzma.LZMAFile`` or ``tarfile.TarFile``, respectively.
962 As an example, the following could be passed for Zstandard
963 decompression using a custom compression dictionary:
964 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
966 storage_options : dict, optional
967 Extra options that make sense for a particular storage connection,
968 e.g. host, port, username, password, etc. For HTTP(S) URLs the
969 key-value pairs are forwarded to ``urllib.request.Request`` as header
970 options. For other URLs (e.g. starting with "s3://", and "gcs://")
971 the key-value pairs are forwarded to ``fsspec.open``. Please see
972 ``fsspec`` and ``urllib`` for more details, and for more examples on
973 storage options refer `here <https://pandas.pydata.org/docs/
974 user_guide/io.html?highlight=storage_options#reading-writing-remote-
975 files>`_.
977 dtype_backend : {'numpy_nullable', 'pyarrow'}
978 Back-end data type applied to the resultant :class:`DataFrame`
979 (still experimental). If not specified, the default behavior
980 is to not use nullable data types. If specified, the behavior
981 is as follows:
983 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
984 * ``"pyarrow"``: returns pyarrow-backed nullable
985 :class:`ArrowDtype` :class:`DataFrame`
987 .. versionadded:: 2.0
989 Returns
990 -------
991 df
992 A DataFrame.
994 See Also
995 --------
996 read_json : Convert a JSON string to pandas object.
997 read_html : Read HTML tables into a list of DataFrame objects.
999 Notes
1000 -----
1001 This method is best designed to import shallow XML documents in
1002 following format which is the ideal fit for the two-dimensions of a
1003 ``DataFrame`` (row by column). ::
1005 <root>
1006 <row>
1007 <column1>data</column1>
1008 <column2>data</column2>
1009 <column3>data</column3>
1010 ...
1011 </row>
1012 <row>
1013 ...
1014 </row>
1015 ...
1016 </root>
1018 As a file format, XML documents can be designed any way including
1019 layout of elements and attributes as long as it conforms to W3C
1020 specifications. Therefore, this method is a convenience handler for
1021 a specific flatter design and not all possible XML structures.
1023 However, for more complex XML documents, ``stylesheet`` allows you to
1024 temporarily redesign original document with XSLT (a special purpose
1025 language) for a flatter version for migration to a DataFrame.
1027 This function will *always* return a single :class:`DataFrame` or raise
1028 exceptions due to issues with XML document, ``xpath``, or other
1029 parameters.
1031 See the :ref:`read_xml documentation in the IO section of the docs
1032 <io.read_xml>` for more information in using this method to parse XML
1033 files to DataFrames.
1035 Examples
1036 --------
1037 >>> from io import StringIO
1038 >>> xml = '''<?xml version='1.0' encoding='utf-8'?>
1039 ... <data xmlns="http://example.com">
1040 ... <row>
1041 ... <shape>square</shape>
1042 ... <degrees>360</degrees>
1043 ... <sides>4.0</sides>
1044 ... </row>
1045 ... <row>
1046 ... <shape>circle</shape>
1047 ... <degrees>360</degrees>
1048 ... <sides/>
1049 ... </row>
1050 ... <row>
1051 ... <shape>triangle</shape>
1052 ... <degrees>180</degrees>
1053 ... <sides>3.0</sides>
1054 ... </row>
1055 ... </data>'''
1057 >>> df = pd.read_xml(StringIO(xml))
1058 >>> df
1059 shape degrees sides
1060 0 square 360 4.0
1061 1 circle 360 NaN
1062 2 triangle 180 3.0
1064 >>> xml = '''<?xml version='1.0' encoding='utf-8'?>
1065 ... <data>
1066 ... <row shape="square" degrees="360" sides="4.0"/>
1067 ... <row shape="circle" degrees="360"/>
1068 ... <row shape="triangle" degrees="180" sides="3.0"/>
1069 ... </data>'''
1071 >>> df = pd.read_xml(StringIO(xml), xpath=".//row")
1072 >>> df
1073 shape degrees sides
1074 0 square 360 4.0
1075 1 circle 360 NaN
1076 2 triangle 180 3.0
1078 >>> xml = '''<?xml version='1.0' encoding='utf-8'?>
1079 ... <doc:data xmlns:doc="https://example.com">
1080 ... <doc:row>
1081 ... <doc:shape>square</doc:shape>
1082 ... <doc:degrees>360</doc:degrees>
1083 ... <doc:sides>4.0</doc:sides>
1084 ... </doc:row>
1085 ... <doc:row>
1086 ... <doc:shape>circle</doc:shape>
1087 ... <doc:degrees>360</doc:degrees>
1088 ... <doc:sides/>
1089 ... </doc:row>
1090 ... <doc:row>
1091 ... <doc:shape>triangle</doc:shape>
1092 ... <doc:degrees>180</doc:degrees>
1093 ... <doc:sides>3.0</doc:sides>
1094 ... </doc:row>
1095 ... </doc:data>'''
1097 >>> df = pd.read_xml(
1098 ... StringIO(xml),
1099 ... xpath="//doc:row",
1100 ... namespaces={"doc": "https://example.com"},
1101 ... )
1102 >>> df
1103 shape degrees sides
1104 0 square 360 4.0
1105 1 circle 360 NaN
1106 2 triangle 180 3.0
1108 >>> xml_data = '''
1109 ... <data>
1110 ... <row>
1111 ... <index>0</index>
1112 ... <a>1</a>
1113 ... <b>2.5</b>
1114 ... <c>True</c>
1115 ... <d>a</d>
1116 ... <e>2019-12-31 00:00:00</e>
1117 ... </row>
1118 ... <row>
1119 ... <index>1</index>
1120 ... <b>4.5</b>
1121 ... <c>False</c>
1122 ... <d>b</d>
1123 ... <e>2019-12-31 00:00:00</e>
1124 ... </row>
1125 ... </data>
1126 ... '''
1128 >>> df = pd.read_xml(
1129 ... StringIO(xml_data), dtype_backend="numpy_nullable", parse_dates=["e"]
1130 ... )
1131 >>> df
1132 index a b c d e
1133 0 0 1 2.5 True a 2019-12-31
1134 1 1 <NA> 4.5 False b 2019-12-31
1135 """
1136 check_dtype_backend(dtype_backend)
1138 return _parse(
1139 path_or_buffer=path_or_buffer,
1140 xpath=xpath,
1141 namespaces=namespaces,
1142 elems_only=elems_only,
1143 attrs_only=attrs_only,
1144 names=names,
1145 dtype=dtype,
1146 converters=converters,
1147 parse_dates=parse_dates,
1148 encoding=encoding,
1149 parser=parser,
1150 stylesheet=stylesheet,
1151 iterparse=iterparse,
1152 compression=compression,
1153 storage_options=storage_options,
1154 dtype_backend=dtype_backend,
1155 )