Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/xml.py: 16%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2:mod:``pandas.io.xml`` is a module for reading XML.
3"""
5from __future__ import annotations
7from functools import partial
8import io
9from os import PathLike
10from typing import (
11 TYPE_CHECKING,
12 Any,
13)
15from pandas._libs import lib
16from pandas.compat._optional import import_optional_dependency
17from pandas.errors import (
18 AbstractMethodError,
19 ParserError,
20)
21from pandas.util._decorators import set_module
22from pandas.util._validators import check_dtype_backend
24from pandas.core.dtypes.common import is_list_like
26from pandas.io.common import (
27 get_handle,
28 infer_compression,
29 is_fsspec_url,
30 is_url,
31 stringify_path,
32)
33from pandas.io.parsers import TextParser
35if TYPE_CHECKING:
36 from collections.abc import (
37 Callable,
38 Sequence,
39 )
40 from typing import IO
41 from xml.etree.ElementTree import Element
43 from lxml import etree
45 from pandas._typing import (
46 CompressionOptions,
47 ConvertersArg,
48 DtypeArg,
49 DtypeBackend,
50 FilePath,
51 ParseDatesArg,
52 ReadBuffer,
53 StorageOptions,
54 XMLParsers,
55 )
57 from pandas import DataFrame
60class _XMLFrameParser:
61 """
62 Internal subclass to parse XML into DataFrames.
64 Parameters
65 ----------
66 path_or_buffer : a valid JSON ``str``, path object or file-like object
67 Any valid string path is acceptable. The string could be a URL. Valid
68 URL schemes include http, ftp, s3, and file.
70 xpath : str or regex
71 The ``XPath`` expression to parse required set of nodes for
72 migration to :class:`~pandas.DataFrame`. ``etree`` supports limited ``XPath``.
74 namespaces : dict
75 The namespaces defined in XML document (``xmlns:namespace='URI'``)
76 as dicts with key being namespace and value the URI.
78 elems_only : bool
79 Parse only the child elements at the specified ``xpath``.
81 attrs_only : bool
82 Parse only the attributes at the specified ``xpath``.
84 names : list
85 Column names for :class:`~pandas.DataFrame` of parsed XML data.
87 dtype : dict
88 Data type for data or columns. E.g. {'a': np.float64,
89 'b': np.int32, 'c': 'Int64'}
91 converters : dict, optional
92 Dict of functions for converting values in certain columns. Keys can
93 either be integers or column labels.
95 parse_dates : bool or list of int or names or list of lists or dict
96 Converts either index or select columns to datetimes
98 encoding : str
99 Encoding of xml object or document.
101 stylesheet : str or file-like
102 URL, file, file-like object, or a raw string containing XSLT,
103 ``etree`` does not support XSLT but retained for consistency.
105 iterparse : dict, optional
106 Dict with row element as key and list of descendant elements
107 and/or attributes as value to be retrieved in iterparsing of
108 XML document.
110 compression : str or dict, default 'infer'
111 For on-the-fly decompression of on-disk data. If 'infer' and
112 'path_or_buffer' is path-like, then detect compression from the
113 following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar',
114 '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression).
115 If using 'zip' or 'tar', the ZIP file must contain only one data
116 file to be read in. Set to ``None`` for no decompression.
117 Can also be a dict with key ``'method'`` set to one of
118 {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``}
119 and other key-value pairs are forwarded to ``zipfile.ZipFile``,
120 ``gzip.GzipFile``, ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``,
121 ``lzma.LZMAFile`` or ``tarfile.TarFile``, respectively.
122 As an example, the following could be passed for Zstandard
123 decompression using a custom compression dictionary:
124 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
126 storage_options : dict, optional
127 Extra options that make sense for a particular storage connection,
128 e.g. host, port, username, password, etc. For HTTP(S) URLs the
129 key-value pairs are forwarded to ``urllib.request.Request`` as header
130 options. For other URLs (e.g. starting with "s3://", and "gcs://")
131 the key-value pairs are forwarded to ``fsspec.open``. Please see
132 ``fsspec`` and ``urllib`` for more details, and for more examples on
133 storage options refer `here <https://pandas.pydata.org/docs/
134 user_guide/io.html?highlight=storage_options#reading-writing-remote-
135 files>`_.
137 See also
138 --------
139 pandas.io.xml._EtreeFrameParser
140 pandas.io.xml._LxmlFrameParser
142 Notes
143 -----
144 To subclass this class effectively you must override the following methods:`
145 * :func:`parse_data`
146 * :func:`_parse_nodes`
147 * :func:`_iterparse_nodes`
148 * :func:`_parse_doc`
149 * :func:`_validate_names`
150 * :func:`_validate_path`
153 See each method's respective documentation for details on their
154 functionality.
155 """
157 def __init__(
158 self,
159 path_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
160 xpath: str,
161 namespaces: dict[str, str] | None,
162 elems_only: bool,
163 attrs_only: bool,
164 names: Sequence[str] | None,
165 dtype: DtypeArg | None,
166 converters: ConvertersArg | None,
167 parse_dates: ParseDatesArg | None,
168 encoding: str | None,
169 stylesheet: FilePath | ReadBuffer[bytes] | ReadBuffer[str] | None,
170 iterparse: dict[str, list[str]] | None,
171 compression: CompressionOptions,
172 storage_options: StorageOptions,
173 ) -> None:
174 self.path_or_buffer = path_or_buffer
175 self.xpath = xpath
176 self.namespaces = namespaces
177 self.elems_only = elems_only
178 self.attrs_only = attrs_only
179 self.names = names
180 self.dtype = dtype
181 self.converters = converters
182 self.parse_dates = parse_dates
183 self.encoding = encoding
184 self.stylesheet = stylesheet
185 self.iterparse = iterparse
186 self.compression: CompressionOptions = compression
187 self.storage_options = storage_options
188 self.xml_doc: Element | etree._Element | etree._XSLTResultTree | None = None
189 self.xsl_doc: etree._Element | None = None
191 def parse_data(self) -> list[dict[str, str | None]]:
192 """
193 Parse xml data.
195 This method will call the other internal methods to
196 validate ``xpath``, names, parse and return specific nodes.
197 """
199 raise AbstractMethodError(self)
201 def _parse_nodes(self, elems: list[Any]) -> list[dict[str, str | None]]:
202 """
203 Parse xml nodes.
205 This method will parse the children and attributes of elements
206 in ``xpath``, conditionally for only elements, only attributes
207 or both while optionally renaming node names.
209 Raises
210 ------
211 ValueError
212 * If only elements and only attributes are specified.
214 Notes
215 -----
216 Namespace URIs will be removed from return node values. Also,
217 elements with missing children or attributes compared to siblings
218 will have optional keys filled with None values.
219 """
221 dicts: list[dict[str, str | None]]
223 if self.elems_only and self.attrs_only:
224 raise ValueError("Either element or attributes can be parsed not both.")
225 if self.elems_only:
226 if self.names:
227 dicts = [
228 {
229 **(
230 {el.tag: el.text}
231 if el.text and not el.text.isspace()
232 else {}
233 ),
234 **{
235 nm: ch.text if ch.text else None
236 for nm, ch in zip(self.names, el.findall("*"), strict=True)
237 },
238 }
239 for el in elems
240 ]
241 else:
242 dicts = [
243 {ch.tag: ch.text if ch.text else None for ch in el.findall("*")}
244 for el in elems
245 ]
247 elif self.attrs_only:
248 dicts = [
249 {k: v if v else None for k, v in el.attrib.items()} for el in elems
250 ]
252 elif self.names:
253 dicts = [
254 {
255 **el.attrib,
256 **({el.tag: el.text} if el.text and not el.text.isspace() else {}),
257 **{
258 nm: ch.text if ch.text else None
259 for nm, ch in zip(self.names, el.findall("*"), strict=False)
260 },
261 }
262 for el in elems
263 ]
265 else:
266 dicts = [
267 {
268 **el.attrib,
269 **({el.tag: el.text} if el.text and not el.text.isspace() else {}),
270 **{ch.tag: ch.text if ch.text else None for ch in el.findall("*")},
271 }
272 for el in elems
273 ]
275 dicts = [
276 {k.split("}")[1] if "}" in k else k: v for k, v in d.items()} for d in dicts
277 ]
279 keys = list(dict.fromkeys([k for d in dicts for k in d.keys()]))
280 dicts = [{k: d[k] if k in d.keys() else None for k in keys} for d in dicts]
282 if self.names:
283 dicts = [dict(zip(self.names, d.values(), strict=True)) for d in dicts]
285 return dicts
287 def _iterparse_nodes(
288 self, iterparse: Callable[..., Any]
289 ) -> list[dict[str, str | None]]:
290 """
291 Iterparse xml nodes.
293 This method will read in local disk, decompressed XML files for elements
294 and underlying descendants using iterparse, a method to iterate through
295 an XML tree without holding entire XML tree in memory.
297 Raises
298 ------
299 TypeError
300 * If ``iterparse`` is not a dict or its dict value is not list-like.
301 ParserError
302 * If ``path_or_buffer`` is not a physical file on disk or file-like object.
303 * If no data is returned from selected items in ``iterparse``.
305 Notes
306 -----
307 Namespace URIs will be removed from return node values. Also,
308 elements with missing children or attributes in submitted list
309 will have optional keys filled with None values.
310 """
312 dicts: list[dict[str, str | None]] = []
313 row: dict[str, str | None] | None = None
315 if not isinstance(self.iterparse, dict):
316 raise TypeError(
317 f"{type(self.iterparse).__name__} is not a valid type for iterparse"
318 )
320 row_node = next(iter(self.iterparse.keys())) if self.iterparse else ""
321 if not is_list_like(self.iterparse[row_node]):
322 raise TypeError(
323 f"{type(self.iterparse[row_node])} is not a valid type "
324 "for value in iterparse"
325 )
327 if (not hasattr(self.path_or_buffer, "read")) and (
328 not isinstance(self.path_or_buffer, (str, PathLike))
329 or is_url(self.path_or_buffer)
330 or is_fsspec_url(self.path_or_buffer)
331 or (
332 isinstance(self.path_or_buffer, str)
333 and self.path_or_buffer.startswith(("<?xml", "<"))
334 )
335 or infer_compression(self.path_or_buffer, "infer") is not None
336 ):
337 raise ParserError(
338 "iterparse is designed for large XML files that are fully extracted on "
339 "local disk and not as compressed files or online sources."
340 )
342 iterparse_repeats = len(self.iterparse[row_node]) != len(
343 set(self.iterparse[row_node])
344 )
346 parser = iterparse(self.path_or_buffer, events=("start", "end"))
347 try:
348 for event, elem in parser:
349 curr_elem = elem.tag.split("}")[1] if "}" in elem.tag else elem.tag
351 if event == "start":
352 if curr_elem == row_node:
353 row = {}
355 if row is not None:
356 if self.names and iterparse_repeats:
357 for col, nm in zip(
358 self.iterparse[row_node], self.names, strict=True
359 ):
360 if curr_elem == col:
361 elem_val = elem.text if elem.text else None
362 if elem_val not in row.values() and nm not in row:
363 row[nm] = elem_val
365 if col in elem.attrib:
366 if (
367 elem.attrib[col] not in row.values()
368 and nm not in row
369 ):
370 row[nm] = elem.attrib[col]
371 else:
372 for col in self.iterparse[row_node]:
373 if curr_elem == col:
374 row[col] = elem.text if elem.text else None
375 if col in elem.attrib:
376 row[col] = elem.attrib[col]
378 if event == "end":
379 if curr_elem == row_node and row is not None:
380 dicts.append(row)
381 row = None
383 elem.clear()
384 if hasattr(elem, "getprevious"):
385 while (
386 elem.getprevious() is not None
387 and elem.getparent() is not None
388 ):
389 del elem.getparent()[0]
390 finally:
391 if hasattr(parser, "close"):
392 parser.close()
394 if dicts == []:
395 raise ParserError("No result from selected items in iterparse.")
397 keys = list(dict.fromkeys([k for d in dicts for k in d.keys()]))
398 dicts = [{k: d[k] if k in d.keys() else None for k in keys} for d in dicts]
400 if self.names:
401 dicts = [dict(zip(self.names, d.values(), strict=True)) for d in dicts]
403 return dicts
405 def _validate_path(self) -> list[Any]:
406 """
407 Validate ``xpath``.
409 This method checks for syntax, evaluation, or empty nodes return.
411 Raises
412 ------
413 SyntaxError
414 * If xpah is not supported or issues with namespaces.
416 ValueError
417 * If xpah does not return any nodes.
418 """
420 raise AbstractMethodError(self)
422 def _validate_names(self) -> None:
423 """
424 Validate names.
426 This method will check if names is a list-like and aligns
427 with length of parse nodes.
429 Raises
430 ------
431 ValueError
432 * If value is not a list and less than length of nodes.
433 """
434 raise AbstractMethodError(self)
436 def _parse_doc(
437 self, raw_doc: FilePath | ReadBuffer[bytes] | ReadBuffer[str]
438 ) -> Element | etree._Element:
439 """
440 Build tree from path_or_buffer.
442 This method will parse XML object into tree
443 either from string/bytes or file location.
444 """
445 raise AbstractMethodError(self)
448class _EtreeFrameParser(_XMLFrameParser):
449 """
450 Internal class to parse XML into DataFrames with the Python
451 standard library XML module: `xml.etree.ElementTree`.
452 """
454 def parse_data(self) -> list[dict[str, str | None]]:
455 from xml.etree.ElementTree import iterparse
457 if self.stylesheet is not None:
458 raise ValueError(
459 "To use stylesheet, you need lxml installed and selected as parser."
460 )
462 elems: list[Any] = []
463 if self.iterparse is None:
464 self.xml_doc = self._parse_doc(self.path_or_buffer)
465 elems = self._validate_path()
467 self._validate_names()
469 xml_dicts: list[dict[str, str | None]] = (
470 self._parse_nodes(elems)
471 if self.iterparse is None
472 else self._iterparse_nodes(iterparse)
473 )
475 return xml_dicts
477 def _validate_path(self) -> list[Any]:
478 """
479 Notes
480 -----
481 ``etree`` supports limited ``XPath``. If user attempts a more complex
482 expression syntax error will raise.
483 """
484 msg = (
485 "xpath does not return any nodes or attributes. "
486 "Be sure to specify in `xpath` the parent nodes of "
487 "children and attributes to parse. "
488 "If document uses namespaces denoted with "
489 "xmlns, be sure to define namespaces and "
490 "use them in xpath."
491 )
492 try:
493 elems = self.xml_doc.findall( # type: ignore[union-attr]
494 self.xpath, namespaces=self.namespaces
495 )
496 children = [ch for el in elems for ch in el.findall("*")]
497 attrs = {k: v for el in elems for k, v in el.attrib.items()}
499 if elems is None:
500 raise ValueError(msg)
502 if elems is not None:
503 if self.elems_only and children == []:
504 raise ValueError(msg)
505 if self.attrs_only and attrs == {}:
506 raise ValueError(msg)
507 if children == [] and attrs == {}:
508 raise ValueError(msg)
510 except (KeyError, SyntaxError) as err:
511 raise SyntaxError(
512 "You have used an incorrect or unsupported XPath "
513 "expression for etree library or you used an "
514 "undeclared namespace prefix."
515 ) from err
517 return elems # pyright: ignore[reportReturnType]
519 def _validate_names(self) -> None:
520 children: list[Any]
522 if self.names:
523 if self.iterparse:
524 children = self.iterparse[next(iter(self.iterparse))]
525 else:
526 parent = self.xml_doc.find( # type: ignore[union-attr]
527 self.xpath, namespaces=self.namespaces
528 )
529 children = parent.findall("*") if parent is not None else []
531 if is_list_like(self.names):
532 if len(self.names) < len(children):
533 raise ValueError(
534 "names does not match length of child elements in xpath."
535 )
536 else:
537 raise TypeError(
538 f"{type(self.names).__name__} is not a valid type for names"
539 )
541 def _parse_doc(
542 self, raw_doc: FilePath | ReadBuffer[bytes] | ReadBuffer[str]
543 ) -> Element:
544 from xml.etree.ElementTree import (
545 XMLParser,
546 parse,
547 )
549 handle_data = get_data_from_filepath(
550 filepath_or_buffer=raw_doc,
551 encoding=self.encoding,
552 compression=self.compression,
553 storage_options=self.storage_options,
554 )
556 with handle_data as xml_data:
557 curr_parser = XMLParser(encoding=self.encoding)
558 document = parse(xml_data, parser=curr_parser)
560 return document.getroot()
563class _LxmlFrameParser(_XMLFrameParser):
564 """
565 Internal class to parse XML into :class:`~pandas.DataFrame` with third-party
566 full-featured XML library, ``lxml``, that supports
567 ``XPath`` 1.0 and XSLT 1.0.
568 """
570 def parse_data(self) -> list[dict[str, str | None]]:
571 """
572 Parse xml data.
574 This method will call the other internal methods to
575 validate ``xpath``, names, optionally parse and run XSLT,
576 and parse original or transformed XML and return specific nodes.
577 """
578 from lxml.etree import iterparse
580 elems: list[Any] = []
581 if self.iterparse is None:
582 self.xml_doc = self._parse_doc(self.path_or_buffer)
584 if self.stylesheet:
585 self.xsl_doc = self._parse_doc(self.stylesheet)
586 self.xml_doc = self._transform_doc()
588 elems = self._validate_path()
590 self._validate_names()
592 xml_dicts: list[dict[str, str | None]] = (
593 self._parse_nodes(elems)
594 if self.iterparse is None
595 # lxml < 6.1 iterparse defaults to resolve_entities=True,
596 # which allows XXE (CVE-2026-41066)
597 else self._iterparse_nodes(partial(iterparse, resolve_entities="internal"))
598 )
600 return xml_dicts
602 def _validate_path(self) -> list[Any]:
603 msg = (
604 "xpath does not return any nodes or attributes. "
605 "Be sure to specify in `xpath` the parent nodes of "
606 "children and attributes to parse. "
607 "If document uses namespaces denoted with "
608 "xmlns, be sure to define namespaces and "
609 "use them in xpath."
610 )
612 elems = self.xml_doc.xpath( # type: ignore[union-attr]
613 self.xpath, namespaces=self.namespaces
614 )
615 children = [ch for el in elems for ch in el.xpath("*")]
616 attrs = {k: v for el in elems for k, v in el.attrib.items()}
618 if elems == []:
619 raise ValueError(msg)
621 if elems != []:
622 if self.elems_only and children == []:
623 raise ValueError(msg)
624 if self.attrs_only and attrs == {}:
625 raise ValueError(msg)
626 if children == [] and attrs == {}:
627 raise ValueError(msg)
629 return elems
631 def _validate_names(self) -> None:
632 children: list[Any]
634 if self.names:
635 if self.iterparse:
636 children = self.iterparse[next(iter(self.iterparse))]
637 else:
638 children = self.xml_doc.xpath( # type: ignore[union-attr]
639 self.xpath + "[1]/*", namespaces=self.namespaces
640 )
642 if is_list_like(self.names):
643 if len(self.names) < len(children):
644 raise ValueError(
645 "names does not match length of child elements in xpath."
646 )
647 else:
648 raise TypeError(
649 f"{type(self.names).__name__} is not a valid type for names"
650 )
652 def _parse_doc(
653 self, raw_doc: FilePath | ReadBuffer[bytes] | ReadBuffer[str]
654 ) -> etree._Element:
655 from lxml.etree import (
656 XMLParser,
657 fromstring,
658 parse,
659 )
661 handle_data = get_data_from_filepath(
662 filepath_or_buffer=raw_doc,
663 encoding=self.encoding,
664 compression=self.compression,
665 storage_options=self.storage_options,
666 )
668 with handle_data as xml_data:
669 curr_parser = XMLParser(encoding=self.encoding, resolve_entities="internal")
671 if isinstance(xml_data, io.StringIO):
672 if self.encoding is None:
673 raise TypeError(
674 "Can not pass encoding None when input is StringIO."
675 )
677 document = fromstring(
678 xml_data.getvalue().encode(self.encoding), parser=curr_parser
679 )
680 else:
681 document = parse(xml_data, parser=curr_parser)
683 return document
685 def _transform_doc(self) -> etree._XSLTResultTree:
686 """
687 Transform original tree using stylesheet.
689 This method will transform original xml using XSLT script into
690 am ideally flatter xml document for easier parsing and migration
691 to Data Frame.
692 """
693 from lxml.etree import XSLT
695 transformer = XSLT(self.xsl_doc)
696 new_doc = transformer(self.xml_doc)
698 return new_doc
701def get_data_from_filepath(
702 filepath_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
703 encoding: str | None,
704 compression: CompressionOptions,
705 storage_options: StorageOptions,
706) -> io.StringIO | io.BytesIO | IO[str] | IO[bytes]:
707 """
708 Extract raw XML data.
710 The method accepts two input types:
711 1. filepath (string-like)
712 2. file-like object (e.g. open file object, StringIO)
713 """
714 filepath_or_buffer = stringify_path(filepath_or_buffer)
715 with get_handle(
716 filepath_or_buffer,
717 "r",
718 encoding=encoding,
719 compression=compression,
720 storage_options=storage_options,
721 ) as handle_obj:
722 return (
723 preprocess_data(handle_obj.handle.read())
724 if hasattr(handle_obj.handle, "read")
725 else handle_obj.handle
726 )
729def preprocess_data(
730 data: str | bytes | io.StringIO | io.BytesIO,
731) -> io.StringIO | io.BytesIO:
732 """
733 Convert extracted raw data.
735 This method will return underlying data of extracted XML content.
736 The data either has a `read` attribute (e.g. a file object or a
737 StringIO/BytesIO) or is a string or bytes that is an XML document.
738 """
740 if isinstance(data, str):
741 data = io.StringIO(data)
743 elif isinstance(data, bytes):
744 data = io.BytesIO(data)
746 return data
749def _data_to_frame(data: list[dict[str, str | None]], **kwargs: Any) -> DataFrame:
750 """
751 Convert parsed data to Data Frame.
753 This method will bind xml dictionary data of keys and values
754 into named columns of Data Frame using the built-in TextParser
755 class that build Data Frame and infers specific dtypes.
756 """
758 tags = next(iter(data))
759 nodes = [list(d.values()) for d in data]
761 try:
762 with TextParser(nodes, names=tags, **kwargs) as tp:
763 return tp.read()
764 except ParserError as err:
765 raise ParserError(
766 "XML document may be too complex for import. "
767 "Try to flatten document and use distinct "
768 "element and attribute names."
769 ) from err
772def _parse(
773 path_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
774 xpath: str,
775 namespaces: dict[str, str] | None,
776 elems_only: bool,
777 attrs_only: bool,
778 names: Sequence[str] | None,
779 dtype: DtypeArg | None,
780 converters: ConvertersArg | None,
781 parse_dates: ParseDatesArg | None,
782 encoding: str | None,
783 parser: XMLParsers,
784 stylesheet: FilePath | ReadBuffer[bytes] | ReadBuffer[str] | None,
785 iterparse: dict[str, list[str]] | None,
786 compression: CompressionOptions,
787 storage_options: StorageOptions,
788 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
789 **kwargs: Any,
790) -> DataFrame:
791 """
792 Call internal parsers.
794 This method will conditionally call internal parsers:
795 LxmlFrameParser and/or EtreeParser.
797 Raises
798 ------
799 ImportError
800 * If lxml is not installed if selected as parser.
802 ValueError
803 * If parser is not lxml or etree.
804 """
806 p: _EtreeFrameParser | _LxmlFrameParser
808 if parser == "lxml":
809 lxml = import_optional_dependency("lxml.etree", errors="ignore")
811 if lxml is not None:
812 p = _LxmlFrameParser(
813 path_or_buffer,
814 xpath,
815 namespaces,
816 elems_only,
817 attrs_only,
818 names,
819 dtype,
820 converters,
821 parse_dates,
822 encoding,
823 stylesheet,
824 iterparse,
825 compression,
826 storage_options,
827 )
828 else:
829 raise ImportError("lxml not found, please install or use the etree parser.")
831 elif parser == "etree":
832 p = _EtreeFrameParser(
833 path_or_buffer,
834 xpath,
835 namespaces,
836 elems_only,
837 attrs_only,
838 names,
839 dtype,
840 converters,
841 parse_dates,
842 encoding,
843 stylesheet,
844 iterparse,
845 compression,
846 storage_options,
847 )
848 else:
849 raise ValueError("Values for parser can only be lxml or etree.")
851 data_dicts = p.parse_data()
853 return _data_to_frame(
854 data=data_dicts,
855 dtype=dtype,
856 converters=converters,
857 parse_dates=parse_dates,
858 dtype_backend=dtype_backend,
859 **kwargs,
860 )
863@set_module("pandas")
864def read_xml(
865 path_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
866 *,
867 xpath: str = "./*",
868 namespaces: dict[str, str] | None = None,
869 elems_only: bool = False,
870 attrs_only: bool = False,
871 names: Sequence[str] | None = None,
872 dtype: DtypeArg | None = None,
873 converters: ConvertersArg | None = None,
874 parse_dates: ParseDatesArg | None = None,
875 # encoding can not be None for lxml and StringIO input
876 encoding: str | None = "utf-8",
877 parser: XMLParsers = "lxml",
878 stylesheet: FilePath | ReadBuffer[bytes] | ReadBuffer[str] | None = None,
879 iterparse: dict[str, list[str]] | None = None,
880 compression: CompressionOptions = "infer",
881 storage_options: StorageOptions | None = None,
882 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
883) -> DataFrame:
884 r"""
885 Read XML document into a :class:`~pandas.DataFrame` object.
887 This function parses an XML document from a file path, URL, or string buffer,
888 and returns the content as a DataFrame. Nodes are selected using an XPath
889 expression, and their child elements and attributes are mapped to columns.
891 Parameters
892 ----------
893 path_or_buffer : str, path object, or file-like object
894 String path, path object (implementing ``os.PathLike[str]``), or file-like
895 object implementing a ``read()`` function. The string can be a path.
896 The string can further be a URL. Valid URL schemes
897 include http, ftp, s3, and file.
899 Certain URL schemes may require additional packages. For example, S3
900 URLs require the ``s3fs`` library. See
901 :ref:`install.optional_dependencies` for a full list.
903 xpath : str, optional, default './\*'
904 The ``XPath`` to parse required set of nodes for migration to
905 :class:`~pandas.DataFrame`.``XPath`` should return a collection of elements
906 and not a single element. Note: The ``etree`` parser supports limited ``XPath``
907 expressions. For more complex ``XPath``, use ``lxml`` which requires
908 installation.
910 namespaces : dict, optional
911 The namespaces defined in XML document as dicts with key being
912 namespace prefix and value the URI. There is no need to include all
913 namespaces in XML, only the ones used in ``xpath`` expression.
914 Note: if XML document uses default namespace denoted as
915 `xmlns='<URI>'` without a prefix, you must assign any temporary
916 namespace prefix such as 'doc' to the URI in order to parse
917 underlying nodes and/or attributes.
919 elems_only : bool, optional, default False
920 Parse only the child elements at the specified ``xpath``. By default,
921 all child elements and non-empty text nodes are returned.
923 attrs_only : bool, optional, default False
924 Parse only the attributes at the specified ``xpath``.
925 By default, all attributes are returned.
927 names : list-like, optional
928 Column names for DataFrame of parsed XML data. Use this parameter to
929 rename original element names and distinguish same named elements and
930 attributes.
932 dtype : Type name or dict of column -> type, optional
933 Data type for data or columns. E.g. {'a': np.float64, 'b': np.int32,
934 'c': 'Int64'}
935 Use ``str`` or ``object`` to preserve and not interpret dtype.
936 If converters are specified, they will be applied INSTEAD
937 of dtype conversion.
939 converters : dict, optional
940 Dict of functions for converting values in certain columns. Keys can either
941 be integers or column labels.
943 parse_dates : bool or list of int or names or list of lists or dict, default False
944 Identifiers to parse index or columns to datetime. The behavior is as follows:
946 * boolean. If True -> try parsing the index.
947 * list of int or names. e.g. If [1, 2, 3] -> try parsing columns 1, 2, 3
948 each as a separate date column.
949 * list of lists. e.g. If [[1, 3]] -> combine columns 1 and 3 and parse as
950 a single date column.
951 * dict, e.g. {'foo' : [1, 3]} -> parse columns 1, 3 as date and call
952 result 'foo'
954 encoding : str, optional, default 'utf-8'
955 Encoding of XML document.
957 parser : {'lxml','etree'}, default 'lxml'
958 Parser module to use for retrieval of data. Only 'lxml' and
959 'etree' are supported. With 'lxml' more complex ``XPath`` searches
960 and ability to use XSLT stylesheet are supported.
962 stylesheet : str, path object or file-like object
963 A URL, file-like object, or a string path containing an XSLT script.
964 This stylesheet should flatten complex, deeply nested XML documents
965 for easier parsing. To use this feature you must have ``lxml`` module
966 installed and specify 'lxml' as ``parser``. The ``xpath`` must
967 reference nodes of transformed XML document generated after XSLT
968 transformation and not the original XML document. Only XSLT 1.0
969 scripts and not later versions is currently supported. This option
970 cannot be combined with ``iterparse`` and is ignored if ``iterparse``
971 is used.
973 iterparse : dict, optional
974 The nodes or attributes to retrieve in iterparsing of XML document
975 as a dict with key being the name of repeating element and value being
976 list of elements or attribute names that are descendants of the repeated
977 element. Note: If this option is used, it will replace ``xpath`` parsing
978 and any ``stylesheet`` is ignored, since XSLT requires the whole tree in
979 memory. Unlike ``xpath``, descendants do not need to relate to each other
980 but can exist anywhere in document under the repeating element. This memory-
981 efficient method should be used for very large XML files (500MB, 1GB, or 5GB+).
982 For example, ``{"row_element": ["child_elem", "attr", "grandchild_elem"]}``.
984 compression : str or dict, default 'infer'
985 For on-the-fly decompression of on-disk data. If 'infer' and
986 'path_or_buffer' is path-like, then detect compression from the
987 following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar',
988 '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression).
989 If using 'zip' or 'tar', the ZIP file must contain only one data
990 file to be read in. Set to ``None`` for no decompression.
991 Can also be a dict with key ``'method'`` set to one of
992 {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``}
993 and other key-value pairs are forwarded to ``zipfile.ZipFile``,
994 ``gzip.GzipFile``, ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``,
995 ``lzma.LZMAFile`` or ``tarfile.TarFile``, respectively.
996 As an example, the following could be passed for Zstandard
997 decompression using a custom compression dictionary:
998 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
1000 storage_options : dict, optional
1001 Extra options that make sense for a particular storage connection,
1002 e.g. host, port, username, password, etc. For HTTP(S) URLs the
1003 key-value pairs are forwarded to ``urllib.request.Request`` as header
1004 options. For other URLs (e.g. starting with "s3://", and "gcs://")
1005 the key-value pairs are forwarded to ``fsspec.open``. Please see
1006 ``fsspec`` and ``urllib`` for more details, and for more examples on
1007 storage options refer `here <https://pandas.pydata.org/docs/
1008 user_guide/io.html?highlight=storage_options#reading-writing-remote-
1009 files>`_.
1011 dtype_backend : {'numpy_nullable', 'pyarrow'}
1012 Back-end data type applied to the resultant :class:`DataFrame`
1013 (still experimental). If not specified, the default behavior
1014 is to not use nullable data types. If specified, the behavior
1015 is as follows:
1017 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
1018 * ``"pyarrow"``: returns pyarrow-backed nullable
1019 :class:`ArrowDtype` :class:`DataFrame`
1021 .. versionadded:: 2.0
1023 Returns
1024 -------
1025 df
1026 A DataFrame.
1028 See Also
1029 --------
1030 read_json : Convert a JSON string to pandas object.
1031 read_html : Read HTML tables into a list of DataFrame objects.
1033 Notes
1034 -----
1035 This method is best designed to import shallow XML documents in
1036 following format which is the ideal fit for the two-dimensions of a
1037 ``DataFrame`` (row by column). ::
1039 <root>
1040 <row>
1041 <column1>data</column1>
1042 <column2>data</column2>
1043 <column3>data</column3>
1044 ...
1045 </row>
1046 <row>
1047 ...
1048 </row>
1049 ...
1050 </root>
1052 As a file format, XML documents can be designed any way including
1053 layout of elements and attributes as long as it conforms to W3C
1054 specifications. Therefore, this method is a convenience handler for
1055 a specific flatter design and not all possible XML structures.
1057 However, for more complex XML documents, ``stylesheet`` allows you to
1058 temporarily redesign original document with XSLT (a special purpose
1059 language) for a flatter version for migration to a DataFrame.
1061 This function will *always* return a single :class:`DataFrame` or raise
1062 exceptions due to issues with XML document, ``xpath``, or other
1063 parameters.
1065 See the :ref:`read_xml documentation in the IO section of the docs
1066 <io.read_xml>` for more information in using this method to parse XML
1067 files to DataFrames.
1069 Examples
1070 --------
1071 >>> from io import StringIO
1072 >>> xml = '''<?xml version='1.0' encoding='utf-8'?>
1073 ... <data xmlns="http://example.com">
1074 ... <row>
1075 ... <shape>square</shape>
1076 ... <degrees>360</degrees>
1077 ... <sides>4.0</sides>
1078 ... </row>
1079 ... <row>
1080 ... <shape>circle</shape>
1081 ... <degrees>360</degrees>
1082 ... <sides/>
1083 ... </row>
1084 ... <row>
1085 ... <shape>triangle</shape>
1086 ... <degrees>180</degrees>
1087 ... <sides>3.0</sides>
1088 ... </row>
1089 ... </data>'''
1091 >>> df = pd.read_xml(StringIO(xml))
1092 >>> df
1093 shape degrees sides
1094 0 square 360 4.0
1095 1 circle 360 NaN
1096 2 triangle 180 3.0
1098 >>> xml = '''<?xml version='1.0' encoding='utf-8'?>
1099 ... <data>
1100 ... <row shape="square" degrees="360" sides="4.0"/>
1101 ... <row shape="circle" degrees="360"/>
1102 ... <row shape="triangle" degrees="180" sides="3.0"/>
1103 ... </data>'''
1105 >>> df = pd.read_xml(StringIO(xml), xpath=".//row")
1106 >>> df
1107 shape degrees sides
1108 0 square 360 4.0
1109 1 circle 360 NaN
1110 2 triangle 180 3.0
1112 >>> xml = '''<?xml version='1.0' encoding='utf-8'?>
1113 ... <doc:data xmlns:doc="https://example.com">
1114 ... <doc:row>
1115 ... <doc:shape>square</doc:shape>
1116 ... <doc:degrees>360</doc:degrees>
1117 ... <doc:sides>4.0</doc:sides>
1118 ... </doc:row>
1119 ... <doc:row>
1120 ... <doc:shape>circle</doc:shape>
1121 ... <doc:degrees>360</doc:degrees>
1122 ... <doc:sides/>
1123 ... </doc:row>
1124 ... <doc:row>
1125 ... <doc:shape>triangle</doc:shape>
1126 ... <doc:degrees>180</doc:degrees>
1127 ... <doc:sides>3.0</doc:sides>
1128 ... </doc:row>
1129 ... </doc:data>'''
1131 >>> df = pd.read_xml(
1132 ... StringIO(xml),
1133 ... xpath="//doc:row",
1134 ... namespaces={"doc": "https://example.com"},
1135 ... )
1136 >>> df
1137 shape degrees sides
1138 0 square 360 4.0
1139 1 circle 360 NaN
1140 2 triangle 180 3.0
1142 >>> xml = '''<?xml version='1.0' encoding='utf-8'?>
1143 ... <data xmlns="http://example.com">
1144 ... <row>
1145 ... <shape>square</shape>
1146 ... <degrees>360</degrees>
1147 ... <sides>4.0</sides>
1148 ... </row>
1149 ... <row>
1150 ... <shape>circle</shape>
1151 ... <degrees>360</degrees>
1152 ... <sides/>
1153 ... </row>
1154 ... <row>
1155 ... <shape>triangle</shape>
1156 ... <degrees>180</degrees>
1157 ... <sides>3.0</sides>
1158 ... </row>
1159 ... </data>'''
1161 >>> df = pd.read_xml(
1162 ... StringIO(xml),
1163 ... xpath="//doc:row",
1164 ... namespaces={"doc": "http://example.com"},
1165 ... )
1166 >>> df
1167 shape degrees sides
1168 0 square 360 4.0
1169 1 circle 360 NaN
1170 2 triangle 180 3.0
1172 >>> xml_data = '''
1173 ... <data>
1174 ... <row>
1175 ... <index>0</index>
1176 ... <a>1</a>
1177 ... <b>2.5</b>
1178 ... <c>True</c>
1179 ... <d>a</d>
1180 ... <e>2019-12-31 00:00:00</e>
1181 ... </row>
1182 ... <row>
1183 ... <index>1</index>
1184 ... <b>4.5</b>
1185 ... <c>False</c>
1186 ... <d>b</d>
1187 ... <e>2019-12-31 00:00:00</e>
1188 ... </row>
1189 ... </data>
1190 ... '''
1192 >>> df = pd.read_xml(
1193 ... StringIO(xml_data), dtype_backend="numpy_nullable", parse_dates=["e"]
1194 ... )
1195 >>> df
1196 index a b c d e
1197 0 0 1 2.5 True a 2019-12-31
1198 1 1 <NA> 4.5 False b 2019-12-31
1199 """
1200 check_dtype_backend(dtype_backend)
1202 return _parse(
1203 path_or_buffer=path_or_buffer,
1204 xpath=xpath,
1205 namespaces=namespaces,
1206 elems_only=elems_only,
1207 attrs_only=attrs_only,
1208 names=names,
1209 dtype=dtype,
1210 converters=converters,
1211 parse_dates=parse_dates,
1212 encoding=encoding,
1213 parser=parser,
1214 stylesheet=stylesheet,
1215 iterparse=iterparse,
1216 compression=compression,
1217 storage_options=storage_options,
1218 dtype_backend=dtype_backend,
1219 )