Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/orc.py: 26%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""orc compat"""
3from __future__ import annotations
5import io
6from typing import (
7 TYPE_CHECKING,
8 Any,
9 Literal,
10)
12from pandas._libs import lib
13from pandas.compat._optional import import_optional_dependency
14from pandas.util._decorators import set_module
15from pandas.util._validators import check_dtype_backend
17from pandas.core.indexes.api import default_index
19from pandas.io._util import arrow_table_to_pandas
20from pandas.io.common import (
21 get_handle,
22 is_fsspec_url,
23)
25if TYPE_CHECKING:
26 import fsspec
27 import pyarrow.fs
29 from pandas._typing import (
30 DtypeBackend,
31 FilePath,
32 ReadBuffer,
33 WriteBuffer,
34 )
36 from pandas.core.frame import DataFrame
39@set_module("pandas")
40def read_orc(
41 path: FilePath | ReadBuffer[bytes],
42 columns: list[str] | None = None,
43 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
44 filesystem: pyarrow.fs.FileSystem | fsspec.spec.AbstractFileSystem | None = None,
45 **kwargs: Any,
46) -> DataFrame:
47 """
48 Load an ORC object from the file path, returning a DataFrame.
50 This method reads an ORC (Optimized Row Columnar) file into a pandas
51 DataFrame using the `pyarrow.orc` library. ORC is a columnar storage format
52 that provides efficient compression and fast retrieval for analytical workloads.
53 It allows reading specific columns, handling different filesystem
54 types (such as local storage, cloud storage via fsspec, or pyarrow filesystem),
55 and supports different data type backends, including `numpy_nullable` and `pyarrow`.
57 Parameters
58 ----------
59 path : str, path object, or file-like object
60 String, path object (implementing ``os.PathLike[str]``), or file-like
61 object implementing a binary ``read()`` function. The string could be a URL.
62 Valid URL schemes include http, ftp, s3, and file. For file URLs, a host is
63 expected. A local file could be:
64 ``file://localhost/path/to/table.orc``.
65 columns : list, default None
66 If not None, only these columns will be read from the file.
67 Output always follows the ordering of the file and not the columns list.
68 This mirrors the original behaviour of
69 :external+pyarrow:py:meth:`pyarrow.orc.ORCFile.read`.
70 dtype_backend : {'numpy_nullable', 'pyarrow'}
71 Back-end data type applied to the resultant :class:`DataFrame`
72 (still experimental). If not specified, the default behavior
73 is to not use nullable data types. If specified, the behavior
74 is as follows:
76 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
77 * ``"pyarrow"``: returns pyarrow-backed nullable
78 :class:`ArrowDtype` :class:`DataFrame`
80 .. versionadded:: 2.0
82 filesystem : fsspec or pyarrow filesystem, default None
83 Filesystem object to use when reading the orc file.
85 .. versionadded:: 2.1.0
87 **kwargs
88 Any additional kwargs are passed to pyarrow.
90 Returns
91 -------
92 DataFrame
93 DataFrame based on the ORC file.
95 See Also
96 --------
97 read_csv : Read a comma-separated values (csv) file into a pandas DataFrame.
98 read_excel : Read an Excel file into a pandas DataFrame.
99 read_spss : Read an SPSS file into a pandas DataFrame.
100 read_sas : Load a SAS file into a pandas DataFrame.
101 read_feather : Load a feather-format object into a pandas DataFrame.
103 Notes
104 -----
105 Before using this function you should read the :ref:`user guide about ORC <io.orc>`
106 and :ref:`install optional dependencies <install.warn_orc>`.
108 If ``path`` is a URI scheme pointing to a local or remote file (e.g. "s3://"),
109 a ``pyarrow.fs`` filesystem will be attempted to read the file. You can also pass a
110 pyarrow or fsspec filesystem object into the filesystem keyword to override this
111 behavior.
113 Examples
114 --------
115 >>> result = pd.read_orc("example_pa.orc") # doctest: +SKIP
116 """
117 # we require a newer version of pyarrow than we support for orc
119 orc = import_optional_dependency("pyarrow.orc")
121 check_dtype_backend(dtype_backend)
123 with get_handle(path, "rb", is_text=False) as handles:
124 source = handles.handle
125 if is_fsspec_url(path) and filesystem is None:
126 pa = import_optional_dependency("pyarrow")
127 pa_fs = import_optional_dependency("pyarrow.fs")
128 try:
129 filesystem, source = pa_fs.FileSystem.from_uri(path)
130 except (TypeError, pa.ArrowInvalid):
131 pass
133 pa_table = orc.read_table(
134 source=source, columns=columns, filesystem=filesystem, **kwargs
135 )
136 return arrow_table_to_pandas(pa_table, dtype_backend=dtype_backend)
139def to_orc(
140 df: DataFrame,
141 path: FilePath | WriteBuffer[bytes] | None = None,
142 *,
143 engine: Literal["pyarrow"] = "pyarrow",
144 index: bool | None = None,
145 engine_kwargs: dict[str, Any] | None = None,
146) -> bytes | None:
147 """
148 Write a DataFrame to the ORC format.
150 Parameters
151 ----------
152 df : DataFrame
153 The dataframe to be written to ORC. Raises NotImplementedError
154 if dtype of one or more columns is category, unsigned integers,
155 intervals, periods or sparse.
156 path : str, file-like object or None, default None
157 If a string, it will be used as Root Directory path
158 when writing a partitioned dataset. By file-like object,
159 we refer to objects with a write() method, such as a file handle
160 (e.g. via builtin open function). If path is None,
161 a bytes object is returned.
162 engine : str, default 'pyarrow'
163 ORC library to use.
164 index : bool, optional
165 If ``True``, include the dataframe's index(es) in the file output. If
166 ``False``, they will not be written to the file.
167 If ``None``, similar to ``infer`` the dataframe's index(es)
168 will be saved. However, instead of being saved as values,
169 the RangeIndex will be stored as a range in the metadata so it
170 doesn't require much space and is faster. Other indexes will
171 be included as columns in the file output.
172 engine_kwargs : dict[str, Any] or None, default None
173 Additional keyword arguments passed to :func:`pyarrow.orc.write_table`.
175 Returns
176 -------
177 bytes if no path argument is provided else None
179 Raises
180 ------
181 NotImplementedError
182 Dtype of one or more columns is category, unsigned integers, interval,
183 period or sparse.
184 ValueError
185 engine is not pyarrow.
187 Notes
188 -----
189 * Before using this function you should read the
190 :ref:`user guide about ORC <io.orc>` and
191 :ref:`install optional dependencies <install.warn_orc>`.
192 * This function requires `pyarrow <https://arrow.apache.org/docs/python/>`_
193 library.
194 * For supported dtypes please refer to `supported ORC features in Arrow
195 <https://arrow.apache.org/docs/cpp/orc.html#data-types>`__.
196 * Currently timezones in datetime columns are not preserved when a
197 dataframe is converted into ORC files.
198 """
199 if index is None:
200 index = df.index.names[0] is not None
201 if engine_kwargs is None:
202 engine_kwargs = {}
204 # validate index
205 # --------------
207 # validate that we have only a default index
208 # raise on anything else as we don't serialize the index
210 if not df.index.equals(default_index(len(df))):
211 raise ValueError(
212 "orc does not support serializing a non-default index for the index; "
213 "you can .reset_index() to make the index into column(s)"
214 )
216 if df.index.name is not None:
217 raise ValueError("orc does not serialize index meta-data on a default index")
219 if engine != "pyarrow":
220 raise ValueError("engine must be 'pyarrow'")
221 pa = import_optional_dependency("pyarrow")
222 orc = import_optional_dependency("pyarrow.orc")
224 was_none = path is None
225 if was_none:
226 path = io.BytesIO()
227 assert path is not None # For mypy
228 with get_handle(path, "wb", is_text=False) as handles:
229 try:
230 orc.write_table(
231 pa.Table.from_pandas(df, preserve_index=index),
232 handles.handle,
233 **engine_kwargs,
234 )
235 except (TypeError, pa.ArrowNotImplementedError) as e:
236 raise NotImplementedError(
237 "The dtype of one or more columns is not supported yet."
238 ) from e
240 if was_none:
241 assert isinstance(path, io.BytesIO) # For mypy
242 return path.getvalue()
243 return None