Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""feather-format compat"""
3from __future__ import annotations
5from typing import (
6 TYPE_CHECKING,
7 Any,
8)
9import warnings
11from pandas._libs import lib
12from pandas.compat._optional import import_optional_dependency
13from pandas.util._decorators import set_module
14from pandas.util._validators import check_dtype_backend
16from pandas.core.api import DataFrame
18from pandas.io._util import (
19 arrow_table_to_pandas,
20 suppress_pyarrow_values_warning,
21)
22from pandas.io.common import get_handle
24if TYPE_CHECKING:
25 from collections.abc import (
26 Hashable,
27 Sequence,
28 )
30 from pandas._typing import (
31 DtypeBackend,
32 FilePath,
33 ReadBuffer,
34 StorageOptions,
35 WriteBuffer,
36 )
39def to_feather(
40 df: DataFrame,
41 path: FilePath | WriteBuffer[bytes],
42 storage_options: StorageOptions | None = None,
43 **kwargs: Any,
44) -> None:
45 """
46 Write a DataFrame to the binary Feather format.
48 Parameters
49 ----------
50 df : DataFrame
51 path : str, path object, or file-like object
52 storage_options : dict, optional
53 Extra options that make sense for a particular storage connection, e.g.
54 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
55 are forwarded to ``urllib.request.Request`` as header options. For other
56 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
57 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
58 details, and for more examples on storage options refer `here
59 <https://pandas.pydata.org/docs/user_guide/io.html?
60 highlight=storage_options#reading-writing-remote-files>`_.
61 **kwargs :
62 Additional keywords passed to `pyarrow.feather.write_feather`.
64 """
65 import_optional_dependency("pyarrow")
66 from pyarrow import feather
68 if not isinstance(df, DataFrame):
69 raise ValueError("feather only support IO with DataFrames")
71 with get_handle(
72 path, "wb", storage_options=storage_options, is_text=False
73 ) as handles:
74 # pyarrow>=24 deprecates feather.write_feather in favor of pyarrow.ipc;
75 # suppress until we migrate the implementation (GH#66177)
76 with warnings.catch_warnings(), suppress_pyarrow_values_warning():
77 warnings.filterwarnings(
78 "ignore",
79 "pyarrow.feather.write_feather is deprecated",
80 FutureWarning,
81 )
82 feather.write_feather(df, handles.handle, **kwargs)
85@set_module("pandas")
86def read_feather(
87 path: FilePath | ReadBuffer[bytes],
88 columns: Sequence[Hashable] | None = None,
89 use_threads: bool = True,
90 storage_options: StorageOptions | None = None,
91 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
92) -> DataFrame:
93 """
94 Load a feather-format object from the file path.
96 This function requires the `pyarrow <https://arrow.apache.org/docs/python/>`_
97 library.
99 Feather is particularly useful for scenarios that require efficient
100 serialization and deserialization of tabular data. It supports
101 schema preservation, making it a reliable choice for use cases
102 such as sharing data between Python and R, or persisting intermediate
103 results during data processing pipelines. This method provides additional
104 flexibility with options for selective column reading, thread parallelism,
105 and choosing the backend for data types.
107 Parameters
108 ----------
109 path : str, path object, or file-like object
110 String, path object (implementing ``os.PathLike[str]``), or file-like
111 object implementing a binary ``read()`` function. The string could be a URL.
112 Valid URL schemes include http, ftp, s3, gs and file. For file URLs, a host is
113 expected. A local file could be: ``file://localhost/path/to/table.feather``.
115 Certain URL schemes may require additional packages. For example, S3
116 URLs require the ``s3fs`` library. See
117 :ref:`install.optional_dependencies` for a full list.
118 columns : sequence, default None
119 If not provided, all columns are read.
120 use_threads : bool, default True
121 Whether to parallelize reading using multiple threads.
122 storage_options : dict, optional
123 Extra options that make sense for a particular storage connection, e.g.
124 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
125 are forwarded to ``urllib.request.Request`` as header options. For other
126 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
127 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
128 details, and for more examples on storage options refer `here
129 <https://pandas.pydata.org/docs/user_guide/io.html?
130 highlight=storage_options#reading-writing-remote-files>`_.
132 dtype_backend : {'numpy_nullable', 'pyarrow'}
133 Back-end data type applied to the resultant :class:`DataFrame`
134 (still experimental). If not specified, the default behavior
135 is to not use nullable data types. If specified, the behavior
136 is as follows:
138 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`.
139 * ``"pyarrow"``: returns pyarrow-backed nullable
140 :class:`ArrowDtype` :class:`DataFrame`
142 .. versionadded:: 2.0
144 Returns
145 -------
146 type of object stored in file
147 DataFrame object stored in the file.
149 See Also
150 --------
151 read_csv : Read a comma-separated values (csv) file into a pandas DataFrame.
152 read_excel : Read an Excel file into a pandas DataFrame.
153 read_spss : Read an SPSS file into a pandas DataFrame.
154 read_orc : Load an ORC object into a pandas DataFrame.
155 read_sas : Read SAS file into a pandas DataFrame.
157 Examples
158 --------
159 >>> df = pd.read_feather("path/to/file.feather") # doctest: +SKIP
160 """
161 import_optional_dependency("pyarrow")
162 from pyarrow import feather
164 # import utils to register the pyarrow extension types
165 import pandas.core.arrays.arrow.extension_types # pyright: ignore[reportUnusedImport] # noqa: F401
167 check_dtype_backend(dtype_backend)
169 with get_handle(
170 path, "rb", storage_options=storage_options, is_text=False
171 ) as handles:
172 # pyarrow>=24 deprecates feather.read_table in favor of pyarrow.ipc;
173 # suppress until we migrate the implementation (GH#66177)
174 with warnings.catch_warnings():
175 warnings.filterwarnings(
176 "ignore",
177 "pyarrow.feather.read_table is deprecated",
178 FutureWarning,
179 )
180 pa_table = feather.read_table(
181 handles.handle, columns=columns, use_threads=bool(use_threads)
182 )
183 return arrow_table_to_pandas(pa_table, dtype_backend=dtype_backend)