Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""feather-format compat"""
3from __future__ import annotations
5from typing import (
6 TYPE_CHECKING,
7 Any,
8)
9import warnings
11import numpy as np
13from pandas._config import using_string_dtype
15from pandas._libs import lib
16from pandas.compat._optional import import_optional_dependency
17from pandas.errors import Pandas4Warning
18from pandas.util._decorators import set_module
19from pandas.util._validators import check_dtype_backend
21from pandas.core.api import DataFrame
22from pandas.core.arrays.string_ import StringDtype
24from pandas.io._util import arrow_table_to_pandas
25from pandas.io.common import get_handle
27if TYPE_CHECKING:
28 from collections.abc import (
29 Hashable,
30 Sequence,
31 )
33 from pandas._typing import (
34 DtypeBackend,
35 FilePath,
36 ReadBuffer,
37 StorageOptions,
38 WriteBuffer,
39 )
42def to_feather(
43 df: DataFrame,
44 path: FilePath | WriteBuffer[bytes],
45 storage_options: StorageOptions | None = None,
46 **kwargs: Any,
47) -> None:
48 """
49 Write a DataFrame to the binary Feather format.
51 Parameters
52 ----------
53 df : DataFrame
54 path : str, path object, or file-like object
55 storage_options : dict, optional
56 Extra options that make sense for a particular storage connection, e.g.
57 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
58 are forwarded to ``urllib.request.Request`` as header options. For other
59 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
60 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
61 details, and for more examples on storage options refer `here
62 <https://pandas.pydata.org/docs/user_guide/io.html?
63 highlight=storage_options#reading-writing-remote-files>`_.
64 **kwargs :
65 Additional keywords passed to `pyarrow.feather.write_feather`.
67 """
68 import_optional_dependency("pyarrow")
69 from pyarrow import feather
71 if not isinstance(df, DataFrame):
72 raise ValueError("feather only support IO with DataFrames")
74 with get_handle(
75 path, "wb", storage_options=storage_options, is_text=False
76 ) as handles:
77 # pyarrow>=24 deprecates feather.write_feather in favor of pyarrow.ipc;
78 # suppress until we migrate the implementation (GH#66169)
79 with warnings.catch_warnings():
80 warnings.filterwarnings(
81 "ignore",
82 "pyarrow.feather.write_feather is deprecated",
83 FutureWarning,
84 )
85 feather.write_feather(df, handles.handle, **kwargs)
88@set_module("pandas")
89def read_feather(
90 path: FilePath | ReadBuffer[bytes],
91 columns: Sequence[Hashable] | None = None,
92 use_threads: bool = True,
93 storage_options: StorageOptions | None = None,
94 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
95) -> DataFrame:
96 """
97 Load a feather-format object from the file path.
99 Feather is particularly useful for scenarios that require efficient
100 serialization and deserialization of tabular data. It supports
101 schema preservation, making it a reliable choice for use cases
102 such as sharing data between Python and R, or persisting intermediate
103 results during data processing pipelines. This method provides additional
104 flexibility with options for selective column reading, thread parallelism,
105 and choosing the backend for data types.
107 Parameters
108 ----------
109 path : str, path object, or file-like object
110 String, path object (implementing ``os.PathLike[str]``), or file-like
111 object implementing a binary ``read()`` function. The string could be a URL.
112 Valid URL schemes include http, ftp, s3, gs and file. For file URLs, a host is
113 expected. A local file could be: ``file://localhost/path/to/table.feather``.
114 columns : sequence, default None
115 If not provided, all columns are read.
116 use_threads : bool, default True
117 Whether to parallelize reading using multiple threads.
118 storage_options : dict, optional
119 Extra options that make sense for a particular storage connection, e.g.
120 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
121 are forwarded to ``urllib.request.Request`` as header options. For other
122 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
123 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
124 details, and for more examples on storage options refer `here
125 <https://pandas.pydata.org/docs/user_guide/io.html?
126 highlight=storage_options#reading-writing-remote-files>`_.
128 dtype_backend : {'numpy_nullable', 'pyarrow'}
129 Back-end data type applied to the resultant :class:`DataFrame`
130 (still experimental). If not specified, the default behavior
131 is to not use nullable data types. If specified, the behavior
132 is as follows:
134 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`.
135 * ``"pyarrow"``: returns pyarrow-backed nullable
136 :class:`ArrowDtype` :class:`DataFrame`
138 .. versionadded:: 2.0
140 Returns
141 -------
142 type of object stored in file
143 DataFrame object stored in the file.
145 See Also
146 --------
147 read_csv : Read a comma-separated values (csv) file into a pandas DataFrame.
148 read_excel : Read an Excel file into a pandas DataFrame.
149 read_spss : Read an SPSS file into a pandas DataFrame.
150 read_orc : Load an ORC object into a pandas DataFrame.
151 read_sas : Read SAS file into a pandas DataFrame.
153 Examples
154 --------
155 >>> df = pd.read_feather("path/to/file.feather") # doctest: +SKIP
156 """
157 import_optional_dependency("pyarrow")
158 from pyarrow import feather
160 # import utils to register the pyarrow extension types
161 import pandas.core.arrays.arrow.extension_types # pyright: ignore[reportUnusedImport] # noqa: F401
163 check_dtype_backend(dtype_backend)
165 with get_handle(
166 path, "rb", storage_options=storage_options, is_text=False
167 ) as handles:
168 if dtype_backend is lib.no_default and not using_string_dtype():
169 with warnings.catch_warnings():
170 warnings.filterwarnings(
171 "ignore",
172 "make_block is deprecated",
173 Pandas4Warning,
174 )
175 # pyarrow>=24 deprecates feather.read_feather in favor of
176 # pyarrow.ipc; suppress until we migrate the implementation
177 # (GH#66169)
178 warnings.filterwarnings(
179 "ignore",
180 "pyarrow.feather.read_feather is deprecated",
181 FutureWarning,
182 )
184 df = feather.read_feather(
185 handles.handle, columns=columns, use_threads=bool(use_threads)
186 )
187 # Convert any StringDtype columns to object dtype (pyarrow always
188 # uses string dtype even when the infer_string option is False)
189 for col, dtype in zip(df.columns, df.dtypes, strict=True):
190 if isinstance(dtype, StringDtype) and dtype.na_value is np.nan:
191 df[col] = df[col].astype("object")
192 return df
194 # pyarrow>=24 deprecates feather.read_table in favor of pyarrow.ipc;
195 # suppress until we migrate the implementation (GH#66169)
196 with warnings.catch_warnings():
197 warnings.filterwarnings(
198 "ignore",
199 "pyarrow.feather.read_table is deprecated",
200 FutureWarning,
201 )
202 pa_table = feather.read_table(
203 handles.handle, columns=columns, use_threads=bool(use_threads)
204 )
205 return arrow_table_to_pandas(pa_table, dtype_backend=dtype_backend)