Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/parquet.py: 18%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""parquet compat"""
3from __future__ import annotations
5import io
6import json
7import os
8from typing import (
9 TYPE_CHECKING,
10 Any,
11 Literal,
12)
13import warnings
14from warnings import (
15 catch_warnings,
16 filterwarnings,
17)
19from pandas._config.config import _global_config as config
21from pandas._libs import lib
22from pandas.compat._optional import import_optional_dependency
23from pandas.errors import (
24 AbstractMethodError,
25 Pandas4Warning,
26)
27from pandas.util._decorators import set_module
28from pandas.util._exceptions import find_stack_level
29from pandas.util._validators import check_dtype_backend
31from pandas import DataFrame
33from pandas.io._util import (
34 arrow_table_to_pandas,
35 suppress_pyarrow_values_warning,
36)
37from pandas.io.common import (
38 IOHandles,
39 check_parent_directory,
40 get_handle,
41 is_fsspec_url,
42 is_url,
43 stringify_path,
44)
46if TYPE_CHECKING:
47 from pandas._typing import (
48 DtypeBackend,
49 FilePath,
50 ParquetCompressionOptions,
51 ReadBuffer,
52 StorageOptions,
53 WriteBuffer,
54 )
57def get_engine(engine: str) -> BaseImpl:
58 """return our implementation"""
59 if engine == "auto":
60 engine = config["io"]["parquet"]["engine"]
62 if engine == "auto":
63 # try engines in this order
64 engine_classes = [PyArrowImpl, FastParquetImpl]
66 error_msgs = ""
67 for engine_class in engine_classes:
68 try:
69 return engine_class()
70 except ImportError as err:
71 error_msgs += "\n - " + str(err)
73 raise ImportError(
74 "Unable to find a usable engine; "
75 "tried using: 'pyarrow', 'fastparquet'.\n"
76 "A suitable version of "
77 "pyarrow or fastparquet is required for parquet "
78 "support.\n"
79 "Trying to import the above resulted in these errors:"
80 f"{error_msgs}"
81 )
83 if engine == "pyarrow":
84 return PyArrowImpl()
85 elif engine == "fastparquet":
86 warnings.warn(
87 "The 'fastparquet' engine is deprecated and will be removed in a "
88 "future version. Use 'pyarrow' instead.",
89 Pandas4Warning,
90 stacklevel=find_stack_level(),
91 )
92 return FastParquetImpl()
94 raise ValueError("engine must be one of 'pyarrow', 'fastparquet'")
97def _get_path_or_handle(
98 path: FilePath | ReadBuffer[bytes] | WriteBuffer[bytes],
99 fs: Any,
100 storage_options: StorageOptions | None = None,
101 mode: str = "rb",
102 is_dir: bool = False,
103) -> tuple[
104 FilePath | ReadBuffer[bytes] | WriteBuffer[bytes], IOHandles[bytes] | None, Any
105]:
106 """File handling for PyArrow."""
107 path_or_handle = stringify_path(path)
108 if fs is not None:
109 pa_fs = import_optional_dependency("pyarrow.fs", errors="ignore")
110 fsspec = import_optional_dependency("fsspec", errors="ignore")
111 if pa_fs is not None and isinstance(fs, pa_fs.FileSystem):
112 if storage_options:
113 raise NotImplementedError(
114 "storage_options not supported with a pyarrow FileSystem."
115 )
116 elif fsspec is not None and isinstance(fs, fsspec.spec.AbstractFileSystem):
117 pass
118 else:
119 raise ValueError(
120 f"filesystem must be a pyarrow or fsspec FileSystem, "
121 f"not a {type(fs).__name__}"
122 )
123 if is_fsspec_url(path_or_handle) and fs is None:
124 pa_error = None
125 if storage_options is None:
126 pa = import_optional_dependency("pyarrow")
127 pa_fs = import_optional_dependency("pyarrow.fs")
129 try:
130 fs, path_or_handle = pa_fs.FileSystem.from_uri(path)
131 except (TypeError, pa.ArrowException):
132 pass
133 except OSError as err:
134 # Only "hdfs:///path" (no host) resolves differently in fsspec
135 # (GH#58078); otherwise this is a real error, e.g. no libhdfs
136 if not str(path_or_handle).startswith("hdfs:///"):
137 raise
138 pa_error = err
139 if fs is None:
140 try:
141 fsspec = import_optional_dependency("fsspec")
142 fs, path_or_handle = fsspec.core.url_to_fs(
143 path_or_handle, **(storage_options or {})
144 )
145 except Exception as err:
146 if pa_error is None:
147 raise
148 # keep pyarrow's error, e.g. a missing JVM
149 raise err from pa_error
150 elif storage_options and (not is_url(path_or_handle) or mode != "rb"):
151 # can't write to a remote url
152 # without making use of fsspec at the moment
153 raise ValueError("storage_options passed with buffer, or non-supported URL")
155 handles = None
156 if (
157 not fs
158 and not is_dir
159 and isinstance(path_or_handle, str)
160 and not os.path.isdir(path_or_handle)
161 ):
162 if is_url(path_or_handle):
163 # pyarrow cannot read non-fsspec URLs (e.g. http/https), so let
164 # get_handle download them into a buffer for pyarrow to consume.
165 handles = get_handle(
166 path_or_handle, mode, is_text=False, storage_options=storage_options
167 )
168 path_or_handle = handles.handle
169 else:
170 # Local path: keep the I/O in pyarrow's C++ layer rather than the
171 # Python one (GH#47702), and open the file only once -- adding a
172 # get_handle open on top would clobber pyarrow's data to 0 bytes
173 # (GH#65810). get_handle would also expand "~" and check the parent
174 # directory on write, so reproduce both below.
175 path_or_handle = os.path.expanduser(path_or_handle)
176 if "w" in mode or "a" in mode or "x" in mode:
177 check_parent_directory(path_or_handle)
178 # Open the destination instead of handing over its path:
179 # write_table deletes a path-like target when the write raises,
180 # destroying a pre-existing file it never managed to open
181 # (GH#69022). pa.OSFile opens it in C++ rather than through
182 # builtins.open, so this stays a single native open.
183 pa = import_optional_dependency("pyarrow")
184 stream = pa.OSFile(path_or_handle, mode)
185 handles = IOHandles(
186 handle=stream,
187 compression={"method": None},
188 created_handles=[stream],
189 )
190 path_or_handle = stream
191 return path_or_handle, handles, fs
194class BaseImpl:
195 @staticmethod
196 def validate_dataframe(df: DataFrame) -> None:
197 if not isinstance(df, DataFrame):
198 raise ValueError("to_parquet only supports IO with DataFrames")
200 def write(self, df: DataFrame, path, compression, **kwargs) -> None:
201 raise AbstractMethodError(self)
203 def read(self, path, columns=None, **kwargs) -> DataFrame:
204 raise AbstractMethodError(self)
207class PyArrowImpl(BaseImpl):
208 def __init__(self) -> None:
209 import_optional_dependency(
210 "pyarrow", extra="pyarrow is required for parquet support."
211 )
212 import pyarrow.parquet
214 # import utils to register the pyarrow extension types
215 import pandas.core.arrays.arrow.extension_types # pyright: ignore[reportUnusedImport] # noqa: F401
217 self.api = pyarrow
219 def write(
220 self,
221 df: DataFrame,
222 path: FilePath | WriteBuffer[bytes],
223 compression: ParquetCompressionOptions = "snappy",
224 index: bool | None = None,
225 storage_options: StorageOptions | None = None,
226 partition_cols: list[str] | None = None,
227 filesystem=None,
228 **kwargs,
229 ) -> None:
230 self.validate_dataframe(df)
232 from_pandas_kwargs: dict[str, Any] = {"schema": kwargs.pop("schema", None)}
233 if index is not None:
234 from_pandas_kwargs["preserve_index"] = index
236 with suppress_pyarrow_values_warning():
237 table = self.api.Table.from_pandas(df, **from_pandas_kwargs)
239 if df.attrs:
240 df_metadata = {"PANDAS_ATTRS": json.dumps(df.attrs)}
241 existing_metadata = table.schema.metadata
242 merged_metadata = {**existing_metadata, **df_metadata}
243 table = table.replace_schema_metadata(merged_metadata)
245 path_or_handle, handles, filesystem = _get_path_or_handle(
246 path,
247 filesystem,
248 storage_options=storage_options,
249 mode="wb",
250 is_dir=partition_cols is not None,
251 )
252 try:
253 if partition_cols is not None:
254 # writes to multiple files under the given path
255 self.api.parquet.write_to_dataset(
256 table,
257 path_or_handle,
258 compression=compression,
259 partition_cols=partition_cols,
260 filesystem=filesystem,
261 **kwargs,
262 )
263 else:
264 # write to single output file
265 self.api.parquet.write_table(
266 table,
267 path_or_handle,
268 compression=compression,
269 filesystem=filesystem,
270 **kwargs,
271 )
272 finally:
273 if handles is not None:
274 handles.close()
276 def read(
277 self,
278 path,
279 columns=None,
280 filters=None,
281 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
282 storage_options: StorageOptions | None = None,
283 filesystem=None,
284 to_pandas_kwargs: dict[str, Any] | None = None,
285 **kwargs,
286 ) -> DataFrame:
287 kwargs["use_pandas_metadata"] = True
289 path_or_handle, handles, filesystem = _get_path_or_handle(
290 path,
291 filesystem,
292 storage_options=storage_options,
293 mode="rb",
294 )
295 try:
296 pa_table = self.api.parquet.read_table(
297 path_or_handle,
298 columns=columns,
299 filesystem=filesystem,
300 filters=filters,
301 **kwargs,
302 )
304 df_metadata = None
305 if pa_table.schema.metadata:
306 if b"PANDAS_ATTRS" in pa_table.schema.metadata:
307 df_metadata = pa_table.schema.metadata[b"PANDAS_ATTRS"]
309 with catch_warnings():
310 filterwarnings(
311 "ignore",
312 "make_block is deprecated",
313 Pandas4Warning,
314 )
315 result = arrow_table_to_pandas(
316 pa_table,
317 dtype_backend=dtype_backend,
318 to_pandas_kwargs=to_pandas_kwargs,
319 )
321 if df_metadata is not None:
322 result.attrs = json.loads(df_metadata)
323 return result
324 finally:
325 if handles is not None:
326 handles.close()
329class FastParquetImpl(BaseImpl):
330 def __init__(self) -> None:
331 # since pandas is a dependency of fastparquet
332 # we need to import on first use
333 fastparquet = import_optional_dependency(
334 "fastparquet", extra="fastparquet is required for parquet support."
335 )
336 self.api = fastparquet
338 def write(
339 self,
340 df: DataFrame,
341 path,
342 compression: Literal["snappy", "gzip", "brotli"] | None = "snappy",
343 index=None,
344 partition_cols=None,
345 storage_options: StorageOptions | None = None,
346 filesystem=None,
347 **kwargs,
348 ) -> None:
349 self.validate_dataframe(df)
351 if "partition_on" in kwargs and partition_cols is not None:
352 raise ValueError(
353 "Cannot use both partition_on and "
354 "partition_cols. Use partition_cols for partitioning data"
355 )
356 if "partition_on" in kwargs:
357 partition_cols = kwargs.pop("partition_on")
359 if partition_cols is not None:
360 kwargs["file_scheme"] = "hive"
362 if filesystem is not None:
363 raise NotImplementedError(
364 "filesystem is not implemented for the fastparquet engine."
365 )
367 # cannot use get_handle as write() does not accept file buffers
368 path = stringify_path(path)
369 if is_fsspec_url(path):
370 fsspec = import_optional_dependency("fsspec")
372 # if filesystem is provided by fsspec, file must be opened in 'wb' mode.
373 kwargs["open_with"] = lambda path, _: fsspec.open(
374 path, "wb", **(storage_options or {})
375 ).open()
376 elif storage_options:
377 raise ValueError(
378 "storage_options passed with file object or non-fsspec file path"
379 )
381 with catch_warnings(record=True):
382 self.api.write(
383 path,
384 df,
385 compression=compression,
386 write_index=index,
387 partition_on=partition_cols,
388 **kwargs,
389 )
391 def read(
392 self,
393 path,
394 columns=None,
395 filters=None,
396 storage_options: StorageOptions | None = None,
397 filesystem=None,
398 to_pandas_kwargs: dict[str, Any] | None = None,
399 **kwargs,
400 ) -> DataFrame:
401 parquet_kwargs: dict[str, Any] = {}
402 dtype_backend = kwargs.pop("dtype_backend", lib.no_default)
403 # We are disabling nullable dtypes for fastparquet pending discussion
404 parquet_kwargs["pandas_nulls"] = False
405 if dtype_backend is not lib.no_default:
406 raise ValueError(
407 "The 'dtype_backend' argument is not supported for the "
408 "fastparquet engine"
409 )
410 if filesystem is not None:
411 raise NotImplementedError(
412 "filesystem is not implemented for the fastparquet engine."
413 )
414 if to_pandas_kwargs is not None:
415 raise NotImplementedError(
416 "to_pandas_kwargs is not implemented for the fastparquet engine."
417 )
418 path = stringify_path(path)
419 handles = None
420 if is_fsspec_url(path):
421 fsspec = import_optional_dependency("fsspec")
423 parquet_kwargs["fs"] = fsspec.open(path, "rb", **(storage_options or {})).fs
424 elif isinstance(path, str) and not os.path.isdir(path):
425 # use get_handle only when we are very certain that it is not a directory
426 # fsspec resources can also point to directories
427 # this branch is used for example when reading from non-fsspec URLs
428 handles = get_handle(
429 path, "rb", is_text=False, storage_options=storage_options
430 )
431 path = handles.handle
433 try:
434 parquet_file = self.api.ParquetFile(path, **parquet_kwargs)
435 with catch_warnings():
436 filterwarnings(
437 "ignore",
438 "make_block is deprecated",
439 Pandas4Warning,
440 )
441 return parquet_file.to_pandas(
442 columns=columns, filters=filters, **kwargs
443 )
444 finally:
445 if handles is not None:
446 handles.close()
449def to_parquet(
450 df: DataFrame,
451 path: FilePath | WriteBuffer[bytes] | None = None,
452 engine: str | lib.NoDefault = lib.no_default,
453 compression: ParquetCompressionOptions = "snappy",
454 index: bool | None = None,
455 storage_options: StorageOptions | None = None,
456 partition_cols: list[str] | None = None,
457 filesystem: Any = None,
458 **kwargs,
459) -> bytes | None:
460 """
461 Write a DataFrame to the parquet format.
463 Parameters
464 ----------
465 df : DataFrame
466 path : str, path object, file-like object, or None, default None
467 String, path object (implementing ``os.PathLike[str]``), or file-like
468 object implementing a binary ``write()`` function. If None, the result
469 is returned as bytes. If a string, it will be used as the root directory
470 path when writing a partitioned dataset. The engine fastparquet does
471 not accept file-like objects.
473 The string could be a URL. Valid URL schemes include http, ftp, s3,
474 gs, and file. For file URLs, a host is expected. A local file could be:
475 ``file://localhost/path/to/table.parquet``. A remote example could be:
476 ``s3://bucket/path/to/table.parquet``.
478 Certain URL schemes may require additional packages. For example, S3
479 URLs require the ``s3fs`` library. See
480 :ref:`install.optional_dependencies` for a full list.
481 engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
482 Parquet library to use. If 'auto', then the option
483 ``io.parquet.engine`` is used. The default ``io.parquet.engine``
484 behavior is to try 'pyarrow', falling back to 'fastparquet' if
485 'pyarrow' is unavailable.
487 When using the ``'pyarrow'`` engine and no storage options are provided
488 and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec``
489 (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first.
490 Use the filesystem keyword with an instantiated fsspec filesystem
491 if you wish to use its implementation.
493 .. deprecated:: 3.1.0
494 The ``'fastparquet'`` and ``'auto'`` engine options are
495 deprecated. Use ``'pyarrow'`` or do not pass ``engine``
496 to use the default.
498 compression : {'snappy', 'gzip', 'brotli', 'lz4', 'zstd', None},
499 default 'snappy'. Name of the compression to use. Use ``None``
500 for no compression.
501 index : bool, default None
502 If ``True``, include the dataframe's index(es) in the file output. If
503 ``False``, they will not be written to the file.
504 If ``None``, similar to ``True`` the dataframe's index(es)
505 will be saved. However, instead of being saved as values,
506 the RangeIndex will be stored as a range in the metadata so it
507 doesn't require much space and is faster. Other indexes will
508 be included as columns in the file output.
509 partition_cols : str or list, optional, default None
510 Column names by which to partition the dataset.
511 Columns are partitioned in the order they are given.
512 Must be None if path is not a string.
513 storage_options : dict, optional
514 Extra options that make sense for a particular storage connection, e.g.
515 host, port, username, password, etc. For HTTP(S) URLs the key-value
516 pairs are forwarded to ``urllib.request.Request`` as header options.
517 For other URLs (e.g. starting with "s3://", and "gcs://") the
518 key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec``
519 and ``urllib`` for more details, and for more examples on storage
520 options refer `here <https://pandas.pydata.org/docs/user_guide/io.html?
521 highlight=storage_options#reading-writing-remote-files>`_.
522 filesystem : fsspec or pyarrow filesystem, default None
523 Filesystem object to use when reading the parquet file. Only implemented
524 for ``engine="pyarrow"``.
526 .. versionadded:: 2.1.0
528 **kwargs
529 Additional keyword arguments passed to the engine:
531 * For ``engine="pyarrow"``: passed to :func:`pyarrow.parquet.write_table`
532 or :func:`pyarrow.parquet.write_to_dataset` (when using partition_cols)
533 * For ``engine="fastparquet"``: passed to :func:`fastparquet.write`
535 Returns
536 -------
537 bytes if no path argument is provided else None
538 """
539 if isinstance(partition_cols, str):
540 partition_cols = [partition_cols]
541 if engine is not lib.no_default and engine == "auto":
542 warnings.warn(
543 "engine='auto' is deprecated and will be removed in a "
544 "future version. Pass engine='pyarrow' or do not pass "
545 "'engine' to use the default.",
546 Pandas4Warning,
547 stacklevel=find_stack_level(),
548 )
549 if engine is lib.no_default:
550 engine = "auto"
551 impl = get_engine(engine)
553 path_or_buf: FilePath | WriteBuffer[bytes] = io.BytesIO() if path is None else path
555 impl.write(
556 df,
557 path_or_buf,
558 compression=compression,
559 index=index,
560 partition_cols=partition_cols,
561 storage_options=storage_options,
562 filesystem=filesystem,
563 **kwargs,
564 )
566 if path is None:
567 assert isinstance(path_or_buf, io.BytesIO)
568 return path_or_buf.getvalue()
569 else:
570 return None
573@set_module("pandas")
574def read_parquet(
575 path: FilePath | ReadBuffer[bytes],
576 engine: str | lib.NoDefault = lib.no_default,
577 columns: list[str] | None = None,
578 storage_options: StorageOptions | None = None,
579 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
580 filesystem: Any = None,
581 filters: list[tuple[Any, ...]] | list[list[tuple[Any, ...]]] | None = None,
582 to_pandas_kwargs: dict[str, Any] | None = None,
583 **kwargs,
584) -> DataFrame:
585 """
586 Load a parquet object from the file path, returning a DataFrame.
588 This function requires the `pyarrow <https://arrow.apache.org/docs/python/>`_
589 library.
591 The function automatically handles reading the data from a parquet file
592 and creates a DataFrame with the appropriate structure.
594 Parameters
595 ----------
596 path : str, path object or file-like object
597 String, path object (implementing ``os.PathLike[str]``), or file-like
598 object implementing a binary ``read()`` function.
599 The string could be a URL. Valid URL schemes include http, ftp, s3,
600 gs, and file. For file URLs, a host is expected. A local file could be:
601 ``file://localhost/path/to/table.parquet``.
603 Certain URL schemes may require additional packages. For example, S3
604 URLs require the ``s3fs`` library. See
605 :ref:`install.optional_dependencies` for a full list.
607 A file URL can also be a path to a directory that contains multiple
608 partitioned parquet files. Both pyarrow and fastparquet support
609 paths to directories as well as file URLs. A directory path could be:
610 ``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``.
611 engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
612 Parquet library to use. If 'auto', then the option
613 ``io.parquet.engine`` is used. The default ``io.parquet.engine``
614 behavior is to try 'pyarrow', falling back to 'fastparquet' if
615 'pyarrow' is unavailable.
617 When using the ``'pyarrow'`` engine and no storage options are provided
618 and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec``
619 (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first.
620 Use the filesystem keyword with an instantiated fsspec filesystem
621 if you wish to use its implementation.
623 .. deprecated:: 3.1.0
624 The ``'fastparquet'`` and ``'auto'`` engine options are
625 deprecated. Use ``'pyarrow'`` or do not pass ``engine``
626 to use the default.
628 columns : list, default=None
629 If not None, only these columns will be read from the file.
630 storage_options : dict, optional
631 Extra options that make sense for a particular storage connection, e.g.
632 host, port, username, password, etc. For HTTP(S) URLs the key-value
633 pairs are forwarded to ``urllib.request.Request`` as header options.
634 For other URLs (e.g. starting with "s3://", and "gcs://") the
635 key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec``
636 and ``urllib`` for more details, and for more examples on storage
637 options refer `here <https://pandas.pydata.org/docs/user_guide/io.html?
638 highlight=storage_options#reading-writing-remote-files>`_.
639 dtype_backend : {'numpy_nullable', 'pyarrow'}
640 Back-end data type applied to the resultant :class:`DataFrame`
641 (still experimental). If not specified, the default behavior
642 is to not use nullable data types. If specified, the behavior
643 is as follows:
645 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
646 * ``"pyarrow"``: returns pyarrow-backed nullable
647 :class:`ArrowDtype` :class:`DataFrame`
649 .. versionadded:: 2.0
651 filesystem : fsspec or pyarrow filesystem, default None
652 Filesystem object to use when reading the parquet file. Only implemented
653 for ``engine="pyarrow"``.
655 .. versionadded:: 2.1.0
657 filters : List[Tuple] or List[List[Tuple]], default None
658 To filter out data.
659 Filter syntax: [[(column, op, val), ...],...]
660 where op is [==, =, >, >=, <, <=, !=, in, not in]
661 The innermost tuples are transposed into a set of filters applied
662 through an `AND` operation.
663 The outer list combines these sets of filters through an `OR`
664 operation.
665 A single list of tuples can also be used, meaning that no `OR`
666 operation between set of filters is to be conducted.
668 Using this argument will NOT result in row-wise filtering of the final
669 partitions unless ``engine="pyarrow"`` is also specified. For
670 other engines, filtering is only performed at the partition level, that is,
671 to prevent the loading of some row-groups and/or files.
673 .. versionadded:: 2.1.0
675 to_pandas_kwargs : dict | None, default None
676 Keyword arguments to pass through to :func:`pyarrow.Table.to_pandas`
677 when ``engine="pyarrow"``.
679 .. versionadded:: 3.0.0
681 **kwargs
682 Additional keyword arguments passed to the engine:
684 * For ``engine="pyarrow"``: passed to :func:`pyarrow.parquet.read_table`
685 * For ``engine="fastparquet"``: passed to
686 :meth:`fastparquet.ParquetFile.to_pandas`
688 Returns
689 -------
690 DataFrame
691 DataFrame based on parquet file.
693 See Also
694 --------
695 DataFrame.to_parquet : Create a parquet object that serializes a DataFrame.
697 Examples
698 --------
699 >>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)})
700 >>> original_df
701 foo bar
702 0 0 5
703 1 1 6
704 2 2 7
705 3 3 8
706 4 4 9
707 >>> df_parquet_bytes = original_df.to_parquet()
708 >>> from io import BytesIO
709 >>> restored_df = pd.read_parquet(BytesIO(df_parquet_bytes))
710 >>> restored_df
711 foo bar
712 0 0 5
713 1 1 6
714 2 2 7
715 3 3 8
716 4 4 9
717 >>> restored_df.equals(original_df)
718 True
719 >>> restored_bar = pd.read_parquet(BytesIO(df_parquet_bytes), columns=["bar"])
720 >>> restored_bar
721 bar
722 0 5
723 1 6
724 2 7
725 3 8
726 4 9
727 >>> restored_bar.equals(original_df[["bar"]])
728 True
730 The function uses `kwargs` that are passed directly to the engine.
731 In the following example, we use the `filters` argument of the pyarrow
732 engine to filter the rows of the DataFrame.
734 Since `pyarrow` is the default engine, we can omit the `engine` argument.
735 Note that the `filters` argument is implemented by the `pyarrow` engine,
736 which can benefit from multithreading and also potentially be more
737 economical in terms of memory.
739 >>> sel = [("foo", ">", 2)]
740 >>> restored_part = pd.read_parquet(BytesIO(df_parquet_bytes), filters=sel)
741 >>> restored_part
742 foo bar
743 0 3 8
744 1 4 9
745 """
747 if engine is not lib.no_default and engine == "auto":
748 warnings.warn(
749 "engine='auto' is deprecated and will be removed in a "
750 "future version. Pass engine='pyarrow' or do not pass "
751 "'engine' to use the default.",
752 Pandas4Warning,
753 stacklevel=find_stack_level(),
754 )
755 if engine is lib.no_default:
756 engine = "auto"
757 impl = get_engine(engine)
758 check_dtype_backend(dtype_backend)
760 return impl.read(
761 path,
762 columns=columns,
763 filters=filters,
764 storage_options=storage_options,
765 dtype_backend=dtype_backend,
766 filesystem=filesystem,
767 to_pandas_kwargs=to_pandas_kwargs,
768 **kwargs,
769 )