Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import ctypes
4import re
5from typing import (
6 Any,
7 overload,
8)
9import warnings
11import numpy as np
13from pandas._config import using_string_dtype
15from pandas.compat._optional import import_optional_dependency
16from pandas.errors import Pandas4Warning
17from pandas.util._decorators import set_module
18from pandas.util._exceptions import find_stack_level
20import pandas as pd
21from pandas.core.interchange.dataframe_protocol import (
22 Buffer,
23 Column,
24 ColumnNullType,
25 DataFrame as DataFrameXchg,
26 DtypeKind,
27)
28from pandas.core.interchange.utils import (
29 ArrowCTypes,
30 Endianness,
31)
33_NP_DTYPES: dict[DtypeKind, dict[int, Any]] = {
34 DtypeKind.INT: {8: np.int8, 16: np.int16, 32: np.int32, 64: np.int64},
35 DtypeKind.UINT: {8: np.uint8, 16: np.uint16, 32: np.uint32, 64: np.uint64},
36 DtypeKind.FLOAT: {32: np.float32, 64: np.float64},
37 DtypeKind.BOOL: {1: bool, 8: bool},
38}
41@set_module("pandas.api.interchange")
42def from_dataframe(df, allow_copy: bool = True) -> pd.DataFrame:
43 """
44 Build a ``pd.DataFrame`` from any DataFrame supporting the interchange protocol.
46 .. note::
48 For new development, we highly recommend using the Arrow C Data Interface
49 alongside the Arrow PyCapsule Interface instead of the interchange protocol.
50 From pandas 3.0 onwards, `from_dataframe` uses the PyCapsule Interface,
51 only falling back to the interchange protocol if that fails.
53 From pandas 4.0 onwards, that fallback will no longer be available and only
54 the PyCapsule Interface will be used.
56 .. warning::
58 Due to severe implementation issues, we recommend only considering using the
59 interchange protocol in the following cases:
61 - converting to pandas: for pandas >= 2.0.3
62 - converting from pandas: for pandas >= 3.0.0
64 Parameters
65 ----------
66 df : DataFrameXchg
67 Object supporting the interchange protocol, i.e. `__dataframe__` method.
68 allow_copy : bool, default: True
69 Whether to allow copying the memory to perform the conversion
70 (if false then zero-copy approach is requested).
72 Returns
73 -------
74 pd.DataFrame
75 A pandas DataFrame built from the provided interchange
76 protocol object.
78 See Also
79 --------
80 pd.DataFrame : DataFrame class which can be created from various input data
81 formats, including objects that support the interchange protocol.
83 Examples
84 --------
85 >>> df_not_necessarily_pandas = pd.DataFrame({"A": [1, 2], "B": [3, 4]})
86 >>> interchange_object = df_not_necessarily_pandas.__dataframe__()
87 >>> interchange_object.column_names()
88 Index(['A', 'B'], dtype='str')
89 >>> df_pandas = pd.api.interchange.from_dataframe(
90 ... interchange_object.select_columns_by_name(["A"])
91 ... )
92 >>> df_pandas
93 A
94 0 1
95 1 2
97 These methods (``column_names``, ``select_columns_by_name``) should work
98 for any dataframe library which implements the interchange protocol.
99 """
100 if isinstance(df, pd.DataFrame):
101 return df
103 if hasattr(df, "__arrow_c_stream__"):
104 try:
105 pa = import_optional_dependency("pyarrow", min_version="14.0.0")
106 except ImportError:
107 # fallback to _from_dataframe
108 warnings.warn(
109 "Conversion using Arrow PyCapsule Interface failed due to "
110 "missing PyArrow>=14 dependency, falling back to (deprecated) "
111 "interchange protocol. We recommend that you install "
112 "PyArrow>=14.0.0.",
113 UserWarning,
114 stacklevel=find_stack_level(),
115 )
116 else:
117 try:
118 return pa.table(df).to_pandas(zero_copy_only=not allow_copy)
119 except pa.ArrowInvalid as e:
120 raise RuntimeError(e) from e
122 if not hasattr(df, "__dataframe__"):
123 raise ValueError("`df` does not support __dataframe__")
125 warnings.warn(
126 "The Dataframe Interchange Protocol is deprecated.\n"
127 "For dataframe-agnostic code, you may want to look into:\n"
128 "- Arrow PyCapsule Interface: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html\n"
129 "- Narwhals: https://github.com/narwhals-dev/narwhals\n",
130 Pandas4Warning,
131 stacklevel=find_stack_level(),
132 )
134 return _from_dataframe(
135 df.__dataframe__(allow_copy=allow_copy), allow_copy=allow_copy
136 )
139def _from_dataframe(df: DataFrameXchg, allow_copy: bool = True) -> pd.DataFrame:
140 """
141 Build a ``pd.DataFrame`` from the DataFrame interchange object.
143 Parameters
144 ----------
145 df : DataFrameXchg
146 Object supporting the interchange protocol, i.e. `__dataframe__` method.
147 allow_copy : bool, default: True
148 Whether to allow copying the memory to perform the conversion
149 (if false then zero-copy approach is requested).
151 Returns
152 -------
153 pd.DataFrame
154 """
155 pandas_dfs = []
156 for chunk in df.get_chunks():
157 pandas_df = protocol_df_chunk_to_pandas(chunk)
158 pandas_dfs.append(pandas_df)
160 if not allow_copy and len(pandas_dfs) > 1:
161 raise RuntimeError(
162 "To join chunks a copy is required which is forbidden by allow_copy=False"
163 )
164 if not pandas_dfs:
165 pandas_df = protocol_df_chunk_to_pandas(df)
166 elif len(pandas_dfs) == 1:
167 pandas_df = pandas_dfs[0]
168 else:
169 pandas_df = pd.concat(pandas_dfs, axis=0, ignore_index=True, copy=False)
171 index_obj = df.metadata.get("pandas.index", None)
172 if index_obj is not None:
173 pandas_df.index = index_obj
175 return pandas_df
178def protocol_df_chunk_to_pandas(df: DataFrameXchg) -> pd.DataFrame:
179 """
180 Convert interchange protocol chunk to ``pd.DataFrame``.
182 Parameters
183 ----------
184 df : DataFrameXchg
186 Returns
187 -------
188 pd.DataFrame
189 """
190 columns: dict[str, Any] = {}
191 buffers = [] # hold on to buffers, keeps memory alive
192 for name in df.column_names():
193 if not isinstance(name, str):
194 raise ValueError(f"Column {name} is not a string")
195 if name in columns:
196 raise ValueError(f"Column {name} is not unique")
197 col = df.get_column_by_name(name)
198 dtype = col.dtype[0]
199 if dtype in (
200 DtypeKind.INT,
201 DtypeKind.UINT,
202 DtypeKind.FLOAT,
203 DtypeKind.BOOL,
204 ):
205 columns[name], buf = primitive_column_to_ndarray(col)
206 elif dtype == DtypeKind.CATEGORICAL:
207 columns[name], buf = categorical_column_to_series(col)
208 elif dtype == DtypeKind.STRING:
209 columns[name], buf = string_column_to_ndarray(col)
210 elif dtype == DtypeKind.DATETIME:
211 columns[name], buf = datetime_column_to_ndarray(col)
212 else:
213 raise NotImplementedError(f"Data type {dtype} not handled yet")
215 buffers.append(buf)
217 pandas_df = pd.DataFrame(columns)
218 pandas_df.attrs["_INTERCHANGE_PROTOCOL_BUFFERS"] = buffers
219 return pandas_df
222def primitive_column_to_ndarray(col: Column) -> tuple[np.ndarray, Any]:
223 """
224 Convert a column holding one of the primitive dtypes to a NumPy array.
226 A primitive type is one of: int, uint, float, bool.
228 Parameters
229 ----------
230 col : Column
232 Returns
233 -------
234 tuple
235 Tuple of np.ndarray holding the data and the memory owner object
236 that keeps the memory alive.
237 """
238 buffers = col.get_buffers()
240 data_buff, data_dtype = buffers["data"]
241 data = buffer_to_ndarray(
242 data_buff, data_dtype, offset=col.offset, length=col.size()
243 )
245 data = set_nulls(data, col, buffers["validity"])
246 return data, buffers
249def categorical_column_to_series(col: Column) -> tuple[pd.Series, Any]:
250 """
251 Convert a column holding categorical data to a pandas Series.
253 Parameters
254 ----------
255 col : Column
257 Returns
258 -------
259 tuple
260 Tuple of pd.Series holding the data and the memory owner object
261 that keeps the memory alive.
262 """
263 categorical = col.describe_categorical
265 if not categorical["is_dictionary"]:
266 raise NotImplementedError("Non-dictionary categoricals not supported yet")
268 cat_column = categorical["categories"]
269 if hasattr(cat_column, "_col"):
270 # Item "Column" of "Optional[Column]" has no attribute "_col"
271 # Item "None" of "Optional[Column]" has no attribute "_col"
272 categories = np.array(cat_column._col) # type: ignore[union-attr]
273 else:
274 raise NotImplementedError(
275 "Interchanging categorical columns isn't supported yet, and our "
276 "fallback of using the `col._col` attribute (a ndarray) failed."
277 )
278 buffers = col.get_buffers()
280 codes_buff, codes_dtype = buffers["data"]
281 codes = buffer_to_ndarray(
282 codes_buff, codes_dtype, offset=col.offset, length=col.size()
283 )
285 # Doing module in order to not get ``IndexError`` for
286 # out-of-bounds sentinel values in `codes`
287 if len(categories) > 0:
288 values = categories[codes % len(categories)]
289 else:
290 values = codes
292 cat = pd.Categorical(
293 values, categories=categories, ordered=categorical["is_ordered"]
294 )
295 data = pd.Series(cat)
297 data = set_nulls(data, col, buffers["validity"])
298 return data, buffers
301def string_column_to_ndarray(col: Column) -> tuple[np.ndarray, Any]:
302 """
303 Convert a column holding string data to a NumPy array.
305 Parameters
306 ----------
307 col : Column
309 Returns
310 -------
311 tuple
312 Tuple of np.ndarray holding the data and the memory owner object
313 that keeps the memory alive.
314 """
315 null_kind, sentinel_val = col.describe_null
317 if null_kind not in (
318 ColumnNullType.NON_NULLABLE,
319 ColumnNullType.USE_BITMASK,
320 ColumnNullType.USE_BYTEMASK,
321 ):
322 raise NotImplementedError(
323 f"{null_kind} null kind is not yet supported for string columns."
324 )
326 buffers = col.get_buffers()
328 assert buffers["offsets"], "String buffers must contain offsets"
329 # Retrieve the data buffer containing the UTF-8 code units
330 data_buff, _ = buffers["data"]
331 # We're going to reinterpret the buffer as uint8, so make sure we can do it safely
332 assert col.dtype[2] in (
333 ArrowCTypes.STRING,
334 ArrowCTypes.LARGE_STRING,
335 ) # format_str == utf-8
336 # Convert the buffers to NumPy arrays. In order to go from STRING to
337 # an equivalent ndarray, we claim that the buffer is uint8 (i.e., a byte array)
338 data_dtype = (
339 DtypeKind.UINT,
340 8,
341 ArrowCTypes.UINT8,
342 Endianness.NATIVE,
343 )
344 # Specify zero offset as we don't want to chunk the string data
345 data = buffer_to_ndarray(data_buff, data_dtype, offset=0, length=data_buff.bufsize)
347 # Retrieve the offsets buffer containing the index offsets demarcating
348 # the beginning and the ending of each string
349 offset_buff, offset_dtype = buffers["offsets"]
350 # Offsets buffer contains start-stop positions of strings in the data buffer,
351 # meaning that it has more elements than in the data buffer, do `col.size() + 1`
352 # here to pass a proper offsets buffer size
353 offsets = buffer_to_ndarray(
354 offset_buff, offset_dtype, offset=col.offset, length=col.size() + 1
355 )
357 null_pos = None
358 if null_kind in (ColumnNullType.USE_BITMASK, ColumnNullType.USE_BYTEMASK):
359 validity = buffers["validity"]
360 if validity is not None:
361 valid_buff, valid_dtype = validity
362 null_pos = buffer_to_ndarray(
363 valid_buff, valid_dtype, offset=col.offset, length=col.size()
364 )
365 if sentinel_val == 0:
366 null_pos = ~null_pos
368 # Assemble the strings from the code units
369 str_list: list[None | float | str] = [None] * col.size()
370 for i in range(col.size()):
371 # Check for missing values
372 if null_pos is not None and null_pos[i]:
373 str_list[i] = np.nan
374 continue
376 # Extract a range of code units
377 units = data[offsets[i] : offsets[i + 1]]
379 # Convert the list of code units to bytes
380 str_bytes = bytes(units)
382 # Create the string
383 string = str_bytes.decode(encoding="utf-8")
385 # Add to our list of strings
386 str_list[i] = string
388 if using_string_dtype():
389 res = pd.Series(str_list, dtype="str")
390 else:
391 res = np.asarray(str_list, dtype="object") # type: ignore[assignment]
393 return res, buffers # type: ignore[return-value]
396def parse_datetime_format_str(format_str, data) -> pd.Series | np.ndarray:
397 """Parse datetime `format_str` to interpret the `data`."""
398 # timestamp 'ts{unit}:tz'
399 timestamp_meta = re.match(r"ts([smun]):(.*)", format_str)
400 if timestamp_meta:
401 unit, tz = timestamp_meta.group(1), timestamp_meta.group(2)
402 if unit != "s":
403 # the format string describes only a first letter of the unit, so
404 # add one extra letter to convert the unit to numpy-style:
405 # 'm' -> 'ms', 'u' -> 'us', 'n' -> 'ns'
406 unit += "s"
407 data = data.astype(f"datetime64[{unit}]")
408 if tz != "":
409 data = pd.Series(data).dt.tz_localize("UTC").dt.tz_convert(tz)
410 return data
412 # date 'td{Days/Ms}'
413 date_meta = re.match(r"td([Dm])", format_str)
414 if date_meta:
415 unit = date_meta.group(1)
416 if unit == "D":
417 # NumPy doesn't support DAY unit, so converting days to seconds
418 # (converting to uint64 to avoid overflow)
419 data = (data.astype(np.uint64) * (24 * 60 * 60)).astype("datetime64[s]")
420 elif unit == "m":
421 data = data.astype("datetime64[ms]")
422 else:
423 raise NotImplementedError(f"Date unit is not supported: {unit}")
424 return data
426 raise NotImplementedError(f"DateTime kind is not supported: {format_str}")
429def datetime_column_to_ndarray(col: Column) -> tuple[np.ndarray | pd.Series, Any]:
430 """
431 Convert a column holding DateTime data to a NumPy array.
433 Parameters
434 ----------
435 col : Column
437 Returns
438 -------
439 tuple
440 Tuple of np.ndarray holding the data and the memory owner object
441 that keeps the memory alive.
442 """
443 buffers = col.get_buffers()
445 _, col_bit_width, format_str, _ = col.dtype
446 dbuf, _ = buffers["data"]
447 # Consider dtype being `uint` to get number of units passed since the 01.01.1970
449 data = buffer_to_ndarray(
450 dbuf,
451 (
452 DtypeKind.INT,
453 col_bit_width,
454 getattr(ArrowCTypes, f"INT{col_bit_width}"),
455 Endianness.NATIVE,
456 ),
457 offset=col.offset,
458 length=col.size(),
459 )
461 data = parse_datetime_format_str(format_str, data) # type: ignore[assignment]
462 data = set_nulls(data, col, buffers["validity"])
463 return data, buffers
466def buffer_to_ndarray(
467 buffer: Buffer,
468 dtype: tuple[DtypeKind, int, str, str],
469 *,
470 length: int,
471 offset: int = 0,
472) -> np.ndarray:
473 """
474 Build a NumPy array from the passed buffer.
476 Parameters
477 ----------
478 buffer : Buffer
479 Buffer to build a NumPy array from.
480 dtype : tuple
481 Data type of the buffer conforming protocol dtypes format.
482 offset : int, default: 0
483 Number of elements to offset from the start of the buffer.
484 length : int, optional
485 If the buffer is a bit-mask, specifies a number of bits to read
486 from the buffer. Has no effect otherwise.
488 Returns
489 -------
490 np.ndarray
492 Notes
493 -----
494 The returned array doesn't own the memory. The caller of this function is
495 responsible for keeping the memory owner object alive as long as
496 the returned NumPy array is being used.
497 """
498 kind, bit_width, _, _ = dtype
500 column_dtype = _NP_DTYPES.get(kind, {}).get(bit_width, None)
501 if column_dtype is None:
502 raise NotImplementedError(f"Conversion for {dtype} is not yet supported.")
504 # TODO: No DLPack yet, so need to construct a new ndarray from the data pointer
505 # and size in the buffer plus the dtype on the column. Use DLPack as NumPy supports
506 # it since https://github.com/numpy/numpy/pull/19083
507 ctypes_type = np.ctypeslib.as_ctypes_type(column_dtype)
509 if bit_width == 1:
510 assert length is not None, "`length` must be specified for a bit-mask buffer."
511 pa = import_optional_dependency("pyarrow")
512 arr = pa.BooleanArray.from_buffers(
513 pa.bool_(),
514 length,
515 [None, pa.foreign_buffer(buffer.ptr, length)],
516 offset=offset,
517 )
518 return np.asarray(arr)
519 else:
520 data_pointer = ctypes.cast(
521 buffer.ptr + (offset * bit_width // 8), ctypes.POINTER(ctypes_type)
522 )
523 if length > 0:
524 return np.ctypeslib.as_array(data_pointer, shape=(length,))
525 return np.array([], dtype=ctypes_type)
528@overload
529def set_nulls(
530 data: np.ndarray,
531 col: Column,
532 validity: tuple[Buffer, tuple[DtypeKind, int, str, str]] | None,
533 allow_modify_inplace: bool = ...,
534) -> np.ndarray: ...
537@overload
538def set_nulls(
539 data: pd.Series,
540 col: Column,
541 validity: tuple[Buffer, tuple[DtypeKind, int, str, str]] | None,
542 allow_modify_inplace: bool = ...,
543) -> pd.Series: ...
546@overload
547def set_nulls(
548 data: np.ndarray | pd.Series,
549 col: Column,
550 validity: tuple[Buffer, tuple[DtypeKind, int, str, str]] | None,
551 allow_modify_inplace: bool = ...,
552) -> np.ndarray | pd.Series: ...
555def set_nulls(
556 data: np.ndarray | pd.Series,
557 col: Column,
558 validity: tuple[Buffer, tuple[DtypeKind, int, str, str]] | None,
559 allow_modify_inplace: bool = True,
560) -> np.ndarray | pd.Series:
561 """
562 Set null values for the data according to the column null kind.
564 Parameters
565 ----------
566 data : np.ndarray or pd.Series
567 Data to set nulls in.
568 col : Column
569 Column object that describes the `data`.
570 validity : tuple(Buffer, dtype) or None
571 The return value of ``col.buffers()``. We do not access the ``col.buffers()``
572 here to not take the ownership of the memory of buffer objects.
573 allow_modify_inplace : bool, default: True
574 Whether to modify the `data` inplace when zero-copy is possible (True) or always
575 modify a copy of the `data` (False).
577 Returns
578 -------
579 np.ndarray or pd.Series
580 Data with the nulls being set.
581 """
582 if validity is None:
583 return data
584 null_kind, sentinel_val = col.describe_null
585 null_pos = None
587 if null_kind == ColumnNullType.USE_SENTINEL:
588 null_pos = pd.Series(data) == sentinel_val
589 elif null_kind in (ColumnNullType.USE_BITMASK, ColumnNullType.USE_BYTEMASK):
590 valid_buff, valid_dtype = validity
591 null_pos = buffer_to_ndarray(
592 valid_buff, valid_dtype, offset=col.offset, length=col.size()
593 )
594 if sentinel_val == 0:
595 null_pos = ~null_pos
596 elif null_kind in (ColumnNullType.NON_NULLABLE, ColumnNullType.USE_NAN):
597 pass
598 else:
599 raise NotImplementedError(f"Null kind {null_kind} is not yet supported.")
601 if null_pos is not None and np.any(null_pos):
602 if not allow_modify_inplace:
603 data = data.copy()
604 try:
605 data[null_pos] = None
606 except TypeError:
607 # TypeError happens if the `data` dtype appears to be non-nullable
608 # in numpy notation (bool, int, uint). If this happens,
609 # cast the `data` to nullable float dtype.
610 data = data.astype(float)
611 data[null_pos] = None
613 return data