Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Sparse accessor"""
3from __future__ import annotations
5from typing import TYPE_CHECKING
7import numpy as np
9from pandas.compat._optional import import_optional_dependency
11from pandas.core.dtypes.cast import find_common_type
12from pandas.core.dtypes.dtypes import SparseDtype
13from pandas.core.dtypes.missing import isna
15from pandas.core.accessor import (
16 PandasDelegate,
17 delegate_names,
18)
19from pandas.core.arrays.sparse.array import SparseArray
21if TYPE_CHECKING:
22 from scipy.sparse import (
23 coo_matrix,
24 spmatrix,
25 )
27 from pandas._typing import IndexLabel
29 from pandas import (
30 DataFrame,
31 Series,
32 )
35class BaseAccessor:
36 _validation_msg = "Can only use the '.sparse' accessor with Sparse data."
38 def __init__(self, data=None) -> None:
39 self._parent = data
40 self._validate(data)
42 def _validate(self, data) -> None:
43 raise NotImplementedError
46@delegate_names(
47 SparseArray, ["npoints", "density", "fill_value", "sp_values"], typ="property"
48)
49class SparseAccessor(BaseAccessor, PandasDelegate):
50 """
51 Accessor for SparseArray from other sparse matrix data types.
53 Provides methods and properties to work with the underlying sparse data
54 in a Series or DataFrame. It allows accessing sparse-specific attributes
55 such as fill value, density, and stored non-fill values, as well as
56 converting to and from scipy sparse matrices.
58 Parameters
59 ----------
60 data : Series or DataFrame
61 The Series or DataFrame to which the SparseAccessor is attached.
63 See Also
64 --------
65 Series.sparse.to_coo : Create a scipy.sparse.coo_matrix from a Series with
66 MultiIndex.
67 Series.sparse.from_coo : Create a Series with sparse values from a
68 scipy.sparse.coo_matrix.
70 Examples
71 --------
72 >>> ser = pd.Series([0, 0, 2, 2, 2], dtype="Sparse[int]")
73 >>> ser.sparse.density
74 0.6
75 >>> ser.sparse.sp_values
76 array([2, 2, 2])
77 """
79 def _validate(self, data) -> None:
80 if not isinstance(data.dtype, SparseDtype):
81 raise AttributeError(self._validation_msg)
83 def _delegate_property_get(self, name: str, *args, **kwargs):
84 return getattr(self._parent.array, name)
86 def _delegate_method(self, name: str, *args, **kwargs):
87 if name == "from_coo":
88 return self.from_coo(*args, **kwargs)
89 elif name == "to_coo":
90 return self.to_coo(*args, **kwargs)
91 else:
92 raise ValueError
94 @classmethod
95 def from_coo(cls, A, dense_index: bool = False) -> Series:
96 """
97 Create a Series with sparse values from a scipy.sparse.coo_matrix.
99 This method takes a ``scipy.sparse.coo_matrix`` (coordinate format) as input and
100 returns a pandas ``Series`` where the non-zero elements are represented as
101 sparse values. The index of the Series can either include only the coordinates
102 of non-zero elements (default behavior) or the full sorted set of coordinates
103 from the matrix if ``dense_index`` is set to `True`.
105 Parameters
106 ----------
107 A : scipy.sparse.coo_matrix
108 The sparse matrix in coordinate format from which the sparse Series
109 will be created.
110 dense_index : bool, default False
111 If False (default), the index consists of only the
112 coords of the non-null entries of the original coo_matrix.
113 If True, the index consists of the full sorted
114 (row, col) coordinates of the coo_matrix.
116 Returns
117 -------
118 s : Series
119 A Series with sparse values.
121 See Also
122 --------
123 DataFrame.sparse.from_spmatrix : Create a new DataFrame from a scipy sparse
124 matrix.
125 scipy.sparse.coo_matrix : A sparse matrix in COOrdinate format.
127 Examples
128 --------
129 >>> from scipy import sparse
131 >>> A = sparse.coo_matrix(
132 ... ([3.0, 1.0, 2.0], ([1, 0, 0], [0, 2, 3])), shape=(3, 4)
133 ... )
134 >>> A
135 <COOrdinate sparse matrix of dtype 'float64'
136 with 3 stored elements and shape (3, 4)>
138 >>> A.todense()
139 matrix([[0., 0., 1., 2.],
140 [3., 0., 0., 0.],
141 [0., 0., 0., 0.]])
143 >>> ss = pd.Series.sparse.from_coo(A)
144 >>> ss
145 0 2 1.0
146 3 2.0
147 1 0 3.0
148 dtype: Sparse[float64, nan]
149 """
150 from pandas import Series
151 from pandas.core.arrays.sparse.scipy_sparse import coo_to_sparse_series
153 result = coo_to_sparse_series(A, dense_index=dense_index)
154 result = Series(result.array, index=result.index, copy=False)
156 return result
158 def to_coo(
159 self, row_levels=(0,), column_levels=(1,), sort_labels: bool = False
160 ) -> tuple[coo_matrix, list[IndexLabel], list[IndexLabel]]:
161 """
162 Create a scipy.sparse.coo_matrix from a Series with MultiIndex.
164 Use row_levels and column_levels to determine the row and column
165 coordinates respectively. row_levels and column_levels are the names
166 (labels) or numbers of the levels. {row_levels, column_levels} must be
167 a partition of the MultiIndex level names (or numbers).
169 Parameters
170 ----------
171 row_levels : tuple/list
172 MultiIndex levels to use for row coordinates, specified by name or index.
173 column_levels : tuple/list
174 MultiIndex levels to use for column coordinates, specified by name or index.
175 sort_labels : bool, default False
176 Sort the row and column labels before forming the sparse matrix.
177 When `row_levels` and/or `column_levels` refer to a single level,
178 set to `True` for a faster execution.
180 Returns
181 -------
182 y : scipy.sparse.coo_matrix
183 The sparse matrix in coordinate format.
184 rows : list (row labels)
185 Labels corresponding to the row coordinates.
186 columns : list (column labels)
187 Labels corresponding to the column coordinates.
189 See Also
190 --------
191 Series.sparse.from_coo : Create a Series with sparse values from a
192 scipy.sparse.coo_matrix.
194 Examples
195 --------
196 >>> s = pd.Series([3.0, np.nan, 1.0, 3.0, np.nan, np.nan])
197 >>> s.index = pd.MultiIndex.from_tuples(
198 ... [
199 ... (1, 2, "a", 0),
200 ... (1, 2, "a", 1),
201 ... (1, 1, "b", 0),
202 ... (1, 1, "b", 1),
203 ... (2, 1, "b", 0),
204 ... (2, 1, "b", 1),
205 ... ],
206 ... names=["A", "B", "C", "D"],
207 ... )
208 >>> s
209 A B C D
210 1 2 a 0 3.0
211 1 NaN
212 1 b 0 1.0
213 1 3.0
214 2 1 b 0 NaN
215 1 NaN
216 dtype: float64
218 >>> ss = s.astype("Sparse")
219 >>> ss
220 A B C D
221 1 2 a 0 3.0
222 1 NaN
223 1 b 0 1.0
224 1 3.0
225 2 1 b 0 NaN
226 1 NaN
227 dtype: Sparse[float64, nan]
229 >>> A, rows, columns = ss.sparse.to_coo(
230 ... row_levels=["A", "B"], column_levels=["C", "D"], sort_labels=True
231 ... )
232 >>> A
233 <COOrdinate sparse matrix of dtype 'float64'
234 with 3 stored elements and shape (3, 4)>
235 >>> A.todense()
236 matrix([[0., 0., 1., 3.],
237 [3., 0., 0., 0.],
238 [0., 0., 0., 0.]])
240 >>> rows
241 [(1, 1), (1, 2), (2, 1)]
242 >>> columns
243 [('a', 0), ('a', 1), ('b', 0), ('b', 1)]
244 """
245 from pandas.core.arrays.sparse.scipy_sparse import sparse_series_to_coo
247 A, rows, columns = sparse_series_to_coo(
248 self._parent, row_levels, column_levels, sort_labels=sort_labels
249 )
250 return A, rows, columns
252 def to_dense(self) -> Series:
253 """
254 Convert a Series from sparse values to dense.
256 Returns
257 -------
258 Series:
259 A Series with the same values, stored as a dense array.
261 Examples
262 --------
263 >>> series = pd.Series(pd.arrays.SparseArray([0, 1, 0]))
264 >>> series
265 0 0
266 1 1
267 2 0
268 dtype: Sparse[int64, 0]
270 >>> series.sparse.to_dense()
271 0 0
272 1 1
273 2 0
274 dtype: int64
275 """
276 from pandas import Series
278 return Series(
279 self._parent.array.to_dense(),
280 index=self._parent.index,
281 name=self._parent.name,
282 copy=False,
283 )
286class SparseFrameAccessor(BaseAccessor, PandasDelegate):
287 """
288 DataFrame accessor for sparse data.
290 It allows users to interact with a `DataFrame` that contains sparse data types
291 (`SparseDtype`). It provides methods and attributes to efficiently work with sparse
292 storage, reducing memory usage while maintaining compatibility with standard pandas
293 operations.
295 Parameters
296 ----------
297 data : scipy.sparse.spmatrix
298 Must be convertible to csc format.
300 See Also
301 --------
302 DataFrame.sparse.density : Ratio of non-sparse points to total (dense) data points.
304 Examples
305 --------
306 >>> df = pd.DataFrame({"a": [1, 2, 0, 0], "b": [3, 0, 0, 4]}, dtype="Sparse[int]")
307 >>> df.sparse.density
308 np.float64(0.5)
309 """
311 def _validate(self, data) -> None:
312 dtypes = data.dtypes
313 if not all(isinstance(t, SparseDtype) for t in dtypes):
314 raise AttributeError(self._validation_msg)
316 @classmethod
317 def from_spmatrix(cls, data, index=None, columns=None) -> DataFrame:
318 """
319 Create a new DataFrame from a scipy sparse matrix.
321 This method converts a scipy sparse matrix into a pandas DataFrame
322 with sparse array columns for memory efficiency.
324 Parameters
325 ----------
326 data : scipy.sparse.spmatrix
327 Must be convertible to csc format.
328 index, columns : Index, optional
329 Row and column labels to use for the resulting DataFrame.
330 Defaults to a RangeIndex.
332 Returns
333 -------
334 DataFrame
335 Each column of the DataFrame is stored as a
336 :class:`arrays.SparseArray`.
338 See Also
339 --------
340 DataFrame.sparse.to_coo : Return the contents of the frame as a
341 sparse SciPy COO matrix.
343 Examples
344 --------
345 >>> import scipy.sparse
346 >>> mat = scipy.sparse.eye(3, dtype=float)
347 >>> pd.DataFrame.sparse.from_spmatrix(mat)
348 0 1 2
349 0 1.0 0.0 0.0
350 1 0.0 1.0 0.0
351 2 0.0 0.0 1.0
352 """
353 from pandas._libs.sparse import IntIndex
355 from pandas import DataFrame
357 data = data.tocsc()
358 index, columns = cls._prep_index(data, index, columns)
359 n_rows, n_columns = data.shape
360 # We need to make sure indices are sorted, as we create
361 # IntIndex with no input validation (i.e. check_integrity=False ).
362 # Indices may already be sorted in scipy in which case this adds
363 # a small overhead.
364 data.sort_indices()
365 indices = data.indices
366 indptr = data.indptr
367 array_data = data.data
368 # scipy's unstored entries are zeros, not missing values, see GH#59212
369 zero = np.array(0, dtype=array_data.dtype).item()
370 dtype = SparseDtype(array_data.dtype, zero)
371 arrays = []
372 for i in range(n_columns):
373 sl = slice(indptr[i], indptr[i + 1])
374 idx = IntIndex(n_rows, indices[sl], check_integrity=False)
375 arr = SparseArray._simple_new(array_data[sl], idx, dtype)
376 arrays.append(arr)
377 return DataFrame._from_arrays(
378 arrays, columns=columns, index=index, verify_integrity=False
379 )
381 def to_dense(self) -> DataFrame:
382 """
383 Convert a DataFrame with sparse values to dense.
385 This method converts all SparseArray columns in the DataFrame to
386 regular NumPy arrays.
388 Returns
389 -------
390 DataFrame
391 A DataFrame with the same values stored as dense arrays.
393 See Also
394 --------
395 DataFrame.sparse.density : Ratio of non-sparse points to total
396 (dense) data points.
398 Examples
399 --------
400 >>> df = pd.DataFrame({"A": pd.arrays.SparseArray([0, 1, 0])})
401 >>> df.sparse.to_dense()
402 A
403 0 0
404 1 1
405 2 0
406 """
407 data = {k: v.array.to_dense() for k, v in self._parent.items()}
408 return self._parent._constructor(
409 data, index=self._parent.index, columns=self._parent.columns
410 )
412 def to_coo(self) -> spmatrix:
413 """
414 Return the contents of the frame as a sparse SciPy COO matrix.
416 This method converts the DataFrame to a scipy COO (Coordinate) sparse
417 matrix format for interoperability with scipy.sparse routines.
419 Returns
420 -------
421 scipy.sparse.spmatrix
422 If the caller is heterogeneous and contains booleans or objects,
423 the result will be of dtype=object. See Notes.
425 Raises
426 ------
427 ValueError
428 If any column's ``fill_value`` is not zero.
430 See Also
431 --------
432 DataFrame.sparse.to_dense : Convert a DataFrame with sparse values to dense.
434 Notes
435 -----
436 The dtype will be the lowest-common-denominator type (implicit
437 upcasting); that is to say if the dtypes (even of numeric types)
438 are mixed, the one that accommodates all will be chosen.
440 e.g. If the dtypes are float16 and float32, dtype will be upcast to
441 float32. By numpy.find_common_type convention, mixing int64 and
442 and uint64 will result in a float64 dtype.
444 Examples
445 --------
446 >>> df = pd.DataFrame({"A": pd.arrays.SparseArray([0, 1, 0, 1])})
447 >>> df.sparse.to_coo()
448 <COOrdinate sparse matrix of dtype 'int64'
449 with 2 stored elements and shape (4, 1)>
450 """
451 import_optional_dependency("scipy")
452 from scipy.sparse import coo_matrix
454 dtype = find_common_type(self._parent.dtypes.to_list())
455 if isinstance(dtype, SparseDtype):
456 dtype = dtype.subtype
458 cols, rows, data = [], [], []
459 for col, (_, ser) in enumerate(self._parent.items()):
460 sp_arr = ser.array
461 # isna first: `pd.NA != 0` is NA, which is not usable as a condition
462 if isna(sp_arr.fill_value) or sp_arr.fill_value != 0:
463 raise ValueError("fill value must be 0 when converting to COO matrix")
465 row = sp_arr.sp_index.indices
466 cols.append(np.repeat(col, len(row)))
467 rows.append(row)
468 data.append(sp_arr.sp_values.astype(dtype, copy=False))
470 cols_arr = np.concatenate(cols)
471 rows_arr = np.concatenate(rows)
472 data_arr = np.concatenate(data)
473 return coo_matrix((data_arr, (rows_arr, cols_arr)), shape=self._parent.shape)
475 @property
476 def density(self) -> float:
477 """
478 Ratio of non-sparse points to total (dense) data points.
480 This property returns the proportion of the data that is not sparse
481 (i.e., not equal to the fill value), as a value between 0 and 1. An
482 empty DataFrame has no density and returns ``nan``.
484 See Also
485 --------
486 DataFrame.sparse.from_spmatrix : Create a new DataFrame from a
487 scipy sparse matrix.
489 Examples
490 --------
491 >>> df = pd.DataFrame({"A": pd.arrays.SparseArray([0, 1, 0, 1])})
492 >>> df.sparse.density
493 np.float64(0.5)
494 """
495 if len(self._parent.columns) == 0:
496 # mean of an empty list is undefined
497 return np.nan
498 return np.mean([column.array.density for _, column in self._parent.items()])
500 @staticmethod
501 def _prep_index(data, index, columns):
502 from pandas.core.indexes.api import (
503 default_index,
504 ensure_index,
505 )
507 N, K = data.shape
508 if index is None:
509 index = default_index(N)
510 else:
511 index = ensure_index(index)
512 if columns is None:
513 columns = default_index(K)
514 else:
515 columns = ensure_index(columns)
517 if len(columns) != K:
518 raise ValueError(f"Column length mismatch: {len(columns)} vs. {K}")
519 if len(index) != N:
520 raise ValueError(f"Index length mismatch: {len(index)} vs. {N}")
521 return index, columns