Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Sparse accessor"""
3from __future__ import annotations
5from typing import TYPE_CHECKING
7import numpy as np
9from pandas.compat._optional import import_optional_dependency
11from pandas.core.dtypes.cast import find_common_type
12from pandas.core.dtypes.dtypes import SparseDtype
14from pandas.core.accessor import (
15 PandasDelegate,
16 delegate_names,
17)
18from pandas.core.arrays.sparse.array import SparseArray
20if TYPE_CHECKING:
21 from scipy.sparse import (
22 coo_matrix,
23 spmatrix,
24 )
26 from pandas import (
27 DataFrame,
28 Series,
29 )
32class BaseAccessor:
33 _validation_msg = "Can only use the '.sparse' accessor with Sparse data."
35 def __init__(self, data=None) -> None:
36 self._parent = data
37 self._validate(data)
39 def _validate(self, data) -> None:
40 raise NotImplementedError
43@delegate_names(
44 SparseArray, ["npoints", "density", "fill_value", "sp_values"], typ="property"
45)
46class SparseAccessor(BaseAccessor, PandasDelegate):
47 """
48 Accessor for SparseSparse from other sparse matrix data types.
50 Parameters
51 ----------
52 data : Series or DataFrame
53 The Series or DataFrame to which the SparseAccessor is attached.
55 See Also
56 --------
57 Series.sparse.to_coo : Create a scipy.sparse.coo_matrix from a Series with
58 MultiIndex.
59 Series.sparse.from_coo : Create a Series with sparse values from a
60 scipy.sparse.coo_matrix.
62 Examples
63 --------
64 >>> ser = pd.Series([0, 0, 2, 2, 2], dtype="Sparse[int]")
65 >>> ser.sparse.density
66 0.6
67 >>> ser.sparse.sp_values
68 array([2, 2, 2])
69 """
71 def _validate(self, data) -> None:
72 if not isinstance(data.dtype, SparseDtype):
73 raise AttributeError(self._validation_msg)
75 def _delegate_property_get(self, name: str, *args, **kwargs):
76 return getattr(self._parent.array, name)
78 def _delegate_method(self, name: str, *args, **kwargs):
79 if name == "from_coo":
80 return self.from_coo(*args, **kwargs)
81 elif name == "to_coo":
82 return self.to_coo(*args, **kwargs)
83 else:
84 raise ValueError
86 @classmethod
87 def from_coo(cls, A, dense_index: bool = False) -> Series:
88 """
89 Create a Series with sparse values from a scipy.sparse.coo_matrix.
91 This method takes a ``scipy.sparse.coo_matrix`` (coordinate format) as input and
92 returns a pandas ``Series`` where the non-zero elements are represented as
93 sparse values. The index of the Series can either include only the coordinates
94 of non-zero elements (default behavior) or the full sorted set of coordinates
95 from the matrix if ``dense_index`` is set to `True`.
97 Parameters
98 ----------
99 A : scipy.sparse.coo_matrix
100 The sparse matrix in coordinate format from which the sparse Series
101 will be created.
102 dense_index : bool, default False
103 If False (default), the index consists of only the
104 coords of the non-null entries of the original coo_matrix.
105 If True, the index consists of the full sorted
106 (row, col) coordinates of the coo_matrix.
108 Returns
109 -------
110 s : Series
111 A Series with sparse values.
113 See Also
114 --------
115 DataFrame.sparse.from_spmatrix : Create a new DataFrame from a scipy sparse
116 matrix.
117 scipy.sparse.coo_matrix : A sparse matrix in COOrdinate format.
119 Examples
120 --------
121 >>> from scipy import sparse
123 >>> A = sparse.coo_matrix(
124 ... ([3.0, 1.0, 2.0], ([1, 0, 0], [0, 2, 3])), shape=(3, 4)
125 ... )
126 >>> A
127 <COOrdinate sparse matrix of dtype 'float64'
128 with 3 stored elements and shape (3, 4)>
130 >>> A.todense()
131 matrix([[0., 0., 1., 2.],
132 [3., 0., 0., 0.],
133 [0., 0., 0., 0.]])
135 >>> ss = pd.Series.sparse.from_coo(A)
136 >>> ss
137 0 2 1.0
138 3 2.0
139 1 0 3.0
140 dtype: Sparse[float64, nan]
141 """
142 from pandas import Series
143 from pandas.core.arrays.sparse.scipy_sparse import coo_to_sparse_series
145 result = coo_to_sparse_series(A, dense_index=dense_index)
146 result = Series(result.array, index=result.index, copy=False)
148 return result
150 def to_coo(
151 self, row_levels=(0,), column_levels=(1,), sort_labels: bool = False
152 ) -> tuple[coo_matrix, list, list]:
153 """
154 Create a scipy.sparse.coo_matrix from a Series with MultiIndex.
156 Use row_levels and column_levels to determine the row and column
157 coordinates respectively. row_levels and column_levels are the names
158 (labels) or numbers of the levels. {row_levels, column_levels} must be
159 a partition of the MultiIndex level names (or numbers).
161 Parameters
162 ----------
163 row_levels : tuple/list
164 MultiIndex levels to use for row coordinates, specified by name or index.
165 column_levels : tuple/list
166 MultiIndex levels to use for column coordinates, specified by name or index.
167 sort_labels : bool, default False
168 Sort the row and column labels before forming the sparse matrix.
169 When `row_levels` and/or `column_levels` refer to a single level,
170 set to `True` for a faster execution.
172 Returns
173 -------
174 y : scipy.sparse.coo_matrix
175 The sparse matrix in coordinate format.
176 rows : list (row labels)
177 Labels corresponding to the row coordinates.
178 columns : list (column labels)
179 Labels corresponding to the column coordinates.
181 See Also
182 --------
183 Series.sparse.from_coo : Create a Series with sparse values from a
184 scipy.sparse.coo_matrix.
186 Examples
187 --------
188 >>> s = pd.Series([3.0, np.nan, 1.0, 3.0, np.nan, np.nan])
189 >>> s.index = pd.MultiIndex.from_tuples(
190 ... [
191 ... (1, 2, "a", 0),
192 ... (1, 2, "a", 1),
193 ... (1, 1, "b", 0),
194 ... (1, 1, "b", 1),
195 ... (2, 1, "b", 0),
196 ... (2, 1, "b", 1),
197 ... ],
198 ... names=["A", "B", "C", "D"],
199 ... )
200 >>> s
201 A B C D
202 1 2 a 0 3.0
203 1 NaN
204 1 b 0 1.0
205 1 3.0
206 2 1 b 0 NaN
207 1 NaN
208 dtype: float64
210 >>> ss = s.astype("Sparse")
211 >>> ss
212 A B C D
213 1 2 a 0 3.0
214 1 NaN
215 1 b 0 1.0
216 1 3.0
217 2 1 b 0 NaN
218 1 NaN
219 dtype: Sparse[float64, nan]
221 >>> A, rows, columns = ss.sparse.to_coo(
222 ... row_levels=["A", "B"], column_levels=["C", "D"], sort_labels=True
223 ... )
224 >>> A
225 <COOrdinate sparse matrix of dtype 'float64'
226 with 3 stored elements and shape (3, 4)>
227 >>> A.todense()
228 matrix([[0., 0., 1., 3.],
229 [3., 0., 0., 0.],
230 [0., 0., 0., 0.]])
232 >>> rows
233 [(1, 1), (1, 2), (2, 1)]
234 >>> columns
235 [('a', 0), ('a', 1), ('b', 0), ('b', 1)]
236 """
237 from pandas.core.arrays.sparse.scipy_sparse import sparse_series_to_coo
239 A, rows, columns = sparse_series_to_coo(
240 self._parent, row_levels, column_levels, sort_labels=sort_labels
241 )
242 return A, rows, columns
244 def to_dense(self) -> Series:
245 """
246 Convert a Series from sparse values to dense.
248 Returns
249 -------
250 Series:
251 A Series with the same values, stored as a dense array.
253 Examples
254 --------
255 >>> series = pd.Series(pd.arrays.SparseArray([0, 1, 0]))
256 >>> series
257 0 0
258 1 1
259 2 0
260 dtype: Sparse[int64, 0]
262 >>> series.sparse.to_dense()
263 0 0
264 1 1
265 2 0
266 dtype: int64
267 """
268 from pandas import Series
270 return Series(
271 self._parent.array.to_dense(),
272 index=self._parent.index,
273 name=self._parent.name,
274 copy=False,
275 )
278class SparseFrameAccessor(BaseAccessor, PandasDelegate):
279 """
280 DataFrame accessor for sparse data.
282 It allows users to interact with a `DataFrame` that contains sparse data types
283 (`SparseDtype`). It provides methods and attributes to efficiently work with sparse
284 storage, reducing memory usage while maintaining compatibility with standard pandas
285 operations.
287 Parameters
288 ----------
289 data : scipy.sparse.spmatrix
290 Must be convertible to csc format.
292 See Also
293 --------
294 DataFrame.sparse.density : Ratio of non-sparse points to total (dense) data points.
296 Examples
297 --------
298 >>> df = pd.DataFrame({"a": [1, 2, 0, 0], "b": [3, 0, 0, 4]}, dtype="Sparse[int]")
299 >>> df.sparse.density
300 np.float64(0.5)
301 """
303 def _validate(self, data) -> None:
304 dtypes = data.dtypes
305 if not all(isinstance(t, SparseDtype) for t in dtypes):
306 raise AttributeError(self._validation_msg)
308 @classmethod
309 def from_spmatrix(cls, data, index=None, columns=None) -> DataFrame:
310 """
311 Create a new DataFrame from a scipy sparse matrix.
313 Parameters
314 ----------
315 data : scipy.sparse.spmatrix
316 Must be convertible to csc format.
317 index, columns : Index, optional
318 Row and column labels to use for the resulting DataFrame.
319 Defaults to a RangeIndex.
321 Returns
322 -------
323 DataFrame
324 Each column of the DataFrame is stored as a
325 :class:`arrays.SparseArray`.
327 See Also
328 --------
329 DataFrame.sparse.to_coo : Return the contents of the frame as a
330 sparse SciPy COO matrix.
332 Examples
333 --------
334 >>> import scipy.sparse
335 >>> mat = scipy.sparse.eye(3, dtype=int)
336 >>> pd.DataFrame.sparse.from_spmatrix(mat)
337 0 1 2
338 0 1 0 0
339 1 0 1 0
340 2 0 0 1
341 """
342 from pandas._libs.sparse import IntIndex
344 from pandas import DataFrame
346 data = data.tocsc()
347 index, columns = cls._prep_index(data, index, columns)
348 n_rows, n_columns = data.shape
349 # We need to make sure indices are sorted, as we create
350 # IntIndex with no input validation (i.e. check_integrity=False ).
351 # Indices may already be sorted in scipy in which case this adds
352 # a small overhead.
353 data.sort_indices()
354 indices = data.indices
355 indptr = data.indptr
356 array_data = data.data
357 dtype = SparseDtype(array_data.dtype)
358 arrays = []
359 for i in range(n_columns):
360 sl = slice(indptr[i], indptr[i + 1])
361 idx = IntIndex(n_rows, indices[sl], check_integrity=False)
362 arr = SparseArray._simple_new(array_data[sl], idx, dtype)
363 arrays.append(arr)
364 return DataFrame._from_arrays(
365 arrays, columns=columns, index=index, verify_integrity=False
366 )
368 def to_dense(self) -> DataFrame:
369 """
370 Convert a DataFrame with sparse values to dense.
372 Returns
373 -------
374 DataFrame
375 A DataFrame with the same values stored as dense arrays.
377 See Also
378 --------
379 DataFrame.sparse.density : Ratio of non-sparse points to total
380 (dense) data points.
382 Examples
383 --------
384 >>> df = pd.DataFrame({"A": pd.arrays.SparseArray([0, 1, 0])})
385 >>> df.sparse.to_dense()
386 A
387 0 0
388 1 1
389 2 0
390 """
391 data = {k: v.array.to_dense() for k, v in self._parent.items()}
392 return self._parent._constructor(
393 data, index=self._parent.index, columns=self._parent.columns
394 )
396 def to_coo(self) -> spmatrix:
397 """
398 Return the contents of the frame as a sparse SciPy COO matrix.
400 Returns
401 -------
402 scipy.sparse.spmatrix
403 If the caller is heterogeneous and contains booleans or objects,
404 the result will be of dtype=object. See Notes.
406 See Also
407 --------
408 DataFrame.sparse.to_dense : Convert a DataFrame with sparse values to dense.
410 Notes
411 -----
412 The dtype will be the lowest-common-denominator type (implicit
413 upcasting); that is to say if the dtypes (even of numeric types)
414 are mixed, the one that accommodates all will be chosen.
416 e.g. If the dtypes are float16 and float32, dtype will be upcast to
417 float32. By numpy.find_common_type convention, mixing int64 and
418 and uint64 will result in a float64 dtype.
420 Examples
421 --------
422 >>> df = pd.DataFrame({"A": pd.arrays.SparseArray([0, 1, 0, 1])})
423 >>> df.sparse.to_coo()
424 <COOrdinate sparse matrix of dtype 'int64'
425 with 2 stored elements and shape (4, 1)>
426 """
427 import_optional_dependency("scipy")
428 from scipy.sparse import coo_matrix
430 dtype = find_common_type(self._parent.dtypes.to_list())
431 if isinstance(dtype, SparseDtype):
432 dtype = dtype.subtype
434 cols, rows, data = [], [], []
435 for col, (_, ser) in enumerate(self._parent.items()):
436 sp_arr = ser.array
438 row = sp_arr.sp_index.indices
439 cols.append(np.repeat(col, len(row)))
440 rows.append(row)
441 data.append(sp_arr.sp_values.astype(dtype, copy=False))
443 cols_arr = np.concatenate(cols)
444 rows_arr = np.concatenate(rows)
445 data_arr = np.concatenate(data)
446 return coo_matrix((data_arr, (rows_arr, cols_arr)), shape=self._parent.shape)
448 @property
449 def density(self) -> float:
450 """
451 Ratio of non-sparse points to total (dense) data points.
453 See Also
454 --------
455 DataFrame.sparse.from_spmatrix : Create a new DataFrame from a
456 scipy sparse matrix.
458 Examples
459 --------
460 >>> df = pd.DataFrame({"A": pd.arrays.SparseArray([0, 1, 0, 1])})
461 >>> df.sparse.density
462 np.float64(0.5)
463 """
464 tmp = np.mean([column.array.density for _, column in self._parent.items()])
465 return tmp
467 @staticmethod
468 def _prep_index(data, index, columns):
469 from pandas.core.indexes.api import (
470 default_index,
471 ensure_index,
472 )
474 N, K = data.shape
475 if index is None:
476 index = default_index(N)
477 else:
478 index = ensure_index(index)
479 if columns is None:
480 columns = default_index(K)
481 else:
482 columns = ensure_index(columns)
484 if len(columns) != K:
485 raise ValueError(f"Column length mismatch: {len(columns)} vs. {K}")
486 if len(index) != N:
487 raise ValueError(f"Index length mismatch: {len(index)} vs. {N}")
488 return index, columns