Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2SparseArray data structure
3"""
5from __future__ import annotations
7from collections import abc
8import numbers
9import operator
10from typing import (
11 TYPE_CHECKING,
12 Any,
13 Literal,
14 Self,
15 cast,
16 overload,
17)
18import warnings
20import numpy as np
22from pandas._config.config import get_option
24from pandas._libs import lib
25import pandas._libs.sparse as splib
26from pandas._libs.sparse import (
27 BlockIndex,
28 IntIndex,
29 SparseIndex,
30)
31from pandas._libs.tslibs import NaT
32from pandas.compat.numpy import function as nv
33from pandas.errors import PerformanceWarning
34from pandas.util._decorators import (
35 doc,
36 set_module,
37)
38from pandas.util._exceptions import find_stack_level
39from pandas.util._validators import (
40 validate_bool_kwarg,
41 validate_insert_loc,
42)
44from pandas.core.dtypes.astype import astype_array
45from pandas.core.dtypes.cast import (
46 find_common_type,
47 maybe_box_datetimelike,
48)
49from pandas.core.dtypes.common import (
50 is_bool_dtype,
51 is_integer,
52 is_list_like,
53 is_object_dtype,
54 is_scalar,
55 is_string_dtype,
56 pandas_dtype,
57)
58from pandas.core.dtypes.dtypes import (
59 DatetimeTZDtype,
60 SparseDtype,
61)
62from pandas.core.dtypes.generic import (
63 ABCIndex,
64 ABCSeries,
65)
66from pandas.core.dtypes.missing import (
67 isna,
68 na_value_for_dtype,
69 notna,
70)
72from pandas.core import arraylike
73import pandas.core.algorithms as algos
74from pandas.core.arraylike import OpsMixin
75from pandas.core.arrays import ExtensionArray
76from pandas.core.base import PandasObject
77import pandas.core.common as com
78from pandas.core.construction import (
79 ensure_wrapped_if_datetimelike,
80 extract_array,
81 sanitize_array,
82)
83from pandas.core.indexers import (
84 check_array_indexer,
85 unpack_tuple_and_ellipses,
86)
87from pandas.core.nanops import check_below_min_count
89from pandas.io.formats import printing
91if TYPE_CHECKING:
92 from collections.abc import (
93 Callable,
94 Sequence,
95 )
96 from types import EllipsisType
97 from typing import (
98 Protocol,
99 type_check_only,
100 )
102 from scipy.sparse import (
103 csc_array,
104 csc_matrix,
105 )
107 @type_check_only
108 class _SparseMatrixLike(Protocol):
109 @property
110 def shape(self, /) -> tuple[int, int]: ...
111 def tocsc(self, /) -> csc_array | csc_matrix: ...
113 from pandas._typing import NumpySorter
115 SparseIndexKind = Literal["integer", "block"]
117 from pandas._typing import (
118 ArrayLike,
119 AstypeArg,
120 Axis,
121 AxisInt,
122 Dtype,
123 NpDtype,
124 PositionalIndexer,
125 Scalar,
126 ScalarIndexer,
127 SequenceIndexer,
128 npt,
129 )
131 from pandas import Series
134# ----------------------------------------------------------------------------
135# Array
137_sparray_doc_kwargs = {"klass": "SparseArray"}
140def _get_fill(arr: SparseArray) -> np.ndarray:
141 """
142 Create a 0-dim ndarray containing the fill value
144 Parameters
145 ----------
146 arr : SparseArray
148 Returns
149 -------
150 fill_value : ndarray
151 0-dim ndarray with just the fill value.
153 Notes
154 -----
155 coerce fill_value to arr dtype if possible
156 int64 SparseArray can have NaN as fill_value if there is no missing
157 """
158 try:
159 return np.asarray(arr.fill_value, dtype=arr.dtype.subtype)
160 except ValueError:
161 return np.asarray(arr.fill_value)
164def _sparse_array_op(
165 left: SparseArray, right: SparseArray, op: Callable, name: str
166) -> SparseArray:
167 """
168 Perform a binary operation between two arrays.
170 Parameters
171 ----------
172 left : Union[SparseArray, ndarray]
173 right : Union[SparseArray, ndarray]
174 op : Callable
175 The binary operation to perform
176 name str
177 Name of the callable.
179 Returns
180 -------
181 SparseArray
182 """
183 if name.startswith("__"):
184 # For lookups in _libs.sparse we need non-dunder op name
185 name = name[2:-2]
187 # dtype used to find corresponding sparse method
188 ltype = left.dtype.subtype
189 rtype = right.dtype.subtype
191 if ltype != rtype:
192 subtype = find_common_type([ltype, rtype])
193 ltype = SparseDtype(subtype, left.fill_value)
194 rtype = SparseDtype(subtype, right.fill_value)
196 left = left.astype(ltype, copy=False)
197 right = right.astype(rtype, copy=False)
198 dtype = ltype.subtype
199 else:
200 dtype = ltype
202 # dtype the result must have
203 result_dtype = None
205 if left.sp_index.ngaps == 0 or right.sp_index.ngaps == 0:
206 with np.errstate(all="ignore"):
207 result = op(left.to_dense(), right.to_dense())
208 fill = op(_get_fill(left), _get_fill(right))
210 if left.sp_index.ngaps == 0:
211 index = left.sp_index
212 else:
213 index = right.sp_index
214 elif left.sp_index.equals(right.sp_index):
215 with np.errstate(all="ignore"):
216 result = op(left.sp_values, right.sp_values)
217 fill = op(_get_fill(left), _get_fill(right))
218 index = left.sp_index
219 else:
220 if name[0] == "r":
221 left, right = right, left
222 name = name[1:]
224 if name in ("and", "or", "xor") and dtype == "bool":
225 opname = f"sparse_{name}_uint8"
226 # to make template simple, cast here
227 left_sp_values = left.sp_values.view(np.uint8)
228 right_sp_values = right.sp_values.view(np.uint8)
229 result_dtype = bool
230 else:
231 opname = f"sparse_{name}_{dtype}"
232 left_sp_values = left.sp_values
233 right_sp_values = right.sp_values
235 if (
236 name in ["floordiv", "mod"]
237 and (right == 0).any()
238 and left.dtype.kind in "iu"
239 ):
240 # Match the non-Sparse Series behavior
241 opname = f"sparse_{name}_float64"
242 left_sp_values = left_sp_values.astype("float64")
243 right_sp_values = right_sp_values.astype("float64")
245 sparse_op = getattr(splib, opname)
247 with np.errstate(all="ignore"):
248 result, index, fill = sparse_op(
249 left_sp_values,
250 left.sp_index,
251 left.fill_value,
252 right_sp_values,
253 right.sp_index,
254 right.fill_value,
255 )
257 if name == "divmod":
258 # result is a 2-tuple
259 # error: Incompatible return value type (got "Tuple[SparseArray,
260 # SparseArray]", expected "SparseArray")
261 return ( # type: ignore[return-value]
262 _wrap_result(name, result[0], index, fill[0], dtype=result_dtype),
263 _wrap_result(name, result[1], index, fill[1], dtype=result_dtype),
264 )
266 if result_dtype is None:
267 result_dtype = result.dtype
269 return _wrap_result(name, result, index, fill, dtype=result_dtype)
272def _wrap_result(
273 name: str, data, sparse_index, fill_value, dtype: Dtype | None = None
274) -> SparseArray:
275 """
276 wrap op result to have correct dtype
277 """
278 if name.startswith("__"):
279 # e.g. __eq__ --> eq
280 name = name[2:-2]
282 if name in ("eq", "ne", "lt", "gt", "le", "ge"):
283 dtype = bool
285 fill_value = lib.item_from_zerodim(fill_value)
287 if is_bool_dtype(dtype):
288 # fill_value may be np.bool_
289 fill_value = bool(fill_value)
290 return SparseArray(
291 data, sparse_index=sparse_index, fill_value=fill_value, dtype=dtype
292 )
295@set_module("pandas.arrays")
296class SparseArray(OpsMixin, PandasObject, ExtensionArray):
297 """
298 An ExtensionArray for storing sparse data.
300 SparseArray efficiently stores data with a high frequency of a
301 specific fill value (e.g., zeros), saving memory by only retaining
302 non-fill elements and their indices. This class is particularly
303 useful for large datasets where most values are redundant.
305 Parameters
306 ----------
307 data : array-like or scalar
308 A dense array of values to store in the SparseArray. This may contain
309 `fill_value`.
310 sparse_index : SparseIndex, optional
311 Index indicating the locations of sparse elements.
312 fill_value : scalar, optional
313 Elements in data that are ``fill_value`` are not stored in the
314 SparseArray. For memory savings, this should be the most common value
315 in `data`. By default, `fill_value` depends on the dtype of `data`:
317 =========== ==========
318 data.dtype na_value
319 =========== ==========
320 float ``np.nan``
321 int ``0``
322 bool False
323 datetime64 ``pd.NaT``
324 timedelta64 ``pd.NaT``
325 =========== ==========
327 The fill value is potentially specified in three ways. In order of
328 precedence, these are
330 1. The `fill_value` argument
331 2. ``dtype.fill_value`` if `fill_value` is None and `dtype` is
332 a ``SparseDtype``
333 3. ``data.dtype.fill_value`` if `fill_value` is None and `dtype`
334 is not a ``SparseDtype`` and `data` is a ``SparseArray``.
336 kind : str
337 Can be 'integer' or 'block', default is 'integer'.
338 The type of storage for sparse locations.
340 * 'block': Stores a `block` and `block_length` for each
341 contiguous *span* of sparse values. This is best when
342 sparse data tends to be clumped together, with large
343 regions of ``fill-value`` values between sparse values.
344 * 'integer': uses an integer to store the location of
345 each sparse value.
347 dtype : np.dtype or SparseDtype, optional
348 The dtype to use for the SparseArray. For numpy dtypes, this
349 determines the dtype of ``self.sp_values``. For SparseDtype,
350 this determines ``self.sp_values`` and ``self.fill_value``.
351 copy : bool, default False
352 Whether to explicitly copy the incoming `data` array.
354 Attributes
355 ----------
356 None
358 Methods
359 -------
360 None
362 See Also
363 --------
364 SparseDtype : Dtype for sparse data.
366 Examples
367 --------
368 >>> from pandas.arrays import SparseArray
369 >>> arr = SparseArray([0, 0, 1, 2])
370 >>> arr
371 [0, 0, 1, 2]
372 Fill: 0
373 IntIndex
374 Indices: array([2, 3], dtype=int32)
375 """
377 _subtyp = "sparse_array" # register ABCSparseArray
378 _hidden_attrs = PandasObject._hidden_attrs | frozenset([])
379 _sparse_index: SparseIndex
380 _sparse_values: np.ndarray
381 _dtype: SparseDtype
383 def __init__(
384 self,
385 data,
386 sparse_index=None,
387 fill_value=None,
388 kind: SparseIndexKind = "integer",
389 dtype: Dtype | None = None,
390 copy: bool = False,
391 ) -> None:
392 if fill_value is None and isinstance(dtype, SparseDtype):
393 fill_value = dtype.fill_value
395 if isinstance(data, type(self)):
396 # disable normal inference on dtype, sparse_index, & fill_value
397 if sparse_index is None:
398 sparse_index = data.sp_index
399 if fill_value is None:
400 fill_value = data.fill_value
401 if dtype is None:
402 dtype = data.dtype
403 # TODO: make kind=None, and use data.kind?
404 data = data.sp_values
406 # Handle use-provided dtype
407 if isinstance(dtype, str):
408 # Two options: dtype='int', regular numpy dtype
409 # or dtype='Sparse[int]', a sparse dtype
410 try:
411 dtype = SparseDtype.construct_from_string(dtype)
412 except TypeError:
413 dtype = pandas_dtype(dtype)
415 if isinstance(dtype, SparseDtype):
416 if fill_value is None:
417 fill_value = dtype.fill_value
418 dtype = dtype.subtype
420 if is_scalar(data):
421 raise TypeError(
422 f"Cannot construct {type(self).__name__} from scalar data. "
423 "Pass a sequence instead."
424 )
426 if dtype is not None:
427 dtype = pandas_dtype(dtype)
429 # TODO: disentangle the fill_value dtype inference from
430 # dtype inference
431 if data is None:
432 # TODO: What should the empty dtype be? Object or float?
434 # error: Argument "dtype" to "array" has incompatible type
435 # "Union[ExtensionDtype, dtype[Any], None]"; expected "Union[dtype[Any],
436 # None, type, _SupportsDType, str, Union[Tuple[Any, int], Tuple[Any,
437 # Union[int, Sequence[int]]], List[Any], _DTypeDict, Tuple[Any, Any]]]"
438 data = np.array([], dtype=dtype) # type: ignore[arg-type]
440 try:
441 data = sanitize_array(data, index=None)
442 except ValueError:
443 # NumPy may raise a ValueError on data like [1, []]
444 # we retry with object dtype here.
445 if dtype is None:
446 dtype = np.dtype(object)
447 data = np.atleast_1d(np.asarray(data, dtype=dtype))
448 else:
449 raise
451 if copy:
452 # TODO: avoid double copy when dtype forces cast.
453 data = data.copy()
455 if fill_value is None:
456 fill_value_dtype = data.dtype if dtype is None else dtype
457 if fill_value_dtype is None:
458 fill_value = np.nan
459 else:
460 fill_value = na_value_for_dtype(fill_value_dtype)
462 if isinstance(data, type(self)) and sparse_index is None:
463 sparse_index = data._sparse_index
464 # error: Argument "dtype" to "asarray" has incompatible type
465 # "Union[ExtensionDtype, dtype[Any], None]"; expected "None"
466 sparse_values = np.asarray(
467 data.sp_values,
468 dtype=dtype, # type: ignore[arg-type]
469 )
470 elif sparse_index is None:
471 data = extract_array(data, extract_numpy=True)
472 if not isinstance(data, np.ndarray):
473 # EA
474 if isinstance(data.dtype, DatetimeTZDtype):
475 warnings.warn(
476 f"Creating SparseArray from {data.dtype} data "
477 "loses timezone information. Cast to object before "
478 "sparse to retain timezone information.",
479 UserWarning,
480 stacklevel=find_stack_level(),
481 )
482 data = np.asarray(data, dtype="datetime64[ns]")
483 if fill_value is NaT:
484 fill_value = np.datetime64("NaT", "ns")
485 data = np.asarray(data)
486 sparse_values, sparse_index, fill_value = _make_sparse(
487 # error: Argument "dtype" to "_make_sparse" has incompatible type
488 # "Union[ExtensionDtype, dtype[Any], None]"; expected
489 # "Optional[dtype[Any]]"
490 data,
491 kind=kind,
492 fill_value=fill_value,
493 dtype=dtype, # type: ignore[arg-type]
494 )
495 else:
496 # error: Argument "dtype" to "asarray" has incompatible type
497 # "Union[ExtensionDtype, dtype[Any], None]"; expected "None"
498 sparse_values = np.asarray(data, dtype=dtype) # type: ignore[arg-type]
499 if len(sparse_values) != sparse_index.npoints:
500 raise AssertionError(
501 f"Non array-like type {type(sparse_values)} must "
502 "have the same length as the index"
503 )
504 self._sparse_index = sparse_index
505 self._sparse_values = sparse_values
506 self._dtype = SparseDtype(sparse_values.dtype, fill_value)
508 @classmethod
509 def _simple_new(
510 cls,
511 sparse_array: np.ndarray,
512 sparse_index: SparseIndex,
513 dtype: SparseDtype,
514 ) -> Self:
515 new = object.__new__(cls)
516 new._sparse_index = sparse_index
517 new._sparse_values = sparse_array
518 new._dtype = dtype
519 return new
521 @classmethod
522 def from_spmatrix(cls, data: _SparseMatrixLike) -> Self:
523 """
524 Create a SparseArray from a scipy.sparse matrix.
526 Parameters
527 ----------
528 data : scipy.sparse.sp_matrix
529 This should be a SciPy sparse matrix where the size
530 of the second dimension is 1. In other words, a
531 sparse matrix with a single column.
533 Returns
534 -------
535 SparseArray
537 Examples
538 --------
539 >>> import scipy.sparse
540 >>> mat = scipy.sparse.coo_matrix((4, 1))
541 >>> pd.arrays.SparseArray.from_spmatrix(mat)
542 [0.0, 0.0, 0.0, 0.0]
543 Fill: 0.0
544 IntIndex
545 Indices: array([], dtype=int32)
546 """
547 length, ncol = data.shape
549 if ncol != 1:
550 raise ValueError(f"'data' must have a single column, not '{ncol}'")
552 # our sparse index classes require that the positions be strictly
553 # increasing. So we need to sort loc, and arr accordingly.
554 data_csc = data.tocsc()
555 data_csc.sort_indices()
556 arr = data_csc.data
557 idx = data_csc.indices
559 zero = np.array(0, dtype=arr.dtype).item()
560 dtype = SparseDtype(arr.dtype, zero)
561 index = IntIndex(length, idx)
563 return cls._simple_new(arr, index, dtype)
565 def __array__(
566 self, dtype: NpDtype | None = None, copy: bool | None = None
567 ) -> np.ndarray:
568 if self.sp_index.ngaps == 0:
569 # Compat for na dtype and int values.
570 if copy is True:
571 return np.array(self.sp_values)
572 else:
573 result = self.sp_values
574 if self._readonly:
575 result = result.view()
576 result.flags.writeable = False
577 return result
579 if copy is False:
580 raise ValueError(
581 "Unable to avoid copy while creating an array as requested."
582 )
584 fill_value = self.fill_value
586 if dtype is None:
587 # Can NumPy represent this type?
588 # If not, `np.result_type` will raise. We catch that
589 # and return object.
590 if self.sp_values.dtype.kind == "M":
591 # However, we *do* special-case the common case of
592 # a datetime64 with pandas NaT.
593 if fill_value is NaT:
594 # Can't put pd.NaT in a datetime64[ns]
595 unit = np.datetime_data(self.sp_values.dtype)[0]
596 fill_value = np.datetime64("NaT", unit) # type: ignore[call-overload]
597 try:
598 dtype = np.result_type(self.sp_values.dtype, type(fill_value))
599 except TypeError:
600 dtype = object
602 out = np.full(self.shape, fill_value, dtype=dtype)
603 out[self.sp_index.indices] = self.sp_values
604 return out
606 def __setitem__(self, key, value) -> None:
607 if self._readonly:
608 raise ValueError("Cannot modify read-only array")
609 # I suppose we could allow setting of non-fill_value elements.
610 # TODO(SparseArray.__setitem__): remove special cases in
611 # ExtensionBlock.where
612 msg = "SparseArray does not support item assignment via setitem"
613 raise TypeError(msg)
615 @classmethod
616 def _from_sequence(
617 cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
618 ) -> Self:
619 return cls(scalars, dtype=dtype)
621 @classmethod
622 def _from_factorized(cls, values, original) -> Self:
623 return cls(values, dtype=original.dtype)
625 def _cast_pointwise_result(self, values):
626 values = np.asarray(values, dtype=object)
627 result = lib.maybe_convert_objects(values, convert_non_numeric=True)
628 if result.dtype.kind == self.dtype.kind:
629 try:
630 # e.g. test_groupby_agg_extension
631 res = type(self)._from_sequence(result, dtype=self.dtype)
632 if ((res == result) | (isna(result) & res.isna())).all():
633 # This does not hold for e.g.
634 # test_arith_frame_with_scalar[0-__truediv__]
635 return res
636 return type(self)._from_sequence(result)
637 except (ValueError, TypeError):
638 return type(self)._from_sequence(result)
639 else:
640 # e.g. test_combine_le avoid casting bools to Sparse[float64, nan]
641 return type(self)._from_sequence(result)
643 # ------------------------------------------------------------------------
644 # Data
645 # ------------------------------------------------------------------------
646 @property
647 def sp_index(self) -> SparseIndex:
648 """
649 The SparseIndex containing the location of non- ``fill_value`` points.
650 """
651 return self._sparse_index
653 @property
654 def sp_values(self) -> np.ndarray:
655 """
656 An ndarray containing the non- ``fill_value`` values.
658 This property returns the actual data values stored in the sparse
659 representation, excluding the values that are equal to the ``fill_value``.
660 The result is an ndarray of the underlying values, preserving the sparse
661 structure by omitting the default ``fill_value`` entries.
663 See Also
664 --------
665 Series.sparse.to_dense : Convert a Series from sparse values to dense.
666 Series.sparse.fill_value : Elements in `data` that are `fill_value` are
667 not stored.
668 Series.sparse.density : The percent of non- ``fill_value`` points, as decimal.
670 Examples
671 --------
672 >>> from pandas.arrays import SparseArray
673 >>> s = SparseArray([0, 0, 1, 0, 2], fill_value=0)
674 >>> s.sp_values
675 array([1, 2])
676 """
677 return self._sparse_values
679 @property
680 def dtype(self) -> SparseDtype:
681 return self._dtype
683 @property
684 def fill_value(self):
685 """
686 Elements in `data` that are `fill_value` are not stored.
688 For memory savings, this should be the most common value in the array.
690 See Also
691 --------
692 SparseDtype : Dtype for data stored in :class:`SparseArray`.
693 Series.value_counts : Return a Series containing counts of unique values.
694 Series.fillna : Fill NA/NaN in a Series with a specified value.
696 Examples
697 --------
698 >>> ser = pd.Series([0, 0, 2, 2, 2], dtype="Sparse[int]")
699 >>> ser.sparse.fill_value
700 0
701 >>> spa_dtype = pd.SparseDtype(dtype=np.int32, fill_value=2)
702 >>> ser = pd.Series([0, 0, 2, 2, 2], dtype=spa_dtype)
703 >>> ser.sparse.fill_value
704 2
705 """
706 return self.dtype.fill_value
708 @fill_value.setter
709 def fill_value(self, value) -> None:
710 self._dtype = SparseDtype(self.dtype.subtype, value)
712 @property
713 def kind(self) -> SparseIndexKind:
714 """
715 The kind of sparse index for this array. One of {'integer', 'block'}.
716 """
717 if isinstance(self.sp_index, IntIndex):
718 return "integer"
719 else:
720 return "block"
722 @property
723 def _valid_sp_values(self) -> np.ndarray:
724 sp_vals = self.sp_values
725 mask = notna(sp_vals)
726 return sp_vals[mask]
728 def __len__(self) -> int:
729 return self.sp_index.length
731 @property
732 def _null_fill_value(self) -> bool:
733 return self._dtype._is_na_fill_value
735 @property
736 def nbytes(self) -> int:
737 return self.sp_values.nbytes + self.sp_index.nbytes
739 @property
740 def density(self) -> float:
741 """
742 The percent of non- ``fill_value`` points, as decimal.
744 See Also
745 --------
746 DataFrame.sparse.from_spmatrix : Create a new DataFrame from a
747 scipy sparse matrix.
749 Examples
750 --------
751 >>> from pandas.arrays import SparseArray
752 >>> s = SparseArray([0, 0, 1, 1, 1], fill_value=0)
753 >>> s.density
754 0.6
755 """
756 return self.sp_index.npoints / self.sp_index.length
758 @property
759 def npoints(self) -> int:
760 """
761 The number of non- ``fill_value`` points.
763 This property returns the number of elements in the sparse series that are
764 not equal to the ``fill_value``. Sparse data structures store only the
765 non-``fill_value`` elements, reducing memory usage when the majority of
766 values are the same.
768 See Also
769 --------
770 Series.sparse.to_dense : Convert a Series from sparse values to dense.
771 Series.sparse.fill_value : Elements in ``data`` that are ``fill_value`` are
772 not stored.
773 Series.sparse.density : The percent of non- ``fill_value`` points, as decimal.
775 Examples
776 --------
777 >>> from pandas.arrays import SparseArray
778 >>> s = SparseArray([0, 0, 1, 1, 1], fill_value=0)
779 >>> s.npoints
780 3
781 """
782 return self.sp_index.npoints
784 # error: Return type "SparseArray" of "isna" incompatible with return type
785 # "ndarray[Any, Any] | ExtensionArraySupportsAnyAll" in supertype "ExtensionArray"
786 def isna(self) -> Self: # type: ignore[override]
787 # If null fill value, we want SparseDtype[bool, true]
788 # to preserve the same memory usage.
789 dtype = SparseDtype(bool, self._null_fill_value)
790 if self._null_fill_value:
791 return type(self)._simple_new(isna(self.sp_values), self.sp_index, dtype)
792 mask = np.full(len(self), False, dtype=np.bool_)
793 mask[self.sp_index.indices] = isna(self.sp_values)
794 return type(self)(mask, fill_value=False, dtype=dtype)
796 def fillna(
797 self,
798 value,
799 limit: int | None = None,
800 copy: bool = True,
801 ) -> Self:
802 """
803 Fill missing values with `value`.
805 Parameters
806 ----------
807 value : scalar
808 limit : int, optional
809 Not supported for SparseArray, must be None.
810 copy: bool, default True
811 Ignored for SparseArray.
813 Returns
814 -------
815 SparseArray
817 Notes
818 -----
819 When `value` is specified, the result's ``fill_value`` depends on
820 ``self.fill_value``. The goal is to maintain low-memory use.
822 If ``self.fill_value`` is NA, the result dtype will be
823 ``SparseDtype(self.dtype, fill_value=value)``. This will preserve
824 amount of memory used before and after filling.
826 When ``self.fill_value`` is not NA, the result dtype will be
827 ``self.dtype``. Again, this preserves the amount of memory used.
828 """
829 if limit is not None:
830 raise ValueError("limit must be None")
831 new_values = np.where(isna(self.sp_values), value, self.sp_values)
833 if self._null_fill_value:
834 # This is essentially just updating the dtype.
835 new_dtype = SparseDtype(self.dtype.subtype, fill_value=value)
836 else:
837 new_dtype = self.dtype
839 return self._simple_new(new_values, self._sparse_index, new_dtype)
841 def shift(self, periods: int = 1, fill_value=None) -> Self:
842 if not len(self) or periods == 0:
843 return self.copy()
845 if isna(fill_value):
846 fill_value = self.dtype.na_value
848 subtype = np.result_type(fill_value, self.dtype.subtype)
850 if subtype != self.dtype.subtype:
851 # just coerce up front
852 arr = self.astype(SparseDtype(subtype, self.fill_value))
853 else:
854 arr = self
856 empty = self._from_sequence(
857 [fill_value] * min(abs(periods), len(self)), dtype=arr.dtype
858 )
860 if periods > 0:
861 a = empty
862 b = arr[:-periods]
863 else:
864 a = arr[abs(periods) :]
865 b = empty
866 return arr._concat_same_type([a, b])
868 def _first_fill_value_loc(self):
869 """
870 Get the location of the first fill value.
872 Returns
873 -------
874 int
875 """
876 if len(self) == 0 or self.sp_index.npoints == len(self):
877 return -1
879 indices = self.sp_index.indices
880 if not len(indices) or indices[0] > 0:
881 return 0
883 # a number larger than 1 should be appended to
884 # the last in case of fill value only appears
885 # in the tail of array
886 diff = np.r_[np.diff(indices), 2]
887 return indices[(diff > 1).argmax()] + 1
889 @doc(ExtensionArray.duplicated)
890 def duplicated(
891 self, keep: Literal["first", "last", False] = "first"
892 ) -> npt.NDArray[np.bool_]:
893 values = np.asarray(self)
894 mask = np.asarray(self.isna())
895 return algos.duplicated(values, keep=keep, mask=mask)
897 def unique(self) -> Self:
898 uniques = algos.unique(self.sp_values)
899 if len(self.sp_values) != len(self):
900 fill_loc = self._first_fill_value_loc()
901 # Inorder to align the behavior of pd.unique or
902 # pd.Series.unique, we should keep the original
903 # order, here we use unique again to find the
904 # insertion place. Since the length of sp_values
905 # is not large, maybe minor performance hurt
906 # is worthwhile to the correctness.
907 insert_loc = len(algos.unique(self.sp_values[:fill_loc]))
908 uniques = np.insert(uniques, insert_loc, self.fill_value)
909 return type(self)._from_sequence(uniques, dtype=self.dtype)
911 def _values_for_factorize(self):
912 # Still override this for hash_pandas_object
913 return np.asarray(self), self.fill_value
915 def factorize(
916 self,
917 use_na_sentinel: bool = True,
918 ) -> tuple[np.ndarray, SparseArray]:
919 # Currently, ExtensionArray.factorize -> Tuple[ndarray, EA]
920 # The sparsity on this is backwards from what Sparse would want. Want
921 # ExtensionArray.factorize -> Tuple[EA, EA]
922 # Given that we have to return a dense array of codes, why bother
923 # implementing an efficient factorize?
924 codes, uniques = algos.factorize(
925 np.asarray(self), use_na_sentinel=use_na_sentinel
926 )
927 uniques_sp = SparseArray(uniques, dtype=self.dtype)
928 return codes, uniques_sp
930 def value_counts(self, dropna: bool = True) -> Series:
931 """
932 Returns a Series containing counts of unique values.
934 Parameters
935 ----------
936 dropna : bool, default True
937 Don't include counts of NaN, even if NaN is in sp_values.
939 Returns
940 -------
941 counts : Series
942 """
943 from pandas import (
944 Index,
945 Series,
946 )
948 keys, counts, _ = algos.value_counts_arraylike(self.sp_values, dropna=dropna)
949 fcounts = self.sp_index.ngaps
950 if fcounts > 0 and (not self._null_fill_value or not dropna):
951 mask = isna(keys) if self._null_fill_value else keys == self.fill_value
952 if mask.any():
953 counts[mask] += fcounts
954 else:
955 # error: Argument 1 to "insert" has incompatible type "Union[
956 # ExtensionArray,ndarray[Any, Any]]"; expected "Union[
957 # _SupportsArray[dtype[Any]], Sequence[_SupportsArray[dtype
958 # [Any]]], Sequence[Sequence[_SupportsArray[dtype[Any]]]],
959 # Sequence[Sequence[Sequence[_SupportsArray[dtype[Any]]]]], Sequence
960 # [Sequence[Sequence[Sequence[_SupportsArray[dtype[Any]]]]]]]"
961 keys = np.insert(keys, 0, self.fill_value) # type: ignore[arg-type]
962 counts = np.insert(counts, 0, fcounts)
964 if not isinstance(keys, ABCIndex):
965 index = Index(keys, copy=False)
966 else:
967 index = keys
968 return Series(counts, index=index, copy=False)
970 # --------
971 # Indexing
972 # --------
973 @overload
974 def __getitem__(self, key: ScalarIndexer) -> Any: ...
976 @overload
977 def __getitem__(
978 self,
979 key: SequenceIndexer | tuple[int | EllipsisType, ...],
980 ) -> Self: ...
982 def __getitem__(
983 self,
984 key: PositionalIndexer | tuple[int | EllipsisType, ...],
985 ) -> Self | Any:
986 if isinstance(key, tuple):
987 key = unpack_tuple_and_ellipses(key)
988 if key is ...:
989 raise ValueError("Cannot slice with Ellipsis")
991 if is_integer(key):
992 return self._get_val_at(key)
993 elif isinstance(key, tuple):
994 data_slice = self.to_dense()[key]
995 elif isinstance(key, slice):
996 if key == slice(None):
997 # to ensure arr[:] (used by view()) does not make a copy
998 result = type(self)._simple_new(
999 self.sp_values, self.sp_index, self.dtype
1000 )
1001 result._readonly = self._readonly
1002 return result
1003 # Avoid densifying when handling contiguous slices
1004 if key.step is None or key.step == 1:
1005 start = 0 if key.start is None else key.start
1006 if start < 0:
1007 start += len(self)
1009 end = len(self) if key.stop is None else key.stop
1010 if end < 0:
1011 end += len(self)
1013 indices = self.sp_index.indices
1014 keep_inds = np.flatnonzero((indices >= start) & (indices < end))
1015 sp_vals = self.sp_values[keep_inds]
1017 sp_index = indices[keep_inds].copy()
1019 # If we've sliced to not include the start of the array, all our indices
1020 # should be shifted. NB: here we are careful to also not shift by a
1021 # negative value for a case like [0, 1][-100:] where the start index
1022 # should be treated like 0
1023 if start > 0:
1024 sp_index -= start
1026 # Length of our result should match applying this slice to a range
1027 # of the length of our original array
1028 new_len = len(range(len(self))[key])
1029 new_sp_index = make_sparse_index(new_len, sp_index, self.kind)
1030 return type(self)._simple_new(sp_vals, new_sp_index, self.dtype)
1031 else:
1032 indices = np.arange(len(self), dtype=np.int32)[key]
1033 return self.take(indices)
1035 elif not is_list_like(key):
1036 # e.g. "foo" or 2.5
1037 # exception message copied from numpy
1038 raise IndexError(
1039 r"only integers, slices (`:`), ellipsis (`...`), numpy.newaxis "
1040 r"(`None`) and integer or boolean arrays are valid indices"
1041 )
1043 else:
1044 if isinstance(key, SparseArray):
1045 # NOTE: If we guarantee that SparseDType(bool)
1046 # has only fill_value - true, false or nan
1047 # (see GH PR 44955)
1048 # we can apply mask very fast:
1049 if is_bool_dtype(key):
1050 if isna(key.fill_value):
1051 return self.take(key.sp_index.indices[key.sp_values])
1052 if not key.fill_value:
1053 return self.take(key.sp_index.indices)
1054 n = len(self)
1055 mask = np.full(n, True, dtype=np.bool_)
1056 mask[key.sp_index.indices] = False
1057 return self.take(np.arange(n)[mask])
1058 else:
1059 key = np.asarray(key)
1061 key = check_array_indexer(self, key)
1063 if com.is_bool_indexer(key):
1064 # mypy doesn't know we have an array here
1065 key = cast(np.ndarray, key)
1066 return self.take(np.arange(len(key), dtype=np.int32)[key])
1067 elif hasattr(key, "__len__"):
1068 return self.take(key)
1069 else:
1070 raise ValueError(f"Cannot slice with '{key}'")
1072 return type(self)(data_slice, kind=self.kind)
1074 def _get_val_at(self, loc):
1075 loc = validate_insert_loc(loc, len(self))
1077 sp_loc = self.sp_index.lookup(loc)
1078 if sp_loc == -1:
1079 return self.fill_value
1080 else:
1081 val = self.sp_values[sp_loc]
1082 val = maybe_box_datetimelike(val, self.sp_values.dtype)
1083 return val
1085 def take(self, indices, *, allow_fill: bool = False, fill_value=None) -> Self:
1086 if is_scalar(indices):
1087 raise ValueError(f"'indices' must be an array, not a scalar '{indices}'.")
1088 indices = np.asarray(indices, dtype=np.int32)
1090 dtype = None
1091 if indices.size == 0:
1092 result = np.array([], dtype="object")
1093 dtype = self.dtype
1094 elif allow_fill:
1095 result = self._take_with_fill(indices, fill_value=fill_value)
1096 else:
1097 return self._take_without_fill(indices)
1099 return type(self)(
1100 result, fill_value=self.fill_value, kind=self.kind, dtype=dtype
1101 )
1103 def _take_with_fill(self, indices, fill_value=None) -> np.ndarray:
1104 if fill_value is None:
1105 fill_value = self.dtype.na_value
1107 if indices.min() < -1:
1108 raise ValueError(
1109 "Invalid value in 'indices'. Must be between -1 "
1110 "and the length of the array."
1111 )
1113 if indices.max() >= len(self):
1114 raise IndexError("out of bounds value in 'indices'.")
1116 if len(self) == 0:
1117 # Empty... Allow taking only if all empty
1118 if (indices == -1).all():
1119 dtype = np.result_type(self.sp_values, type(fill_value))
1120 taken = np.empty_like(indices, dtype=dtype)
1121 taken.fill(fill_value)
1122 return taken
1123 else:
1124 raise IndexError("cannot do a non-empty take from an empty axes.")
1126 # sp_indexer may be -1 for two reasons
1127 # 1.) we took for an index of -1 (new)
1128 # 2.) we took a value that was self.fill_value (old)
1129 sp_indexer = self.sp_index.lookup_array(indices)
1130 new_fill_indices = indices == -1
1131 old_fill_indices = (sp_indexer == -1) & ~new_fill_indices
1133 if self.sp_index.npoints == 0 and old_fill_indices.all():
1134 # We've looked up all valid points on an all-sparse array.
1135 taken = np.full(
1136 sp_indexer.shape, fill_value=self.fill_value, dtype=self.dtype.subtype
1137 )
1139 elif self.sp_index.npoints == 0:
1140 # Use the old fill_value unless we took for an index of -1
1141 _dtype = np.result_type(self.dtype.subtype, type(fill_value))
1142 taken = np.full(sp_indexer.shape, fill_value=fill_value, dtype=_dtype)
1143 taken[old_fill_indices] = self.fill_value
1144 else:
1145 taken = self.sp_values.take(sp_indexer)
1147 # Fill in two steps.
1148 # Old fill values
1149 # New fill values
1150 # potentially coercing to a new dtype at each stage.
1152 m0 = sp_indexer[old_fill_indices] < 0
1153 m1 = sp_indexer[new_fill_indices] < 0
1155 result_type = taken.dtype
1157 if m0.any():
1158 result_type = np.result_type(result_type, type(self.fill_value))
1159 taken = taken.astype(result_type)
1160 taken[old_fill_indices] = self.fill_value
1162 if m1.any():
1163 result_type = np.result_type(result_type, type(fill_value))
1164 taken = taken.astype(result_type)
1165 taken[new_fill_indices] = fill_value
1167 return taken
1169 def _take_without_fill(self, indices) -> Self:
1170 to_shift = indices < 0
1172 n = len(self)
1174 if (indices.max() >= n) or (indices.min() < -n):
1175 if n == 0:
1176 raise IndexError("cannot do a non-empty take from an empty axes.")
1177 raise IndexError("out of bounds value in 'indices'.")
1179 if to_shift.any():
1180 indices = indices.copy()
1181 indices[to_shift] += n
1183 sp_indexer = self.sp_index.lookup_array(indices)
1184 value_mask = sp_indexer != -1
1185 new_sp_values = self.sp_values[sp_indexer[value_mask]]
1187 value_indices = np.flatnonzero(value_mask).astype(np.int32, copy=False)
1189 new_sp_index = make_sparse_index(len(indices), value_indices, kind=self.kind)
1190 return type(self)._simple_new(new_sp_values, new_sp_index, dtype=self.dtype)
1192 def searchsorted(
1193 self,
1194 v: ArrayLike | object,
1195 side: Literal["left", "right"] = "left",
1196 sorter: NumpySorter | None = None,
1197 ) -> npt.NDArray[np.intp] | np.intp:
1198 if get_option("performance_warnings"):
1199 msg = "searchsorted requires high memory usage."
1200 warnings.warn(msg, PerformanceWarning, stacklevel=find_stack_level())
1201 v = np.asarray(v)
1202 return np.asarray(self, dtype=self.dtype.subtype).searchsorted(v, side, sorter)
1204 def copy(self) -> Self:
1205 values = self.sp_values.copy()
1206 return self._simple_new(values, self.sp_index, self.dtype)
1208 @classmethod
1209 def _concat_same_type(cls, to_concat: Sequence[Self]) -> Self:
1210 fill_value = to_concat[0].fill_value
1212 values = []
1213 length = 0
1215 if to_concat:
1216 sp_kind = to_concat[0].kind
1217 else:
1218 sp_kind = "integer"
1220 sp_index: SparseIndex
1221 if sp_kind == "integer":
1222 indices = []
1224 for arr in to_concat:
1225 int_idx = arr.sp_index.indices.copy()
1226 int_idx += length # TODO: wraparound
1227 length += arr.sp_index.length
1229 values.append(arr.sp_values)
1230 indices.append(int_idx)
1232 data = np.concatenate(values)
1233 indices_arr = np.concatenate(indices)
1234 sp_index = IntIndex(length, indices_arr)
1236 else:
1237 # when concatenating block indices, we don't claim that you'll
1238 # get an identical index as concatenating the values and then
1239 # creating a new index. We don't want to spend the time trying
1240 # to merge blocks across arrays in `to_concat`, so the resulting
1241 # BlockIndex may have more blocks.
1242 blengths = []
1243 blocs = []
1245 for arr in to_concat:
1246 block_idx = arr.sp_index.to_block_index()
1248 values.append(arr.sp_values)
1249 blocs.append(block_idx.blocs.copy() + length)
1250 blengths.append(block_idx.blengths)
1251 length += arr.sp_index.length
1253 data = np.concatenate(values)
1254 blocs_arr = np.concatenate(blocs)
1255 blengths_arr = np.concatenate(blengths)
1257 sp_index = BlockIndex(length, blocs_arr, blengths_arr)
1259 return cls(data, sparse_index=sp_index, fill_value=fill_value)
1261 def astype(self, dtype: AstypeArg | None = None, copy: bool = True):
1262 """
1263 Change the dtype of a SparseArray.
1265 The output will always be a SparseArray. To convert to a dense
1266 ndarray with a certain dtype, use :meth:`numpy.asarray`.
1268 Parameters
1269 ----------
1270 dtype : np.dtype or ExtensionDtype
1271 For SparseDtype, this changes the dtype of
1272 ``self.sp_values`` and the ``self.fill_value``.
1274 For other dtypes, this only changes the dtype of
1275 ``self.sp_values``.
1277 copy : bool, default True
1278 Whether to ensure a copy is made, even if not necessary.
1280 Returns
1281 -------
1282 SparseArray
1284 Examples
1285 --------
1286 >>> arr = pd.arrays.SparseArray([0, 0, 1, 2])
1287 >>> arr
1288 [0, 0, 1, 2]
1289 Fill: 0
1290 IntIndex
1291 Indices: array([2, 3], dtype=int32)
1293 >>> arr.astype(pd.SparseDtype(np.dtype("int32")))
1294 [0, 0, 1, 2]
1295 Fill: 0
1296 IntIndex
1297 Indices: array([2, 3], dtype=int32)
1299 Using a NumPy dtype with a different kind (e.g. float) will coerce
1300 just ``self.sp_values``.
1302 >>> arr.astype(pd.SparseDtype(np.dtype("float64")))
1303 ... # doctest: +NORMALIZE_WHITESPACE
1304 [nan, nan, 1.0, 2.0]
1305 Fill: nan
1306 IntIndex
1307 Indices: array([2, 3], dtype=int32)
1309 Using a SparseDtype, you can also change the fill value as well.
1311 >>> arr.astype(pd.SparseDtype("float64", fill_value=0.0))
1312 ... # doctest: +NORMALIZE_WHITESPACE
1313 [0.0, 0.0, 1.0, 2.0]
1314 Fill: 0.0
1315 IntIndex
1316 Indices: array([2, 3], dtype=int32)
1317 """
1318 if dtype == self._dtype:
1319 if not copy:
1320 return self
1321 else:
1322 return self.copy()
1324 future_dtype = pandas_dtype(dtype)
1325 if not isinstance(future_dtype, SparseDtype):
1326 # GH#34457
1327 values = np.asarray(self)
1328 values = ensure_wrapped_if_datetimelike(values)
1329 return astype_array(values, dtype=future_dtype, copy=False)
1331 dtype = self.dtype.update_dtype(dtype)
1332 subtype = pandas_dtype(dtype._subtype_with_str)
1333 subtype = cast(np.dtype, subtype) # ensured by update_dtype
1334 values = ensure_wrapped_if_datetimelike(self.sp_values)
1335 sp_values = astype_array(values, subtype, copy=copy)
1336 sp_values = np.asarray(sp_values)
1338 return self._simple_new(sp_values, self.sp_index, dtype)
1340 def map(self, mapper, na_action: Literal["ignore"] | None = None) -> Self:
1341 """
1342 Map categories using an input mapping or function.
1344 Parameters
1345 ----------
1346 mapper : dict, Series, callable
1347 The correspondence from old values to new.
1348 na_action : {None, 'ignore'}, default None
1349 If 'ignore', propagate NA values, without passing them to the
1350 mapping correspondence.
1352 Returns
1353 -------
1354 SparseArray
1355 The output array will have the same density as the input.
1356 The output fill value will be the result of applying the
1357 mapping to ``self.fill_value``
1359 Examples
1360 --------
1361 >>> arr = pd.arrays.SparseArray([0, 1, 2])
1362 >>> arr.map(lambda x: x + 10)
1363 [10, 11, 12]
1364 Fill: 10
1365 IntIndex
1366 Indices: array([1, 2], dtype=int32)
1368 >>> arr.map({0: 10, 1: 11, 2: 12})
1369 [10, 11, 12]
1370 Fill: 10
1371 IntIndex
1372 Indices: array([1, 2], dtype=int32)
1374 >>> arr.map(pd.Series([10, 11, 12], index=[0, 1, 2]))
1375 [10, 11, 12]
1376 Fill: 10
1377 IntIndex
1378 Indices: array([1, 2], dtype=int32)
1379 """
1380 is_map = isinstance(mapper, (abc.Mapping, ABCSeries))
1382 fill_val = self.fill_value
1384 if na_action is None or notna(fill_val):
1385 fill_val = mapper.get(fill_val, fill_val) if is_map else mapper(fill_val)
1387 def func(sp_val):
1388 new_sp_val = mapper.get(sp_val, None) if is_map else mapper(sp_val)
1389 # check identity and equality because nans are not equal to each other
1390 if new_sp_val is fill_val or new_sp_val == fill_val:
1391 msg = "fill value in the sparse values not supported"
1392 raise ValueError(msg)
1393 return new_sp_val
1395 sp_values = [func(x) for x in self.sp_values]
1397 return type(self)(sp_values, sparse_index=self.sp_index, fill_value=fill_val)
1399 def to_dense(self) -> np.ndarray:
1400 """
1401 Convert SparseArray to a NumPy array.
1403 Returns
1404 -------
1405 arr : NumPy array
1406 """
1407 return np.asarray(self, dtype=self.sp_values.dtype)
1409 def _where(self, mask, value):
1410 # NB: may not preserve dtype, e.g. result may be Sparse[float64]
1411 # while self is Sparse[int64]
1412 naive_implementation = np.where(mask, self, value)
1413 dtype = SparseDtype(naive_implementation.dtype, fill_value=self.fill_value)
1414 result = type(self)._from_sequence(naive_implementation, dtype=dtype)
1415 return result
1417 # ------------------------------------------------------------------------
1418 # IO
1419 # ------------------------------------------------------------------------
1420 def __setstate__(self, state) -> None:
1421 """Necessary for making this object picklable"""
1422 if isinstance(state, tuple):
1423 # Compat for pandas < 0.24.0
1424 nd_state, (fill_value, sp_index) = state
1425 sparse_values = np.array([])
1426 sparse_values.__setstate__(nd_state)
1428 self._sparse_values = sparse_values
1429 self._sparse_index = sp_index
1430 self._dtype = SparseDtype(sparse_values.dtype, fill_value)
1431 else:
1432 self.__dict__.update(state)
1434 def nonzero(self) -> tuple[npt.NDArray[np.int32]]:
1435 if self.fill_value == 0:
1436 return (self.sp_index.indices,)
1437 else:
1438 return (self.sp_index.indices[self.sp_values != 0],)
1440 # ------------------------------------------------------------------------
1441 # Reductions
1442 # ------------------------------------------------------------------------
1444 def _reduce(
1445 self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
1446 ):
1447 method = getattr(self, name, None)
1449 if method is None:
1450 raise TypeError(f"cannot perform {name} with type {self.dtype}")
1452 if skipna:
1453 arr = self
1454 else:
1455 arr = self.dropna()
1457 result = getattr(arr, name)(**kwargs)
1459 if keepdims:
1460 return type(self)([result], dtype=self.dtype)
1461 else:
1462 return result
1464 def all(self, axis=None, *args, **kwargs):
1465 """
1466 Tests whether all elements evaluate True
1468 Returns
1469 -------
1470 all : bool
1472 See Also
1473 --------
1474 numpy.all
1475 """
1476 nv.validate_all(args, kwargs)
1478 values = self.sp_values
1480 if len(values) != len(self) and not np.all(self.fill_value):
1481 return False
1483 return values.all()
1485 def any(self, axis: AxisInt = 0, *args, **kwargs) -> bool:
1486 """
1487 Tests whether at least one of elements evaluate True
1489 Returns
1490 -------
1491 any : bool
1493 See Also
1494 --------
1495 numpy.any
1496 """
1497 nv.validate_any(args, kwargs)
1499 values = self.sp_values
1501 if len(values) != len(self) and np.any(self.fill_value):
1502 return True
1504 return values.any().item()
1506 def sum(
1507 self,
1508 axis: AxisInt = 0,
1509 min_count: int = 0,
1510 skipna: bool = True,
1511 *args,
1512 **kwargs,
1513 ) -> Scalar:
1514 """
1515 Sum of non-NA/null values
1517 Parameters
1518 ----------
1519 axis : int, default 0
1520 Not Used. NumPy compatibility.
1521 min_count : int, default 0
1522 The required number of valid values to perform the summation. If fewer
1523 than ``min_count`` valid values are present, the result will be the missing
1524 value indicator for subarray type.
1525 *args, **kwargs
1526 Not Used. NumPy compatibility.
1528 Returns
1529 -------
1530 scalar
1531 """
1532 nv.validate_sum(args, kwargs)
1533 valid_vals = self._valid_sp_values
1534 sp_sum = valid_vals.sum()
1535 has_na = self.sp_index.ngaps > 0 and not self._null_fill_value
1537 if has_na and not skipna:
1538 return na_value_for_dtype(self.dtype.subtype, compat=False)
1540 if self._null_fill_value:
1541 if check_below_min_count(valid_vals.shape, None, min_count):
1542 return na_value_for_dtype(self.dtype.subtype, compat=False)
1543 return sp_sum
1544 else:
1545 nsparse = self.sp_index.ngaps
1546 if check_below_min_count(valid_vals.shape, None, min_count - nsparse):
1547 return na_value_for_dtype(self.dtype.subtype, compat=False)
1548 return sp_sum + self.fill_value * nsparse
1550 def cumsum(self, axis: AxisInt = 0, *args, **kwargs) -> SparseArray:
1551 """
1552 Cumulative sum of non-NA/null values.
1554 When performing the cumulative summation, any non-NA/null values will
1555 be skipped. The resulting SparseArray will preserve the locations of
1556 NaN values, but the fill value will be `np.nan` regardless.
1558 Parameters
1559 ----------
1560 axis : int or None
1561 Axis over which to perform the cumulative summation. If None,
1562 perform cumulative summation over flattened array.
1564 Returns
1565 -------
1566 cumsum : SparseArray
1567 """
1568 nv.validate_cumsum(args, kwargs)
1570 if axis is not None and axis >= self.ndim: # Mimic ndarray behaviour.
1571 raise ValueError(f"axis(={axis}) out of bounds")
1573 if not self._null_fill_value:
1574 return SparseArray(self.to_dense(), fill_value=np.nan).cumsum()
1576 return SparseArray(
1577 self.sp_values.cumsum(),
1578 sparse_index=self.sp_index,
1579 fill_value=self.fill_value,
1580 )
1582 def mean(self, axis: Axis = 0, *args, **kwargs):
1583 """
1584 Mean of non-NA/null values
1586 Returns
1587 -------
1588 mean : float
1589 """
1590 nv.validate_mean(args, kwargs)
1591 valid_vals = self._valid_sp_values
1592 sp_sum = valid_vals.sum()
1593 ct = len(valid_vals)
1595 if self._null_fill_value:
1596 return sp_sum / ct
1597 else:
1598 nsparse = self.sp_index.ngaps
1599 return (sp_sum + self.fill_value * nsparse) / (ct + nsparse)
1601 def max(self, *, axis: AxisInt | None = None, skipna: bool = True):
1602 """
1603 Max of array values, ignoring NA values if specified.
1605 Parameters
1606 ----------
1607 axis : int, default 0
1608 Not Used. NumPy compatibility.
1609 skipna : bool, default True
1610 Whether to ignore NA values.
1612 Returns
1613 -------
1614 scalar
1615 """
1616 nv.validate_minmax_axis(axis, self.ndim)
1617 return self._min_max("max", skipna=skipna)
1619 def min(self, *, axis: AxisInt | None = None, skipna: bool = True):
1620 """
1621 Min of array values, ignoring NA values if specified.
1623 Parameters
1624 ----------
1625 axis : int, default 0
1626 Not Used. NumPy compatibility.
1627 skipna : bool, default True
1628 Whether to ignore NA values.
1630 Returns
1631 -------
1632 scalar
1633 """
1634 nv.validate_minmax_axis(axis, self.ndim)
1635 return self._min_max("min", skipna=skipna)
1637 def _min_max(self, kind: Literal["min", "max"], skipna: bool) -> Scalar:
1638 """
1639 Min/max of non-NA/null values
1641 Parameters
1642 ----------
1643 kind : {"min", "max"}
1644 skipna : bool
1646 Returns
1647 -------
1648 scalar
1649 """
1650 valid_vals = self._valid_sp_values
1651 has_nonnull_fill_vals = not self._null_fill_value and self.sp_index.ngaps > 0
1653 if len(valid_vals) > 0:
1654 sp_min_max = getattr(valid_vals, kind)()
1656 # If a non-null fill value is currently present, it might be the min/max
1657 if has_nonnull_fill_vals:
1658 func = max if kind == "max" else min
1659 return func(sp_min_max, self.fill_value)
1660 elif skipna:
1661 return sp_min_max
1662 elif self.sp_index.ngaps == 0:
1663 # No NAs present
1664 return sp_min_max
1665 else:
1666 return na_value_for_dtype(self.dtype.subtype, compat=False)
1667 elif has_nonnull_fill_vals:
1668 return self.fill_value
1669 else:
1670 return na_value_for_dtype(self.dtype.subtype, compat=False)
1672 def _argmin_argmax(self, kind: Literal["argmin", "argmax"]) -> int:
1673 values = self._sparse_values
1674 index = self._sparse_index.indices
1675 mask = np.asarray(isna(values))
1676 func = np.argmax if kind == "argmax" else np.argmin
1678 idx = np.arange(values.shape[0])
1679 non_nans = values[~mask]
1680 non_nan_idx = idx[~mask]
1682 _candidate = non_nan_idx[func(non_nans)]
1683 candidate = index[_candidate]
1685 if isna(self.fill_value):
1686 return candidate
1687 if kind == "argmin" and self[candidate] < self.fill_value:
1688 return candidate
1689 if kind == "argmax" and self[candidate] > self.fill_value:
1690 return candidate
1691 _loc = self._first_fill_value_loc()
1692 if _loc == -1:
1693 # fill_value doesn't exist
1694 return candidate
1695 else:
1696 return _loc
1698 def argmax(self, skipna: bool = True) -> int:
1699 validate_bool_kwarg(skipna, "skipna")
1700 if not skipna and self._hasna:
1701 raise ValueError("Encountered an NA value with skipna=False")
1702 return self._argmin_argmax("argmax")
1704 def argmin(self, skipna: bool = True) -> int:
1705 validate_bool_kwarg(skipna, "skipna")
1706 if not skipna and self._hasna:
1707 raise ValueError("Encountered an NA value with skipna=False")
1708 return self._argmin_argmax("argmin")
1710 # ------------------------------------------------------------------------
1711 # Ufuncs
1712 # ------------------------------------------------------------------------
1714 _HANDLED_TYPES = (np.ndarray, numbers.Number)
1716 def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
1717 out = kwargs.get("out", ())
1719 for x in inputs + out:
1720 if not isinstance(x, (*self._HANDLED_TYPES, SparseArray)):
1721 return NotImplemented
1723 # for binary ops, use our custom dunder methods
1724 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(
1725 self, ufunc, method, *inputs, **kwargs
1726 )
1727 if result is not NotImplemented:
1728 return result
1730 if "out" in kwargs:
1731 # e.g. tests.arrays.sparse.test_arithmetics.test_ndarray_inplace
1732 res = arraylike.dispatch_ufunc_with_out(
1733 self, ufunc, method, *inputs, **kwargs
1734 )
1735 return res
1737 if method == "reduce":
1738 result = arraylike.dispatch_reduction_ufunc(
1739 self, ufunc, method, *inputs, **kwargs
1740 )
1741 if result is not NotImplemented:
1742 # e.g. tests.series.test_ufunc.TestNumpyReductions
1743 return result
1745 if len(inputs) == 1:
1746 # No alignment necessary.
1747 sp_values = getattr(ufunc, method)(self.sp_values, **kwargs)
1748 fill_value = getattr(ufunc, method)(self.fill_value, **kwargs)
1750 if ufunc.nout > 1:
1751 # multiple outputs. e.g. modf
1752 arrays = tuple(
1753 self._simple_new(
1754 sp_value, self.sp_index, SparseDtype(sp_value.dtype, fv)
1755 )
1756 for sp_value, fv in zip(sp_values, fill_value, strict=True)
1757 )
1758 return arrays
1759 elif method == "reduce":
1760 # e.g. reductions
1761 return sp_values
1763 return self._simple_new(
1764 sp_values, self.sp_index, SparseDtype(sp_values.dtype, fill_value)
1765 )
1767 new_inputs = tuple(np.asarray(x) for x in inputs)
1768 result = getattr(ufunc, method)(*new_inputs, **kwargs)
1769 if out:
1770 if len(out) == 1:
1771 out = out[0]
1772 return out
1774 if ufunc.nout > 1:
1775 return tuple(type(self)(x) for x in result)
1776 elif method == "at":
1777 # no return value
1778 return None
1779 else:
1780 return type(self)(result)
1782 # ------------------------------------------------------------------------
1783 # Ops
1784 # ------------------------------------------------------------------------
1786 def _arith_method(self, other, op):
1787 op_name = op.__name__
1789 if isinstance(other, SparseArray):
1790 return _sparse_array_op(self, other, op, op_name)
1792 elif is_scalar(other):
1793 with np.errstate(all="ignore"):
1794 fill = op(_get_fill(self), np.asarray(other))
1795 result = op(self.sp_values, other)
1797 if op_name == "divmod":
1798 left, right = result
1799 lfill, rfill = fill
1800 return (
1801 _wrap_result(op_name, left, self.sp_index, lfill),
1802 _wrap_result(op_name, right, self.sp_index, rfill),
1803 )
1805 return _wrap_result(op_name, result, self.sp_index, fill)
1807 else:
1808 other = np.asarray(other)
1809 with np.errstate(all="ignore"):
1810 if len(self) != len(other):
1811 raise AssertionError(
1812 f"length mismatch: {len(self)} vs. {len(other)}"
1813 )
1814 if not isinstance(other, SparseArray):
1815 dtype = getattr(other, "dtype", None)
1816 other = SparseArray(other, fill_value=self.fill_value, dtype=dtype)
1817 return _sparse_array_op(self, other, op, op_name)
1819 def _cmp_method(self, other, op) -> SparseArray:
1820 if not is_scalar(other) and not isinstance(other, type(self)):
1821 # convert list-like to ndarray
1822 other = np.asarray(other)
1824 if isinstance(other, np.ndarray):
1825 # TODO: make this more flexible than just ndarray...
1826 other = SparseArray(other, fill_value=self.fill_value)
1828 if isinstance(other, SparseArray):
1829 if len(self) != len(other):
1830 raise ValueError(
1831 f"operands have mismatched length {len(self)} and {len(other)}"
1832 )
1834 op_name = op.__name__.strip("_")
1835 return _sparse_array_op(self, other, op, op_name)
1836 else:
1837 # scalar
1838 fill_value = op(self.fill_value, other)
1839 result = np.full(len(self), fill_value, dtype=np.bool_)
1840 result[self.sp_index.indices] = op(self.sp_values, other)
1842 return type(self)(
1843 result,
1844 fill_value=fill_value,
1845 dtype=np.bool_,
1846 )
1848 _logical_method = _cmp_method
1850 def _unary_method(self, op) -> SparseArray:
1851 fill_value = op(np.array(self.fill_value)).item()
1852 dtype = SparseDtype(self.dtype.subtype, fill_value)
1853 # NOTE: if fill_value doesn't change
1854 # we just have to apply op to sp_values
1855 if isna(self.fill_value) or fill_value == self.fill_value:
1856 values = op(self.sp_values)
1857 return type(self)._simple_new(values, self.sp_index, self.dtype)
1858 # In the other case we have to recalc indexes
1859 return type(self)(op(self.to_dense()), dtype=dtype)
1861 def __pos__(self) -> SparseArray:
1862 return self._unary_method(operator.pos)
1864 def __neg__(self) -> SparseArray:
1865 return self._unary_method(operator.neg)
1867 def __invert__(self) -> SparseArray:
1868 return self._unary_method(operator.invert)
1870 def __abs__(self) -> SparseArray:
1871 return self._unary_method(operator.abs)
1873 # ----------
1874 # Formatting
1875 # -----------
1876 def __repr__(self) -> str:
1877 pp_str = printing.pprint_thing(self)
1878 pp_fill = printing.pprint_thing(self.fill_value)
1879 pp_index = printing.pprint_thing(self.sp_index)
1880 return f"{pp_str}\nFill: {pp_fill}\n{pp_index}"
1882 # error: Return type "None" of "_formatter" incompatible with return
1883 # type "Callable[[Any], str | None]" in supertype "ExtensionArray"
1884 def _formatter(self, boxed: bool = False) -> None: # type: ignore[override]
1885 # Defer to the formatter from the GenericArrayFormatter calling us.
1886 # This will infer the correct formatter from the dtype of the values.
1887 return None
1890def _make_sparse(
1891 arr: np.ndarray,
1892 kind: SparseIndexKind = "block",
1893 fill_value=None,
1894 dtype: np.dtype | None = None,
1895):
1896 """
1897 Convert ndarray to sparse format
1899 Parameters
1900 ----------
1901 arr : ndarray
1902 kind : {'block', 'integer'}
1903 fill_value : NaN or another value
1904 dtype : np.dtype, optional
1905 copy : bool, default False
1907 Returns
1908 -------
1909 (sparse_values, index, fill_value) : (ndarray, SparseIndex, Scalar)
1910 """
1911 assert isinstance(arr, np.ndarray)
1913 if arr.ndim > 1:
1914 raise TypeError("expected dimension <= 1 data")
1916 if fill_value is None:
1917 fill_value = na_value_for_dtype(arr.dtype)
1919 if isna(fill_value):
1920 mask = notna(arr)
1921 else:
1922 # cast to object comparison to be safe
1923 if is_string_dtype(arr.dtype):
1924 arr = arr.astype(object)
1926 if is_object_dtype(arr.dtype):
1927 # element-wise equality check method in numpy doesn't treat
1928 # each element type, eg. 0, 0.0, and False are treated as
1929 # same. So we have to check the both of its type and value.
1930 mask = splib.make_mask_object_ndarray(arr, fill_value)
1931 else:
1932 mask = arr != fill_value
1934 length = len(arr)
1935 if length != len(mask):
1936 # the arr is a SparseArray
1937 indices = mask.sp_index.indices
1938 else:
1939 indices = mask.nonzero()[0].astype(np.int32)
1941 index = make_sparse_index(length, indices, kind)
1942 sparsified_values = arr[mask]
1943 if dtype is not None:
1944 sparsified_values = ensure_wrapped_if_datetimelike(sparsified_values)
1945 sparsified_values = astype_array(sparsified_values, dtype=dtype)
1946 sparsified_values = np.asarray(sparsified_values)
1948 # TODO: copy
1949 return sparsified_values, index, fill_value
1952@overload
1953def make_sparse_index(length: int, indices, kind: Literal["block"]) -> BlockIndex: ...
1956@overload
1957def make_sparse_index(length: int, indices, kind: Literal["integer"]) -> IntIndex: ...
1960def make_sparse_index(length: int, indices, kind: SparseIndexKind) -> SparseIndex:
1961 index: SparseIndex
1962 if kind == "block":
1963 locs, lens = splib.get_blocks(indices)
1964 index = BlockIndex(length, locs, lens)
1965 elif kind == "integer":
1966 index = IntIndex(length, indices)
1967 else: # pragma: no cover
1968 raise ValueError("must be block or integer type")
1969 return index