Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/arrays/sparse/array.py: 15%

1"""

2SparseArray data structure

3"""

5from __future__ import annotations

7from collections import abc

8import numbers

9import operator

10from typing import (

11 TYPE_CHECKING,

12 Any,

13 Literal,

14 Self,

15 cast,

16 overload,

17)

18import warnings

20import numpy as np

22from pandas._config.config import _global_config as config

24from pandas._libs import (

25 lib,

26 missing as libmissing,

27)

28import pandas._libs.sparse as splib

29from pandas._libs.sparse import (

30 BlockIndex,

31 IntIndex,

32 SparseIndex,

33)

34from pandas._libs.tslibs import (

35 NaT,

36 Timedelta,

37 Timestamp,

38 is_supported_dtype,

39)

40from pandas.compat import PYPY

41from pandas.compat.numpy import function as nv

42from pandas.errors import (

43 Pandas4Warning,

44 PerformanceWarning,

45)

46from pandas.util._decorators import (

47 set_module,

48)

49from pandas.util._exceptions import find_stack_level

50from pandas.util._validators import (

51 validate_bool_kwarg,

52)

54from pandas.core.dtypes.astype import astype_array

55from pandas.core.dtypes.cast import (

56 can_hold_element,

57 construct_1d_object_array_from_listlike,

58 find_common_type,

59 maybe_box_datetimelike,

60 maybe_promote,

61)

62from pandas.core.dtypes.common import (

63 is_bool_dtype,

64 is_integer,

65 is_list_like,

66 is_object_dtype,

67 is_scalar,

68 is_string_dtype,

69 pandas_dtype,

70)

71from pandas.core.dtypes.dtypes import (

72 BaseMaskedDtype,

73 DatetimeTZDtype,

74 SparseDtype,

75)

76from pandas.core.dtypes.generic import (

77 ABCIndex,

78 ABCSeries,

79)

80from pandas.core.dtypes.missing import (

81 is_valid_na_for_dtype,

82 isna,

83 na_value_for_dtype,

84 notna,

85)

87from pandas.core import (

88 arraylike,

89 nanops,

90 ops,

91)

92import pandas.core.algorithms as algos

93from pandas.core.arraylike import OpsMixin

94from pandas.core.arrays import ExtensionArray

95from pandas.core.base import PandasObject

96import pandas.core.common as com

97from pandas.core.construction import (

98 ensure_wrapped_if_datetimelike,

99 extract_array,

100 sanitize_array,

101)

102from pandas.core.indexers import (

103 check_array_indexer,

104 unpack_tuple_and_ellipses,

105)

106from pandas.core.nanops import (

107 check_below_min_count,

108 dt64_any_all_msg,

109 na_accum_func,

110)

111from pandas.core.ops.array_ops import (

112 arithmetic_op,

113 comparison_op,

114 logical_op,

115)

116

117if TYPE_CHECKING:

118 from collections.abc import (

119 Callable,

120 Sequence,

121 )

122 from typing import (

123 Protocol,

124 type_check_only,

125 )

126

127 from scipy.sparse import (

128 csc_array,

129 csc_matrix,

130 )

131

132 @type_check_only

133 class _SparseMatrixLike(Protocol):

134 @property

135 def shape(self, /) -> tuple[int, int]: ...

136 def tocsc(self, /) -> csc_array | csc_matrix: ...

137

138 from pandas._typing import NumpySorter

139

140 SparseIndexKind = Literal["integer", "block"]

141

142 from pandas._typing import (

143 ArrayLike,

144 AstypeArg,

145 Axis,

146 AxisInt,

147 Dtype,

148 NpDtype,

149 PositionalIndexer,

150 Scalar,

151 ScalarIndexer,

152 SequenceIndexer,

153 SortKind,

154 npt,

155 )

156

157 from pandas import Series

158

159

160# Reductions whose SparseArray method takes skipna and must be handed it directly.

161_skipna_aware_reductions = frozenset(

162 {

163 "sum",

164 "prod",

165 "mean",

166 "median",

167 "var",

168 "std",

169 "sem",

170 "skew",

171 "kurt",

172 "min",

173 "max",

174 "any",

175 "all",

176 "argmin",

177 "argmax",

178 }

179)

180

181# Reductions whose result is a position in the array, not one of its values.

182_positional_reductions = frozenset({"argmin", "argmax"})

183

184# Reductions of complex input that give a real result.

185_complex_to_real_reductions = frozenset({"var", "std", "sem", "skew", "kurt"})

186

187# Reductions that give a bool rather than a value of the array's own dtype.

188_boolean_reductions = frozenset({"any", "all"})

189

190# Op names, dunders stripped, whose result is bool rather than either operand's dtype.

191_comparison_ops = frozenset({"eq", "ne", "lt", "gt", "le", "ge"})

192

193

194# ----------------------------------------------------------------------------

195# Array

196

197

198def _get_fill(arr: SparseArray) -> np.ndarray:

199 """

200 Create a 0-dim ndarray containing the fill value

201

202 Parameters

203 ----------

204 arr : SparseArray

205

206 Returns

207 -------

208 fill_value : ndarray

209 0-dim ndarray with just the fill value.

210

211 Notes

212 -----

213 coerce fill_value to arr dtype if possible

214 int64 SparseArray can have NaN as fill_value if there is no missing

215 """

216 try:

217 return np.asarray(arr.fill_value, dtype=arr.dtype.subtype)

218 except ValueError:

219 return np.asarray(arr.fill_value)

220

221

222def _has_dense_fill(arr: SparseArray) -> bool:

223 """

224 Whether _get_fill can express arr's fill value as an array at all.

225

226 Its fallback catches only ValueError, so a pd.NA fill escapes as a TypeError;

227 such an operand keeps the object-cast path, which carries the fill through.

228 """

229 try:

230 with np.errstate(all="ignore"):

231 _get_fill(arr)

232 except TypeError:

233 return False

234 return True

235

236

237def _sparse_array_op(

238 left: SparseArray, right: SparseArray, op: Callable[..., Any], name: str

239) -> SparseArray:

240 """

241 Perform a binary operation between two arrays.

242

243 Parameters

244 ----------

245 left : Union[SparseArray, ndarray]

246 right : Union[SparseArray, ndarray]

247 op : Callable

248 The binary operation to perform

249 name str

250 Name of the callable.

251

252 Returns

253 -------

254 SparseArray

255 """

256 if name.startswith("__"):

257 # For lookups in _libs.sparse we need non-dunder op name

258 name = name[2:-2]

259

260 # dtype used to find corresponding sparse method

261 ltype = left.dtype.subtype

262 rtype = right.dtype.subtype

263

264 if ltype != rtype:

265 subtype = find_common_type([ltype, rtype])

266 if (

267 is_object_dtype(subtype)

268 and not (is_object_dtype(ltype) or is_object_dtype(rtype))

269 and _has_dense_fill(left)

270 and _has_dense_fill(right)

271 ):

272 # GH#68562 no common subtype, e.g. int64 and m8[us]: casting both

273 # to object would leave an object result where the dense op

274 # promotes to m8[us]

275 return _dense_array_op(left, right, op, name)

276

277 ltype = SparseDtype(subtype, left.fill_value)

278 rtype = SparseDtype(subtype, right.fill_value)

279

280 left = left.astype(ltype, copy=False)

281 right = right.astype(rtype, copy=False)

282 dtype = ltype.subtype

283 else:

284 dtype = ltype

285

286 # dtype the result must have

287 result_dtype = None

288

289 if left.sp_index.ngaps == 0 or right.sp_index.ngaps == 0:

290 with np.errstate(all="ignore"):

291 result = op(left.to_dense(), right.to_dense())

292 fill = op(_get_fill(left), _get_fill(right))

293

294 if left.sp_index.ngaps == 0:

295 index = left.sp_index

296 else:

297 index = right.sp_index

298 elif left.sp_index.equals(right.sp_index):

299 with np.errstate(all="ignore"):

300 result = op(left.sp_values, right.sp_values)

301 fill = op(_get_fill(left), _get_fill(right))

302 index = left.sp_index

303 else:

304 if name[0] == "r":

305 left, right = right, left

306 name = name[1:]

307

308 if name in ("and", "or", "xor") and dtype == "bool":

309 opname = f"sparse_{name}_uint8"

310 # to make template simple, cast here

311 left_sp_values = left.sp_values.view(np.uint8)

312 right_sp_values = right.sp_values.view(np.uint8)

313 result_dtype = bool

314 else:

315 opname = f"sparse_{name}_{dtype}"

316 left_sp_values = left.sp_values

317 right_sp_values = right.sp_values

318

319 if (

320 name in ["floordiv", "mod"]

321 and (right == 0).any()

322 and left.dtype.kind in "iu"

323 ):

324 # Match the non-Sparse Series behavior

325 opname = f"sparse_{name}_float64"

326 left_sp_values = left_sp_values.astype("float64")

327 right_sp_values = right_sp_values.astype("float64")

328

329 sparse_op = getattr(splib, opname)

330

331 with np.errstate(all="ignore"):

332 result, index, fill = sparse_op(

333 left_sp_values,

334 left.sp_index,

335 left.fill_value,

336 right_sp_values,

337 right.sp_index,

338 right.fill_value,

339 )

340

341 if name == "divmod":

342 # result is a 2-tuple

343 # error: Incompatible return value type (got "Tuple[SparseArray,

344 # SparseArray]", expected "SparseArray")

345 return ( # type: ignore[return-value]

346 _wrap_result(name, result[0], index, fill[0], dtype=result_dtype),

347 _wrap_result(name, result[1], index, fill[1], dtype=result_dtype),

348 )

349

350 if result_dtype is None:

351 result_dtype = result.dtype

352

353 return _wrap_result(name, result, index, fill, dtype=result_dtype)

354

355

356def _dense_array_op(

357 left: SparseArray, right: SparseArray, op: Callable[..., Any], name: str

358) -> SparseArray:

359 """

360 Densify both operands, run the non-sparse op, and re-sparsify on the fill value.

361

362 For subtypes with no common type only the non-sparse op knows the result

363 dtype, and only it applies pandas' rules for the pairs numpy would otherwise

364 accept (e.g. int64 + M8[us]).

365 """

366 if name in {"and", "rand", "or", "ror", "xor", "rxor"}:

367 dense_op = logical_op

368 elif name in _comparison_ops:

369 dense_op = comparison_op

370 else:

371 dense_op = arithmetic_op

372

373 lvalues = ensure_wrapped_if_datetimelike(left.to_dense())

374 rvalues = ensure_wrapped_if_datetimelike(right.to_dense())

375

376 with np.errstate(all="ignore"):

377 # length-1 arrays, not scalars, so the fill values promote the way the

378 # values did

379 lfill = ensure_wrapped_if_datetimelike(_get_fill(left).reshape(1))

380 rfill = ensure_wrapped_if_datetimelike(_get_fill(right).reshape(1))

381 result = dense_op(lvalues, rvalues, op)

382 fill = dense_op(lfill, rfill, op)

383

384 if name in ("divmod", "rdivmod"):

385 # error: Incompatible return value type (got "Tuple[SparseArray,

386 # SparseArray]", expected "SparseArray")

387 return ( # type: ignore[return-value]

388 _wrap_dense_result(name, result[0], fill[0], left.kind),

389 _wrap_dense_result(name, result[1], fill[1], left.kind),

390 )

391 return _wrap_dense_result(name, result, fill, left.kind)

392

393

394def _wrap_dense_result(name: str, result, fill, kind: SparseIndexKind) -> SparseArray:

395 """

396 Sparsify one _dense_array_op result on the matching entry of its fill array.

397 """

398 values = np.asarray(result)

399 # np.asarray so a datetimelike op's Timestamp/Timedelta fill is spelled the

400 # way every other _sparse_array_op path spells it

401 fill_value = np.asarray(fill)[0]

402 if values.dtype.kind in "mM" and isna(fill_value):

403 # _get_fill gives up the subtype for a fill value it cannot hold -- the

404 # np.nan a float operand contributes -- so this can be the wrong NA

405 fill_value = SparseDtype(values.dtype).fill_value

406 return _wrap_result(name, values, None, fill_value, values.dtype, kind)

407

408

409def _as_sparse_operand(

410 values: np.ndarray, fill_value, dtype: Dtype | None = None

411) -> SparseArray:

412 """

413 Wrap a non-sparse operand for _sparse_array_op, reusing the other operand's

414 fill_value where that is a valid value for this operand's subtype.

415 """

416 # GH#68466 fill_value need not suit the operand's subtype; check it here

417 # rather than catch the construction, which can warn before it raises.

418 try:

419 SparseDtype(dtype if dtype is not None else values.dtype, fill_value)

420 except (TypeError, ValueError):

421 # leave any real incompatibility for op() to report

422 return SparseArray(values, dtype=dtype)

423 return SparseArray(values, fill_value=fill_value, dtype=dtype)

424

425

426def _wrap_result(

427 name: str,

428 data,

429 sparse_index,

430 fill_value,

431 dtype: Dtype | None = None,

432 kind: SparseIndexKind = "integer",

433) -> SparseArray:

434 """

435 wrap op result to have correct dtype

436 """

437 if name.startswith("__"):

438 # e.g. __eq__ --> eq

439 name = name[2:-2]

440

441 if name in _comparison_ops:

442 dtype = bool

443

444 fill_value = lib.item_from_zerodim(fill_value)

445

446 if is_bool_dtype(dtype):

447 # fill_value may be np.bool_

448 fill_value = bool(fill_value)

449 if sparse_index is None:

450 # a dense result, e.g. from _wrap_dense_result: sparsify it here

451 return SparseArray(data, fill_value=fill_value, dtype=dtype, kind=kind)

452

453 data, sparse_index = _prune_fill_value(data, sparse_index, fill_value)

454

455 if dtype is not None:

456 data = np.asarray(data, dtype=dtype) # type: ignore[arg-type]

457 else:

458 data = np.asarray(data)

459 sparse_dtype = SparseDtype(data.dtype, fill_value)

460 return SparseArray._simple_new(data, sparse_index, sparse_dtype)

461

462

463def _differs_from_fill(values: np.ndarray, fill_value) -> np.ndarray:

464 """

465 Which of ``values`` would not densify as ``fill_value``.

466 """

467 if isna(fill_value):

468 return notna(values)

469 mask = values != fill_value

470 if values.dtype.kind == "f" and fill_value == 0:

471 # -0.0 == 0.0 but the two densify differently, so a stored signed zero

472 # is a real point

473 mask |= np.signbit(values) != np.signbit(fill_value)

474 return mask

475

476

477def _prune_fill_value(

478 sp_values: np.ndarray, sparse_index: SparseIndex, fill_value

479) -> tuple[np.ndarray, SparseIndex]:

480 """

481 Drop stored values that densify as fill_value, so sp_index holds only real points.

482

483 GH#45126 the ops reuse an operand's sparse index or the union of both, either

484 of which can cover a position the op turned into the fill value.

485 """

486 if sp_values.dtype.kind == "c":

487 # isna() is True when either part is NaN, so it cannot decide a complex

488 # on its own

489 mask = _differs_from_fill(sp_values.real, np.real(fill_value)) | (

490 _differs_from_fill(sp_values.imag, np.imag(fill_value))

491 )

492 elif sp_values.dtype == object:

493 if isna(fill_value):

494 # object can hold several NAs at once, and only the fill's own spelling

495 # densifies back as the fill

496 mask = np.array(

497 [not libmissing.is_matching_na(x, fill_value) for x in sp_values],

498 dtype=bool,

499 )

500 else:

501 # 0, 0.0 and False compare equal, so use _make_sparse's type-aware check

502 mask = splib.make_mask_object_ndarray(sp_values, fill_value)

503 else:

504 mask = _differs_from_fill(sp_values, fill_value)

505

506 if mask.all():

507 return sp_values, sparse_index

508

509 kind: SparseIndexKind = "integer" if isinstance(sparse_index, IntIndex) else "block"

510 indices = sparse_index.indices[mask]

511 return sp_values[mask], make_sparse_index(sparse_index.length, indices, kind)

512

513

514_BOOL_SPARSE_DTYPE_FALSE_FILL = SparseDtype(bool, False)

515_BOOL_SPARSE_DTYPE_TRUE_FILL = SparseDtype(bool, True)

516

517

518def _can_hold_for_fill(dtype: np.dtype, fill_value) -> bool:

519 """

520 Whether a dense ``dtype`` array can hold ``fill_value`` without promoting.

521 """

522 dummy = ensure_wrapped_if_datetimelike(np.empty(0, dtype=dtype))

523 return can_hold_element(dummy, fill_value)

524

525

526def _unbox_for_fill(dtype: np.dtype, fill_value):

527 """

528 ``fill_value`` unboxed — for a datetimelike ``dtype``, as a numpy scalar in

529 that dtype's own unit. Only valid once ``_can_hold_for_fill`` has accepted it.

530 """

531 if dtype.kind not in "mM":

532 # an object dtype holds these as-is; only a datetimelike array needs

533 # the numpy scalar, and only it rejects np.nan in place of NaT

534 return fill_value

535 if isna(fill_value):

536 # a unitless NaT is deprecated as of numpy 2.5

537 return dtype.type("NaT", np.datetime_data(dtype)[0])

538 # numpy routes a boxed scalar through the stdlib datetime protocol and

539 # truncates; can_hold_element has already ruled out a lossy conversion

540 box = Timestamp if dtype.kind == "M" else Timedelta

541 return box(fill_value).asm8.astype(dtype)

542

543

544def _promote_for_fill(dtype: np.dtype, fill_value) -> tuple[np.dtype, Any]:

545 """

546 Dense dtype wide enough to hold ``fill_value``, and ``fill_value`` unboxed —

547 for a datetimelike ``dtype``, as a numpy scalar in that dtype's own unit.

548

549 ``maybe_promote`` is no good on its own: its datetime64 arm widens to ``M8[ns]``

550 whenever the fill value's unit differs, so a ``Timestamp`` would pull a

551 ``Sparse[M8[s]]`` up to nanoseconds.

552 """

553 if _can_hold_for_fill(dtype, fill_value):

554 return dtype, _unbox_for_fill(dtype, fill_value)

555 return maybe_promote(dtype, fill_value)

556

557

558@set_module("pandas.arrays")

559class SparseArray(OpsMixin, PandasObject, ExtensionArray):

560 """

561 An ExtensionArray for storing sparse data.

562

563 SparseArray efficiently stores data with a high frequency of a

564 specific fill value (e.g., zeros), saving memory by only retaining

565 non-fill elements and their indices. This class is particularly

566 useful for large datasets where most values are redundant.

567

568 Parameters

569 ----------

570 data : array-like or scalar

571 A dense array of values to store in the SparseArray. This may contain

572 `fill_value`.

573 sparse_index : SparseIndex, optional

574 Index indicating the locations of sparse elements.

575

576 .. deprecated:: 3.1.0

577 Use :meth:`SparseArray.from_indices` instead.

578 fill_value : scalar, optional

579 Elements in data that are ``fill_value`` are not stored in the

580 SparseArray. For memory savings, this should be the most common value

581 in `data`. By default, `fill_value` depends on the dtype of `data`:

582

583 =========== ==========

584 data.dtype na_value

585 =========== ==========

586 float ``np.nan``

587 int ``0``

588 bool False

589 datetime64 ``np.datetime64("NaT")``

590 timedelta64 ``np.timedelta64("NaT")``

591 =========== ==========

592

593 The fill value is potentially specified in three ways. In order of

594 precedence, these are

595

596 1. The `fill_value` argument

597 2. ``dtype.fill_value`` if `fill_value` is None and `dtype` is

598 a ``SparseDtype``

599 3. ``data.dtype.fill_value`` if `fill_value` is None and `dtype`

600 is not a ``SparseDtype`` and `data` is a ``SparseArray``.

601

602 kind : str

603 Can be 'integer' or 'block', default is 'integer'.

604 The type of storage for sparse locations.

605

606 * 'block': Stores a `block` and `block_length` for each

607 contiguous *span* of sparse values. This is best when

608 sparse data tends to be clumped together, with large

609 regions of ``fill-value`` values between sparse values.

610 * 'integer': uses an integer to store the location of

611 each sparse value.

612

613 dtype : np.dtype or SparseDtype, optional

614 The dtype to use for the SparseArray. For numpy dtypes, this

615 determines the dtype of ``self.sp_values``. For SparseDtype,

616 this determines ``self.sp_values`` and ``self.fill_value``.

617 copy : bool, default False

618 Whether to explicitly copy the incoming `data` array.

619

620 Attributes

621 ----------

622 None

623

624 Methods

625 -------

626 None

627

628 See Also

629 --------

630 SparseDtype : Dtype for sparse data.

631

632 Examples

633 --------

634 >>> from pandas.arrays import SparseArray

635 >>> arr = SparseArray([0, 0, 1, 2])

636 >>> arr

637 <SparseArray>

638 [0, 0, 1, 2]

639 Length: 4, dtype: Sparse[int64, 0]

640 """

641

642 _subtyp = "sparse_array" # register ABCSparseArray

643 _hidden_attrs = PandasObject._hidden_attrs | frozenset([])

644 _sparse_index: SparseIndex

645 _sparse_values: np.ndarray

646 _dtype: SparseDtype

647

648 def __init__(

649 self,

650 data,

651 sparse_index=lib.no_default,

652 fill_value=None,

653 kind: SparseIndexKind = "integer",

654 dtype: Dtype | None = None,

655 copy: bool = False,

656 ) -> None:

657 if sparse_index is not lib.no_default:

658 warnings.warn(

659 "The 'sparse_index' parameter of SparseArray.__init__ is "

660 "deprecated and will be removed in a future version. "

661 "Use SparseArray.from_indices instead.",

662 Pandas4Warning,

663 stacklevel=find_stack_level(),

664 )

665 else:

666 sparse_index = None

667

668 if fill_value is None and isinstance(dtype, SparseDtype):

669 fill_value = dtype.fill_value

670

671 if isinstance(data, type(self)):

672 # disable normal inference on dtype, sparse_index, & fill_value

673 if sparse_index is None:

674 sparse_index = data.sp_index

675 if fill_value is None:

676 fill_value = data.fill_value

677 if dtype is None:

678 dtype = data.dtype

679 # TODO: make kind=None, and use data.kind?

680 data = data.sp_values

681

682 # Handle use-provided dtype

683 if isinstance(dtype, str):

684 # Two options: dtype='int', regular numpy dtype

685 # or dtype='Sparse[int]', a sparse dtype

686 try:

687 dtype = SparseDtype.construct_from_string(dtype)

688 except TypeError:

689 dtype = pandas_dtype(dtype)

690

691 if isinstance(dtype, SparseDtype):

692 if fill_value is None:

693 fill_value = dtype.fill_value

694 dtype = dtype.subtype

695

696 if is_scalar(data):

697 raise TypeError(

698 f"Cannot construct {type(self).__name__} from scalar data. "

699 "Pass a sequence instead."

700 )

701

702 if dtype is not None:

703 dtype = pandas_dtype(dtype)

704

705 # TODO: disentangle the fill_value dtype inference from

706 # dtype inference

707 if data is None:

708 # TODO: What should the empty dtype be? Object or float?

709

710 # error: Argument "dtype" to "array" has incompatible type

711 # "Union[ExtensionDtype, dtype[Any], None]"; expected "Union[dtype[Any],

712 # None, type, _SupportsDType, str, Union[Tuple[Any, int], Tuple[Any,

713 # Union[int, Sequence[int]]], List[Any], _DTypeDict, Tuple[Any, Any]]]"

714 data = np.array([], dtype=dtype) # type: ignore[arg-type]

715

716 try:

717 data = sanitize_array(data, index=None)

718 except ValueError:

719 # NumPy may raise a ValueError on data like [1, []]

720 # we retry with object dtype here.

721 if dtype is None:

722 dtype = np.dtype(object)

723 data = np.atleast_1d(np.asarray(data, dtype=dtype))

724 else:

725 raise

726

727 if lib.is_np_dtype(data.dtype, "mM") and not is_supported_dtype(data.dtype):

728 # GH#68522 cast to a resolution we can hold, and reject a multiplier

729 # unit, the way the dense constructors do

730 data = np.asarray(ensure_wrapped_if_datetimelike(data))

731

732 if copy:

733 # TODO: avoid double copy when dtype forces cast.

734 data = data.copy()

735

736 if fill_value is None:

737 fill_value_dtype = data.dtype if dtype is None else dtype

738 if fill_value_dtype is None:

739 fill_value = np.nan

740 else:

741 fill_value = na_value_for_dtype(fill_value_dtype)

742

743 if isinstance(data, type(self)) and sparse_index is None:

744 sparse_index = data._sparse_index

745 # error: Argument "dtype" to "asarray" has incompatible type

746 # "Union[ExtensionDtype, dtype[Any], None]"; expected "None"

747 sparse_values = np.asarray(

748 data.sp_values,

749 dtype=dtype, # type: ignore[arg-type]

750 )

751 elif sparse_index is None:

752 data = extract_array(data, extract_numpy=True)

753 if not isinstance(data, np.ndarray):

754 # EA

755 if isinstance(data.dtype, DatetimeTZDtype):

756 warnings.warn(

757 f"Creating SparseArray from {data.dtype} data "

758 "loses timezone information. Cast to object before "

759 "sparse to retain timezone information.",

760 UserWarning,

761 stacklevel=find_stack_level(),

762 )

763 data = np.asarray(data, dtype="datetime64[ns]")

764 if fill_value is NaT:

765 fill_value = np.datetime64("NaT", "ns")

766 data = np.asarray(data)

767 sparse_values, sparse_index, fill_value = _make_sparse(

768 # error: Argument "dtype" to "_make_sparse" has incompatible type

769 # "Union[ExtensionDtype, dtype[Any], None]"; expected

770 # "Optional[dtype[Any]]"

771 data,

772 kind=kind,

773 fill_value=fill_value,

774 dtype=dtype, # type: ignore[arg-type]

775 )

776 else:

777 # error: Argument "dtype" to "asarray" has incompatible type

778 # "Union[ExtensionDtype, dtype[Any], None]"; expected "None"

779 sparse_values = np.asarray(data, dtype=dtype) # type: ignore[arg-type]

780 if len(sparse_values) != sparse_index.npoints:

781 raise AssertionError(

782 f"Non array-like type {type(sparse_values)} must "

783 "have the same length as the index"

784 )

785 self._sparse_index = sparse_index

786 self._sparse_values = sparse_values

787 self._dtype = SparseDtype(sparse_values.dtype, fill_value)

788

789 @classmethod

790 def _simple_new(

791 cls,

792 sparse_array: np.ndarray,

793 sparse_index: SparseIndex,

794 dtype: SparseDtype,

795 ) -> Self:

796 new = object.__new__(cls)

797 new._sparse_index = sparse_index

798 new._sparse_values = sparse_array

799 new._dtype = dtype

800 return new

801

802 @classmethod

803 def from_spmatrix(cls, data: _SparseMatrixLike) -> Self:

804 """

805 Create a SparseArray from a scipy.sparse matrix.

806

807 Parameters

808 ----------

809 data : scipy.sparse.sp_matrix

810 This should be a SciPy sparse matrix where the size

811 of the second dimension is 1. In other words, a

812 sparse matrix with a single column.

813

814 Returns

815 -------

816 SparseArray

817

818 Examples

819 --------

820 >>> import scipy.sparse

821 >>> mat = scipy.sparse.coo_matrix((4, 1))

822 >>> pd.arrays.SparseArray.from_spmatrix(mat)

823 <SparseArray>

824 [0.0, 0.0, 0.0, 0.0]

825 Length: 4, dtype: Sparse[float64, 0.0]

826 """

827 length, ncol = data.shape

828

829 if ncol != 1:

830 raise ValueError(f"'data' must have a single column, not '{ncol}'")

831

832 # our sparse index classes require that the positions be strictly

833 # increasing. So we need to sort loc, and arr accordingly.

834 data_csc = data.tocsc()

835 data_csc.sort_indices()

836 arr = data_csc.data

837 idx = data_csc.indices

838

839 zero = np.array(0, dtype=arr.dtype).item()

840 dtype = SparseDtype(arr.dtype, zero)

841 index = IntIndex(length, idx)

842

843 return cls._simple_new(arr, index, dtype)

844

845 @classmethod

846 def from_indices(

847 cls,

848 data,

849 indices,

850 length: int,

851 fill_value=None,

852 kind: SparseIndexKind = "integer",

853 dtype: Dtype | None = None,

854 ) -> Self:

855 """

856 Create a SparseArray from an array of sparse values and their indices.

857

858 This allows constructing a SparseArray without materializing a dense

859 array, which is useful for large arrays where most values are the

860 fill value.

861

862 Parameters

863 ----------

864 data : array-like

865 The sparse (non-fill) values.

866 indices : array-like of int

867 The indices in the dense array where the sparse values are located.

868 Must be strictly increasing and within ``[0, length)``.

869 length : int

870 The total length of the resulting SparseArray.

871 fill_value : scalar, optional

872 The fill value to use for positions not in ``indices``.

873 By default, uses ``np.nan`` for float dtypes and the appropriate

874 NA value for other dtypes.

875 kind : {{'integer', 'block'}}, default 'integer'

876 The type of sparse index to use.

877 dtype : np.dtype or ExtensionDtype, optional

878 The dtype for the sparse values.

879

880 Returns

881 -------

882 SparseArray

883

884 See Also

885 --------

886 SparseArray.from_spmatrix : Create a SparseArray from a scipy.sparse matrix.

887

888 Examples

889 --------

890 >>> SparseArray.from_indices(

891 ... [1, 2, 3], indices=[1, 3, 5], length=7, fill_value=0

892 ... )

893 <SparseArray>

894 [0, 1, 0, 2, 0, 3, 0]

895 Length: 7, dtype: Sparse[int64, 0]

896

897 This avoids materializing a dense array of length 7.

898 """

899 sparse_values = np.asarray(

900 sanitize_array(data, index=None), # type: ignore[arg-type]

901 dtype=dtype, # type: ignore[arg-type]

902 )

903 indices = np.asarray(indices, dtype=np.int32)

904

905 if len(sparse_values) != len(indices):

906 raise ValueError(

907 f"Length of data ({len(sparse_values)}) must match "

908 f"length of indices ({len(indices)})."

909 )

910

911 sparse_index = make_sparse_index(length, indices, kind)

912

913 if fill_value is None:

914 fill_value = na_value_for_dtype(sparse_values.dtype)

915

916 sparse_dtype = SparseDtype(sparse_values.dtype, fill_value)

917 return cls._simple_new(sparse_values, sparse_index, sparse_dtype)

918

919 def __array__(

920 self, dtype: NpDtype | None = None, copy: bool | None = None

921 ) -> np.ndarray:

922 if (

923 dtype is not None

924 and np.dtype(dtype) == object

925 and self.sp_values.dtype.kind in "mM"

926 ):

927 # numpy renders datetime64/timedelta64 as ints when casting to

928 # object; box them ourselves, see test_array_object_datetimelike

929 if copy is False:

930 raise ValueError(

931 "Unable to avoid copy while creating an array as requested."

932 )

933 dense = ensure_wrapped_if_datetimelike(np.asarray(self))

934 return np.asarray(dense, dtype=object)

935

936 if self.sp_index.ngaps == 0:

937 # Compat for na dtype and int values.

938 if copy is True:

939 return np.array(self.sp_values)

940 else:

941 result = self.sp_values

942 if self._readonly:

943 result = result.view()

944 result.flags.writeable = False

945 return result

946

947 if copy is False:

948 raise ValueError(

949 "Unable to avoid copy while creating an array as requested."

950 )

951

952 fill_value = self.fill_value

953

954 if self.sp_values.dtype.kind in "mM":

955 # np.result_type gives object for a Timestamp/Timedelta fill value

956 # and raises for np.nan; the helper also puts the fill in the

957 # subtype's unit, see test_to_dense_datetimelike_fill_other_unit

958 inferred, fill_value = _promote_for_fill(self.sp_values.dtype, fill_value)

959 if dtype is None:

960 dtype = inferred

961 elif dtype is None:

962 # Can NumPy represent this type?

963 # If not, `np.result_type` will raise. We catch that

964 # and return object.

965 try:

966 dtype = np.result_type(self.sp_values.dtype, type(fill_value))

967 except TypeError:

968 dtype = object

969

970 out = np.full(self.shape, fill_value, dtype=dtype)

971 out[self.sp_index.indices] = self.sp_values

972 return out

973

974 def __setitem__(self, key, value) -> None:

975 if self._readonly:

976 raise ValueError("Cannot modify read-only array")

977 # Only positions already held in sp_values could be written in place,

978 # and which those are depends on the value being set, so SparseDtype

979 # is _is_immutable, see GH#21818.

980 msg = "SparseArray does not support item assignment via setitem"

981 raise TypeError(msg)

982

983 def sort(

984 self,

985 *,

986 ascending: bool = True,

987 kind: SortKind = "quicksort",

988 na_position: str = "last",

989 ) -> None:

990 raise NotImplementedError("SparseArray does not support in-place sort")

991

992 @classmethod

993 def _from_sequence(

994 cls, scalars, *, dtype: Dtype | None = None, copy: bool = False

995 ) -> Self:

996 return cls(scalars, dtype=dtype)

997

998 @classmethod

999 def _from_factorized(cls, values, original) -> Self:

1000 return cls(values, dtype=original.dtype)

1001

1002 def _cast_pointwise_result(self, values):

1003 if not (isinstance(values, np.ndarray) and values.dtype == object):

1004 values = construct_1d_object_array_from_listlike(values)

1005 result = lib.maybe_convert_objects(values, convert_non_numeric=True)

1006 if result.dtype.kind == self.dtype.kind:

1007 try:

1008 # e.g. test_groupby_agg_extension

1009 res = type(self)._from_sequence(result, dtype=self.dtype)

1010 if ((res == result) | (isna(result) & res.isna())).all():

1011 # This does not hold for e.g.

1012 # test_arith_frame_with_scalar[0-__truediv__]

1013 return res

1014 return type(self)._from_sequence(result)

1015 except (ValueError, TypeError):

1016 return type(self)._from_sequence(result)

1017 else:

1018 # e.g. test_combine_le avoid casting bools to Sparse[float64, nan]

1019 return type(self)._from_sequence(result)

1020

1021 # ------------------------------------------------------------------------

1022 # Data

1023 # ------------------------------------------------------------------------

1024 @property

1025 def sp_index(self) -> SparseIndex:

1026 """

1027 The SparseIndex containing the location of non- ``fill_value`` points.

1028 """

1029 return self._sparse_index

1030

1031 @property

1032 def sp_values(self) -> np.ndarray:

1033 """

1034 An ndarray containing the non- ``fill_value`` values.

1035

1036 This property returns the actual data values stored in the sparse

1037 representation, excluding the values that are equal to the ``fill_value``.

1038 The result is an ndarray of the underlying values, preserving the sparse

1039 structure by omitting the default ``fill_value`` entries.

1040

1041 See Also

1042 --------

1043 Series.sparse.to_dense : Convert a Series from sparse values to dense.

1044 Series.sparse.fill_value : Elements in `data` that are `fill_value` are

1045 not stored.

1046 Series.sparse.density : The percent of non- ``fill_value`` points, as decimal.

1047

1048 Examples

1049 --------

1050 >>> from pandas.arrays import SparseArray

1051 >>> s = SparseArray([0, 0, 1, 0, 2], fill_value=0)

1052 >>> s.sp_values

1053 array([1, 2])

1054 """

1055 return self._sparse_values

1056

1057 @property

1058 def dtype(self) -> SparseDtype:

1059 return self._dtype

1060

1061 @property

1062 def fill_value(self):

1063 """

1064 Elements in `data` that are `fill_value` are not stored.

1065

1066 For memory savings, this should be the most common value in the array.

1067

1068 See Also

1069 --------

1070 SparseDtype : Dtype for data stored in :class:`SparseArray`.

1071 Series.value_counts : Return a Series containing counts of unique values.

1072 Series.fillna : Fill NA/NaN in a Series with a specified value.

1073

1074 Examples

1075 --------

1076 >>> ser = pd.Series([0, 0, 2, 2, 2], dtype="Sparse[int]")

1077 >>> ser.sparse.fill_value

1078 0

1079 >>> spa_dtype = pd.SparseDtype(dtype=np.int32, fill_value=2)

1080 >>> ser = pd.Series([0, 0, 2, 2, 2], dtype=spa_dtype)

1081 >>> ser.sparse.fill_value

1082 2

1083 """

1084 return self.dtype.fill_value

1085

1086 @fill_value.setter

1087 def fill_value(self, value) -> None:

1088 self._dtype = SparseDtype(self.dtype.subtype, value)

1089

1090 @property

1091 def kind(self) -> SparseIndexKind:

1092 """

1093 The kind of sparse index for this array. One of {'integer', 'block'}.

1094 """

1095 if isinstance(self.sp_index, IntIndex):

1096 return "integer"

1097 else:

1098 return "block"

1099

1100 @property

1101 def _valid_sp_values(self) -> np.ndarray:

1102 sp_vals = self.sp_values

1103 mask = notna(sp_vals)

1104 return sp_vals[mask]

1105

1106 def __len__(self) -> int:

1107 return self.sp_index.length

1108

1109 @property

1110 def _null_fill_value(self) -> bool:

1111 return self._dtype._is_na_fill_value

1112

1113 @property

1114 def nbytes(self) -> int:

1115 return self.sp_values.nbytes + self.sp_index.nbytes

1116

1117 def memory_usage(self, deep: bool = False) -> int:

1118 """

1119 Memory usage of the stored values and the sparse index, in bytes.

1120

1121 Positions holding ``fill_value`` are not stored.

1122

1123 Parameters

1124 ----------

1125 deep : bool, default False

1126 Introspect the data deeply by interrogating an ``object`` subtype

1127 for system-level memory consumption. Has no effect on PyPy.

1128

1129 Returns

1130 -------

1131 int

1132 Bytes consumed.

1133

1134 See Also

1135 --------

1136 SparseArray.nbytes : Memory usage without deep introspection.

1137 Series.memory_usage : Memory usage of a Series.

1138 DataFrame.memory_usage : Memory usage of each column in a DataFrame.

1139

1140 Examples

1141 --------

1142 >>> arr = pd.arrays.SparseArray(["a", "a", "b"], fill_value="a")

1143 >>> arr.memory_usage()

1144 12

1145 >>> arr.memory_usage(deep=True) > arr.memory_usage()

1146 True

1147 """

1148 result = self.nbytes

1149 if deep and not PYPY and self.sp_values.dtype == object:

1150 # deep introspection relies on sys.getsizeof, which always

1151 # raises TypeError on PyPy (GH#46176)

1152 return result + lib.memory_usage_of_objects(self.sp_values)

1153 return result

1154

1155 @property

1156 def density(self) -> float:

1157 """

1158 The percent of non- ``fill_value`` points, as decimal.

1159

1160 This is calculated as the number of non-fill-value points divided by

1161 the total length of the array. A density of 1.0 means no values are

1162 the fill value, while 0.0 means all values are the fill value. An

1163 empty array has no density and returns ``nan``.

1164

1165 See Also

1166 --------

1167 DataFrame.sparse.from_spmatrix : Create a new DataFrame from a

1168 scipy sparse matrix.

1169

1170 Examples

1171 --------

1172 >>> from pandas.arrays import SparseArray

1173 >>> s = SparseArray([0, 0, 1, 1, 1], fill_value=0)

1174 >>> s.density

1175 0.6

1176 """

1177 if self.sp_index.length == 0:

1178 # 0 / 0 is undefined

1179 return np.nan

1180 return self.sp_index.npoints / self.sp_index.length

1181

1182 @property

1183 def npoints(self) -> int:

1184 """

1185 The number of non- ``fill_value`` points.

1186

1187 This property returns the number of elements in the sparse series that are

1188 not equal to the ``fill_value``. Sparse data structures store only the

1189 non-``fill_value`` elements, reducing memory usage when the majority of

1190 values are the same.

1191

1192 See Also

1193 --------

1194 Series.sparse.to_dense : Convert a Series from sparse values to dense.

1195 Series.sparse.fill_value : Elements in ``data`` that are ``fill_value`` are

1196 not stored.

1197 Series.sparse.density : The percent of non- ``fill_value`` points, as decimal.

1198

1199 Examples

1200 --------

1201 >>> from pandas.arrays import SparseArray

1202 >>> s = SparseArray([0, 0, 1, 1, 1], fill_value=0)

1203 >>> s.npoints

1204 3

1205 """

1206 return self.sp_index.npoints

1207

1208 # error: Return type "SparseArray" of "isna" incompatible with return type

1209 # "ndarray[Any, Any] | ExtensionArrayNaResult" in supertype "ExtensionArray"

1210 def isna(self) -> Self: # type: ignore[override]

1211 # If null fill value, we want SparseDtype[bool, true]

1212 # to preserve the same memory usage.

1213 if self._null_fill_value:

1214 return type(self)._simple_new(

1215 isna(self.sp_values),

1216 self.sp_index,

1217 _BOOL_SPARSE_DTYPE_TRUE_FILL,

1218 )

1219 # GH#41023 - avoid densify-then-resparsify round-trip.

1220 # The NA positions are exactly the subset of sp_index where sp_values

1221 # are NA; we can construct the sparse index directly.

1222 sp_mask = isna(self.sp_values)

1223 na_indices = self.sp_index.indices[sp_mask]

1224 new_sp_index = make_sparse_index(len(self), na_indices, self.kind)

1225 return type(self)._simple_new(

1226 np.ones(len(na_indices), dtype=bool),

1227 new_sp_index,

1228 _BOOL_SPARSE_DTYPE_FALSE_FILL,

1229 )

1230

1231 def fillna(

1232 self,

1233 value,

1234 limit: int | None = None,

1235 copy: bool = True,

1236 ) -> Self:

1237 """

1238 Fill missing values with `value`.

1239

1240 Parameters

1241 ----------

1242 value : scalar

1243 limit : int, optional

1244 Not supported for SparseArray, must be None.

1245 copy: bool, default True

1246 Ignored for SparseArray.

1247

1248 Returns

1249 -------

1250 SparseArray

1251

1252 Notes

1253 -----

1254 When `value` is specified, the result's ``fill_value`` depends on

1255 ``self.fill_value``. The goal is to maintain low-memory use.

1256

1257 If ``self.fill_value`` is NA, the result dtype will be

1258 ``SparseDtype(self.dtype, fill_value=value)``. This will preserve

1259 amount of memory used before and after filling.

1260

1261 When ``self.fill_value`` is not NA, the result dtype will be

1262 ``self.dtype``. Again, this preserves the amount of memory used.

1263 """

1264 if isinstance(value, dict):

1265 raise TypeError(

1266 "ExtensionArray.fillna does not support filling with a dict. "

1267 "Use Series.fillna instead."

1268 )

1269 if limit is not None:

1270 raise ValueError("limit must be None")

1271

1272 subtype = self.sp_values.dtype

1273 if (

1274 subtype.kind in "mM"

1275 and is_scalar(value)

1276 and _can_hold_for_fill(subtype, value)

1277 ):

1278 # np.where resolves a scalar it cannot match against an M8/m8 array

1279 # as object, or in the scalar's own unit, either way leaving

1280 # sp_values disagreeing with the dtype

1281 value = _unbox_for_fill(subtype, value)

1282

1283 new_values = np.where(isna(self.sp_values), value, self.sp_values)

1284

1285 if self._null_fill_value:

1286 # This is essentially just updating the dtype.

1287 new_dtype = SparseDtype(self.dtype.subtype, fill_value=value)

1288 else:

1289 new_dtype = self.dtype

1290

1291 return self._simple_new(new_values, self._sparse_index, new_dtype)

1292

1293 def shift(self, periods: int = 1, fill_value=None) -> Self:

1294 if not len(self) or periods == 0:

1295 return self.copy()

1296

1297 if isna(fill_value):

1298 fill_value = self.dtype.na_value

1299

1300 subtype, fill_value = _promote_for_fill(self.dtype.subtype, fill_value)

1301

1302 if subtype != self.dtype.subtype:

1303 # just coerce up front

1304 arr = self.astype(SparseDtype(subtype, self.fill_value))

1305 else:

1306 arr = self

1307

1308 empty = self._from_sequence(

1309 [fill_value] * min(abs(periods), len(self)), dtype=arr.dtype

1310 )

1311

1312 if periods > 0:

1313 a = empty

1314 b = arr[:-periods]

1315 else:

1316 a = arr[abs(periods) :]

1317 b = empty

1318 return arr._concat_same_type([a, b])

1319

1320 def _first_fill_value_loc(self):

1321 """

1322 Get the location of the first fill value.

1323

1324 Returns

1325 -------

1326 int

1327 """

1328 if len(self) == 0 or self.sp_index.npoints == len(self):

1329 return -1

1330

1331 indices = self.sp_index.indices

1332 if not len(indices) or indices[0] > 0:

1333 return 0

1334

1335 # a number larger than 1 should be appended to

1336 # the last in case of fill value only appears

1337 # in the tail of array

1338 diff = np.r_[np.diff(indices), 2]

1339 return indices[(diff > 1).argmax()] + 1

1340

1341 def duplicated(

1342 self, keep: Literal["first", "last", False] = "first"

1343 ) -> npt.NDArray[np.bool_]:

1344 """

1345 Return boolean ndarray denoting duplicate values.

1346

1347 Parameters

1348 ----------

1349 keep : {'first', 'last', False}, default 'first'

1350 - ``first`` : Mark duplicates as ``True`` except for the first occurrence.

1351 - ``last`` : Mark duplicates as ``True`` except for the last occurrence.

1352 - False : Mark all duplicates as ``True``.

1353

1354 Returns

1355 -------

1356 ndarray[bool]

1357 With true in indices where elements are duplicated and false otherwise.

1358

1359 See Also

1360 --------

1361 DataFrame.duplicated : Return boolean Series denoting

1362 duplicate rows.

1363 Series.duplicated : Indicate duplicate Series values.

1364 api.extensions.ExtensionArray.unique : Compute the ExtensionArray

1365 of unique values.

1366

1367 Examples

1368 --------

1369 >>> pd.array([1, 1, 2, 3, 3], dtype=pd.SparseDtype(np.int64)).duplicated()

1370 array([False, True, False, False, True])

1371 """

1372

1373 values = np.asarray(self)

1374 mask = np.asarray(self.isna())

1375 return algos.duplicated(values, keep=keep, mask=mask)

1376

1377 def unique(self) -> Self:

1378 uniques = algos.unique(self.sp_values)

1379 if len(self.sp_values) != len(self):

1380 fill_loc = self._first_fill_value_loc()

1381 # Inorder to align the behavior of pd.unique or

1382 # pd.Series.unique, we should keep the original

1383 # order, here we use unique again to find the

1384 # insertion place. Since the length of sp_values

1385 # is not large, maybe minor performance hurt

1386 # is worthwhile to the correctness.

1387 insert_loc = len(algos.unique(self.sp_values[:fill_loc]))

1388 uniques = np.insert(uniques, insert_loc, self.fill_value)

1389 return type(self)._from_sequence(uniques, dtype=self.dtype)

1390

1391 def _values_for_factorize(self):

1392 # Still override this for hash_pandas_object

1393 return np.asarray(self), self.fill_value

1394

1395 def factorize(

1396 self,

1397 use_na_sentinel: bool = True,

1398 ) -> tuple[np.ndarray, SparseArray]:

1399 # Currently, ExtensionArray.factorize -> Tuple[ndarray, EA]

1400 # The sparsity on this is backwards from what Sparse would want. Want

1401 # ExtensionArray.factorize -> Tuple[EA, EA]

1402 # Given that we have to return a dense array of codes, why bother

1403 # implementing an efficient factorize?

1404 codes, uniques = algos.factorize(

1405 np.asarray(self), use_na_sentinel=use_na_sentinel

1406 )

1407 uniques_sp = SparseArray(uniques, dtype=self.dtype)

1408 return codes, uniques_sp

1409

1410 def value_counts(self, dropna: bool = True) -> Series:

1411 """

1412 Returns a Series containing counts of unique values.

1413

1414 Parameters

1415 ----------

1416 dropna : bool, default True

1417 Don't include counts of NaN, even if NaN is in sp_values.

1418

1419 Returns

1420 -------

1421 counts : Series

1422 """

1423 from pandas import (

1424 Index,

1425 Series,

1426 )

1427

1428 keys, counts, _ = algos.value_counts_arraylike(self.sp_values, dropna=dropna)

1429 fcounts = self.sp_index.ngaps

1430 if fcounts > 0 and (not self._null_fill_value or not dropna):

1431 if self._null_fill_value:

1432 mask = isna(keys)

1433 elif keys.dtype == object:

1434 # GH#68586 an object key such as pd.NA compares to a non-bool,

1435 # which mask.any() cannot consume

1436 mask = ops.comp_method_OBJECT_ARRAY(operator.eq, keys, self.fill_value)

1437 else:

1438 mask = keys == self.fill_value

1439 if mask.any():

1440 counts[mask] += fcounts

1441 else:

1442 # error: Argument 1 to "insert" has incompatible type "Union[

1443 # ExtensionArray,ndarray[Any, Any]]"; expected "Union[

1444 # _SupportsArray[dtype[Any]], Sequence[_SupportsArray[dtype

1445 # [Any]]], Sequence[Sequence[_SupportsArray[dtype[Any]]]],

1446 # Sequence[Sequence[Sequence[_SupportsArray[dtype[Any]]]]], Sequence

1447 # [Sequence[Sequence[Sequence[_SupportsArray[dtype[Any]]]]]]]"

1448 keys = np.insert(keys, 0, self.fill_value) # type: ignore[arg-type]

1449 counts = np.insert(counts, 0, fcounts)

1450

1451 if not isinstance(keys, ABCIndex):

1452 index = Index(keys, copy=False)

1453 else:

1454 index = keys

1455 return Series(counts, index=index, copy=False)

1456

1457 # --------

1458 # Indexing

1459 # --------

1460 @overload

1461 def __getitem__(self, key: ScalarIndexer) -> Any: ...

1462

1463 @overload

1464 def __getitem__(self, key: SequenceIndexer) -> Self: ...

1465

1466 def __getitem__(self, key: PositionalIndexer) -> Self | Any:

1467 if isinstance(key, tuple):

1468 key = unpack_tuple_and_ellipses(key)

1469 if key is ...:

1470 raise ValueError("Cannot slice with Ellipsis")

1471

1472 if is_integer(key):

1473 return self._get_val_at(key)

1474 elif isinstance(key, tuple):

1475 data_slice = self.to_dense()[key]

1476 elif isinstance(key, slice):

1477 if key == slice(None):

1478 # to ensure arr[:] (used by view()) does not make a copy

1479 result = type(self)._simple_new(

1480 self.sp_values, self.sp_index, self.dtype

1481 )

1482 result._readonly = self._readonly

1483 return result

1484 # Avoid densifying when handling contiguous slices

1485 if key.step is None or key.step == 1:

1486 start = 0 if key.start is None else key.start

1487 if start < 0:

1488 start += len(self)

1489

1490 end = len(self) if key.stop is None else key.stop

1491 if end < 0:

1492 end += len(self)

1493

1494 indices = self.sp_index.indices

1495 keep_inds = np.flatnonzero((indices >= start) & (indices < end))

1496 sp_vals = self.sp_values[keep_inds]

1497

1498 sp_index = indices[keep_inds].copy()

1499

1500 # If we've sliced to not include the start of the array, all our indices

1501 # should be shifted. NB: here we are careful to also not shift by a

1502 # negative value for a case like [0, 1][-100:] where the start index

1503 # should be treated like 0

1504 if start > 0:

1505 sp_index -= start

1506

1507 # Length of our result should match applying this slice to a range

1508 # of the length of our original array

1509 new_len = len(range(len(self))[key])

1510 new_sp_index = make_sparse_index(new_len, sp_index, self.kind)

1511 return type(self)._simple_new(sp_vals, new_sp_index, self.dtype)

1512 else:

1513 indices = np.arange(len(self), dtype=np.int32)[key]

1514 return self.take(indices)

1515

1516 elif not is_list_like(key):

1517 # e.g. "foo" or 2.5

1518 # exception message copied from numpy

1519 raise IndexError(

1520 r"only integers, slices (`:`), ellipsis (`...`), numpy.newaxis "

1521 r"(`None`) and integer or boolean arrays are valid indices"

1522 )

1523

1524 else:

1525 if isinstance(key, SparseArray):

1526 if is_bool_dtype(key):

1527 if len(key) != len(self):

1528 # the fast path below skips check_array_indexer, which

1529 # would raise here

1530 raise IndexError(

1531 f"Boolean index has wrong length: "

1532 f"{len(key)} instead of {len(self)}"

1533 )

1534 # GH#45284 a stored value may equal the fill value, so select

1535 # on sp_values rather than on which positions are stored

1536 if isna(key.fill_value) or not key.fill_value:

1537 return self.take(key.sp_index.indices[key.sp_values])

1538 mask = np.full(len(self), True, dtype=np.bool_)

1539 mask[key.sp_index.indices] = key.sp_values

1540 return self.take(np.flatnonzero(mask))

1541 else:

1542 key = np.asarray(key)

1543

1544 key = check_array_indexer(self, key)

1545

1546 if com.is_bool_indexer(key):

1547 # mypy doesn't know we have an array here

1548 key = cast("np.ndarray", key)

1549 return self.take(np.arange(len(key), dtype=np.int32)[key])

1550 elif hasattr(key, "__len__"):

1551 return self.take(key)

1552 else:

1553 raise ValueError(f"Cannot slice with '{key}'")

1554

1555 return type(self)(data_slice, kind=self.kind)

1556

1557 def _get_val_at(self, loc):

1558 n = len(self)

1559 if loc < 0:

1560 loc += n

1561

1562 if loc >= n or loc < 0:

1563 raise IndexError(

1564 f"index is out of bounds: must be an integer between -{n} and {n - 1}"

1565 )

1566

1567 sp_loc = self.sp_index.lookup(loc)

1568 if sp_loc == -1:

1569 return self.fill_value

1570 else:

1571 val = self.sp_values[sp_loc]

1572 val = maybe_box_datetimelike(val, self.sp_values.dtype)

1573 return val

1574

1575 def take(self, indices, *, allow_fill: bool = False, fill_value=None) -> Self:

1576 if is_scalar(indices):

1577 raise ValueError(f"'indices' must be an array, not a scalar '{indices}'.")

1578 indices = np.asarray(indices, dtype=np.int32)

1579

1580 dtype = None

1581 if indices.size == 0:

1582 result = np.array([], dtype="object")

1583 dtype = self.dtype

1584 elif allow_fill:

1585 result = self._take_with_fill(indices, fill_value=fill_value)

1586 else:

1587 return self._take_without_fill(indices)

1588

1589 return type(self)(

1590 result, fill_value=self.fill_value, kind=self.kind, dtype=dtype

1591 )

1592

1593 def _take_with_fill(self, indices, fill_value=None) -> np.ndarray:

1594 if fill_value is None:

1595 fill_value = self.dtype.na_value

1596

1597 if indices.min() < -1:

1598 raise ValueError(

1599 "Invalid value in 'indices'. Must be between -1 "

1600 "and the length of the array."

1601 )

1602

1603 if indices.max() >= len(self):

1604 raise IndexError("out of bounds value in 'indices'.")

1605

1606 if len(self) == 0:

1607 # Empty... Allow taking only if all empty

1608 if (indices == -1).all():

1609 dtype, new_fill = _promote_for_fill(self.sp_values.dtype, fill_value)

1610 taken = np.empty_like(indices, dtype=dtype)

1611 taken.fill(new_fill)

1612 return taken

1613 else:

1614 raise IndexError("cannot do a non-empty take from an empty axes.")

1615

1616 # sp_indexer may be -1 for two reasons

1617 # 1.) we took for an index of -1 (new)

1618 # 2.) we took a value that was self.fill_value (old)

1619 sp_indexer = self.sp_index.lookup_array(indices)

1620 new_fill_indices = indices == -1

1621 old_fill_indices = (sp_indexer == -1) & ~new_fill_indices

1622

1623 if self.sp_index.npoints == 0 and old_fill_indices.all():

1624 # We've looked up all valid points on an all-sparse array.

1625 _dtype, old_fill = _promote_for_fill(self.dtype.subtype, self.fill_value)

1626 taken = np.full(sp_indexer.shape, old_fill, dtype=_dtype)

1627

1628 elif self.sp_index.npoints == 0:

1629 # Use the old fill_value unless we took for an index of -1

1630 _dtype, new_fill = _promote_for_fill(self.dtype.subtype, fill_value)

1631 _dtype, old_fill = _promote_for_fill(_dtype, self.fill_value)

1632 taken = np.full(sp_indexer.shape, new_fill, dtype=_dtype)

1633 taken[old_fill_indices] = old_fill

1634 else:

1635 taken = self.sp_values.take(sp_indexer)

1636

1637 # Fill in two steps.

1638 # Old fill values

1639 # New fill values

1640 # potentially coercing to a new dtype at each stage.

1641

1642 m0 = sp_indexer[old_fill_indices] < 0

1643 m1 = sp_indexer[new_fill_indices] < 0

1644

1645 result_type = taken.dtype

1646

1647 if m0.any():

1648 result_type, old_fill = _promote_for_fill(result_type, self.fill_value)

1649 taken = taken.astype(result_type, copy=False)

1650 taken[old_fill_indices] = old_fill

1651

1652 if m1.any():

1653 result_type, new_fill = _promote_for_fill(result_type, fill_value)

1654 taken = taken.astype(result_type, copy=False)

1655 taken[new_fill_indices] = new_fill

1656

1657 return taken

1658

1659 def _take_without_fill(self, indices) -> Self:

1660 to_shift = indices < 0

1661

1662 n = len(self)

1663

1664 if (indices.max() >= n) or (indices.min() < -n):

1665 if n == 0:

1666 raise IndexError("cannot do a non-empty take from an empty axes.")

1667 raise IndexError("out of bounds value in 'indices'.")

1668

1669 if to_shift.any():

1670 indices = indices.copy()

1671 indices[to_shift] += n

1672

1673 sp_indexer = self.sp_index.lookup_array(indices)

1674 value_mask = sp_indexer != -1

1675 new_sp_values = self.sp_values[sp_indexer[value_mask]]

1676

1677 value_indices = np.flatnonzero(value_mask).astype(np.int32, copy=False)

1678

1679 new_sp_index = make_sparse_index(len(indices), value_indices, kind=self.kind)

1680 return type(self)._simple_new(new_sp_values, new_sp_index, dtype=self.dtype)

1681

1682 def searchsorted(

1683 self,

1684 v: ArrayLike | object,

1685 side: Literal["left", "right"] = "left",

1686 sorter: NumpySorter | None = None,

1687 ) -> npt.NDArray[np.intp] | np.intp:

1688 if config["mode"]["performance_warnings"]:

1689 msg = "searchsorted requires high memory usage."

1690 warnings.warn(msg, PerformanceWarning, stacklevel=find_stack_level())

1691 v = np.asarray(v)

1692 return np.asarray(self, dtype=self.dtype.subtype).searchsorted(v, side, sorter)

1693

1694 def copy(self) -> Self:

1695 values = self.sp_values.copy()

1696 return self._simple_new(values, self.sp_index, self.dtype)

1697

1698 @classmethod

1699 def _concat_same_type(cls, to_concat: Sequence[Self]) -> Self:

1700 fill_value = to_concat[0].fill_value

1701

1702 values = []

1703 length = 0

1704

1705 if to_concat:

1706 sp_kind = to_concat[0].kind

1707 else:

1708 sp_kind = "integer"

1709

1710 sp_index: SparseIndex

1711 if sp_kind == "integer":

1712 indices = []

1713

1714 for arr in to_concat:

1715 int_idx = arr.sp_index.indices.copy()

1716 int_idx += length # TODO: wraparound

1717 length += arr.sp_index.length

1718

1719 values.append(arr.sp_values)

1720 indices.append(int_idx)

1721

1722 data = np.concatenate(values)

1723 indices_arr = np.concatenate(indices)

1724 sp_index = IntIndex(length, indices_arr)

1725

1726 else:

1727 # when concatenating block indices, we don't claim that you'll

1728 # get an identical index as concatenating the values and then

1729 # creating a new index. We don't want to spend the time trying

1730 # to merge blocks across arrays in `to_concat`, so the resulting

1731 # BlockIndex may have more blocks.

1732 blengths = []

1733 blocs = []

1734

1735 for arr in to_concat:

1736 block_idx = arr.sp_index.to_block_index()

1737

1738 values.append(arr.sp_values)

1739 blocs.append(block_idx.blocs.copy() + length)

1740 blengths.append(block_idx.blengths)

1741 length += arr.sp_index.length

1742

1743 data = np.concatenate(values)

1744 blocs_arr = np.concatenate(blocs)

1745 blengths_arr = np.concatenate(blengths)

1746

1747 sp_index = BlockIndex(length, blocs_arr, blengths_arr)

1748

1749 sparse_dtype = SparseDtype(data.dtype, fill_value)

1750 return cls._simple_new(data, sp_index, sparse_dtype)

1751

1752 def insert(self, loc: int, item) -> Self:

1753 if not is_valid_na_for_dtype(item, self.dtype):

1754 # GH#69028 our _from_sequence casts to the subtype instead of

1755 # raising, so validate here; Index.insert widens on the raise

1756 if not _can_hold_for_fill(self.dtype.subtype, item):

1757 raise TypeError(f"Invalid value '{item!s}' for dtype '{self.dtype}'")

1758 return super().insert(loc, item)

1759

1760 def astype(self, dtype: AstypeArg | None = None, copy: bool = True):

1761 """

1762 Change the dtype of a SparseArray.

1763

1764 Parameters

1765 ----------

1766 dtype : np.dtype or ExtensionDtype

1767 For SparseDtype, this changes the subtype and the

1768 ``fill_value``, preserving the values.

1769

1770 For any other dtype, the array is densified and cast to it.

1771

1772 copy : bool, default True

1773 Whether to ensure a copy is made, even if not necessary.

1774

1775 Returns

1776 -------

1777 np.ndarray or pandas.api.extensions.ExtensionArray

1778 A SparseArray for a SparseDtype, another ExtensionArray for any

1779 other ExtensionDtype, otherwise a dense ndarray of ``dtype``.

1780

1781 Examples

1782 --------

1783 >>> arr = pd.arrays.SparseArray([0, 0, 1, 2])

1784 >>> arr

1785 <SparseArray>

1786 [0, 0, 1, 2]

1787 Length: 4, dtype: Sparse[int64, 0]

1788

1789 >>> arr.astype(pd.SparseDtype(np.dtype("int32")))

1790 <SparseArray>

1791 [0, 0, 1, 2]

1792 Length: 4, dtype: Sparse[int32, 0]

1793

1794 Changing the subtype can change the fill value too -- here ``0``

1795 becomes ``nan``, the default for ``float64``. The values are never

1796 changed to match a new fill value, so a fill value absent from the

1797 data leaves nothing to compress.

1798

1799 >>> arr.astype(pd.SparseDtype(np.dtype("float64")))

1800 <SparseArray>

1801 [0.0, 0.0, 1.0, 2.0]

1802 Length: 4, dtype: Sparse[float64, nan]

1803

1804 >>> arr.astype(pd.SparseDtype("float64", fill_value=0.0))

1805 <SparseArray>

1806 [0.0, 0.0, 1.0, 2.0]

1807 Length: 4, dtype: Sparse[float64, 0.0]

1808 """

1809 if dtype == self._dtype:

1810 if not copy:

1811 return self

1812 else:

1813 return self.copy()

1814

1815 future_dtype = pandas_dtype(dtype)

1816 if not isinstance(future_dtype, SparseDtype):

1817 # GH#34457

1818 values = self._densify()

1819 values = ensure_wrapped_if_datetimelike(values)

1820 return astype_array(values, dtype=future_dtype, copy=False)

1821

1822 dtype = self.dtype.update_dtype(dtype)

1823

1824 # GH#49631: update_dtype resolves the target to the subtype's default

1825 # fill_value (e.g. 0 for int64) instead of converting the source

1826 # fill_value. Only fire when the target fill is the subtype default, so

1827 # an explicitly-requested non-default fill_value is respected.

1828 if (

1829 not dtype._is_na_fill_value

1830 and self.dtype.subtype.kind in "mM"

1831 and dtype.fill_value == na_value_for_dtype(dtype.subtype)

1832 ):

1833 # build from the source subtype so a boxed fill value converts too

1834 fv_arr = ensure_wrapped_if_datetimelike(np.zeros(1, self.dtype.subtype))

1835 fv_arr[0] = self.fill_value

1836 converted_fv = np.asarray(astype_array(fv_arr, dtype.subtype))

1837 dtype = SparseDtype(dtype.subtype, fill_value=converted_fv[0])

1838 # dtype.fill_value is now the converted self.fill_value, so the gaps

1839 # keep their value and the check below would only cost a densify

1840 converted_fill = True

1841 else:

1842 converted_fill = False

1843

1844 subtype = pandas_dtype(dtype._subtype_with_str)

1845 subtype = cast("np.dtype", subtype) # ensured by update_dtype

1846

1847 if not converted_fill and not self._fill_value_matches(

1848 dtype.fill_value, subtype

1849 ):

1850 # GH#35795 the gaps hold self.fill_value, so keeping sp_index while

1851 # adopting a different fill_value would silently change those values

1852 values = ensure_wrapped_if_datetimelike(self._densify())

1853 values = np.asarray(astype_array(values, subtype, copy=False))

1854 return type(self)(values, kind=self.kind, dtype=dtype)

1855

1856 values = ensure_wrapped_if_datetimelike(self.sp_values)

1857 sp_values = astype_array(values, subtype, copy=copy)

1858 sp_values = np.asarray(sp_values)

1859

1860 return self._simple_new(sp_values, self.sp_index, dtype)

1861

1862 def _fill_value_matches(self, fill_value, subtype) -> bool:

1863 """Whether fill_value is the value the gaps already hold under subtype."""

1864 if isna(self.fill_value) or isna(fill_value):

1865 return bool(isna(self.fill_value) and isna(fill_value))

1866 if is_object_dtype(self.dtype.subtype) or is_object_dtype(subtype):

1867 # _make_sparse tells 0, 0.0 and False apart for object data, so this

1868 # has to as well, see make_mask_object_ndarray

1869 return type(self.fill_value) is type(fill_value) and bool(

1870 self.fill_value == fill_value

1871 )

1872 return bool(self.fill_value == fill_value)

1873

1874 def map(self, mapper, na_action: Literal["ignore"] | None = None) -> Self:

1875 """

1876 Map categories using an input mapping or function.

1877

1878 Parameters

1879 ----------

1880 mapper : dict, Series, callable

1881 The correspondence from old values to new.

1882 na_action : {None, 'ignore'}, default None

1883 If 'ignore', propagate NA values, without passing them to the

1884 mapping correspondence.

1885

1886 Returns

1887 -------

1888 SparseArray

1889 The output array will have the same density as the input.

1890 The output fill value will be the result of applying the

1891 mapping to ``self.fill_value``

1892

1893 Examples

1894 --------

1895 >>> arr = pd.arrays.SparseArray([0, 1, 2])

1896 >>> arr.map(lambda x: x + 10)

1897 <SparseArray>

1898 [10, 11, 12]

1899 Length: 3, dtype: Sparse[int64, 10]

1900

1901 >>> arr.map({0: 10, 1: 11, 2: 12})

1902 <SparseArray>

1903 [10, 11, 12]

1904 Length: 3, dtype: Sparse[int64, 10]

1905

1906 >>> arr.map(pd.Series([10, 11, 12], index=[0, 1, 2]))

1907 <SparseArray>

1908 [10, 11, 12]

1909 Length: 3, dtype: Sparse[int64, 10]

1910 """

1911 is_map = isinstance(mapper, (abc.Mapping, ABCSeries))

1912

1913 fill_val = self.fill_value

1914

1915 if na_action is None or notna(fill_val):

1916 fill_val = mapper.get(fill_val, fill_val) if is_map else mapper(fill_val)

1917

1918 def func(sp_val):

1919 new_sp_val = mapper.get(sp_val, None) if is_map else mapper(sp_val)

1920 # check identity as well as equality because NA values are not equal to

1921 # each other, and GH#68586 comparing a stored one gives a non-bool.

1922 # is_scalar first: isna on a list-valued mapper result is not a bool

1923 if new_sp_val is fill_val or (

1924 not (is_scalar(new_sp_val) and isna(new_sp_val))

1925 and new_sp_val == fill_val

1926 ):

1927 msg = "fill value in the sparse values not supported"

1928 raise ValueError(msg)

1929 return new_sp_val

1930

1931 sp_values = np.asarray(

1932 sanitize_array([func(x) for x in self.sp_values], index=None)

1933 )

1934

1935 sparse_dtype = SparseDtype(sp_values.dtype, fill_val)

1936 return type(self)._simple_new(sp_values, self.sp_index, sparse_dtype)

1937

1938 def _groupby_op(

1939 self,

1940 *,

1941 how: str,

1942 has_dropped_na: bool,

1943 min_count: int,

1944 ngroups: int,

1945 ids: npt.NDArray[np.intp],

1946 **kwargs,

1947 ):

1948 # first/last are handled by the base class to preserve EA type

1949 if how in ["first", "last"]:

1950 return self._groupby_first_last(

1951 how=how,

1952 min_count=min_count,

1953 ngroups=ngroups,

1954 ids=ids,

1955 skipna=kwargs.get("skipna", True),

1956 )

1957

1958 npvalues = self._densify()

1959

1960 if npvalues.dtype.kind in "mM":

1961 # Defer to DatetimeArray/TimedeltaArray, which reject the

1962 # operations that are invalid for their dtype.

1963 return ensure_wrapped_if_datetimelike(npvalues)._groupby_op(

1964 how=how,

1965 has_dropped_na=has_dropped_na,

1966 min_count=min_count,

1967 ngroups=ngroups,

1968 ids=ids,

1969 **kwargs,

1970 )

1971

1972 from pandas.core.groupby.ops import WrappedCythonOp

1973

1974 op = WrappedCythonOp(how=how, has_dropped_na=has_dropped_na)

1975

1976 res_values = op._cython_op_ndim_compat(

1977 npvalues,

1978 min_count=min_count,

1979 ngroups=ngroups,

1980 comp_ids=ids,

1981 mask=None,

1982 **kwargs,

1983 )

1984

1985 return res_values

1986

1987 def to_dense(self) -> np.ndarray:

1988 """

1989 Convert SparseArray to a NumPy array.

1990

1991 Returns

1992 -------

1993 arr : NumPy array

1994 """

1995 return np.asarray(self, dtype=self.sp_values.dtype)

1996

1997 def _densify(self) -> np.ndarray:

1998 """

1999 Dense values, keeping the dense dtype the subtype would have.

2000

2001 ``to_dense`` is no good for this because it casts the fill_value to the

2002 subtype, turning the NAs of a ``Sparse[int64, nan]`` into 0s; ``np.asarray``

2003 is no good either because it promotes on ``type(fill_value)`` and so widens

2004 e.g. ``Sparse[uint64]`` to float64.

2005

2006 With no gaps this is ``sp_values`` itself (or a read-only view of it), not

2007 a copy, so callers must not write to the result.

2008 """

2009 if self.sp_index.ngaps == 0:

2010 result = self.sp_values

2011 if self._readonly:

2012 result = result.view()

2013 result.flags.writeable = False

2014 return result

2015

2016 npdtype = self.sp_values.dtype

2017 fill_value = self.fill_value

2018 if isna(fill_value):

2019 # SparseDtype._check_fill_value guarantees that a non-NA

2020 # fill_value fits in the subtype, so only an NA needs promoting.

2021 npdtype, fill_value = maybe_promote(npdtype, fill_value)

2022 else:

2023 fill_value = _unbox_for_fill(npdtype, fill_value)

2024 npvalues = np.full(self.shape, fill_value, dtype=npdtype)

2025 npvalues[self.sp_index.indices] = self.sp_values

2026 return npvalues

2027

2028 def _where(self, mask, value):

2029 # NB: may not preserve dtype, e.g. result may be Sparse[float64]

2030 # while self is Sparse[int64]

2031 naive_implementation = np.where(mask, self, value)

2032 dtype = SparseDtype(naive_implementation.dtype, fill_value=self.fill_value)

2033 result = type(self)._from_sequence(naive_implementation, dtype=dtype)

2034 return result

2035

2036 # ------------------------------------------------------------------------

2037 # IO

2038 # ------------------------------------------------------------------------

2039 def __setstate__(self, state) -> None:

2040 """Necessary for making this object picklable"""

2041 if isinstance(state, tuple):

2042 # Compat for pandas < 0.24.0

2043 nd_state, (fill_value, sp_index) = state

2044 sparse_values = np.array([])

2045 sparse_values.__setstate__(nd_state)

2046

2047 self._sparse_values = sparse_values

2048 self._sparse_index = sp_index

2049 self._dtype = SparseDtype(sparse_values.dtype, fill_value)

2050 else:

2051 self.__dict__.update(state)

2052

2053 def nonzero(self) -> tuple[npt.NDArray[np.int32]]:

2054 if self.fill_value == 0:

2055 return (self.sp_index.indices,)

2056 else:

2057 return (self.sp_index.indices[self.sp_values != 0],)

2058

2059 # ------------------------------------------------------------------------

2060 # Reductions

2061 # ------------------------------------------------------------------------

2062

2063 def _reduce(

2064 self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs

2065 ):

2066 method = getattr(self, name, None)

2067

2068 if method is None:

2069 raise TypeError(f"cannot perform {name} with type {self.dtype}")

2070

2071 if name in _skipna_aware_reductions:

2072 # these methods handle skipna themselves; dropping NAs beforehand

2073 # would hide the NA from their skipna=False short-circuit

2074 result = method(skipna=skipna, **kwargs)

2075 else:

2076 if skipna:

2077 arr = self

2078 else:

2079 arr = self.dropna()

2080 result = getattr(arr, name)(**kwargs)

2081

2082 if keepdims:

2083 dtype = self.dtype

2084 result_dtype = np.asarray(result).dtype

2085 if name in _positional_reductions:

2086 # intp over result_dtype, which varies with which branch of

2087 # _argmin_argmax answered. See test_frame_idxmin_idxmax

2088 dtype = SparseDtype(np.intp)

2089 elif name in _boolean_reductions:

2090 # see test_any_all_keepdims_is_boolean

2091 dtype = SparseDtype(bool)

2092 elif name in _complex_to_real_reductions and dtype.subtype.kind == "c":

2093 # np.result_type below would widen the real result straight back to

2094 # complex, and a complex fill value has no real counterpart. Gated on

2095 # the reduction, not the result dtype; see

2096 # test_frame_complex_reduction_na_keeps_complex

2097 dtype = SparseDtype(result_dtype)

2098 elif dtype.subtype.kind in "biufc":

2099 # The reduction is not always closed over self.dtype, e.g. the

2100 # mean of an integer column, the sum of a narrow one, or the

2101 # NaN that min_count inserts; casting the result back to

2102 # self.dtype would silently truncate it.

2103 subtype = np.result_type(dtype.subtype, result)

2104 if subtype != dtype.subtype:

2105 fill_value = self.fill_value

2106 if notna(fill_value):

2107 # keep the fill value so that reducing a frame of

2108 # sparse columns does not mix fill values

2109 fill_value = subtype.type(fill_value).item()

2110 dtype = SparseDtype(subtype, fill_value)

2111 elif dtype.subtype.kind in "mM" and (

2112 result is NaT or isinstance(result, (Timestamp, Timedelta))

2113 ):

2114 # datetimelike reductions are closed over the subtype except std,

2115 # which turns a datetime64 column into a timedelta64 one. Go by

2116 # the reduction, not the result: a boxed result no longer says which.

2117 if dtype.subtype.kind == "M" and name in _complex_to_real_reductions:

2118 unit = np.datetime_data(dtype.subtype)[0]

2119 dtype = SparseDtype(np.dtype(f"m8[{unit}]"))

2120 if result is NaT:

2121 # a boxed NaT is inferred as datetime64, so a timedelta64

2122 # subtype needs the numpy NaT back

2123 result = na_value_for_dtype(dtype.subtype, compat=False)

2124 return type(self)([result], dtype=dtype)

2125 else:

2126 return result

2127

2128 @property

2129 def _na_scalar(self) -> Scalar:

2130 """

2131 The NA a reduction returns, boxed as NaT for a datetimelike subtype.

2132 """

2133 return maybe_box_datetimelike(

2134 na_value_for_dtype(self.dtype.subtype, compat=False), self.dtype.subtype

2135 )

2136

2137 def _dense_reduce(self, name: str, *, skipna: bool, **kwargs):

2138 """

2139 Compute a reduction that has no sparse-aware kernel on the dense values.

2140 """

2141 result = getattr(nanops, f"nan{name}")(self._densify(), skipna=skipna, **kwargs)

2142 return maybe_box_datetimelike(result, self.dtype.subtype)

2143

2144 def all(self, axis=None, *args, skipna: bool = True, **kwargs) -> bool:

2145 """

2146 Tests whether all elements evaluate True

2147

2148 Parameters

2149 ----------

2150 axis : int, default None

2151 Not Used. NumPy compatibility.

2152 skipna : bool, default True

2153 Exclude NA/null values. If False, NA is evaluated for truthiness

2154 like any other value.

2155 *args, **kwargs

2156 Not Used. NumPy compatibility.

2157

2158 Returns

2159 -------

2160 all : bool

2161

2162 See Also

2163 --------

2164 numpy.all

2165 """

2166 nv.validate_all(args, kwargs)

2167 skipna = validate_bool_kwarg(skipna, "skipna")

2168

2169 if self.dtype.subtype.kind == "M":

2170 # GH#34479: match nanops.nanall on the dense values

2171 raise TypeError(dt64_any_all_msg("all"))

2172

2173 values = self.sp_values

2174 fill_value = self.fill_value

2175 if skipna:

2176 # NaN and NaT are truthy, which is already what skipping them means for

2177 # all() (see nanops.nanall); only a falsy object None needs the mask

2178 if values.dtype.kind == "O":

2179 values = self._valid_sp_values

2180 if isna(fill_value):

2181 fill_value = True

2182 elif isna(fill_value):

2183 # gaps count as what _densify puts there: the promoted NA. NaN and NaT

2184 # are truthy; a pd.NA survives the promotion and raises like dense

2185 fill_value = maybe_promote(self.sp_values.dtype, fill_value)[1]

2186

2187 # not np.all(): an object pd.NA fill then raises like dense; see

2188 # test_any_all_na_fill_value

2189 if self.sp_index.ngaps > 0 and not fill_value:

2190 return False

2191

2192 return values.all().item()

2193

2194 def any(self, axis: AxisInt = 0, *args, skipna: bool = True, **kwargs) -> bool:

2195 """

2196 Tests whether at least one of elements evaluate True

2197

2198 Parameters

2199 ----------

2200 axis : int, default 0

2201 Not Used. NumPy compatibility.

2202 skipna : bool, default True

2203 Exclude NA/null values. If False, NA is evaluated for truthiness

2204 like any other value.

2205 *args, **kwargs

2206 Not Used. NumPy compatibility.

2207

2208 Returns

2209 -------

2210 any : bool

2211

2212 See Also

2213 --------

2214 numpy.any

2215 """

2216 nv.validate_any(args, kwargs)

2217 skipna = validate_bool_kwarg(skipna, "skipna")

2218

2219 if self.dtype.subtype.kind == "M":

2220 # GH#34479: match nanops.nanany on the dense values

2221 raise TypeError(dt64_any_all_msg("any"))

2222

2223 values = self.sp_values

2224 fill_value = self.fill_value

2225 if skipna:

2226 # unlike all(), a truthy NaN does change the answer (see nanops.nanany),

2227 # so every subtype that can hold NA needs the mask

2228 if values.dtype.kind not in "biu":

2229 values = self._valid_sp_values

2230 if isna(fill_value):

2231 fill_value = False

2232 elif isna(fill_value):

2233 # see the fill-value note in all()

2234 fill_value = maybe_promote(self.sp_values.dtype, fill_value)[1]

2235

2236 # not np.any(); see the note in all()

2237 if self.sp_index.ngaps > 0 and fill_value:

2238 return True

2239

2240 return values.any().item()

2241

2242 def sum(

2243 self,

2244 axis: AxisInt = 0,

2245 min_count: int = 0,

2246 skipna: bool = True,

2247 *args,

2248 **kwargs,

2249 ) -> Scalar:

2250 """

2251 Sum of non-NA/null values

2252

2253 Parameters

2254 ----------

2255 axis : int, default 0

2256 Not Used. NumPy compatibility.

2257 min_count : int, default 0

2258 The required number of valid values to perform the summation. If fewer

2259 than ``min_count`` valid values are present, the result will be the missing

2260 value indicator for subarray type.

2261 skipna : bool, default True

2262 Exclude NA/null values. If False and NA is present, return NA.

2263 *args, **kwargs

2264 Not Used. NumPy compatibility.

2265

2266 Returns

2267 -------

2268 scalar

2269 """

2270 nv.validate_sum(args, kwargs)

2271 skipna = validate_bool_kwarg(skipna, "skipna")

2272

2273 if self.dtype.subtype.kind == "O" and not self._null_fill_value:

2274 # the fill_value * nsparse adjustment below assumes a commutative +,

2275 # which str and bytes do not have; an NA fill is skipped, not added,

2276 # so it needs none. See test_sum_object_fill_value.

2277 return self._dense_reduce("sum", skipna=skipna, min_count=min_count)

2278

2279 valid_vals = self._valid_sp_values

2280 sp_sum = valid_vals.sum()

2281

2282 if not skipna and self._hasna:

2283 return self._na_scalar

2284

2285 if self._null_fill_value:

2286 if check_below_min_count(valid_vals.shape, None, min_count):

2287 return self._na_scalar

2288 result = sp_sum

2289 else:

2290 nsparse = self.sp_index.ngaps

2291 if check_below_min_count(valid_vals.shape, None, min_count - nsparse):

2292 return self._na_scalar

2293 result = sp_sum + self.fill_value * nsparse

2294 return maybe_box_datetimelike(result, self.dtype.subtype)

2295

2296 def _accumulate(self, name: str, *, skipna: bool = True, **kwargs) -> SparseArray:

2297 accum_func = {

2298 "cumsum": np.cumsum,

2299 "cumprod": np.cumprod,

2300 "cummin": np.minimum.accumulate,

2301 "cummax": np.maximum.accumulate,

2302 }.get(name)

2303 if accum_func is None:

2304 raise NotImplementedError(f"cannot perform {name} with type {self.dtype}")

2305

2306 if name == "cumsum" and skipna and self.dtype.subtype.kind in "biufc":

2307 # numeric cumsum keeps the NA gaps as gaps instead of densifying;

2308 # other subtypes go dense to match its NA handling

2309 return self.cumsum(**kwargs)

2310

2311 result: ExtensionArray | np.ndarray

2312 values = ensure_wrapped_if_datetimelike(self._densify())

2313 if isinstance(values, ExtensionArray):

2314 # datetimelike subtypes have their own accumulations

2315 result = values._accumulate(name, skipna=skipna, **kwargs)

2316 else:

2317 result = na_accum_func(values, accum_func, skipna=skipna)

2318

2319 # like cumsum, the result's fill value is NA regardless of our own

2320 fill_value = na_value_for_dtype(result.dtype, compat=False)

2321 return type(self)(result, fill_value=fill_value)

2322

2323 def prod(

2324 self,

2325 *,

2326 axis: AxisInt = 0,

2327 min_count: int = 0,

2328 skipna: bool = True,

2329 **kwargs,

2330 ) -> Scalar:

2331 """

2332 Product of non-NA/null values.

2333

2334 Parameters

2335 ----------

2336 axis : int, default 0

2337 Not Used. NumPy compatibility.

2338 min_count : int, default 0

2339 The required number of valid values to perform the multiplication. If

2340 fewer than ``min_count`` valid values are present, the result will be

2341 the missing value indicator for subarray type.

2342 skipna : bool, default True

2343 Exclude NA/null values. If False and NA is present, return NA.

2344 **kwargs

2345 Not Used. NumPy compatibility.

2346

2347 Returns

2348 -------

2349 scalar

2350

2351 See Also

2352 --------

2353 SparseArray.sum : Sum of non-NA/null values.

2354 numpy.prod : Product of array elements over a given axis.

2355

2356 Examples

2357 --------

2358 >>> arr = pd.arrays.SparseArray([1, 2, 3])

2359 >>> arr.prod()

2360 np.int64(6)

2361 """

2362 nv.validate_prod((), kwargs)

2363 skipna = validate_bool_kwarg(skipna, "skipna")

2364 return self._dense_reduce("prod", skipna=skipna, min_count=min_count)

2365

2366 def cumsum(self, axis: AxisInt = 0, *args, **kwargs) -> SparseArray:

2367 """

2368 Cumulative sum of non-NA/null values.

2369

2370 When performing the cumulative summation, any non-NA/null values will

2371 be skipped. The resulting SparseArray will preserve the locations of

2372 NaN values, but the fill value will be `np.nan` regardless.

2373

2374 Parameters

2375 ----------

2376 axis : int or None

2377 Axis over which to perform the cumulative summation. If None,

2378 perform cumulative summation over flattened array.

2379

2380 Returns

2381 -------

2382 cumsum : SparseArray

2383 """

2384 nv.validate_cumsum(args, kwargs)

2385

2386 if axis is not None and axis >= self.ndim: # Mimic ndarray behaviour.

2387 raise ValueError(f"axis(={axis}) out of bounds")

2388

2389 if not self._null_fill_value:

2390 return SparseArray(self.to_dense(), fill_value=np.nan).cumsum()

2391

2392 sp_values = self.sp_values

2393 mask = isna(sp_values)

2394 if not mask.any():

2395 result = sp_values.cumsum()

2396 else:

2397 # a stored NA would otherwise propagate into every later entry,

2398 # see GH#68972

2399 filled = sp_values.copy()

2400 filled[mask] = 0

2401 result = filled.cumsum()

2402 # the dtype's own NA, not the stored object: nanops.na_accum_func

2403 # drops an imaginary-only NaN's real part too

2404 result[mask] = na_value_for_dtype(sp_values.dtype, compat=False)

2405

2406 sparse_dtype = SparseDtype(result.dtype, self.fill_value)

2407 return type(self)._simple_new(result, self.sp_index, sparse_dtype)

2408

2409 def mean(self, axis: Axis = 0, *args, skipna: bool = True, **kwargs):

2410 """

2411 Mean of non-NA/null values.

2412

2413 Parameters

2414 ----------

2415 axis : int, default 0

2416 Not Used. NumPy compatibility.

2417 skipna : bool, default True

2418 Exclude NA/null values. If False and NA is present, return NA.

2419 *args, **kwargs

2420 Not Used. NumPy compatibility.

2421

2422 Returns

2423 -------

2424 scalar

2425 """

2426 nv.validate_mean(args, kwargs)

2427 skipna = validate_bool_kwarg(skipna, "skipna")

2428 valid_vals = self._valid_sp_values

2429 sp_sum = valid_vals.sum()

2430 ct = len(valid_vals)

2431

2432 nsparse = 0 if self._null_fill_value else self.sp_index.ngaps

2433 nobs = ct + nsparse

2434

2435 if not nobs:

2436 # nobs == 0 is exactly "every entry is NA", empty arrays included; the

2437 # division and the fill_value adjustment both warn or raise there (GH#68484)

2438 return self._na_scalar

2439

2440 # Compute the reduction before the skipna=False NA short-circuit so

2441 # unsupported dtypes still raise during the operation validation.

2442 if self._null_fill_value:

2443 mean = sp_sum / nobs

2444 else:

2445 mean = (sp_sum + self.fill_value * nsparse) / nobs

2446

2447 if not skipna and self._hasna:

2448 return self._na_scalar

2449

2450 return maybe_box_datetimelike(mean, self.dtype.subtype)

2451

2452 def median(

2453 self,

2454 *,

2455 axis: AxisInt = 0,

2456 skipna: bool = True,

2457 **kwargs,

2458 ):

2459 """

2460 Median of non-NA/null values.

2461

2462 Parameters

2463 ----------

2464 axis : int, default 0

2465 Not Used. NumPy compatibility.

2466 skipna : bool, default True

2467 Exclude NA/null values. If False and NA is present, return NA.

2468 **kwargs

2469 Not Used. NumPy compatibility.

2470

2471 Returns

2472 -------

2473 scalar

2474

2475 See Also

2476 --------

2477 Series.median : Equivalent method for Series.

2478

2479 Examples

2480 --------

2481 >>> arr = pd.arrays.SparseArray([1, 2, 3, 4])

2482 >>> arr.median()

2483 2.5

2484 """

2485 nv.validate_median((), kwargs)

2486 skipna = validate_bool_kwarg(skipna, "skipna")

2487 return self._dense_reduce("median", skipna=skipna)

2488

2489 def var(

2490 self,

2491 *,

2492 axis: AxisInt = 0,

2493 ddof: int = 1,

2494 skipna: bool = True,

2495 **kwargs,

2496 ):

2497 """

2498 Unbiased variance of non-NA/null values.

2499

2500 Parameters

2501 ----------

2502 axis : int, default 0

2503 Not Used. NumPy compatibility.

2504 ddof : int, default 1

2505 Delta degrees of freedom. The divisor used in calculations is

2506 ``N - ddof``, where ``N`` is the number of non-NA/null values.

2507 skipna : bool, default True

2508 Exclude NA/null values. If False and NA is present, return NA.

2509 **kwargs

2510 Not Used. NumPy compatibility.

2511

2512 Returns

2513 -------

2514 scalar

2515

2516 See Also

2517 --------

2518 Series.var : Equivalent method for Series.

2519

2520 Examples

2521 --------

2522 >>> arr = pd.arrays.SparseArray([1, 2, 3, 4])

2523 >>> arr.var()

2524 1.6666666666666667

2525 """

2526 nv.validate_stat_ddof_func((), kwargs, fname="var")

2527 skipna = validate_bool_kwarg(skipna, "skipna")

2528 return self._dense_reduce("var", skipna=skipna, ddof=ddof)

2529

2530 def std(

2531 self,

2532 *,

2533 axis: AxisInt = 0,

2534 ddof: int = 1,

2535 skipna: bool = True,

2536 **kwargs,

2537 ):

2538 """

2539 Sample standard deviation of non-NA/null values.

2540

2541 Parameters

2542 ----------

2543 axis : int, default 0

2544 Not Used. NumPy compatibility.

2545 ddof : int, default 1

2546 Delta degrees of freedom. The divisor used in calculations is

2547 ``N - ddof``, where ``N`` is the number of non-NA/null values.

2548 skipna : bool, default True

2549 Exclude NA/null values. If False and NA is present, return NA.

2550 **kwargs

2551 Not Used. NumPy compatibility.

2552

2553 Returns

2554 -------

2555 scalar

2556

2557 See Also

2558 --------

2559 Series.std : Equivalent method for Series.

2560

2561 Examples

2562 --------

2563 >>> arr = pd.arrays.SparseArray([1, 2, 3, 4])

2564 >>> arr.std()

2565 1.2909944487358056

2566 """

2567 nv.validate_stat_ddof_func((), kwargs, fname="std")

2568 skipna = validate_bool_kwarg(skipna, "skipna")

2569 return self._dense_reduce("std", skipna=skipna, ddof=ddof)

2570

2571 def sem(

2572 self,

2573 *,

2574 axis: AxisInt = 0,

2575 ddof: int = 1,

2576 skipna: bool = True,

2577 **kwargs,

2578 ):

2579 """

2580 Standard error of the mean of non-NA/null values.

2581

2582 Parameters

2583 ----------

2584 axis : int, default 0

2585 Not Used. NumPy compatibility.

2586 ddof : int, default 1

2587 Delta degrees of freedom. The divisor used in calculations is

2588 ``N - ddof``, where ``N`` is the number of non-NA/null values.

2589 skipna : bool, default True

2590 Exclude NA/null values. If False and NA is present, return NA.

2591 **kwargs

2592 Not Used. NumPy compatibility.

2593

2594 Returns

2595 -------

2596 scalar

2597

2598 See Also

2599 --------

2600 Series.sem : Equivalent method for Series.

2601

2602 Examples

2603 --------

2604 >>> arr = pd.arrays.SparseArray([1, 2, 3, 4])

2605 >>> arr.sem()

2606 np.float64(0.6454972243679028)

2607 """

2608 nv.validate_stat_ddof_func((), kwargs, fname="sem")

2609 skipna = validate_bool_kwarg(skipna, "skipna")

2610 return self._dense_reduce("sem", skipna=skipna, ddof=ddof)

2611

2612 def skew(

2613 self,

2614 *,

2615 axis: AxisInt = 0,

2616 skipna: bool = True,

2617 **kwargs,

2618 ):

2619 """

2620 Unbiased skew of non-NA/null values.

2621

2622 Parameters

2623 ----------

2624 axis : int, default 0

2625 Not Used. NumPy compatibility.

2626 skipna : bool, default True

2627 Exclude NA/null values. If False and NA is present, return NA.

2628 **kwargs

2629 Not Used. NumPy compatibility.

2630

2631 Returns

2632 -------

2633 scalar

2634

2635 See Also

2636 --------

2637 Series.skew : Equivalent method for Series.

2638

2639 Examples

2640 --------

2641 >>> arr = pd.arrays.SparseArray([1, 2, 3, 4])

2642 >>> arr.skew()

2643 np.float64(0.0)

2644 """

2645 nv.validate_stat_ddof_func((), kwargs, fname="skew")

2646 skipna = validate_bool_kwarg(skipna, "skipna")

2647 return self._dense_reduce("skew", skipna=skipna)

2648

2649 def kurt(

2650 self,

2651 *,

2652 axis: AxisInt = 0,

2653 skipna: bool = True,

2654 **kwargs,

2655 ):

2656 """

2657 Unbiased kurtosis of non-NA/null values.

2658

2659 Parameters

2660 ----------

2661 axis : int, default 0

2662 Not Used. NumPy compatibility.

2663 skipna : bool, default True

2664 Exclude NA/null values. If False and NA is present, return NA.

2665 **kwargs

2666 Not Used. NumPy compatibility.

2667

2668 Returns

2669 -------

2670 scalar

2671

2672 See Also

2673 --------

2674 Series.kurt : Equivalent method for Series.

2675

2676 Examples

2677 --------

2678 >>> arr = pd.arrays.SparseArray([1, 2, 3, 4])

2679 >>> arr.kurt()

2680 np.float64(-1.200000000000001)

2681 """

2682 nv.validate_stat_ddof_func((), kwargs, fname="kurt")

2683 skipna = validate_bool_kwarg(skipna, "skipna")

2684 return self._dense_reduce("kurt", skipna=skipna)

2685

2686 def max(self, *, axis: AxisInt | None = None, skipna: bool = True):

2687 """

2688 Max of array values, ignoring NA values if specified.

2689

2690 Parameters

2691 ----------

2692 axis : int, default 0

2693 Not Used. NumPy compatibility.

2694 skipna : bool, default True

2695 Whether to ignore NA values.

2696

2697 Returns

2698 -------

2699 scalar

2700 """

2701 nv.validate_minmax_axis(axis, self.ndim)

2702 return self._min_max("max", skipna=skipna)

2703

2704 def min(self, *, axis: AxisInt | None = None, skipna: bool = True):

2705 """

2706 Min of array values, ignoring NA values if specified.

2707

2708 Parameters

2709 ----------

2710 axis : int, default 0

2711 Not Used. NumPy compatibility.

2712 skipna : bool, default True

2713 Whether to ignore NA values.

2714

2715 Returns

2716 -------

2717 scalar

2718 """

2719 nv.validate_minmax_axis(axis, self.ndim)

2720 return self._min_max("min", skipna=skipna)

2721

2722 def _min_max(self, kind: Literal["min", "max"], skipna: bool) -> Scalar:

2723 """

2724 Min/max of non-NA/null values

2725

2726 Parameters

2727 ----------

2728 kind : {"min", "max"}

2729 skipna : bool

2730

2731 Returns

2732 -------

2733 scalar

2734 """

2735 valid_vals = self._valid_sp_values

2736 has_nonnull_fill_vals = not self._null_fill_value and self.sp_index.ngaps > 0

2737

2738 if len(valid_vals) > 0:

2739 sp_min_max = getattr(valid_vals, kind)()

2740

2741 if not skipna and self._hasna:

2742 return self._na_scalar

2743

2744 # If a non-null fill value is currently present, it might be the min/max

2745 if has_nonnull_fill_vals:

2746 func = max if kind == "max" else min

2747 result = func(sp_min_max, self.fill_value)

2748 else:

2749 # A present NA with skipna=False is handled above, so the min/max

2750 # of the valid sparse values is the result.

2751 result = sp_min_max

2752 elif not skipna and self._hasna:

2753 return self._na_scalar

2754 elif has_nonnull_fill_vals:

2755 result = self.fill_value

2756 else:

2757 return self._na_scalar

2758 return maybe_box_datetimelike(result, self.dtype.subtype)

2759

2760 def _argmin_argmax(self, kind: Literal["argmin", "argmax"]) -> int:

2761 values = self._sparse_values

2762 index = self._sparse_index.indices

2763 mask = np.asarray(isna(values))

2764 func = np.argmax if kind == "argmax" else np.argmin

2765

2766 idx = np.arange(values.shape[0])

2767 non_nans = values[~mask]

2768 non_nan_idx = idx[~mask]

2769

2770 if len(non_nans) == 0 and not self._null_fill_value and self.sp_index.ngaps:

2771 # No stored value survives the mask, so the fill value is the only

2772 # candidate; if it is NA or holds no position, the all-NA check

2773 # below raises. GH#68462

2774 return self._first_fill_value_loc()

2775

2776 if not len(non_nans) and len(self) and self.isna().all():

2777 # mask covers sp_values only, which is empty when every value is

2778 # an NA fill value, so the all-NA check has to look at the array

2779 raise ValueError("Encountered all NA values")

2780

2781 _candidate = non_nan_idx[func(non_nans)]

2782 candidate = index[_candidate]

2783

2784 if isna(self.fill_value):

2785 return candidate

2786 if kind == "argmin" and self[candidate] < self.fill_value:

2787 return candidate

2788 if kind == "argmax" and self[candidate] > self.fill_value:

2789 return candidate

2790 _loc = self._first_fill_value_loc()

2791 if _loc == -1:

2792 # fill_value doesn't exist

2793 return candidate

2794 else:

2795 return _loc

2796

2797 def argmax(self, skipna: bool = True) -> int:

2798 validate_bool_kwarg(skipna, "skipna")

2799 if not skipna and self._hasna:

2800 raise ValueError("Encountered an NA value with skipna=False")

2801 return self._argmin_argmax("argmax")

2802

2803 def argmin(self, skipna: bool = True) -> int:

2804 validate_bool_kwarg(skipna, "skipna")

2805 if not skipna and self._hasna:

2806 raise ValueError("Encountered an NA value with skipna=False")

2807 return self._argmin_argmax("argmin")

2808

2809 # ------------------------------------------------------------------------

2810 # Ufuncs

2811 # ------------------------------------------------------------------------

2812

2813 _HANDLED_TYPES = (np.ndarray, numbers.Number)

2814

2815 def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):

2816 # this path never reaches ExtensionArray.__array_ufunc__, and a datetimelike

2817 # scalar is not in _HANDLED_TYPES, so this has to precede that loop

2818 ops.disallow_datetimelike_logical_ufunc(ufunc, inputs)

2819

2820 out = kwargs.get("out", ())

2821

2822 for x in inputs + out:

2823 if not isinstance(x, (*self._HANDLED_TYPES, SparseArray)):

2824 return NotImplemented

2825

2826 # for binary ops, use our custom dunder methods

2827 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(

2828 self, ufunc, method, *inputs, **kwargs

2829 )

2830 if result is not NotImplemented:

2831 return result

2832

2833 if "out" in kwargs:

2834 # e.g. tests.arrays.sparse.test_arithmetics.test_ndarray_inplace

2835 res = arraylike.dispatch_ufunc_with_out(

2836 self, ufunc, method, *inputs, **kwargs

2837 )

2838 return res

2839

2840 if method == "reduce":

2841 result = arraylike.dispatch_reduction_ufunc(

2842 self, ufunc, method, *inputs, **kwargs

2843 )

2844 if result is not NotImplemented:

2845 # e.g. tests.series.test_ufunc.TestNumpyReductions

2846 return result

2847

2848 if len(inputs) == 1:

2849 if method == "reduce":

2850 # GH#68453 reducing sp_values alone would drop the gaps

2851 return ufunc.reduce(self._densify(), **kwargs)

2852

2853 if method == "accumulate":

2854 # GH#68566 numpy cannot accumulate the scalar fill_value; like

2855 # _accumulate, the densified result's fill value is NA regardless

2856 result = ufunc.accumulate(self._densify(), **kwargs)

2857 fill_value = na_value_for_dtype(result.dtype, compat=False)

2858 return type(self)(result, fill_value=fill_value)

2859

2860 # No alignment necessary.

2861 sp_values = getattr(ufunc, method)(self.sp_values, **kwargs)

2862 fill_value = getattr(ufunc, method)(self.fill_value, **kwargs)

2863

2864 if ufunc.nout > 1:

2865 # multiple outputs. e.g. modf

2866 arrays = tuple(

2867 self._simple_new(

2868 sp_value, self.sp_index, SparseDtype(sp_value.dtype, fv)

2869 )

2870 for sp_value, fv in zip(sp_values, fill_value, strict=True)

2871 )

2872 return arrays

2873

2874 return self._simple_new(

2875 sp_values, self.sp_index, SparseDtype(sp_values.dtype, fill_value)

2876 )

2877

2878 new_inputs = tuple(np.asarray(x) for x in inputs)

2879 result = getattr(ufunc, method)(*new_inputs, **kwargs)

2880 if out:

2881 if len(out) == 1:

2882 out = out[0]

2883 return out

2884

2885 if ufunc.nout > 1:

2886 return tuple(type(self)(x) for x in result)

2887 elif method == "at":

2888 # no return value

2889 return None

2890 else:

2891 return type(self)(result)

2892

2893 # ------------------------------------------------------------------------

2894 # Ops

2895 # ------------------------------------------------------------------------

2896

2897 def _arith_method(self, other, op):

2898 op_name = op.__name__

2899

2900 if isinstance(other, SparseArray):

2901 return _sparse_array_op(self, other, op, op_name)

2902

2903 elif is_scalar(other):

2904 with np.errstate(all="ignore"):

2905 fill = op(_get_fill(self), np.asarray(other))

2906 result = op(self.sp_values, other)

2907

2908 if op_name == "divmod":

2909 left, right = result

2910 lfill, rfill = fill

2911 return (

2912 _wrap_result(op_name, left, self.sp_index, lfill),

2913 _wrap_result(op_name, right, self.sp_index, rfill),

2914 )

2915

2916 return _wrap_result(op_name, result, self.sp_index, fill)

2917

2918 else:

2919 if not isinstance(

2920 other, (list, np.ndarray, ExtensionArray)

2921 ) and not ops.has_castable_attr(other):

2922 warnings.warn(

2923 f"Operation with {type(other).__name__} is deprecated. "

2924 "In a future version these will be treated as scalar-like. "

2925 "To retain the old behavior, explicitly wrap in a Series "

2926 "instead.",

2927 Pandas4Warning,

2928 stacklevel=find_stack_level(),

2929 )

2930

2931 other = np.asarray(other)

2932 with np.errstate(all="ignore"):

2933 if len(self) != len(other):

2934 raise AssertionError(

2935 f"length mismatch: {len(self)} vs. {len(other)}"

2936 )

2937 if not isinstance(other, SparseArray):

2938 dtype = getattr(other, "dtype", None)

2939 other = _as_sparse_operand(other, self.fill_value, dtype=dtype)

2940 return _sparse_array_op(self, other, op, op_name)

2941

2942 def _cmp_method(self, other, op) -> SparseArray:

2943 if isinstance(getattr(other, "dtype", None), BaseMaskedDtype):

2944 # GH#68579 defer to the masked operand's reflected comparison, which

2945 # keeps its NA semantics; densifying here loses them.

2946 return NotImplemented

2947

2948 return self._cmp_or_logical_op(other, op)

2949

2950 def _cmp_or_logical_op(self, other, op) -> SparseArray:

2951 # Shared by _cmp_method and _logical_method's fast path.

2952 if (

2953 is_list_like(other)

2954 and not isinstance(other, (list, np.ndarray, ExtensionArray))

2955 and not ops.has_castable_attr(other)

2956 ):

2957 warnings.warn(

2958 f"Operation with {type(other).__name__} is deprecated. "

2959 "In a future version these will be treated as scalar-like. "

2960 "To retain the old behavior, explicitly wrap in a Series "

2961 "instead.",

2962 Pandas4Warning,

2963 stacklevel=find_stack_level(),

2964 )

2965 if not is_scalar(other) and not isinstance(other, type(self)):

2966 # convert list-like to ndarray

2967 other = np.asarray(other)

2968

2969 if isinstance(other, np.ndarray):

2970 # TODO: make this more flexible than just ndarray...

2971 other = _as_sparse_operand(other, self.fill_value)

2972

2973 op_name = op.__name__.strip("_")

2974 # _logical_method routes and/or/xor here for the sparse fast path, and the

2975 # dense route below only knows the comparison operators

2976 is_cmp = op_name in ("eq", "ne", "lt", "gt", "le", "ge")

2977

2978 if isinstance(other, SparseArray):

2979 if len(self) != len(other):

2980 raise ValueError(

2981 f"operands have mismatched length {len(self)} and {len(other)}"

2982 )

2983

2984 if is_cmp and object in (self.dtype.subtype, other.dtype.subtype):

2985 # GH#68586 _sparse_array_op casts both operands to their common

2986 # subtype, so object on either side reaches a kernel splib lacks

2987 return self._cmp_method_dense(other.to_dense(), other.fill_value, op)

2988

2989 return _sparse_array_op(self, other, op, op_name)

2990 elif other is libmissing.NA:

2991 # GH#68579 NA has no truth value, so the scalar arm below cannot

2992 # build a boolean result; answer as the dense path does.

2993 fill_value = op is operator.ne

2994 if self.dtype.subtype.kind in "mM":

2995 result = ops.invalid_comparison(self, other, op)

2996 else:

2997 result = np.full(len(self), fill_value, dtype=np.bool_)

2998 return type(self)(result, fill_value=fill_value, dtype=np.bool_)

2999 elif is_cmp and self.dtype.subtype == object:

3000 # GH#68586 the np.bool_ buffer below cannot hold the result of comparing

3001 # e.g. pd.NA, so use the kernel the dense path uses -- but only the

3002 # stored values need it, so this stays O(nnz) rather than O(len)

3003 sp_values = ops.comparison_op(self.sp_values, other, op)

3004 fill_value = self._cmp_fill_value(other, op)

3005 return _wrap_result(op_name, sp_values, self.sp_index, fill_value)

3006 else:

3007 # scalar

3008 fill_value = op(self.fill_value, other)

3009 result = np.full(len(self), fill_value, dtype=np.bool_)

3010 result[self.sp_index.indices] = op(self.sp_values, other)

3011

3012 return type(self)(

3013 result,

3014 fill_value=fill_value,

3015 dtype=np.bool_,

3016 )

3017

3018 def _cmp_fill_value(self, other, op) -> bool:

3019 # GH#68586 through the object kernel so that an NA fill value compares

3020 # False instead of making the bool() below raise

3021 return bool(

3022 ops.comparison_op(np.array([self.fill_value], dtype=object), other, op)[0]

3023 )

3024

3025 def _cmp_method_dense(self, rvalues, rfill, op) -> SparseArray:

3026 # GH#68586 splib has no object comparison kernels, so an object subtype on

3027 # either side is computed densely; unlike the scalar arm this cannot work

3028 # off sp_values, because the two sparse indexes need not line up.

3029 result = ops.comparison_op(np.asarray(self), rvalues, op)

3030 return type(self)(

3031 result, fill_value=self._cmp_fill_value(rfill, op), dtype=np.bool_

3032 )

3033

3034 def _logical_method(self, other, op):

3035 other_dtype = getattr(other, "dtype", None)

3036 if isinstance(other_dtype, BaseMaskedDtype) and other_dtype.kind == "b":

3037 # GH#68483 defer to the masked operand's reflected op, which keeps

3038 # its Kleene NA semantics; densifying here loses them.

3039 # Boolean only -- the other masked dtypes reach _arith_method,

3040 # which cannot consume a SparseArray.

3041 return NotImplemented

3042

3043 # GH#32119 the sparse fast path (see _sparse_array_op / splib) only

3044 # implements and/or/xor for boolean and integer subtypes. When

3045 # alignment upcasts an operand to object/float -- e.g. NA introduced

3046 # by reindexing a boolean SparseArray to a longer index -- fall back

3047 # to a dense computation so the result matches the non-sparse path.

3048 if not self._logical_needs_dense(other):

3049 return self._cmp_or_logical_op(other, op)

3050

3051 lvalues = np.asarray(self)

3052 # GH#68569 keep an EA operand boxed so logical_op dispatches to it as the dense

3053 # path does; densifying flattened e.g. a Categorical to object.

3054 rvalues = extract_array(other, extract_numpy=True)

3055 if isinstance(rvalues, SparseArray):

3056 # A sparse operand is the exception: dispatching would re-enter here with

3057 # the two swapped, and logical_op treats its left and right differently

3058 # -- see test_logical_op_both_sparse_matches_dense.

3059 rvalues = np.asarray(rvalues)

3060 result = logical_op(lvalues, rvalues, op)

3061 if not isinstance(result, np.ndarray):

3062 # GH#68569 the operand's own op answered in its dtype, as it does densely.

3063 # Re-wrapping a nullable result as sparse would give it an NA fill value

3064 # that later logical ops raise on.

3065 return result

3066 return type(self)(result)

3067

3068 def _logical_needs_dense(self, other) -> bool:

3069 # The and/or/xor sparse kernels only support boolean/integer subtypes;

3070 # everything else (object/float produced by NA upcasting) must be

3071 # computed densely. Inspect dtypes without materializing so the fast

3072 # path still receives the original operand (and its warnings).

3073 if self.dtype.subtype.kind not in "biu":

3074 return True

3075 other_dtype = getattr(other, "dtype", None)

3076 if other_dtype is None:

3077 # scalar or a dtype-less sequence (e.g. list); the fast path

3078 # handles these, including raising on length mismatch

3079 return False

3080 if isinstance(other_dtype, SparseDtype):

3081 return other_dtype.subtype.kind not in "biu"

3082 return other_dtype.kind not in "biu"

3083

3084 def _unary_method(self, op) -> SparseArray:

3085 fill_value = op(np.array(self.fill_value)).item()

3086 dtype = SparseDtype(self.dtype.subtype, fill_value)

3087 # NOTE: if fill_value doesn't change

3088 # we just have to apply op to sp_values

3089 if isna(self.fill_value) or fill_value == self.fill_value:

3090 values = op(self.sp_values)

3091 return type(self)._simple_new(values, self.sp_index, self.dtype)

3092 # In the other case we have to recalc indexes

3093 return type(self)(op(self.to_dense()), dtype=dtype)

3094

3095 def __pos__(self) -> SparseArray:

3096 return self._unary_method(operator.pos)

3097

3098 def __neg__(self) -> SparseArray:

3099 return self._unary_method(operator.neg)

3100

3101 def __invert__(self) -> SparseArray:

3102 return self._unary_method(operator.invert)

3103

3104 def __abs__(self) -> SparseArray:

3105 return self._unary_method(operator.abs)

3106

3107 # ----------

3108 # Formatting

3109 # -----------

3110 # PandasObject (via OpsMixin) comes before ExtensionArray in the MRO,

3111 # and PandasObject.__repr__ falls back to object.__repr__. Explicitly

3112 # resolve to the ExtensionArray version.

3113 __repr__ = ExtensionArray.__repr__

3114

3115 def _formatter(self, boxed: bool = False) -> Callable[[Any], str | None]:

3116 # Use str to avoid np.int64(...) wrapping in repr output.

3117 return str

3118

3119

3120def _make_sparse(

3121 arr: np.ndarray,

3122 kind: SparseIndexKind = "block",

3123 fill_value=None,

3124 dtype: np.dtype | None = None,

3125):

3126 """

3127 Convert ndarray to sparse format

3128

3129 Parameters

3130 ----------

3131 arr : ndarray

3132 kind : {'block', 'integer'}

3133 fill_value : NaN or another value

3134 dtype : np.dtype, optional

3135

3136 Returns

3137 -------

3138 (sparse_values, index, fill_value) : (ndarray, SparseIndex, Scalar)

3139 """

3140 assert isinstance(arr, np.ndarray)

3141

3142 if arr.ndim > 1:

3143 raise TypeError("expected dimension <= 1 data")

3144

3145 if fill_value is None:

3146 fill_value = na_value_for_dtype(arr.dtype)

3147

3148 if isna(fill_value):

3149 mask = notna(arr)

3150 else:

3151 # cast to object comparison to be safe

3152 if is_string_dtype(arr.dtype):

3153 arr = arr.astype(object)

3154

3155 if is_object_dtype(arr.dtype):

3156 # element-wise equality check method in numpy doesn't treat

3157 # each element type, eg. 0, 0.0, and False are treated as

3158 # same. So we have to check the both of its type and value.

3159 mask = splib.make_mask_object_ndarray(arr, fill_value)

3160 else:

3161 mask = arr != fill_value

3162

3163 length = len(arr)

3164 if length != len(mask):

3165 # the arr is a SparseArray

3166 indices = mask.sp_index.indices

3167 else:

3168 indices = mask.nonzero()[0].astype(np.int32)

3169

3170 index = make_sparse_index(length, indices, kind)

3171 sparsified_values = arr[mask]

3172 if dtype is not None:

3173 sparsified_values = ensure_wrapped_if_datetimelike(sparsified_values)

3174 sparsified_values = astype_array(sparsified_values, dtype=dtype)

3175 sparsified_values = np.asarray(sparsified_values)

3176

3177 # TODO: copy

3178 return sparsified_values, index, fill_value

3179

3180

3181@overload

3182def make_sparse_index(length: int, indices, kind: Literal["block"]) -> BlockIndex: ...

3183

3184

3185@overload

3186def make_sparse_index(length: int, indices, kind: Literal["integer"]) -> IntIndex: ...

3187

3188

3189def make_sparse_index(length: int, indices, kind: SparseIndexKind) -> SparseIndex:

3190 index: SparseIndex

3191 if kind == "block":

3192 locs, lens = splib.get_blocks(indices)

3193 index = BlockIndex(length, locs, lens)

3194 elif kind == "integer":

3195 index = IntIndex(length, indices)

3196 else: # pragma: no cover

3197 raise ValueError("must be block or integer type")

3198 return index