Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from typing import (
4 TYPE_CHECKING,
5 Any,
6 Literal,
7 Self,
8 cast,
9 overload,
10)
11import warnings
13import numpy as np
15from pandas._config import (
16 is_nan_na,
17 using_python_scalars,
18)
20from pandas._libs import (
21 algos as libalgos,
22 lib,
23 missing as libmissing,
24)
25from pandas._libs.tslibs import is_supported_dtype
26from pandas.compat import (
27 IS64,
28 is_platform_windows,
29)
30from pandas.errors import AbstractMethodError
32from pandas.core.dtypes.astype import astype_is_view
33from pandas.core.dtypes.base import ExtensionDtype
34from pandas.core.dtypes.cast import (
35 maybe_downcast_to_dtype,
36)
37from pandas.core.dtypes.common import (
38 is_bool,
39 is_integer_dtype,
40 is_list_like,
41 is_scalar,
42 is_string_dtype,
43 pandas_dtype,
44)
45from pandas.core.dtypes.dtypes import (
46 ArrowDtype,
47 BaseMaskedDtype,
48)
49from pandas.core.dtypes.missing import (
50 array_equivalent,
51 is_valid_na_for_dtype,
52 isna,
53 notna,
54)
56from pandas.core import (
57 algorithms as algos,
58 arraylike,
59 missing,
60 nanops,
61 ops,
62)
63from pandas.core.algorithms import (
64 factorize_array,
65 isin,
66 map_array,
67 mode,
68 take,
69)
70from pandas.core.array_algos import (
71 masked_accumulations,
72 masked_reductions,
73)
74from pandas.core.array_algos.quantile import quantile_with_mask
75from pandas.core.array_algos.transforms import shift
76from pandas.core.arraylike import OpsMixin
77from pandas.core.arrays._utils import to_numpy_dtype_inference
78from pandas.core.arrays.base import ExtensionArray
79from pandas.core.construction import (
80 array as pd_array,
81 ensure_wrapped_if_datetimelike,
82 extract_array,
83)
84from pandas.core.indexers import (
85 check_array_indexer,
86 getitem_returns_view,
87)
88from pandas.core.ops import invalid_comparison
89from pandas.core.util.hashing import hash_array
91if TYPE_CHECKING:
92 from collections.abc import Callable
93 from collections.abc import (
94 Iterator,
95 Sequence,
96 )
97 from pandas import Series
98 from pandas.core.arrays import BooleanArray
99 from pandas._typing import (
100 NumpySorter,
101 NumpyValueArrayLike,
102 ArrayLike,
103 AstypeArg,
104 AxisInt,
105 DtypeObj,
106 FillnaOptions,
107 InterpolateOptions,
108 NpDtype,
109 PositionalIndexer,
110 Scalar,
111 ScalarIndexer,
112 SequenceIndexer,
113 Shape,
114 npt,
115 )
116 from pandas._libs.missing import NAType
117 from pandas.core.arrays import FloatingArray
119from pandas.compat.numpy import function as nv
122class BaseMaskedArray(OpsMixin, ExtensionArray):
123 """
124 Base class for masked arrays (which use _data and _mask to store the data).
126 numpy based
127 """
129 # our underlying data and mask are each ndarrays
130 _data: np.ndarray
131 _mask: npt.NDArray[np.bool_]
133 @classmethod
134 def _simple_new(cls, values: np.ndarray, mask: npt.NDArray[np.bool_]) -> Self:
135 result = BaseMaskedArray.__new__(cls)
136 result._data = values
137 result._mask = mask
138 return result
140 def __init__(
141 self, values: np.ndarray, mask: npt.NDArray[np.bool_], copy: bool = False
142 ) -> None:
143 # values is supposed to already be validated in the subclass
144 if not (isinstance(mask, np.ndarray) and mask.dtype == np.bool_):
145 raise TypeError(
146 "mask should be boolean numpy array. Use "
147 "the 'pd.array' function instead"
148 )
149 if values.shape != mask.shape:
150 raise ValueError("values.shape must match mask.shape")
152 if copy:
153 values = values.copy()
154 mask = mask.copy()
156 self._data = values
157 self._mask = mask
159 @classmethod
160 def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False) -> Self:
161 values, mask = cls._coerce_to_array(scalars, dtype=dtype, copy=copy)
162 return cls(values, mask)
164 def _cast_pointwise_result(self, values) -> ArrayLike:
165 if isna(values).all():
166 return type(self)._from_sequence(values, dtype=self.dtype)
167 values = np.asarray(values, dtype=object)
168 result = lib.maybe_convert_objects(values, convert_to_nullable_dtype=True)
169 lkind = self.dtype.kind
170 rkind = result.dtype.kind
171 if (lkind in "iu" and rkind in "iu") or (lkind == rkind == "f"):
172 result = cast(BaseMaskedArray, result)
173 new_data = maybe_downcast_to_dtype(
174 result._data, dtype=self.dtype.numpy_dtype
175 )
176 result = type(result)(new_data, result._mask)
177 return result
179 @classmethod
180 def _empty(cls, shape: Shape, dtype: ExtensionDtype) -> Self:
181 """
182 Create an ExtensionArray with the given shape and dtype.
184 See also
185 --------
186 ExtensionDtype.empty
187 ExtensionDtype.empty is the 'official' public version of this API.
188 """
189 dtype = cast(BaseMaskedDtype, dtype)
190 values: np.ndarray = np.empty(shape, dtype=dtype.type)
191 values.fill(dtype._internal_fill_value)
192 mask = np.ones(shape, dtype=bool)
193 result = cls(values, mask)
194 if not isinstance(result, cls) or dtype != result.dtype:
195 raise NotImplementedError(
196 f"Default 'empty' implementation is invalid for dtype='{dtype}'"
197 )
198 return result
200 def _formatter(self, boxed: bool = False) -> Callable[[Any], str | None]:
201 # NEP 51: https://github.com/numpy/numpy/pull/22449
202 return str
204 @property
205 def dtype(self) -> BaseMaskedDtype:
206 raise AbstractMethodError(self)
208 @overload
209 def __getitem__(self, item: ScalarIndexer) -> Any: ...
211 @overload
212 def __getitem__(self, item: SequenceIndexer) -> Self: ...
214 def __getitem__(self, item: PositionalIndexer) -> Self | Any:
215 item = check_array_indexer(self, item)
217 newmask = self._mask[item]
218 if is_bool(newmask):
219 # This is a scalar indexing
220 if newmask:
221 return self.dtype.na_value
222 return self._data[item]
224 result = self._simple_new(self._data[item], newmask)
225 if getitem_returns_view(self, item):
226 result._readonly = self._readonly
227 return result
229 def _pad_or_backfill(
230 self,
231 *,
232 method: FillnaOptions,
233 limit: int | None = None,
234 limit_area: Literal["inside", "outside"] | None = None,
235 copy: bool = True,
236 ) -> Self:
237 mask = self._mask
239 if mask.any():
240 func = missing.get_fill_func(method, ndim=self.ndim)
242 npvalues = self._data.T
243 new_mask = mask.T
244 if copy:
245 npvalues = npvalues.copy()
246 new_mask = new_mask.copy()
247 elif limit_area is not None:
248 mask = mask.copy()
249 func(npvalues, limit=limit, mask=new_mask)
251 if limit_area is not None and not mask.all():
252 mask = mask.T
253 neg_mask = ~mask
254 first = neg_mask.argmax()
255 last = len(neg_mask) - neg_mask[::-1].argmax() - 1
256 if limit_area == "inside":
257 new_mask[:first] |= mask[:first]
258 new_mask[last + 1 :] |= mask[last + 1 :]
259 elif limit_area == "outside":
260 new_mask[first + 1 : last] |= mask[first + 1 : last]
262 if copy:
263 return self._simple_new(npvalues.T, new_mask.T)
264 else:
265 return self
266 elif copy:
267 new_values = self.copy()
268 else:
269 new_values = self
270 return new_values
272 def fillna(self, value, limit: int | None = None, copy: bool = True) -> Self:
273 """
274 Fill NA/NaN values using the specified method.
276 Parameters
277 ----------
278 value : scalar, array-like
279 If a scalar value is passed it is used to fill all missing values.
280 Alternatively, an array-like "value" can be given. It's expected
281 that the array-like have the same length as 'self'.
282 limit : int, default None
283 The maximum number of entries where NA values will be filled.
284 copy : bool, default True
285 Whether to make a copy of the data before filling. If False, then
286 the original should be modified and no new memory should be allocated.
287 For ExtensionArray subclasses that cannot do this, it is at the
288 author's discretion whether to ignore "copy=False" or to raise.
290 Returns
291 -------
292 ExtensionArray
293 With NA/NaN filled.
295 See Also
296 --------
297 api.extensions.ExtensionArray.dropna : Return ExtensionArray without
298 NA values.
299 api.extensions.ExtensionArray.isna : A 1-D array indicating if
300 each value is missing.
302 Examples
303 --------
304 >>> arr = pd.array([np.nan, np.nan, 2, 3, np.nan, np.nan])
305 >>> arr.fillna(0)
306 <IntegerArray>
307 [0, 0, 2, 3, 0, 0]
308 Length: 6, dtype: Int64
309 """
310 mask = self._mask
311 if limit is not None and limit < len(self):
312 modify = mask.cumsum() > limit
313 if modify.any():
314 # Only copy mask if necessary
315 mask = mask.copy()
316 mask[modify] = False
318 value = missing.check_value_size(value, mask, len(self))
320 if mask.any():
321 # fill with value
322 if copy:
323 new_values = self.copy()
324 else:
325 new_values = self[:]
326 new_values[mask] = value
327 elif copy:
328 new_values = self.copy()
329 else:
330 new_values = self[:]
331 return new_values
333 @classmethod
334 def _coerce_to_array(
335 cls, values, *, dtype: DtypeObj, copy: bool = False
336 ) -> tuple[np.ndarray, np.ndarray]:
337 raise AbstractMethodError(cls)
339 def _validate_setitem_value(self, value):
340 """
341 Check if we have a scalar that we can cast losslessly.
343 Raises
344 ------
345 TypeError
346 """
347 kind = self.dtype.kind
348 # TODO: get this all from np_can_hold_element?
349 if kind == "b":
350 if lib.is_bool(value):
351 return value
353 elif kind == "f":
354 if lib.is_integer(value) or lib.is_float(value):
355 return value
357 elif lib.is_integer(value) or (lib.is_float(value) and value.is_integer()):
358 return value
359 # TODO: unsigned checks
361 # Note: without the "str" here, the f-string rendering raises in
362 # py38 builds.
363 raise TypeError(f"Invalid value '{value!s}' for dtype '{self.dtype}'")
365 def __setitem__(self, key, value) -> None:
366 if self._readonly:
367 raise ValueError("Cannot modify read-only array")
369 key = check_array_indexer(self, key)
371 if is_scalar(value):
372 if is_valid_na_for_dtype(value, self.dtype) and not (
373 lib.is_float(value) and not is_nan_na()
374 ):
375 self._mask[key] = True
376 else:
377 value = self._validate_setitem_value(value)
378 self._data[key] = value
379 self._mask[key] = False
380 return
382 value, mask = self._coerce_to_array(value, dtype=self.dtype)
384 self._data[key] = value
385 self._mask[key] = mask
387 def __contains__(self, key) -> bool:
388 if isna(key) and key is not self.dtype.na_value:
389 # GH#52840
390 if lib.is_float(key) and is_nan_na():
391 key = self.dtype.na_value
392 elif self._data.dtype.kind == "f" and lib.is_float(key):
393 return bool((np.isnan(self._data) & ~self._mask).any())
395 return bool(super().__contains__(key))
397 def __iter__(self) -> Iterator:
398 if self.ndim == 1:
399 if not self._hasna:
400 for val in self._data:
401 yield val
402 else:
403 na_value = self.dtype.na_value
404 for isna_, val in zip(self._mask, self._data, strict=True):
405 if isna_:
406 yield na_value
407 else:
408 yield val
409 else:
410 for i in range(len(self)):
411 yield self[i]
413 def __len__(self) -> int:
414 return len(self._data)
416 @property
417 def shape(self) -> Shape:
418 return self._data.shape
420 @property
421 def ndim(self) -> int:
422 return self._data.ndim
424 def swapaxes(self, axis1, axis2) -> Self:
425 data = self._data.swapaxes(axis1, axis2)
426 mask = self._mask.swapaxes(axis1, axis2)
427 return self._simple_new(data, mask)
429 def delete(self, loc, axis: AxisInt = 0) -> Self:
430 data = np.delete(self._data, loc, axis=axis)
431 mask = np.delete(self._mask, loc, axis=axis)
432 return self._simple_new(data, mask)
434 def reshape(self, *args, **kwargs) -> Self:
435 data = self._data.reshape(*args, **kwargs)
436 mask = self._mask.reshape(*args, **kwargs)
437 return self._simple_new(data, mask)
439 def ravel(self, *args, **kwargs) -> Self:
440 # TODO: need to make sure we have the same order for data/mask
441 data = self._data.ravel(*args, **kwargs)
442 mask = self._mask.ravel(*args, **kwargs)
443 return type(self)(data, mask)
445 def shift(self, periods: int = 1, fill_value=None) -> Self:
446 # NB: shift is always along axis=0
447 axis = 0
448 if fill_value is None:
449 new_data = shift(self._data, periods, axis, 0)
450 new_mask = shift(self._mask, periods, axis, True)
451 else:
452 new_data = shift(self._data, periods, axis, fill_value)
453 new_mask = shift(self._mask, periods, axis, False)
454 return type(self)(new_data, new_mask)
456 @property
457 def T(self) -> Self:
458 return self._simple_new(self._data.T, self._mask.T)
460 def round(self, decimals: int = 0, *args, **kwargs):
461 """
462 Round each value in the array a to the given number of decimals.
464 Parameters
465 ----------
466 decimals : int, default 0
467 Number of decimal places to round to. If decimals is negative,
468 it specifies the number of positions to the left of the decimal point.
469 *args, **kwargs
470 Additional arguments and keywords have no effect but might be
471 accepted for compatibility with NumPy.
473 Returns
474 -------
475 NumericArray
476 Rounded values of the NumericArray.
478 See Also
479 --------
480 numpy.around : Round values of an np.array.
481 DataFrame.round : Round values of a DataFrame.
482 Series.round : Round values of a Series.
483 """
484 if self.dtype.kind == "b":
485 return self
486 nv.validate_round(args, kwargs)
487 values = np.round(self._data, decimals=decimals, **kwargs)
489 # Usually we'll get same type as self, but ndarray[bool] casts to float
490 return self._maybe_mask_result(values, self._mask.copy())
492 # ------------------------------------------------------------------
493 # Unary Methods
495 def __invert__(self) -> Self:
496 return self._simple_new(~self._data, self._mask.copy())
498 def __neg__(self) -> Self:
499 return self._simple_new(-self._data, self._mask.copy())
501 def __pos__(self) -> Self:
502 return self.copy()
504 def __abs__(self) -> Self:
505 return self._simple_new(abs(self._data), self._mask.copy())
507 # ------------------------------------------------------------------
509 def _values_for_json(self) -> np.ndarray:
510 return np.asarray(self, dtype=object)
512 def to_numpy(
513 self,
514 dtype: npt.DTypeLike | None = None,
515 copy: bool = False,
516 na_value: object = lib.no_default,
517 ) -> np.ndarray:
518 """
519 Convert to a NumPy Array.
521 By default converts to an object-dtype NumPy array. Specify the `dtype` and
522 `na_value` keywords to customize the conversion.
524 Parameters
525 ----------
526 dtype : dtype, default object
527 The numpy dtype to convert to.
528 copy : bool, default False
529 Whether to ensure that the returned value is a not a view on
530 the array. Note that ``copy=False`` does not *ensure* that
531 ``to_numpy()`` is no-copy. Rather, ``copy=True`` ensure that
532 a copy is made, even if not strictly necessary. This is typically
533 only possible when no missing values are present and `dtype`
534 is the equivalent numpy dtype.
535 na_value : scalar, optional
536 Scalar missing value indicator to use in numpy array. Defaults
537 to the native missing value indicator of this array (pd.NA).
539 Returns
540 -------
541 numpy.ndarray
543 Examples
544 --------
545 An object-dtype is the default result
547 >>> a = pd.array([True, False, pd.NA], dtype="boolean")
548 >>> a.to_numpy()
549 array([True, False, <NA>], dtype=object)
551 When no missing values are present, an equivalent dtype can be used.
553 >>> pd.array([True, False], dtype="boolean").to_numpy(dtype="bool")
554 array([ True, False])
555 >>> pd.array([1, 2], dtype="Int64").to_numpy("int64")
556 array([1, 2])
558 However, requesting such dtype will raise a ValueError if
559 missing values are present and the default missing value :attr:`NA`
560 is used.
562 >>> a = pd.array([True, False, pd.NA], dtype="boolean")
563 >>> a
564 <BooleanArray>
565 [True, False, <NA>]
566 Length: 3, dtype: boolean
568 >>> a.to_numpy(dtype="bool")
569 Traceback (most recent call last):
570 ...
571 ValueError: cannot convert to bool numpy array in presence of missing values
573 Specify a valid `na_value` instead
575 >>> a.to_numpy(dtype="bool", na_value=False)
576 array([ True, False, False])
577 """
578 hasna = self._hasna
579 dtype, na_value = to_numpy_dtype_inference(self, dtype, na_value, hasna)
580 if dtype is None:
581 dtype = np.dtype(object)
583 if hasna:
584 if (
585 dtype != np.dtype(object)
586 and not is_string_dtype(dtype)
587 and na_value is libmissing.NA
588 ):
589 raise ValueError(
590 f"cannot convert to '{dtype}'-dtype NumPy array "
591 "with missing values. Specify an appropriate 'na_value' "
592 "for this dtype."
593 )
594 # don't pass copy to astype -> always need a copy since we are mutating
595 with warnings.catch_warnings():
596 warnings.filterwarnings("ignore", category=RuntimeWarning)
597 data = self._data.astype(dtype)
598 data[self._mask] = na_value
599 else:
600 with warnings.catch_warnings():
601 warnings.filterwarnings("ignore", category=RuntimeWarning)
602 data = self._data.astype(dtype, copy=copy)
603 if self._readonly and not copy and astype_is_view(self.dtype, dtype):
604 data = data.view()
605 data.flags.writeable = False
606 return data
608 def tolist(self) -> list:
609 """
610 Return a list of the values.
612 These are each a scalar type, which is a Python scalar
613 (for str, int, float) or a pandas scalar
614 (for Timestamp/Timedelta/Interval/Period)
616 Returns
617 -------
618 list
619 Python list of values in array.
621 See Also
622 --------
623 Index.to_list: Return a list of the values in the Index.
624 Series.to_list: Return a list of the values in the Series.
626 Examples
627 --------
628 >>> arr = pd.array([1, 2, 3])
629 >>> arr.tolist()
630 [1, 2, 3]
631 """
632 if self.ndim > 1:
633 return [x.tolist() for x in self]
634 dtype = None if self._hasna else self._data.dtype
635 return self.to_numpy(dtype=dtype, na_value=libmissing.NA).tolist()
637 @overload
638 def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
640 @overload
641 def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
643 @overload
644 def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
646 def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
647 dtype = pandas_dtype(dtype)
649 if dtype == self.dtype:
650 if copy:
651 return self.copy()
652 return self
654 # if we are astyping to another nullable masked dtype, we can fastpath
655 if isinstance(dtype, BaseMaskedDtype):
656 # TODO deal with NaNs for FloatingArray case
657 with warnings.catch_warnings():
658 warnings.filterwarnings("ignore", category=RuntimeWarning)
659 # TODO: Is rounding what we want long term?
660 data = self._data.astype(dtype.numpy_dtype, copy=copy)
661 # mask is copied depending on whether the data was copied, and
662 # not directly depending on the `copy` keyword
663 mask = self._mask if data is self._data else self._mask.copy()
664 cls = dtype.construct_array_type()
665 return cls(data, mask, copy=False)
667 if isinstance(dtype, ExtensionDtype):
668 eacls = dtype.construct_array_type()
669 return eacls._from_sequence(self, dtype=dtype, copy=copy)
671 na_value: float | np.datetime64 | lib.NoDefault
673 # coerce
674 if dtype.kind == "f":
675 # In astype, we consider dtype=float to also mean na_value=np.nan
676 na_value = np.nan
677 elif dtype.kind == "M":
678 unit = np.datetime_data(dtype)[0]
679 na_value = np.datetime64("NaT", unit) # type: ignore[call-overload]
680 else:
681 na_value = lib.no_default
683 # to_numpy will also raise, but we get somewhat nicer exception messages here
684 if dtype.kind in "iu" and self._hasna:
685 raise ValueError("cannot convert NA to integer")
686 if dtype.kind == "b" and self._hasna:
687 # careful: astype_nansafe converts np.nan to True
688 raise ValueError("cannot convert float NaN to bool")
690 data = self.to_numpy(dtype=dtype, na_value=na_value, copy=copy)
691 return data
693 __array_priority__ = 1000 # higher than ndarray so ops dispatch to us
695 def __array__(
696 self, dtype: NpDtype | None = None, copy: bool | None = None
697 ) -> np.ndarray:
698 """
699 the array interface, return my values
700 We return an object array here to preserve our scalar values
701 """
702 if copy is False:
703 if not self._hasna:
704 # special case, here we can simply return the underlying data
705 result = np.array(self._data, dtype=dtype, copy=copy)
706 # If the ExtensionArray is readonly, make the numpy array readonly too
707 if self._readonly:
708 result = result.view()
709 result.flags.writeable = False
710 return result
711 raise ValueError(
712 "Unable to avoid copy while creating an array as requested."
713 )
715 if copy is None:
716 copy = False # The NumPy copy=False meaning is different here.
717 return self.to_numpy(dtype=dtype, copy=copy)
719 _HANDLED_TYPES: tuple[type, ...]
721 def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
722 # For MaskedArray inputs, we apply the ufunc to ._data
723 # and mask the result.
725 out = kwargs.get("out", ())
727 for x in inputs + out:
728 if not isinstance(x, (*self._HANDLED_TYPES, BaseMaskedArray)):
729 return NotImplemented
731 # for binary ops, use our custom dunder methods
732 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(
733 self, ufunc, method, *inputs, **kwargs
734 )
735 if result is not NotImplemented:
736 return result
738 if "out" in kwargs:
739 # e.g. test_ufunc_with_out
740 return arraylike.dispatch_ufunc_with_out(
741 self, ufunc, method, *inputs, **kwargs
742 )
744 if method == "reduce":
745 result = arraylike.dispatch_reduction_ufunc(
746 self, ufunc, method, *inputs, **kwargs
747 )
748 if result is not NotImplemented:
749 return result
751 mask = np.zeros(len(self), dtype=bool)
752 inputs2 = []
753 for x in inputs:
754 if isinstance(x, BaseMaskedArray):
755 mask |= x._mask
756 inputs2.append(x._data)
757 else:
758 inputs2.append(x)
760 def reconstruct(x: np.ndarray):
761 # we don't worry about scalar `x` here, since we
762 # raise for reduce up above.
763 from pandas.core.arrays import (
764 BooleanArray,
765 FloatingArray,
766 IntegerArray,
767 )
769 if x.dtype.kind == "b":
770 m = mask.copy()
771 return BooleanArray(x, m)
772 elif x.dtype.kind in "iu":
773 m = mask.copy()
774 return IntegerArray(x, m)
775 elif x.dtype.kind == "f":
776 m = mask.copy()
777 if x.dtype == np.float16:
778 # reached in e.g. np.sqrt on BooleanArray
779 # we don't support float16
780 x = x.astype(np.float32)
781 if is_nan_na():
782 m[np.isnan(x)] = True
783 return FloatingArray(x, m)
784 else:
785 x[mask] = np.nan
786 return x
788 result = getattr(ufunc, method)(*inputs2, **kwargs)
789 if ufunc.nout > 1:
790 # e.g. np.divmod
791 return tuple(reconstruct(x) for x in result)
792 elif method == "reduce":
793 # e.g. np.add.reduce; test_ufunc_reduce_raises
794 if self._mask.any():
795 return self._na_value
796 return result
797 else:
798 return reconstruct(result)
800 def __arrow_array__(self, type=None):
801 """
802 Convert myself into a pyarrow Array.
803 """
804 import pyarrow as pa
806 return pa.array(self._data, mask=self._mask, type=type)
808 @property
809 def _hasna(self) -> bool:
810 # Note: this is expensive right now! The hope is that we can
811 # make this faster by having an optional mask, but not have to change
812 # source code using it..
814 return bool(self._mask.any())
816 def _propagate_mask(
817 self, mask: npt.NDArray[np.bool_] | None, other
818 ) -> npt.NDArray[np.bool_]:
819 if mask is None:
820 mask = self._mask.copy() # TODO: need test for BooleanArray needing a copy
821 if other is libmissing.NA:
822 # GH#45421 don't alter inplace
823 mask = mask | True
824 elif is_list_like(other) and len(other) == len(mask):
825 mask = mask | isna(other)
826 else:
827 mask = self._mask | mask
828 return mask
830 def _arith_method(self, other, op):
831 op_name = op.__name__
832 omask = None
834 if (
835 not hasattr(other, "dtype")
836 and is_list_like(other)
837 and len(other) == len(self)
838 ):
839 # Try inferring masked dtype instead of casting to object
840 other = pd_array(other)
841 other = extract_array(other, extract_numpy=True)
843 if isinstance(other, BaseMaskedArray):
844 other, omask = other._data, other._mask
846 elif is_list_like(other):
847 if not isinstance(other, ExtensionArray):
848 other = np.asarray(other)
849 if other.ndim > 1:
850 raise NotImplementedError("can only perform ops with 1-d structures")
852 # We wrap the non-masked arithmetic logic used for numpy dtypes
853 # in Series/Index arithmetic ops.
854 other = ops.maybe_prepare_scalar_for_op(other, (len(self),))
855 pd_op = ops.get_array_op(op)
856 other = ensure_wrapped_if_datetimelike(other)
858 if isinstance(other, ExtensionArray) and isinstance(other.dtype, ArrowDtype):
859 # GH#58602
860 return NotImplemented
862 if op_name in {"pow", "rpow"} and isinstance(other, np.bool_):
863 # Avoid DeprecationWarning: In future, it will be an error
864 # for 'np.bool_' scalars to be interpreted as an index
865 # e.g. test_array_scalar_like_equivalence
866 other = bool(other)
868 mask = self._propagate_mask(omask, other)
870 if other is libmissing.NA:
871 result = np.ones_like(self._data)
872 if self.dtype.kind == "b":
873 if op_name in {
874 "floordiv",
875 "rfloordiv",
876 "pow",
877 "rpow",
878 "truediv",
879 "rtruediv",
880 }:
881 # GH#41165 Try to match non-masked Series behavior
882 # This is still imperfect GH#46043
883 raise NotImplementedError(
884 f"operator '{op_name}' not implemented for bool dtypes"
885 )
886 if op_name in {"mod", "rmod"}:
887 dtype = "int8"
888 else:
889 dtype = "bool"
890 result = result.astype(dtype)
891 elif "truediv" in op_name and self.dtype.kind != "f":
892 # The actual data here doesn't matter since the mask
893 # will be all-True, but since this is division, we want
894 # to end up with floating dtype.
895 result = result.astype(np.float64)
896 elif op_name in {"divmod", "rdivmod"}:
897 # GH#62196
898 res = self._maybe_mask_result(result, mask)
899 return res, res.copy()
900 else:
901 # Make sure we do this before the "pow" mask checks
902 # to get an expected exception message on shape mismatch.
903 if self.dtype.kind in "iu" and op_name in ["floordiv", "mod"]:
904 # TODO(GH#30188) ATM we don't match the behavior of non-masked
905 # types with respect to floordiv-by-zero
906 pd_op = op
908 with np.errstate(all="ignore"):
909 result = pd_op(self._data, other)
911 if op_name == "pow":
912 # 1 ** x is 1.
913 mask = np.where((self._data == 1) & ~self._mask, False, mask)
914 # x ** 0 is 1.
915 if omask is not None:
916 mask = np.where((other == 0) & ~omask, False, mask)
917 elif other is not libmissing.NA:
918 mask = np.where(other == 0, False, mask)
920 elif op_name == "rpow":
921 # 1 ** x is 1.
922 if omask is not None:
923 mask = np.where((other == 1) & ~omask, False, mask)
924 elif other is not libmissing.NA:
925 mask = np.where(other == 1, False, mask)
926 # x ** 0 is 1.
927 mask = np.where((self._data == 0) & ~self._mask, False, mask)
929 return self._maybe_mask_result(result, mask)
931 _logical_method = _arith_method
933 def _cmp_method(self, other, op) -> BooleanArray:
934 from pandas.core.arrays import BooleanArray
936 mask = None
938 if isinstance(other, ExtensionArray) and isinstance(other.dtype, ArrowDtype):
939 # GH#58602
940 return NotImplemented
942 elif isinstance(other, BaseMaskedArray):
943 other, mask = other._data, other._mask
945 elif is_list_like(other):
946 other = np.asarray(other)
947 if other.ndim > 1:
948 raise NotImplementedError("can only perform ops with 1-d structures")
949 if len(self) != len(other):
950 raise ValueError("Lengths must match to compare")
952 if other is libmissing.NA:
953 # numpy does not handle pd.NA well as "other" scalar (it returns
954 # a scalar False instead of an array)
955 # This may be fixed by NA.__array_ufunc__. Revisit this check
956 # once that's implemented.
957 result = np.zeros(self._data.shape, dtype="bool")
958 mask = np.ones(self._data.shape, dtype="bool")
959 else:
960 with warnings.catch_warnings():
961 # numpy may show a FutureWarning or DeprecationWarning:
962 # elementwise comparison failed; returning scalar instead,
963 # but in the future will perform elementwise comparison
964 # before returning NotImplemented. We fall back to the correct
965 # behavior today, so that should be fine to ignore.
966 warnings.filterwarnings("ignore", "elementwise", FutureWarning)
967 warnings.filterwarnings("ignore", "elementwise", DeprecationWarning)
968 method = getattr(self._data, f"__{op.__name__}__")
969 result = method(other)
971 if result is NotImplemented:
972 result = invalid_comparison(self._data, other, op)
974 mask = self._propagate_mask(mask, other)
975 return BooleanArray(result, mask, copy=False)
977 def _maybe_mask_result(
978 self, result: np.ndarray | tuple[np.ndarray, np.ndarray], mask: np.ndarray
979 ):
980 """
981 Parameters
982 ----------
983 result : array-like or tuple[array-like]
984 mask : array-like bool
985 """
986 if isinstance(result, tuple):
987 # i.e. divmod
988 div, mod = result
989 return (
990 self._maybe_mask_result(div, mask),
991 self._maybe_mask_result(mod, mask),
992 )
994 if result.dtype.kind == "f":
995 from pandas.core.arrays import FloatingArray
997 if is_nan_na():
998 mask[np.isnan(result)] = True
1000 return FloatingArray(result, mask, copy=False)
1002 elif result.dtype.kind == "b":
1003 from pandas.core.arrays import BooleanArray
1005 return BooleanArray(result, mask, copy=False)
1007 elif lib.is_np_dtype(result.dtype, "m") and is_supported_dtype(result.dtype):
1008 # e.g. test_numeric_arr_mul_tdscalar_numexpr_path
1009 from pandas.core.arrays import TimedeltaArray
1011 unit = np.datetime_data(result.dtype)[0]
1012 result[mask] = np.timedelta64("NaT", unit) # type: ignore[call-overload]
1014 if not isinstance(result, TimedeltaArray):
1015 return TimedeltaArray._simple_new(result, dtype=result.dtype)
1017 return result
1019 elif result.dtype.kind in "iu":
1020 from pandas.core.arrays import IntegerArray
1022 return IntegerArray(result, mask, copy=False)
1024 elif result.dtype == object:
1025 result[mask] = self.dtype.na_value
1026 return result
1027 else:
1028 result[mask] = np.nan
1029 return result
1031 def isna(self) -> np.ndarray:
1032 return self._mask.copy()
1034 @property
1035 def _na_value(self):
1036 return self.dtype.na_value
1038 @property
1039 def nbytes(self) -> int:
1040 return self._data.nbytes + self._mask.nbytes
1042 @classmethod
1043 def _concat_same_type(
1044 cls,
1045 to_concat: Sequence[Self],
1046 axis: AxisInt = 0,
1047 ) -> Self:
1048 data = np.concatenate([x._data for x in to_concat], axis=axis)
1049 mask = np.concatenate([x._mask for x in to_concat], axis=axis)
1050 return cls(data, mask)
1052 def _hash_pandas_object(
1053 self, *, encoding: str, hash_key: str, categorize: bool
1054 ) -> npt.NDArray[np.uint64]:
1055 hashed_array = hash_array(
1056 self._data, encoding=encoding, hash_key=hash_key, categorize=categorize
1057 )
1058 hashed_array[self.isna()] = hash(self.dtype.na_value)
1059 return hashed_array
1061 def take(
1062 self,
1063 indexer,
1064 *,
1065 allow_fill: bool = False,
1066 fill_value: Scalar | None = None,
1067 axis: AxisInt = 0,
1068 ) -> Self:
1069 # we always fill with 1 internally
1070 # to avoid upcasting
1071 data_fill_value = (
1072 self.dtype._internal_fill_value if isna(fill_value) else fill_value
1073 )
1074 result = take(
1075 self._data,
1076 indexer,
1077 fill_value=data_fill_value,
1078 allow_fill=allow_fill,
1079 axis=axis,
1080 )
1082 mask = take(
1083 self._mask, indexer, fill_value=True, allow_fill=allow_fill, axis=axis
1084 )
1086 # if we are filling
1087 # we only fill where the indexer is null
1088 # not existing missing values
1089 # TODO(jreback) what if we have a non-na float as a fill value?
1090 if allow_fill and notna(fill_value):
1091 fill_mask = np.asarray(indexer) == -1
1092 result[fill_mask] = fill_value
1093 mask = mask ^ fill_mask
1095 return self._simple_new(result, mask)
1097 # error: Return type "BooleanArray" of "isin" incompatible with return type
1098 # "ndarray" in supertype "ExtensionArray"
1099 def isin(self, values: ArrayLike) -> BooleanArray: # type: ignore[override]
1100 from pandas.core.arrays import BooleanArray
1102 # algorithms.isin will eventually convert values to an ndarray, so no extra
1103 # cost to doing it here first
1104 values_arr = np.asarray(values)
1105 result = isin(self._data, values_arr)
1107 if self._hasna:
1108 values_have_NA = values_arr.dtype == object and any(
1109 val is self.dtype.na_value for val in values_arr
1110 )
1112 # For now, NA does not propagate so set result according to presence of NA,
1113 # see https://github.com/pandas-dev/pandas/pull/38379 for some discussion
1114 result[self._mask] = values_have_NA
1116 mask = np.zeros(self._data.shape, dtype=bool)
1117 return BooleanArray(result, mask, copy=False)
1119 def copy(self) -> Self:
1120 data = self._data.copy()
1121 mask = self._mask.copy()
1122 return self._simple_new(data, mask)
1124 def _rank(
1125 self,
1126 *,
1127 axis: AxisInt = 0,
1128 method: str = "average",
1129 na_option: str = "keep",
1130 ascending: bool = True,
1131 pct: bool = False,
1132 ):
1133 # GH#62043 Avoid going through copy-making ensure_data in algorithms.rank
1134 if axis != 0 or self.ndim != 1:
1135 raise NotImplementedError
1137 from pandas.core.arrays import FloatingArray
1139 data = self._data
1140 if data.dtype.kind == "b":
1141 data = data.view("uint8")
1143 result = libalgos.rank_1d(
1144 data,
1145 is_datetimelike=False,
1146 ties_method=method,
1147 ascending=ascending,
1148 na_option=na_option,
1149 pct=pct,
1150 mask=self.isna(),
1151 )
1152 if na_option in ["top", "bottom"]:
1153 mask = np.zeros(self.shape, dtype=bool)
1154 else:
1155 mask = self._mask.copy()
1157 if method != "average" and not pct:
1158 if na_option not in ["top", "bottom"]:
1159 result[self._mask] = 0 # avoid warning on casting
1160 result = result.astype("uint64", copy=False)
1161 from pandas.core.arrays import IntegerArray
1163 return IntegerArray(result, mask=mask)
1165 return FloatingArray(result, mask=mask)
1167 def duplicated(
1168 self, keep: Literal["first", "last", False] = "first"
1169 ) -> npt.NDArray[np.bool_]:
1170 """
1171 Return boolean ndarray denoting duplicate values.
1173 Parameters
1174 ----------
1175 keep : {'first', 'last', False}, default 'first'
1176 - ``first`` : Mark duplicates as ``True`` except for the first occurrence.
1177 - ``last`` : Mark duplicates as ``True`` except for the last occurrence.
1178 - False : Mark all duplicates as ``True``.
1180 Returns
1181 -------
1182 ndarray[bool]
1183 With true in indices where elements are duplicated and false otherwise.
1185 See Also
1186 --------
1187 DataFrame.duplicated : Return boolean Series denoting
1188 duplicate rows.
1189 Series.duplicated : Indicate duplicate Series values.
1190 api.extensions.ExtensionArray.unique : Compute the ExtensionArray
1191 of unique values.
1193 Examples
1194 --------
1195 >>> pd.array([1, 1, 2, 3, 3], dtype="Int64").duplicated()
1196 array([False, True, False, False, True])
1197 """
1198 values = self._data
1199 mask = self._mask
1200 return algos.duplicated(values, keep=keep, mask=mask)
1202 def unique(self) -> Self:
1203 """
1204 Compute the BaseMaskedArray of unique values.
1206 Returns
1207 -------
1208 uniques : BaseMaskedArray
1209 """
1210 uniques, mask = algos.unique_with_mask(self._data, self._mask)
1211 return self._simple_new(uniques, mask)
1213 def searchsorted(
1214 self,
1215 value: NumpyValueArrayLike | ExtensionArray,
1216 side: Literal["left", "right"] = "left",
1217 sorter: NumpySorter | None = None,
1218 ) -> npt.NDArray[np.intp] | np.intp:
1219 """
1220 Find indices where elements should be inserted to maintain order.
1222 Find the indices into a sorted array `self` (a) such that, if the
1223 corresponding elements in `value` were inserted before the indices,
1224 the order of `self` would be preserved.
1226 Assuming that `self` is sorted:
1228 ====== ================================
1229 `side` returned index `i` satisfies
1230 ====== ================================
1231 left ``self[i-1] < value <= self[i]``
1232 right ``self[i-1] <= value < self[i]``
1233 ====== ================================
1235 Parameters
1236 ----------
1237 value : array-like, list or scalar
1238 Value(s) to insert into `self`.
1239 side : {'left', 'right'}, optional
1240 If 'left', the index of the first suitable location found is given.
1241 If 'right', return the last such index. If there is no suitable
1242 index, return either 0 or N (where N is the length of `self`).
1243 sorter : 1-D array-like, optional
1244 Optional array of integer indices that sort array a into ascending
1245 order. They are typically the result of argsort.
1247 Returns
1248 -------
1249 array of ints or int
1250 If value is array-like, array of insertion points.
1251 If value is scalar, a single integer.
1253 See Also
1254 --------
1255 numpy.searchsorted : Similar method from NumPy.
1257 Examples
1258 --------
1259 >>> arr = pd.array([1, 2, 3, 5])
1260 >>> arr.searchsorted([4])
1261 array([3])
1262 """
1263 if self._hasna:
1264 raise ValueError(
1265 "searchsorted requires array to be sorted, which is impossible "
1266 "with NAs present."
1267 )
1268 if isinstance(value, ExtensionArray):
1269 value = value.astype(object)
1270 # Base class searchsorted would cast to object, which is *much* slower.
1271 return self._data.searchsorted(value, side=side, sorter=sorter)
1273 def factorize(
1274 self,
1275 use_na_sentinel: bool = True,
1276 ) -> tuple[np.ndarray, ExtensionArray]:
1277 """
1278 Encode the extension array as an enumerated type.
1280 Parameters
1281 ----------
1282 use_na_sentinel : bool, default True
1283 If True, the sentinel -1 will be used for NaN values. If False,
1284 NaN values will be encoded as non-negative integers and will not drop the
1285 NaN from the uniques of the values.
1287 Returns
1288 -------
1289 codes : ndarray
1290 An integer NumPy array that's an indexer into the original
1291 ExtensionArray.
1292 uniques : ExtensionArray
1293 An ExtensionArray containing the unique values of `self`.
1295 .. note::
1297 uniques will *not* contain an entry for the NA value of
1298 the ExtensionArray if there are any missing values present
1299 in `self`.
1301 See Also
1302 --------
1303 factorize : Top-level factorize method that dispatches here.
1305 Notes
1306 -----
1307 :meth:`pandas.factorize` offers a `sort` keyword as well.
1309 Examples
1310 --------
1311 >>> idx1 = pd.PeriodIndex(
1312 ... ["2014-01", "2014-01", "2014-02", "2014-02", "2014-03", "2014-03"],
1313 ... freq="M",
1314 ... )
1315 >>> arr, idx = idx1.factorize()
1316 >>> arr
1317 array([0, 0, 1, 1, 2, 2])
1318 >>> idx
1319 PeriodIndex(['2014-01', '2014-02', '2014-03'], dtype='period[M]')
1320 """
1321 arr = self._data
1322 mask = self._mask
1324 # Use a sentinel for na; recode and add NA to uniques if necessary below
1325 codes, uniques = factorize_array(arr, use_na_sentinel=True, mask=mask)
1327 # check that factorize_array correctly preserves dtype.
1328 assert uniques.dtype == self.dtype.numpy_dtype, (uniques.dtype, self.dtype)
1330 has_na = mask.any()
1331 if use_na_sentinel or not has_na:
1332 size = len(uniques)
1333 else:
1334 # Make room for an NA value
1335 size = len(uniques) + 1
1336 uniques_mask = np.zeros(size, dtype=bool)
1337 if not use_na_sentinel and has_na:
1338 na_index = mask.argmax()
1339 # Insert na with the proper code
1340 if na_index == 0:
1341 na_code = np.intp(0)
1342 else:
1343 na_code = codes[:na_index].max() + 1
1344 codes[codes >= na_code] += 1
1345 codes[codes == -1] = na_code
1346 # dummy value for uniques; not used since uniques_mask will be True
1347 uniques = np.insert(uniques, na_code, 0)
1348 uniques_mask[na_code] = True
1349 uniques_ea = self._simple_new(uniques, uniques_mask)
1351 return codes, uniques_ea
1353 def _values_for_argsort(self) -> np.ndarray:
1354 """
1355 Return values for sorting.
1357 Returns
1358 -------
1359 ndarray
1360 The transformed values should maintain the ordering between values
1361 within the array.
1363 See Also
1364 --------
1365 ExtensionArray.argsort : Return the indices that would sort this array.
1367 Notes
1368 -----
1369 The caller is responsible for *not* modifying these values in-place, so
1370 it is safe for implementers to give views on ``self``.
1372 Functions that use this (e.g. ``ExtensionArray.argsort``) should ignore
1373 entries with missing values in the original array (according to
1374 ``self.isna()``). This means that the corresponding entries in the returned
1375 array don't need to be modified to sort correctly.
1377 Examples
1378 --------
1379 In most cases, this is the underlying Numpy array of the ``ExtensionArray``:
1381 >>> arr = pd.array([1, 2, 3])
1382 >>> arr._values_for_argsort()
1383 array([1, 2, 3])
1384 """
1385 return self._data
1387 def value_counts(self, dropna: bool = True) -> Series:
1388 """
1389 Returns a Series containing counts of each unique value.
1391 Parameters
1392 ----------
1393 dropna : bool, default True
1394 Don't include counts of missing values.
1396 Returns
1397 -------
1398 counts : Series
1400 See Also
1401 --------
1402 Series.value_counts
1403 """
1404 from pandas import (
1405 Index,
1406 Series,
1407 )
1408 from pandas.arrays import IntegerArray
1410 keys, value_counts, na_counter = algos.value_counts_arraylike(
1411 self._data, dropna=dropna, mask=self._mask
1412 )
1413 mask_index = np.zeros((len(value_counts),), dtype=np.bool_)
1414 mask = mask_index.copy()
1416 if na_counter > 0:
1417 mask_index[-1] = True
1419 arr = IntegerArray(value_counts, mask)
1420 index = Index(
1421 self.dtype.construct_array_type()(
1422 keys, # type: ignore[arg-type]
1423 mask_index,
1424 ),
1425 copy=False,
1426 )
1427 return Series(arr, index=index, name="count", copy=False)
1429 def _mode(self, dropna: bool = True) -> Self:
1430 result, res_mask = mode(self._data, dropna=dropna, mask=self._mask)
1431 result = type(self)(result, res_mask)
1432 return result[result.argsort()]
1434 def equals(self, other) -> bool:
1435 """
1436 Return if another array is equivalent to this array.
1438 Equivalent means that both arrays have the same shape and dtype, and
1439 all values compare equal. Missing values in the same location are
1440 considered equal (in contrast with normal equality).
1442 Parameters
1443 ----------
1444 other : ExtensionArray
1445 Array to compare to this Array.
1447 Returns
1448 -------
1449 boolean
1450 Whether the arrays are equivalent.
1452 See Also
1453 --------
1454 numpy.array_equal : Equivalent method for numpy array.
1455 Series.equals : Equivalent method for Series.
1456 DataFrame.equals : Equivalent method for DataFrame.
1458 Examples
1459 --------
1460 >>> arr1 = pd.array([1, 2, np.nan])
1461 >>> arr2 = pd.array([1, 2, np.nan])
1462 >>> arr1.equals(arr2)
1463 True
1465 >>> arr1 = pd.array([1, 3, np.nan])
1466 >>> arr2 = pd.array([1, 2, np.nan])
1467 >>> arr1.equals(arr2)
1468 False
1469 """
1470 if type(self) != type(other):
1471 return False
1472 if other.dtype != self.dtype:
1473 return False
1475 # GH#44382 if e.g. self[1] is np.nan and other[1] is pd.NA, we are NOT
1476 # equal.
1477 if not np.array_equal(self._mask, other._mask):
1478 return False
1480 left = self._data[~self._mask]
1481 right = other._data[~other._mask]
1482 return array_equivalent(left, right, strict_nan=True, dtype_equal=True)
1484 def _quantile(
1485 self, qs: npt.NDArray[np.float64], interpolation: str
1486 ) -> BaseMaskedArray:
1487 """
1488 Dispatch to quantile_with_mask, needed because we do not have
1489 _from_factorized.
1491 Notes
1492 -----
1493 We assume that all impacted cases are 1D-only.
1494 """
1495 res = quantile_with_mask(
1496 self._data,
1497 mask=self._mask,
1498 # TODO(GH#40932): na_value_for_dtype(self.dtype.numpy_dtype)
1499 # instead of np.nan
1500 fill_value=np.nan,
1501 qs=qs,
1502 interpolation=interpolation,
1503 )
1505 if self._hasna:
1506 # Our result mask is all-False unless we are all-NA, in which
1507 # case it is all-True.
1508 if self.ndim == 2:
1509 # I think this should be out_mask=self.isna().all(axis=1)
1510 # but am holding off until we have tests
1511 raise NotImplementedError
1512 if self.isna().all():
1513 out_mask = np.ones(res.shape, dtype=bool)
1515 if is_integer_dtype(self.dtype):
1516 # We try to maintain int dtype if possible for not all-na case
1517 # as well
1518 res = np.zeros(res.shape, dtype=self.dtype.numpy_dtype)
1519 else:
1520 out_mask = np.zeros(res.shape, dtype=bool)
1521 else:
1522 out_mask = np.zeros(res.shape, dtype=bool)
1523 return self._maybe_mask_result(res, mask=out_mask)
1525 # ------------------------------------------------------------------
1526 # Reductions
1528 def _reduce(
1529 self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
1530 ):
1531 if name in {"any", "all", "min", "max", "sum", "prod", "mean", "var", "std"}:
1532 result = getattr(self, name)(skipna=skipna, **kwargs)
1533 else:
1534 # median, skew, kurt, sem
1535 data = self._data
1536 mask = self._mask
1537 op = getattr(nanops, f"nan{name}")
1538 axis = kwargs.pop("axis", None)
1539 result = op(data, axis=axis, skipna=skipna, mask=mask, **kwargs)
1541 if keepdims:
1542 if isna(result):
1543 return self._wrap_na_result(name=name, axis=0, mask_size=(1,))
1544 else:
1545 if using_python_scalars():
1546 result = np.array([result])
1547 else:
1548 result = result.reshape(1)
1549 mask = np.zeros(1, dtype=bool)
1550 return self._maybe_mask_result(result, mask)
1552 if isna(result):
1553 return libmissing.NA
1554 else:
1555 return result
1557 def _wrap_reduction_result(self, name: str, result, *, skipna, axis):
1558 if isinstance(result, np.ndarray):
1559 if skipna:
1560 # we only retain mask for all-NA rows/columns
1561 mask = self._mask.all(axis=axis)
1562 else:
1563 mask = self._mask.any(axis=axis)
1565 return self._maybe_mask_result(result, mask)
1566 return result
1568 def _wrap_na_result(self, *, name, axis, mask_size):
1569 mask = np.ones(mask_size, dtype=bool)
1571 float_dtyp = "float32" if self.dtype == "Float32" else "float64"
1572 if name in ["mean", "median", "var", "std", "skew", "kurt", "sem"]:
1573 np_dtype = float_dtyp
1574 elif name in ["min", "max"] or self.dtype.itemsize == 8:
1575 np_dtype = self.dtype.numpy_dtype.name
1576 else:
1577 is_windows_or_32bit = is_platform_windows() or not IS64
1578 int_dtyp = "int32" if is_windows_or_32bit else "int64"
1579 uint_dtyp = "uint32" if is_windows_or_32bit else "uint64"
1580 np_dtype = {"b": int_dtyp, "i": int_dtyp, "u": uint_dtyp, "f": float_dtyp}[
1581 self.dtype.kind
1582 ]
1584 value = np.array([1], dtype=np_dtype)
1585 return self._maybe_mask_result(value, mask=mask)
1587 def _wrap_min_count_reduction_result(
1588 self, name: str, result, *, skipna, min_count, axis
1589 ):
1590 if min_count == 0 and isinstance(result, np.ndarray):
1591 return self._maybe_mask_result(result, np.zeros(result.shape, dtype=bool))
1592 return self._wrap_reduction_result(name, result, skipna=skipna, axis=axis)
1594 def sum(
1595 self,
1596 *,
1597 skipna: bool = True,
1598 min_count: int = 0,
1599 axis: AxisInt | None = 0,
1600 **kwargs,
1601 ):
1602 nv.validate_sum((), kwargs)
1604 result = masked_reductions.sum(
1605 self._data,
1606 self._mask,
1607 skipna=skipna,
1608 min_count=min_count,
1609 axis=axis,
1610 )
1611 return self._wrap_min_count_reduction_result(
1612 "sum", result, skipna=skipna, min_count=min_count, axis=axis
1613 )
1615 def prod(
1616 self,
1617 *,
1618 skipna: bool = True,
1619 min_count: int = 0,
1620 axis: AxisInt | None = 0,
1621 **kwargs,
1622 ):
1623 nv.validate_prod((), kwargs)
1625 result = masked_reductions.prod(
1626 self._data,
1627 self._mask,
1628 skipna=skipna,
1629 min_count=min_count,
1630 axis=axis,
1631 )
1632 return self._wrap_min_count_reduction_result(
1633 "prod", result, skipna=skipna, min_count=min_count, axis=axis
1634 )
1636 def mean(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
1637 nv.validate_mean((), kwargs)
1638 result = masked_reductions.mean(
1639 self._data,
1640 self._mask,
1641 skipna=skipna,
1642 axis=axis,
1643 )
1644 return self._wrap_reduction_result("mean", result, skipna=skipna, axis=axis)
1646 def var(
1647 self, *, skipna: bool = True, axis: AxisInt | None = 0, ddof: int = 1, **kwargs
1648 ):
1649 nv.validate_stat_ddof_func((), kwargs, fname="var")
1650 result = masked_reductions.var(
1651 self._data,
1652 self._mask,
1653 skipna=skipna,
1654 axis=axis,
1655 ddof=ddof,
1656 )
1657 return self._wrap_reduction_result("var", result, skipna=skipna, axis=axis)
1659 def std(
1660 self, *, skipna: bool = True, axis: AxisInt | None = 0, ddof: int = 1, **kwargs
1661 ):
1662 nv.validate_stat_ddof_func((), kwargs, fname="std")
1663 result = masked_reductions.std(
1664 self._data,
1665 self._mask,
1666 skipna=skipna,
1667 axis=axis,
1668 ddof=ddof,
1669 )
1670 return self._wrap_reduction_result("std", result, skipna=skipna, axis=axis)
1672 def min(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
1673 nv.validate_min((), kwargs)
1674 result = masked_reductions.min(
1675 self._data,
1676 self._mask,
1677 skipna=skipna,
1678 axis=axis,
1679 )
1680 return self._wrap_reduction_result("min", result, skipna=skipna, axis=axis)
1682 def max(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
1683 nv.validate_max((), kwargs)
1684 result = masked_reductions.max(
1685 self._data,
1686 self._mask,
1687 skipna=skipna,
1688 axis=axis,
1689 )
1690 return self._wrap_reduction_result("max", result, skipna=skipna, axis=axis)
1692 def map(self, mapper, na_action: Literal["ignore"] | None = None):
1693 return map_array(self.to_numpy(), mapper, na_action=na_action)
1695 @overload
1696 def any(
1697 self, *, skipna: Literal[True] = ..., axis: AxisInt | None = ..., **kwargs
1698 ) -> np.bool_: ...
1700 @overload
1701 def any(
1702 self, *, skipna: bool, axis: AxisInt | None = ..., **kwargs
1703 ) -> np.bool_ | NAType: ...
1705 def any(
1706 self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs
1707 ) -> np.bool_ | NAType:
1708 """
1709 Return whether any element is truthy.
1711 Returns False unless there is at least one element that is truthy.
1712 By default, NAs are skipped. If ``skipna=False`` is specified and
1713 missing values are present, similar :ref:`Kleene logic <boolean.kleene>`
1714 is used as for logical operations.
1716 Parameters
1717 ----------
1718 skipna : bool, default True
1719 Exclude NA values. If the entire array is NA and `skipna` is
1720 True, then the result will be False, as for an empty array.
1721 If `skipna` is False, the result will still be True if there is
1722 at least one element that is truthy, otherwise NA will be returned
1723 if there are NA's present.
1724 axis : int, optional, default 0
1725 **kwargs : any, default None
1726 Additional keywords have no effect but might be accepted for
1727 compatibility with NumPy.
1729 Returns
1730 -------
1731 bool or :attr:`pandas.NA`
1733 See Also
1734 --------
1735 numpy.any : Numpy version of this method.
1736 BaseMaskedArray.all : Return whether all elements are truthy.
1738 Examples
1739 --------
1740 The result indicates whether any element is truthy (and by default
1741 skips NAs):
1743 >>> pd.array([True, False, True]).any()
1744 np.True_
1745 >>> pd.array([True, False, pd.NA]).any()
1746 np.True_
1747 >>> pd.array([False, False, pd.NA]).any()
1748 np.False_
1749 >>> pd.array([], dtype="boolean").any()
1750 np.False_
1751 >>> pd.array([pd.NA], dtype="boolean").any()
1752 np.False_
1753 >>> pd.array([pd.NA], dtype="Float64").any()
1754 np.False_
1756 With ``skipna=False``, the result can be NA if this is logically
1757 required (whether ``pd.NA`` is True or False influences the result):
1759 >>> pd.array([True, False, pd.NA]).any(skipna=False)
1760 np.True_
1761 >>> pd.array([1, 0, pd.NA]).any(skipna=False)
1762 np.True_
1763 >>> pd.array([False, False, pd.NA]).any(skipna=False)
1764 <NA>
1765 >>> pd.array([0, 0, pd.NA]).any(skipna=False)
1766 <NA>
1767 """
1768 nv.validate_any((), kwargs)
1770 values = self._data.copy()
1771 np.putmask(values, self._mask, self.dtype._falsey_value)
1772 result = values.any()
1773 if skipna:
1774 return result
1775 elif result or len(self) == 0 or not self._mask.any():
1776 return result
1777 else:
1778 return self.dtype.na_value
1780 @overload
1781 def all(
1782 self, *, skipna: Literal[True] = ..., axis: AxisInt | None = ..., **kwargs
1783 ) -> np.bool_: ...
1785 @overload
1786 def all(
1787 self, *, skipna: bool, axis: AxisInt | None = ..., **kwargs
1788 ) -> np.bool_ | NAType: ...
1790 def all(
1791 self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs
1792 ) -> np.bool_ | NAType:
1793 """
1794 Return whether all elements are truthy.
1796 Returns True unless there is at least one element that is falsey.
1797 By default, NAs are skipped. If ``skipna=False`` is specified and
1798 missing values are present, similar :ref:`Kleene logic <boolean.kleene>`
1799 is used as for logical operations.
1801 Parameters
1802 ----------
1803 skipna : bool, default True
1804 Exclude NA values. If the entire array is NA and `skipna` is
1805 True, then the result will be True, as for an empty array.
1806 If `skipna` is False, the result will still be False if there is
1807 at least one element that is falsey, otherwise NA will be returned
1808 if there are NA's present.
1809 axis : int, optional, default 0
1810 **kwargs : any, default None
1811 Additional keywords have no effect but might be accepted for
1812 compatibility with NumPy.
1814 Returns
1815 -------
1816 bool or :attr:`pandas.NA`
1818 See Also
1819 --------
1820 numpy.all : Numpy version of this method.
1821 BooleanArray.any : Return whether any element is truthy.
1823 Examples
1824 --------
1825 The result indicates whether all elements are truthy (and by default
1826 skips NAs):
1828 >>> pd.array([True, True, pd.NA]).all()
1829 np.True_
1830 >>> pd.array([1, 1, pd.NA]).all()
1831 np.True_
1832 >>> pd.array([True, False, pd.NA]).all()
1833 np.False_
1834 >>> pd.array([], dtype="boolean").all()
1835 np.True_
1836 >>> pd.array([pd.NA], dtype="boolean").all()
1837 np.True_
1838 >>> pd.array([pd.NA], dtype="Float64").all()
1839 np.True_
1841 With ``skipna=False``, the result can be NA if this is logically
1842 required (whether ``pd.NA`` is True or False influences the result):
1844 >>> pd.array([True, True, pd.NA]).all(skipna=False)
1845 <NA>
1846 >>> pd.array([1, 1, pd.NA]).all(skipna=False)
1847 <NA>
1848 >>> pd.array([True, False, pd.NA]).all(skipna=False)
1849 np.False_
1850 >>> pd.array([1, 0, pd.NA]).all(skipna=False)
1851 np.False_
1852 """
1853 nv.validate_all((), kwargs)
1855 values = self._data.copy()
1856 np.putmask(values, self._mask, self.dtype._truthy_value)
1857 result = values.all(axis=axis)
1859 if skipna:
1860 return result # type: ignore[return-value]
1861 elif not result or len(self) == 0 or not self._mask.any():
1862 return result # type: ignore[return-value]
1863 else:
1864 return self.dtype.na_value
1866 def interpolate(
1867 self,
1868 *,
1869 method: InterpolateOptions,
1870 axis: int,
1871 index,
1872 limit,
1873 limit_direction,
1874 limit_area,
1875 copy: bool,
1876 **kwargs,
1877 ) -> FloatingArray:
1878 """
1879 See NDFrame.interpolate.__doc__.
1880 """
1881 # NB: we return type(self) even if copy=False
1882 if self.dtype.kind == "f":
1883 if copy:
1884 data = self._data.copy()
1885 mask = self._mask.copy()
1886 else:
1887 data = self._data
1888 mask = self._mask
1889 elif self.dtype.kind in "iu":
1890 copy = True
1891 data = self._data.astype("f8")
1892 mask = self._mask.copy()
1893 else:
1894 raise NotImplementedError(
1895 f"interpolate is not implemented for dtype={self.dtype}"
1896 )
1898 missing.interpolate_2d_inplace(
1899 data,
1900 method=method,
1901 axis=0,
1902 index=index,
1903 limit=limit,
1904 limit_direction=limit_direction,
1905 limit_area=limit_area,
1906 mask=mask,
1907 **kwargs,
1908 )
1909 if not copy:
1910 return self # type: ignore[return-value]
1911 if self.dtype.kind == "f":
1912 return type(self)._simple_new(data, mask) # type: ignore[return-value]
1913 else:
1914 from pandas.core.arrays import FloatingArray
1916 return FloatingArray._simple_new(data, mask)
1918 def _accumulate(
1919 self, name: str, *, skipna: bool = True, **kwargs
1920 ) -> BaseMaskedArray:
1921 data = self._data
1922 mask = self._mask
1924 op = getattr(masked_accumulations, name)
1925 data, mask = op(data, mask, skipna=skipna, **kwargs)
1927 return self._simple_new(data, mask)
1929 # ------------------------------------------------------------------
1930 # GroupBy Methods
1932 def _groupby_op(
1933 self,
1934 *,
1935 how: str,
1936 has_dropped_na: bool,
1937 min_count: int,
1938 ngroups: int,
1939 ids: npt.NDArray[np.intp],
1940 **kwargs,
1941 ):
1942 from pandas.core.groupby.ops import WrappedCythonOp
1944 kind = WrappedCythonOp.get_kind_from_how(how)
1945 op = WrappedCythonOp(how=how, kind=kind, has_dropped_na=has_dropped_na)
1947 # libgroupby functions are responsible for NOT altering mask
1948 mask = self._mask
1949 if op.kind != "aggregate":
1950 result_mask = mask.copy()
1951 else:
1952 result_mask = np.zeros(ngroups, dtype=bool)
1954 if how == "rank" and kwargs.get("na_option") in ["top", "bottom"]:
1955 result_mask[:] = False
1957 res_values = op._cython_op_ndim_compat(
1958 self._data,
1959 min_count=min_count,
1960 ngroups=ngroups,
1961 comp_ids=ids,
1962 mask=mask,
1963 result_mask=result_mask,
1964 **kwargs,
1965 )
1967 if op.how == "ohlc":
1968 arity = op._cython_arity.get(op.how, 1)
1969 result_mask = np.tile(result_mask, (arity, 1)).T
1971 if op.how in ["idxmin", "idxmax"]:
1972 # Result values are indexes to take, keep as ndarray
1973 return res_values
1974 else:
1975 # res_values should already have the correct dtype, we just need to
1976 # wrap in a MaskedArray
1977 return self._maybe_mask_result(res_values, result_mask)
1980def transpose_homogeneous_masked_arrays(
1981 masked_arrays: Sequence[BaseMaskedArray],
1982) -> list[BaseMaskedArray]:
1983 """Transpose masked arrays in a list, but faster.
1985 Input should be a list of 1-dim masked arrays of equal length and all have the
1986 same dtype. The caller is responsible for ensuring validity of input data.
1987 """
1988 masked_arrays = list(masked_arrays)
1989 dtype = masked_arrays[0].dtype
1991 values = [arr._data.reshape(1, -1) for arr in masked_arrays]
1992 transposed_values = np.concatenate(
1993 values,
1994 axis=0,
1995 out=np.empty(
1996 (len(masked_arrays), len(masked_arrays[0])),
1997 order="F",
1998 dtype=dtype.numpy_dtype,
1999 ),
2000 )
2002 masks = [arr._mask.reshape(1, -1) for arr in masked_arrays]
2003 transposed_masks = np.concatenate(
2004 masks, axis=0, out=np.empty_like(transposed_values, dtype=bool)
2005 )
2007 arr_type = dtype.construct_array_type()
2008 transposed_arrays: list[BaseMaskedArray] = []
2009 for i in range(transposed_values.shape[1]):
2010 transposed_arr = arr_type(transposed_values[:, i], mask=transposed_masks[:, i])
2011 transposed_arrays.append(transposed_arr)
2013 return transposed_arrays