Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2An interface for extending pandas with custom arrays.
4.. warning::
6 This is an experimental API and subject to breaking changes
7 without warning.
8"""
10from __future__ import annotations
12import operator
13from typing import (
14 TYPE_CHECKING,
15 Any,
16 ClassVar,
17 Literal,
18 Self,
19 cast,
20 overload,
21)
22import warnings
24import numpy as np
26from pandas._libs import (
27 algos as libalgos,
28 groupby as libgroupby,
29 lib,
30)
31from pandas.compat import set_function_name
32from pandas.compat.numpy import function as nv
33from pandas.errors import AbstractMethodError
34from pandas.util._decorators import (
35 cache_readonly,
36 set_module,
37)
38from pandas.util._exceptions import find_stack_level
39from pandas.util._validators import (
40 validate_bool_kwarg,
41 validate_insert_loc,
42)
44from pandas.core.dtypes.astype import astype_is_view
45from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
46from pandas.core.dtypes.common import (
47 is_bool_dtype,
48 is_float_dtype,
49 is_integer,
50 is_integer_dtype,
51 is_list_like,
52 is_object_dtype,
53 is_scalar,
54 pandas_dtype,
55)
56from pandas.core.dtypes.dtypes import ExtensionDtype
57from pandas.core.dtypes.generic import (
58 ABCDataFrame,
59 ABCIndex,
60 ABCSeries,
61)
62from pandas.core.dtypes.missing import isna
64from pandas.core import (
65 arraylike,
66 missing,
67 ops,
68 roperator,
69)
70from pandas.core.algorithms import (
71 duplicated,
72 factorize_array,
73 is_monotonic,
74 isin,
75 map_array,
76 mode,
77 rank,
78 unique,
79)
80from pandas.core.array_algos.quantile import quantile_with_mask
81from pandas.core.missing import _fill_limit_area_1d
82from pandas.core.sorting import (
83 nargminmax,
84 nargsort,
85)
87if TYPE_CHECKING:
88 from collections.abc import (
89 Callable,
90 Iterator,
91 Sequence,
92 )
94 from pandas._libs.missing import NAType
95 from pandas._typing import (
96 ArrayLike,
97 AstypeArg,
98 AxisInt,
99 Dtype,
100 DtypeObj,
101 FillnaOptions,
102 InterpolateOptions,
103 NumpySorter,
104 NumpyValueArrayLike,
105 PositionalIndexer,
106 RankMethod,
107 RankNaOption,
108 ScalarIndexer,
109 SequenceIndexer,
110 Shape,
111 SortKind,
112 TakeIndexer,
113 npt,
114 )
116 from pandas import (
117 Index,
118 Series,
119 )
122@set_module("pandas.api.extensions")
123class ExtensionArray:
124 """
125 Abstract base class for custom 1-D array types.
127 pandas will recognize instances of this class as proper arrays
128 with a custom type and will not attempt to coerce them to objects. They
129 may be stored directly inside a :class:`DataFrame` or :class:`Series`.
131 Attributes
132 ----------
133 dtype
134 nbytes
135 ndim
136 shape
138 Methods
139 -------
140 argsort
141 astype
142 copy
143 count
144 dropna
145 duplicated
146 factorize
147 fillna
148 equals
149 insert
150 interpolate
151 isin
152 isna
153 item
154 ravel
155 repeat
156 searchsorted
157 shift
158 sort
159 take
160 tolist
161 unique
162 view
163 _accumulate
164 _concat_same_type
165 _explode
166 _formatter
167 _from_factorized
168 _from_sequence
169 _from_sequence_of_strings
170 _hash_pandas_object
171 _is_monotonic_decreasing
172 _is_monotonic_increasing
173 _pad_or_backfill
174 _reduce
175 _values_for_argsort
176 _values_for_factorize
178 See Also
179 --------
180 api.extensions.ExtensionDtype : A custom data type, to be paired with an
181 ExtensionArray.
182 api.extensions.ExtensionArray.dtype : An instance of ExtensionDtype.
184 Notes
185 -----
186 The interface includes the following abstract methods that must be
187 implemented by subclasses:
189 * _from_sequence
190 * _from_factorized
191 * __getitem__
192 * __len__
193 * __eq__
194 * dtype
195 * nbytes
196 * isna
197 * take
198 * copy
199 * _concat_same_type
200 * interpolate
202 A default repr displaying the type, (truncated) data, length,
203 and dtype is provided. It can be customized or replaced
204 by overriding:
206 * __repr__ : A default repr for the ExtensionArray.
207 * _formatter : Print scalars inside a Series or DataFrame.
209 Some methods require casting the ExtensionArray to an ndarray of Python
210 objects with ``self.astype(object)``, which may be expensive. When
211 performance is a concern, we highly recommend overriding the following
212 methods:
214 * fillna
215 * _pad_or_backfill
216 * dropna
217 * unique
218 * factorize / _values_for_factorize
219 * argsort, argmax, argmin / _values_for_argsort
220 * searchsorted
221 * map
223 The remaining methods implemented on this class should be performant,
224 as they only compose abstract methods. Still, a more efficient
225 implementation may be available, and these methods can be overridden.
227 One can implement methods to handle array accumulations or reductions.
229 * _accumulate
230 * _reduce
232 One can implement methods to handle parsing from strings that will be used
233 in methods such as ``pandas.io.parsers.read_csv``.
235 * _from_sequence_of_strings
237 This class does not inherit from 'abc.ABCMeta' for performance reasons.
238 Methods and properties required by the interface raise
239 ``pandas.errors.AbstractMethodError`` and no ``register`` method is
240 provided for registering virtual subclasses.
242 ExtensionArrays are limited to 1 dimension.
244 They may be backed by none, one, or many NumPy arrays. For example,
245 ``pandas.Categorical`` is an extension array backed by two arrays,
246 one for codes and one for categories. An array of IPv6 address may
247 be backed by a NumPy structured array with two fields, one for the
248 lower 64 bits and one for the upper 64 bits. Or they may be backed
249 by some other storage type, like Python lists. Pandas makes no
250 assumptions on how the data are stored, just that it can be converted
251 to a NumPy array.
252 The ExtensionArray interface does not impose any rules on how this data
253 is stored. However, currently, the backing data cannot be stored in
254 attributes called ``.values`` or ``._values`` to ensure full compatibility
255 with pandas internals. But other names as ``.data``, ``._data``,
256 ``._items``, ... can be freely used.
258 If implementing NumPy's ``__array_ufunc__`` interface, pandas expects
259 that
261 1. You defer by returning ``NotImplemented`` when any Series are present
262 in `inputs`. Pandas will extract the arrays and call the ufunc again.
263 2. You define a ``_HANDLED_TYPES`` tuple as an attribute on the class.
264 Pandas inspect this to determine whether the ufunc is valid for the
265 types present.
267 See :ref:`extending.extension.ufunc` for more.
269 By default, ExtensionArrays are not hashable. Immutable subclasses may
270 override this behavior.
272 Examples
273 --------
274 Please see the following:
276 https://github.com/pandas-dev/pandas/blob/main/pandas/tests/extension/list/array.py
277 """
279 # '_typ' is for pandas.core.dtypes.generic.ABCExtensionArray.
280 # Don't override this.
281 _typ = "extension"
283 # similar to __array_priority__, positions ExtensionArray after Index,
284 # Series, and DataFrame. EA subclasses may override to choose which EA
285 # subclass takes priority. If overriding, the value should always be
286 # strictly less than 2000 to be below Index.__pandas_priority__.
287 __pandas_priority__ = 1000
289 _readonly = False
291 # ------------------------------------------------------------------------
292 # Constructors
293 # ------------------------------------------------------------------------
295 @classmethod
296 def _from_sequence(
297 cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
298 ) -> Self:
299 """
300 Construct a new ExtensionArray from a sequence of scalars.
302 This is a required method for ExtensionArray subclasses. It is used
303 to create a new instance of the ExtensionArray from Python scalars
304 or objects that can be converted to the array's scalar type.
306 Parameters
307 ----------
308 scalars : Sequence
309 Each element will be an instance of the scalar type for this
310 array, ``cls.dtype.type`` or be converted into this type in this method.
311 dtype : dtype, optional
312 Construct for this particular dtype. This should be a Dtype
313 compatible with the ExtensionArray.
314 copy : bool, default False
315 If True, copy the underlying data.
317 Returns
318 -------
319 ExtensionArray
321 See Also
322 --------
323 api.extensions.ExtensionArray._from_sequence_of_strings : Construct a new
324 ExtensionArray from a sequence of strings.
325 api.extensions.ExtensionArray._hash_pandas_object : Hook for
326 hash_pandas_object.
328 Examples
329 --------
330 >>> pd.arrays.IntegerArray._from_sequence([4, 5])
331 <IntegerArray>
332 [4, 5]
333 Length: 2, dtype: Int64
334 """
335 raise AbstractMethodError(cls)
337 @classmethod
338 def _from_sequence_of_strings(
339 cls, strings, *, dtype: ExtensionDtype, copy: bool = False
340 ) -> Self:
341 """
342 Construct a new ExtensionArray from a sequence of strings.
344 This method is used to parse string data into the appropriate
345 scalar type for the ExtensionArray. It is commonly used when
346 reading data from text files via parsers like ``read_csv``.
348 Parameters
349 ----------
350 strings : Sequence
351 Each element will be an instance of the scalar type for this
352 array, ``cls.dtype.type``.
353 dtype : ExtensionDtype
354 Construct for this particular dtype. This should be a Dtype
355 compatible with the ExtensionArray.
356 copy : bool, default False
357 If True, copy the underlying data.
359 Returns
360 -------
361 ExtensionArray
363 See Also
364 --------
365 api.extensions.ExtensionArray._from_sequence : Construct a new ExtensionArray
366 from a sequence of scalars.
367 api.extensions.ExtensionArray._from_factorized : Reconstruct an ExtensionArray
368 after factorization.
370 Examples
371 --------
372 >>> pd.arrays.IntegerArray._from_sequence_of_strings(
373 ... ["1", "2", "3"], dtype=pd.Int64Dtype()
374 ... )
375 <IntegerArray>
376 [1, 2, 3]
377 Length: 3, dtype: Int64
378 """
379 raise AbstractMethodError(cls)
381 @classmethod
382 def _from_factorized(cls, values, original):
383 """
384 Reconstruct an ExtensionArray after factorization.
386 This method reverses the encoding applied by :meth:`factorize`,
387 recreating an ExtensionArray from the unique values and original
388 array metadata.
390 Parameters
391 ----------
392 values : ndarray
393 An integer ndarray with the factorized values.
394 original : ExtensionArray
395 The original ExtensionArray that factorize was called on.
397 See Also
398 --------
399 factorize : Top-level factorize method that dispatches here.
400 ExtensionArray.factorize : Encode the extension array as an enumerated type.
402 Examples
403 --------
404 >>> interv_arr = pd.arrays.IntervalArray(
405 ... [pd.Interval(0, 1), pd.Interval(1, 5), pd.Interval(1, 5)]
406 ... )
407 >>> codes, uniques = pd.factorize(interv_arr)
408 >>> pd.arrays.IntervalArray._from_factorized(uniques, interv_arr)
409 <IntervalArray>
410 [(0, 1], (1, 5]]
411 Length: 2, dtype: interval[int64, right]
412 """
413 raise AbstractMethodError(cls)
415 @classmethod
416 def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
417 """
418 Strict analogue to _from_sequence, allowing only sequences of scalars
419 that should be specifically inferred to the given dtype.
421 Parameters
422 ----------
423 scalars : sequence
424 dtype : ExtensionDtype
426 Raises
427 ------
428 TypeError or ValueError
430 Notes
431 -----
432 This is called in a try/except block when casting the result of a
433 pointwise operation in ExtensionArray._cast_pointwise_result.
434 """
435 try:
436 return cls._from_sequence(scalars, dtype=dtype, copy=False)
437 except (ValueError, TypeError):
438 raise
439 except Exception:
440 warnings.warn(
441 "_from_scalars should only raise ValueError or TypeError. "
442 "Consider overriding _from_scalars where appropriate.",
443 stacklevel=find_stack_level(),
444 )
445 raise
447 def _cast_pointwise_result(self, values) -> ArrayLike:
448 """
449 Construct an ExtensionArray after a pointwise operation.
451 Cast the result of a pointwise operation (e.g. Series.map) to an
452 array. This is not required to return an ExtensionArray of the same
453 type as self or of the same dtype. It can also return another
454 ExtensionArray of the same "family" if you implement multiple
455 ExtensionArrays/Dtypes that are interoperable (e.g. if you have float
456 array with units, this method can return an int array with units).
458 If converting to your own ExtensionArray is not possible, this method
459 falls back to returning an array with the default type inference.
460 If you only need to cast to `self.dtype`, it is recommended to override
461 `_from_scalars` instead of this method.
463 Parameters
464 ----------
465 values : sequence
467 Returns
468 -------
469 ExtensionArray or ndarray
470 """
471 try:
472 return type(self)._from_scalars(values, dtype=self.dtype)
473 except (ValueError, TypeError):
474 if not (isinstance(values, np.ndarray) and values.dtype == object):
475 values = construct_1d_object_array_from_listlike(values)
476 return lib.maybe_convert_objects(values, convert_non_numeric=True)
478 # ------------------------------------------------------------------------
479 # Must be a Sequence
480 # ------------------------------------------------------------------------
481 @overload
482 def __getitem__(self, item: ScalarIndexer) -> Any: ...
484 @overload
485 def __getitem__(self, item: SequenceIndexer) -> Self: ...
487 def __getitem__(self, item: PositionalIndexer) -> Self | Any:
488 """
489 Select a subset of self.
491 Parameters
492 ----------
493 item : int, slice, or ndarray
494 * int: The position in 'self' to get.
496 * slice: A slice object, where 'start', 'stop', and 'step' are
497 integers or None
499 * ndarray: A 1-d boolean NumPy ndarray the same length as 'self'
501 * list[int]: A list of int
503 Returns
504 -------
505 item : scalar or ExtensionArray
507 Notes
508 -----
509 For scalar ``item``, return a scalar value suitable for the array's
510 type. This should be an instance of ``self.dtype.type``.
512 For slice ``key``, return an instance of ``ExtensionArray``, even
513 if the slice is length 0 or 1.
515 For a boolean mask, return an instance of ``ExtensionArray``, filtered
516 to the values where ``item`` is True.
517 """
518 raise AbstractMethodError(self)
520 def __setitem__(self, key, value) -> None:
521 """
522 Set one or more values inplace.
524 This method is not required to satisfy the pandas extension array
525 interface.
527 Parameters
528 ----------
529 key : int, ndarray, or slice
530 When called from, e.g. ``Series.__setitem__``, ``key`` will be
531 one of
533 * scalar int
534 * ndarray of integers.
535 * boolean ndarray
536 * slice object
538 value : ExtensionDtype.type, Sequence[ExtensionDtype.type], or object
539 value or values to be set of ``key``.
541 Returns
542 -------
543 None
545 Raises
546 ------
547 ValueError
548 If the array is readonly and modification is attempted.
549 """
550 # Some notes to the ExtensionArray implementer who may have ended up
551 # here. While this method is not required for the interface, if you
552 # *do* choose to implement __setitem__, then some semantics should be
553 # observed:
554 #
555 # * Setting multiple values : ExtensionArrays should support setting
556 # multiple values at once, 'key' will be a sequence of integers and
557 # 'value' will be a same-length sequence.
558 #
559 # * Broadcasting : For a sequence 'key' and a scalar 'value',
560 # each position in 'key' should be set to 'value'.
561 #
562 # * Coercion : Most users will expect basic coercion to work. For
563 # example, a string like '2018-01-01' is coerced to a datetime
564 # when setting on a datetime64ns array. In general, if the
565 # __init__ method coerces that value, then so should __setitem__
566 # Note, also, that Series/DataFrame.where internally use __setitem__
567 # on a copy of the data.
568 # Check if the array is readonly
569 if self._readonly:
570 raise ValueError("Cannot modify read-only array")
572 raise NotImplementedError(f"{type(self)} does not implement __setitem__.")
574 def __len__(self) -> int:
575 """
576 Length of this array
578 Returns
579 -------
580 length : int
581 """
582 raise AbstractMethodError(self)
584 def __iter__(self) -> Iterator[Any]:
585 """
586 Iterate over elements of the array.
587 """
588 # This needs to be implemented so that pandas recognizes extension
589 # arrays as list-like. The default implementation makes successive
590 # calls to ``__getitem__``, which may be slower than necessary.
591 for i in range(len(self)):
592 yield self[i]
594 def __contains__(self, item: object) -> bool | np.bool_:
595 """
596 Return for `item in self`.
597 """
598 # GH37867
599 # comparisons of any item to pd.NA always return pd.NA, so e.g. "a" in [pd.NA]
600 # would raise a TypeError. The implementation below works around that.
601 if is_scalar(item) and isna(item):
602 if not self._can_hold_na:
603 return False
604 elif item is self.dtype.na_value or isinstance(item, self.dtype.type):
605 return self._hasna
606 else:
607 return False
608 else:
609 # error: Item "ExtensionArray" of "Union[ExtensionArray, ndarray]" has no
610 # attribute "any"
611 return (item == self).any() # type: ignore[union-attr]
613 # error: Signature of "__eq__" incompatible with supertype "object"
614 def __eq__(self, other: object) -> ArrayLike: # type: ignore[override]
615 """
616 Return for `self == other` (element-wise equality).
617 """
618 # Implementer note: this should return a boolean numpy ndarray or
619 # a boolean ExtensionArray.
620 # When `other` is one of Series, Index, or DataFrame, this method should
621 # return NotImplemented (to ensure that those objects are responsible for
622 # first unpacking the arrays, and then dispatch the operation to the
623 # underlying arrays)
624 raise AbstractMethodError(self)
626 # error: Signature of "__ne__" incompatible with supertype "object"
627 def __ne__(self, other: object) -> ArrayLike: # type: ignore[override]
628 """
629 Return for `self != other` (element-wise in-equality).
630 """
631 # error: Unsupported operand type for ~ ("ExtensionArray")
632 return ~(self == other) # type: ignore[operator]
634 def item(self, index: int | None = None):
635 """
636 Return the array element at the specified position as a Python scalar.
638 Analogous to :meth:`numpy.ndarray.item`, this converts a single
639 element to a native Python object.
641 Parameters
642 ----------
643 index : int, optional
644 Position of the element. If not provided, the array must contain
645 exactly one element.
647 Returns
648 -------
649 scalar
650 The element at the specified position.
652 Raises
653 ------
654 ValueError
655 If no index is provided and the array does not have exactly
656 one element.
657 IndexError
658 If the specified position is out of bounds.
660 See Also
661 --------
662 numpy.ndarray.item : Return the item of an array as a scalar.
664 Examples
665 --------
666 >>> arr = pd.array([1], dtype="Int64")
667 >>> arr.item()
668 np.int64(1)
670 >>> arr = pd.array([1, 2, 3], dtype="Int64")
671 >>> arr.item(0)
672 np.int64(1)
673 >>> arr.item(2)
674 np.int64(3)
675 """
676 if index is None:
677 if len(self) != 1:
678 raise ValueError(
679 "can only convert an array of size 1 to a Python scalar"
680 )
681 return self[0]
682 else:
683 if not is_integer(index):
684 raise TypeError(f"index must be an integer, got {type(index)}")
685 return self[index]
687 def to_numpy(
688 self,
689 dtype: npt.DTypeLike | None = None,
690 copy: bool = False,
691 na_value: object = lib.no_default,
692 ) -> np.ndarray:
693 """
694 Convert to a NumPy ndarray.
696 This is similar to :meth:`numpy.asarray`, but may provide additional control
697 over how the conversion is done.
699 Parameters
700 ----------
701 dtype : str or numpy.dtype, optional
702 The dtype to pass to :meth:`numpy.asarray`.
703 copy : bool, default False
704 Whether to ensure that the returned value is a not a view on
705 another array. Note that ``copy=False`` does not *ensure* that
706 ``to_numpy()`` is no-copy. Rather, ``copy=True`` ensure that
707 a copy is made, even if not strictly necessary.
708 na_value : Any, optional
709 The value to use for missing values. The default value depends
710 on `dtype` and the type of the array.
712 Returns
713 -------
714 numpy.ndarray
715 """
716 result = np.asarray(self, dtype=dtype)
717 if copy or na_value is not lib.no_default:
718 result = result.copy()
719 elif self._readonly and astype_is_view(self.dtype, result.dtype):
720 # If the ExtensionArray is readonly, make the numpy array readonly too
721 result = result.view()
722 result.flags.writeable = False
724 if na_value is not lib.no_default:
725 result[self.isna()] = na_value # type: ignore[index]
727 return result
729 # ------------------------------------------------------------------------
730 # Required attributes
731 # ------------------------------------------------------------------------
733 @property
734 def dtype(self) -> ExtensionDtype:
735 """
736 An instance of ExtensionDtype.
738 This property returns the dtype object associated with this
739 ExtensionArray. The dtype describes the type of data stored
740 in the array.
742 See Also
743 --------
744 api.extensions.ExtensionDtype : Base class for extension dtypes.
745 api.extensions.ExtensionArray : Base class for extension array types.
746 api.extensions.ExtensionArray.dtype : The dtype of an ExtensionArray.
747 Series.dtype : The dtype of a Series.
748 DataFrame.dtype : The dtype of a DataFrame.
750 Examples
751 --------
752 >>> pd.array([1, 2, 3]).dtype
753 Int64Dtype()
754 """
755 raise AbstractMethodError(self)
757 @property
758 def shape(self) -> Shape:
759 """
760 Return a tuple of the array dimensions.
762 For ExtensionArrays, this is always a 1-tuple containing the
763 length of the array, since ExtensionArrays are 1-dimensional.
765 See Also
766 --------
767 numpy.ndarray.shape : Similar attribute which returns the shape of an array.
768 DataFrame.shape : Return a tuple representing the dimensionality of the
769 DataFrame.
770 Series.shape : Return a tuple representing the dimensionality of the Series.
772 Examples
773 --------
774 >>> arr = pd.array([1, 2, 3])
775 >>> arr.shape
776 (3,)
777 """
778 return (len(self),)
780 @property
781 def size(self) -> int:
782 """
783 The number of elements in the array.
784 """
785 # error: Incompatible return value type (got "signedinteger[_64Bit]",
786 # expected "int") [return-value]
787 return np.prod(self.shape) # type: ignore[return-value]
789 @property
790 def ndim(self) -> int:
791 """
792 Extension Arrays are only allowed to be 1-dimensional.
794 This property always returns 1 for ExtensionArrays, as they
795 are constrained to a single dimension by design.
797 See Also
798 --------
799 ExtensionArray.shape: Return a tuple of the array dimensions.
800 ExtensionArray.size: The number of elements in the array.
802 Examples
803 --------
804 >>> arr = pd.array([1, 2, 3])
805 >>> arr.ndim
806 1
807 """
808 return 1
810 @property
811 def nbytes(self) -> int:
812 """
813 The number of bytes needed to store this object in memory.
815 If computing this is expensive, implementations may return an
816 approximate lower bound on the number of bytes needed.
818 See Also
819 --------
820 ExtensionArray.shape: Return a tuple of the array dimensions.
821 ExtensionArray.size: The number of elements in the array.
823 Examples
824 --------
825 >>> pd.array([1, 2, 3]).nbytes
826 27
827 """
828 # If this is expensive to compute, return an approximate lower bound
829 # on the number of bytes needed.
830 raise AbstractMethodError(self)
832 # ------------------------------------------------------------------------
833 # Additional Methods
834 # ------------------------------------------------------------------------
836 @overload
837 def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
839 @overload
840 def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
842 @overload
843 def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
845 def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
846 """
847 Cast to a NumPy array or ExtensionArray with 'dtype'.
849 If the target dtype is an ExtensionDtype, the result is an
850 ExtensionArray. Otherwise, the result is a NumPy ndarray.
852 Parameters
853 ----------
854 dtype : str or dtype
855 Typecode or data-type to which the array is cast.
856 copy : bool, default True
857 Whether to copy the data, even if not necessary. If False,
858 a copy is made only if the old dtype does not match the
859 new dtype.
861 Returns
862 -------
863 np.ndarray or pandas.api.extensions.ExtensionArray
864 An ``ExtensionArray`` if ``dtype`` is ``ExtensionDtype``,
865 otherwise a Numpy ndarray with ``dtype`` for its dtype.
867 See Also
868 --------
869 Series.astype : Cast a Series to a different dtype.
870 DataFrame.astype : Cast a DataFrame to a different dtype.
871 api.extensions.ExtensionArray : Base class for ExtensionArray objects.
872 core.arrays.DatetimeArray._from_sequence : Create a DatetimeArray from a
873 sequence.
874 core.arrays.TimedeltaArray._from_sequence : Create a TimedeltaArray from
875 a sequence.
877 Examples
878 --------
879 >>> arr = pd.array([1, 2, 3])
880 >>> arr
881 <IntegerArray>
882 [1, 2, 3]
883 Length: 3, dtype: Int64
885 Casting to another ``ExtensionDtype`` returns an ``ExtensionArray``:
887 >>> arr1 = arr.astype("Float64")
888 >>> arr1
889 <FloatingArray>
890 [1.0, 2.0, 3.0]
891 Length: 3, dtype: Float64
892 >>> arr1.dtype
893 Float64Dtype()
895 Otherwise, we will get a Numpy ndarray:
897 >>> arr2 = arr.astype("float64")
898 >>> arr2
899 array([1., 2., 3.])
900 >>> arr2.dtype
901 dtype('float64')
902 """
903 dtype = pandas_dtype(dtype)
904 if dtype == self.dtype:
905 if not copy:
906 return self
907 else:
908 return self.copy()
910 if isinstance(dtype, ExtensionDtype):
911 cls = dtype.construct_array_type()
912 return cls._from_sequence(self, dtype=dtype, copy=copy)
914 elif lib.is_np_dtype(dtype, "M"):
915 from pandas.core.arrays import DatetimeArray
917 return DatetimeArray._from_sequence(self, dtype=dtype, copy=copy)
919 elif lib.is_np_dtype(dtype, "m"):
920 from pandas.core.arrays import TimedeltaArray
922 return TimedeltaArray._from_sequence(self, dtype=dtype, copy=copy)
924 if not copy:
925 return np.asarray(self, dtype=dtype)
926 else:
927 return np.array(self, dtype=dtype, copy=copy)
929 def isna(self) -> np.ndarray | ExtensionArrayNaResult:
930 """
931 A 1-D array indicating if each value is missing.
933 This method returns a boolean array where ``True`` indicates that
934 the corresponding value is missing (NA/NaN).
936 Returns
937 -------
938 numpy.ndarray or pandas.api.extensions.ExtensionArray
939 In most cases, this should return a NumPy ndarray. For
940 exceptional cases like ``SparseArray``, where returning
941 an ndarray would be expensive, an ExtensionArray may be
942 returned.
944 See Also
945 --------
946 ExtensionArray.dropna: Return ExtensionArray without NA values.
947 ExtensionArray.fillna: Fill NA/NaN values using the specified method.
949 Notes
950 -----
951 If returning an ExtensionArray, then
953 * ``na_values._is_boolean`` should be True
954 * ``na_values`` should implement :func:`ExtensionArray._reduce`
955 * ``na_values`` should implement :func:`ExtensionArray._accumulate`
956 * ``na_values.any``, ``na_values.all``, ``na_values.sum``, and
957 ``na_values.__invert__`` should be implemented
959 Examples
960 --------
961 >>> arr = pd.array([1, 2, np.nan, np.nan])
962 >>> arr.isna()
963 array([False, False, True, True])
964 """
965 raise AbstractMethodError(self)
967 @property
968 def _hasna(self) -> bool:
969 # GH#22680
970 """
971 Equivalent to `self.isna().any()`.
973 Some ExtensionArray subclasses may be able to optimize this check.
974 """
975 return bool(self.isna().any())
977 def _values_for_argsort(self) -> np.ndarray:
978 """
979 Return values for sorting.
981 This method provides the underlying values that should be used
982 when sorting the ExtensionArray. The returned values should
983 maintain the same relative ordering as the original array.
985 Returns
986 -------
987 ndarray
988 The transformed values should maintain the ordering between values
989 within the array.
991 See Also
992 --------
993 ExtensionArray.argsort : Return the indices that would sort this array.
995 Notes
996 -----
997 The caller is responsible for *not* modifying these values in-place, so
998 it is safe for implementers to give views on ``self``.
1000 Functions that use this (e.g. ``ExtensionArray.argsort``) should ignore
1001 entries with missing values in the original array (according to
1002 ``self.isna()``). This means that the corresponding entries in the returned
1003 array don't need to be modified to sort correctly.
1005 Examples
1006 --------
1007 In most cases, this is the underlying Numpy array of the ``ExtensionArray``:
1009 >>> arr = pd.array([1, 2, 3])
1010 >>> arr._values_for_argsort()
1011 array([1, 2, 3])
1012 """
1013 # Note: this is used in `ExtensionArray.argsort/argmin/argmax`.
1014 return np.array(self)
1016 def argsort(
1017 self,
1018 *,
1019 ascending: bool = True,
1020 kind: SortKind = "quicksort",
1021 na_position: str = "last",
1022 **kwargs,
1023 ) -> np.ndarray:
1024 """
1025 Return the indices that would sort this array.
1027 This method computes the indices that would produce a sorted
1028 version of the array when used with indexing.
1030 Parameters
1031 ----------
1032 ascending : bool, default True
1033 Whether the indices should result in an ascending
1034 or descending sort.
1035 kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional
1036 Sorting algorithm.
1037 na_position : {'first', 'last'}, default 'last'
1038 If ``'first'``, put ``NaN`` values at the beginning.
1039 If ``'last'``, put ``NaN`` values at the end.
1040 **kwargs
1041 Passed through to :func:`numpy.argsort`.
1043 Returns
1044 -------
1045 np.ndarray[np.intp]
1046 Array of indices that sort ``self``. If NaN values are contained,
1047 NaN values are placed at the end.
1049 See Also
1050 --------
1051 numpy.argsort : Sorting implementation used internally.
1053 Examples
1054 --------
1055 >>> arr = pd.array([3, 1, 2, 5, 4])
1056 >>> arr.argsort()
1057 array([1, 2, 0, 4, 3])
1058 """
1059 # Implementer note: You have two places to override the behavior of
1060 # argsort.
1061 # 1. _values_for_argsort : construct the values passed to np.argsort
1062 # 2. argsort : total control over sorting. In case of overriding this,
1063 # it is recommended to also override argmax/argmin
1064 ascending = nv.validate_argsort_with_ascending(ascending, (), kwargs)
1066 values = self._values_for_argsort()
1067 return nargsort(
1068 values,
1069 kind=kind,
1070 ascending=ascending,
1071 na_position=na_position,
1072 mask=np.asarray(self.isna()),
1073 )
1075 def sort(
1076 self,
1077 *,
1078 ascending: bool = True,
1079 kind: SortKind = "quicksort",
1080 na_position: str = "last",
1081 ) -> None:
1082 """
1083 Sort the array in-place.
1085 Reorders the elements of the array using :meth:`argsort` and writes the
1086 sorted result back through ``self[:]``. Subclasses backed by immutable
1087 storage may override this method to raise ``NotImplementedError``.
1089 Parameters
1090 ----------
1091 ascending : bool, default True
1092 Whether to sort in ascending order.
1093 kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, default 'quicksort'
1094 Sorting algorithm.
1095 na_position : {'first', 'last'}, default 'last'
1096 If 'first', put NaN values at the beginning.
1097 If 'last', put NaN values at the end.
1099 Returns
1100 -------
1101 None
1102 This method mutates the array in place and does not return a value.
1104 See Also
1105 --------
1106 ExtensionArray.argsort : Return the indices that would sort this array.
1108 Examples
1109 --------
1110 >>> arr = pd.array([3, 1, 2, 5, 4])
1111 >>> arr.sort()
1112 >>> arr
1113 <IntegerArray>
1114 [1, 2, 3, 4, 5]
1115 Length: 5, dtype: Int64
1116 """
1117 sort_indices = self.argsort(
1118 ascending=ascending, kind=kind, na_position=na_position
1119 )
1120 self[:] = self.take(sort_indices)
1122 def argmin(self, skipna: bool = True) -> int:
1123 """
1124 Return the index of minimum value.
1126 In case of multiple occurrences of the minimum value, the index
1127 corresponding to the first occurrence is returned.
1129 Parameters
1130 ----------
1131 skipna : bool, default True
1133 Returns
1134 -------
1135 int
1137 See Also
1138 --------
1139 ExtensionArray.argmax : Return the index of the maximum value.
1141 Examples
1142 --------
1143 >>> arr = pd.array([3, 1, 2, 5, 4])
1144 >>> arr.argmin()
1145 np.int64(1)
1146 """
1147 # Implementer note: You have two places to override the behavior of
1148 # argmin.
1149 # 1. _values_for_argsort : construct the values used in nargminmax
1150 # 2. argmin itself : total control over sorting.
1151 validate_bool_kwarg(skipna, "skipna")
1152 if not skipna and self._hasna:
1153 raise ValueError("Encountered an NA value with skipna=False")
1154 return cast("int", nargminmax(self, "argmin"))
1156 def argmax(self, skipna: bool = True) -> int:
1157 """
1158 Return the index of maximum value.
1160 In case of multiple occurrences of the maximum value, the index
1161 corresponding to the first occurrence is returned.
1163 Parameters
1164 ----------
1165 skipna : bool, default True
1167 Returns
1168 -------
1169 int
1171 See Also
1172 --------
1173 ExtensionArray.argmin : Return the index of the minimum value.
1175 Examples
1176 --------
1177 >>> arr = pd.array([3, 1, 2, 5, 4])
1178 >>> arr.argmax()
1179 np.int64(3)
1180 """
1181 # Implementer note: You have two places to override the behavior of
1182 # argmax.
1183 # 1. _values_for_argsort : construct the values used in nargminmax
1184 # 2. argmax itself : total control over sorting.
1185 validate_bool_kwarg(skipna, "skipna")
1186 if not skipna and self._hasna:
1187 raise ValueError("Encountered an NA value with skipna=False")
1188 return cast("int", nargminmax(self, "argmax"))
1190 def interpolate(
1191 self,
1192 *,
1193 method: InterpolateOptions,
1194 axis: int,
1195 index: Index,
1196 limit,
1197 limit_direction,
1198 limit_area,
1199 copy: bool,
1200 **kwargs,
1201 ) -> Self:
1202 """
1203 Fill NaN values using an interpolation method.
1205 This method fills missing values in the array by interpolating
1206 between existing values using the specified interpolation technique.
1208 Parameters
1209 ----------
1210 method : str, default 'linear'
1211 Interpolation technique to use. One of:
1212 * 'linear': Ignore the index and treat the values as equally spaced.
1213 This is the only method supported on MultiIndexes.
1214 * 'time': Works on daily and higher resolution data to interpolate
1215 given length of interval.
1216 * 'index', 'values': use the actual numerical values of the index.
1217 * 'nearest', 'zero', 'slinear', 'quadratic', 'cubic', 'barycentric',
1218 'polynomial': Passed to scipy.interpolate.interp1d, whereas 'spline'
1219 is passed to scipy.interpolate.UnivariateSpline. These methods use
1220 the numerical values of the index.
1221 Both 'polynomial' and 'spline' require that you also specify an
1222 order (int), e.g. arr.interpolate(method='polynomial', order=5).
1223 * 'krogh', 'piecewise_polynomial', 'spline', 'pchip', 'akima',
1224 'cubicspline': Wrappers around the SciPy interpolation methods
1225 of similar names. See Notes.
1226 * 'from_derivatives': Refers to scipy.interpolate.BPoly.from_derivatives.
1227 axis : int
1228 Axis to interpolate along. For 1-dimensional data, use 0.
1229 index : Index
1230 Index to use for interpolation.
1231 limit : int or None
1232 Maximum number of consecutive NaNs to fill. Must be greater than 0.
1233 limit_direction : {'forward', 'backward', 'both'}
1234 Consecutive NaNs will be filled in this direction.
1235 limit_area : {'inside', 'outside'} or None
1236 If limit is specified, consecutive NaNs will be filled with this
1237 restriction.
1238 * None: No fill restriction.
1239 * 'inside': Only fill NaNs surrounded by valid values (interpolate).
1240 * 'outside': Only fill NaNs outside valid values (extrapolate).
1241 copy : bool
1242 If True, a copy of the object is returned with interpolated values.
1243 **kwargs : optional
1244 Keyword arguments to pass on to the interpolating function.
1246 Returns
1247 -------
1248 ExtensionArray
1249 An ExtensionArray with interpolated values.
1251 See Also
1252 --------
1253 Series.interpolate : Interpolate values in a Series.
1254 DataFrame.interpolate : Interpolate values in a DataFrame.
1256 Notes
1257 -----
1258 - All parameters must be specified as keyword arguments.
1259 - The 'krogh', 'piecewise_polynomial', 'spline', 'pchip' and 'akima'
1260 methods are wrappers around the respective SciPy implementations of
1261 similar names. These use the actual numerical values of the index.
1263 Examples
1264 --------
1265 Interpolating values in a NumPy array:
1267 >>> arr = pd.arrays.NumpyExtensionArray(np.array([0, 1, np.nan, 3]))
1268 >>> arr.interpolate(
1269 ... method="linear",
1270 ... limit=3,
1271 ... limit_direction="forward",
1272 ... index=pd.Index(range(len(arr))),
1273 ... fill_value=1,
1274 ... copy=False,
1275 ... axis=0,
1276 ... limit_area="inside",
1277 ... )
1278 <NumpyExtensionArray>
1279 [0.0, 1.0, 2.0, 3.0]
1280 Length: 4, dtype: float64
1282 Interpolating values in a FloatingArray:
1284 >>> arr = pd.array([1.0, pd.NA, 3.0, 4.0, pd.NA, 6.0], dtype="Float64")
1285 >>> arr.interpolate(
1286 ... method="linear",
1287 ... axis=0,
1288 ... index=pd.Index(range(len(arr))),
1289 ... limit=None,
1290 ... limit_direction="both",
1291 ... limit_area=None,
1292 ... copy=True,
1293 ... )
1294 <FloatingArray>
1295 [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
1296 Length: 6, dtype: Float64
1297 """
1298 # NB: we return type(self) even if copy=False
1299 raise NotImplementedError(
1300 f"{type(self).__name__} does not implement interpolate"
1301 )
1303 def _pad_or_backfill(
1304 self,
1305 *,
1306 method: FillnaOptions,
1307 limit: int | None = None,
1308 limit_area: Literal["inside", "outside"] | None = None,
1309 copy: bool = True,
1310 ) -> Self:
1311 """
1312 Pad or backfill values, used by Series/DataFrame ffill and bfill.
1314 This method propagates the last valid observation forward (pad/ffill)
1315 or the next valid observation backward (backfill/bfill) to fill NaN
1316 values.
1318 Parameters
1319 ----------
1320 method : {'backfill', 'bfill', 'pad', 'ffill'}
1321 Method to use for filling holes in reindexed Series:
1323 * pad / ffill: propagate last valid observation forward to next valid.
1324 * backfill / bfill: use NEXT valid observation to fill gap.
1326 limit : int, default None
1327 This is the maximum number of consecutive
1328 NaN values to forward/backward fill. In other words, if there is
1329 a gap with more than this number of consecutive NaNs, it will only
1330 be partially filled. If method is not specified, this is the
1331 maximum number of entries along the entire axis where NaNs will be
1332 filled.
1334 limit_area : {'inside', 'outside'} or None, default None
1335 Specifies which area to limit filling.
1336 - 'inside': Limit the filling to the area within the gaps.
1337 - 'outside': Limit the filling to the area outside the gaps.
1338 If `None`, no limitation is applied.
1340 copy : bool, default True
1341 Whether to make a copy of the data before filling. If False, then
1342 the original should be modified and no new memory should be allocated.
1343 For ExtensionArray subclasses that cannot do this, it is at the
1344 author's discretion whether to ignore "copy=False" or to raise.
1345 The base class implementation ignores the keyword if any NAs are
1346 present.
1348 Returns
1349 -------
1350 Same type as self
1351 The filled array with the same type as the original.
1353 See Also
1354 --------
1355 Series.ffill : Forward fill missing values.
1356 Series.bfill : Backward fill missing values.
1357 DataFrame.ffill : Forward fill missing values in DataFrame.
1358 DataFrame.bfill : Backward fill missing values in DataFrame.
1359 api.types.isna : Check for missing values.
1360 api.types.isnull : Check for missing values.
1362 Examples
1363 --------
1364 >>> arr = pd.array([np.nan, np.nan, 2, 3, np.nan, np.nan])
1365 >>> arr._pad_or_backfill(method="backfill", limit=1)
1366 <IntegerArray>
1367 [<NA>, 2, 2, 3, <NA>, <NA>]
1368 Length: 6, dtype: Int64
1369 """
1370 mask = self.isna()
1372 if mask.any():
1373 # NB: the base class does not respect the "copy" keyword
1374 meth = missing.clean_fill_method(method)
1376 npmask = np.asarray(mask)
1377 if limit_area is not None and not npmask.all():
1378 _fill_limit_area_1d(npmask, limit_area)
1379 if meth == "pad":
1380 indexer = libalgos.get_fill_indexer(npmask, limit=limit)
1381 return self.take(indexer, allow_fill=True)
1382 else:
1383 # i.e. meth == "backfill"
1384 indexer = libalgos.get_fill_indexer(npmask[::-1], limit=limit)[::-1]
1385 return self[::-1].take(indexer, allow_fill=True)
1387 else:
1388 if not copy:
1389 return self
1390 new_values = self.copy()
1391 return new_values
1393 def fillna(
1394 self,
1395 value: object | ArrayLike,
1396 limit: int | None = None,
1397 copy: bool = True,
1398 ) -> Self:
1399 """
1400 Fill NA/NaN values using the specified method.
1402 This method replaces missing values in the array with a scalar value
1403 or corresponding values from an array-like object.
1405 Parameters
1406 ----------
1407 value : scalar, array-like
1408 If a scalar value is passed it is used to fill all missing values.
1409 Alternatively, an array-like "value" can be given. It's expected
1410 that the array-like have the same length as 'self'.
1411 limit : int, default None
1412 The maximum number of entries where NA values will be filled.
1413 copy : bool, default True
1414 Whether to make a copy of the data before filling. If False, then
1415 the original should be modified and no new memory should be allocated.
1416 For ExtensionArray subclasses that cannot do this, it is at the
1417 author's discretion whether to ignore "copy=False" or to raise.
1419 Returns
1420 -------
1421 ExtensionArray
1422 With NA/NaN filled.
1424 See Also
1425 --------
1426 api.extensions.ExtensionArray.dropna : Return ExtensionArray without
1427 NA values.
1428 api.extensions.ExtensionArray.isna : A 1-D array indicating if
1429 each value is missing.
1431 Examples
1432 --------
1433 >>> arr = pd.array([np.nan, np.nan, 2, 3, np.nan, np.nan])
1434 >>> arr.fillna(0)
1435 <IntegerArray>
1436 [0, 0, 2, 3, 0, 0]
1437 Length: 6, dtype: Int64
1438 """
1439 mask = self.isna()
1440 if limit is not None and limit < len(self):
1441 # isna can return an ExtensionArray, we're assuming that comparisons
1442 # are implemented.
1443 # mypy doesn't like that mask can be an EA which need not have `cumsum`
1444 modify = mask.cumsum() > limit # type: ignore[union-attr]
1445 if modify.any():
1446 # Only copy mask if necessary
1447 mask = mask.copy()
1448 mask[modify] = False
1449 # error: Argument 2 to "check_value_size" has incompatible type
1450 # "ExtensionArray"; expected "ndarray"
1451 value = missing.check_value_size(
1452 value,
1453 mask, # type: ignore[arg-type]
1454 len(self),
1455 )
1457 if mask.any():
1458 # fill with value
1459 if not copy:
1460 new_values = self[:]
1461 else:
1462 new_values = self.copy()
1463 new_values[mask] = value
1464 elif not copy:
1465 new_values = self[:]
1466 else:
1467 new_values = self.copy()
1468 return new_values
1470 def dropna(self) -> Self:
1471 """
1472 Return ExtensionArray without NA values.
1474 This method removes all missing values from the array, returning
1475 a new ExtensionArray containing only non-NA values.
1477 Returns
1478 -------
1479 Self
1480 An ExtensionArray of the same type as the original but with all
1481 NA values removed.
1483 See Also
1484 --------
1485 Series.dropna : Remove missing values from a Series.
1486 DataFrame.dropna : Remove missing values from a DataFrame.
1487 api.extensions.ExtensionArray.isna : Check for missing values in
1488 an ExtensionArray.
1490 Examples
1491 --------
1492 >>> pd.array([1, 2, np.nan]).dropna()
1493 <IntegerArray>
1494 [1, 2]
1495 Length: 2, dtype: Int64
1496 """
1497 # isna can return an ExtensionArrayNaResult
1498 return self[~self.isna()] # type: ignore[index]
1500 def duplicated(
1501 self, keep: Literal["first", "last", False] = "first"
1502 ) -> npt.NDArray[np.bool_]:
1503 """
1504 Return boolean ndarray denoting duplicate values.
1506 This method identifies duplicate values in the array. The `keep`
1507 parameter controls which occurrences are marked as duplicates.
1509 Parameters
1510 ----------
1511 keep : {'first', 'last', False}, default 'first'
1512 - ``first`` : Mark duplicates as ``True`` except for the first occurrence.
1513 - ``last`` : Mark duplicates as ``True`` except for the last occurrence.
1514 - False : Mark all duplicates as ``True``.
1516 Returns
1517 -------
1518 ndarray[bool]
1519 With true in indices where elements are duplicated and false otherwise.
1521 See Also
1522 --------
1523 DataFrame.duplicated : Return boolean Series denoting
1524 duplicate rows.
1525 Series.duplicated : Indicate duplicate Series values.
1526 api.extensions.ExtensionArray.unique : Compute the ExtensionArray
1527 of unique values.
1529 Examples
1530 --------
1531 >>> pd.array([1, 1, 2, 3, 3], dtype="Int64").duplicated()
1532 array([False, True, False, False, True])
1533 """
1534 mask = self.isna().astype(np.bool_, copy=False)
1535 return duplicated(values=self, keep=keep, mask=mask)
1537 def shift(self, periods: int = 1, fill_value: object = None) -> ExtensionArray:
1538 """
1539 Shift values by desired number.
1541 Newly introduced missing values are filled with
1542 ``self.dtype.na_value``.
1544 Parameters
1545 ----------
1546 periods : int, default 1
1547 The number of periods to shift. Negative values are allowed
1548 for shifting backwards.
1550 fill_value : object, optional
1551 The scalar value to use for newly introduced missing values.
1552 The default is ``self.dtype.na_value``.
1554 Returns
1555 -------
1556 ExtensionArray
1557 Shifted.
1559 See Also
1560 --------
1561 api.extensions.ExtensionArray.transpose : Return a transposed view on
1562 this array.
1563 api.extensions.ExtensionArray.factorize : Encode the extension array as an
1564 enumerated type.
1566 Notes
1567 -----
1568 If ``self`` is empty or ``periods`` is 0, a copy of ``self`` is
1569 returned.
1571 If ``periods > len(self)``, then an array of size
1572 len(self) is returned, with all values filled with
1573 ``self.dtype.na_value``.
1575 For 2-dimensional ExtensionArrays, we are always shifting along axis=0.
1577 Examples
1578 --------
1579 >>> arr = pd.array([1, 2, 3])
1580 >>> arr.shift(2)
1581 <IntegerArray>
1582 [<NA>, <NA>, 1]
1583 Length: 3, dtype: Int64
1584 """
1585 # Note: this implementation assumes that `self.dtype.na_value` can be
1586 # stored in an instance of your ExtensionArray with `self.dtype`.
1587 if not len(self) or periods == 0:
1588 return self.copy()
1590 if isna(fill_value):
1591 fill_value = self.dtype.na_value
1593 empty = self._from_sequence(
1594 [fill_value] * min(abs(periods), len(self)), dtype=self.dtype
1595 )
1596 if periods > 0:
1597 a = empty
1598 b = self[:-periods]
1599 else:
1600 a = self[abs(periods) :]
1601 b = empty
1602 return self._concat_same_type([a, b])
1604 def unique(self) -> Self:
1605 """
1606 Compute the ExtensionArray of unique values.
1608 This method returns a new ExtensionArray containing the distinct
1609 values from the original array, preserving the order of first
1610 appearance.
1612 Returns
1613 -------
1614 pandas.api.extensions.ExtensionArray
1615 With unique values from the input array.
1617 See Also
1618 --------
1619 Index.unique: Return unique values in the index.
1620 Series.unique: Return unique values of Series object.
1621 unique: Return unique values based on a hash table.
1623 Examples
1624 --------
1625 >>> arr = pd.array([1, 2, 3, 1, 2, 3])
1626 >>> arr.unique()
1627 <IntegerArray>
1628 [1, 2, 3]
1629 Length: 3, dtype: Int64
1630 """
1631 uniques = unique(self.astype(object))
1632 return self._from_sequence(uniques, dtype=self.dtype)
1634 def searchsorted(
1635 self,
1636 value: NumpyValueArrayLike | ExtensionArray,
1637 side: Literal["left", "right"] = "left",
1638 sorter: NumpySorter | None = None,
1639 ) -> npt.NDArray[np.intp] | np.intp:
1640 """
1641 Find indices where elements should be inserted to maintain order.
1643 Find the indices into a sorted array `self` (a) such that, if the
1644 corresponding elements in `value` were inserted before the indices,
1645 the order of `self` would be preserved.
1647 Assuming that `self` is sorted:
1649 ====== ================================
1650 `side` returned index `i` satisfies
1651 ====== ================================
1652 left ``self[i-1] < value <= self[i]``
1653 right ``self[i-1] <= value < self[i]``
1654 ====== ================================
1656 Parameters
1657 ----------
1658 value : array-like, list or scalar
1659 Value(s) to insert into `self`.
1660 side : {'left', 'right'}, optional
1661 If 'left', the index of the first suitable location found is given.
1662 If 'right', return the last such index. If there is no suitable
1663 index, return either 0 or N (where N is the length of `self`).
1664 sorter : 1-D array-like, optional
1665 Optional array of integer indices that sort array a into ascending
1666 order. They are typically the result of argsort.
1668 Returns
1669 -------
1670 array of ints or int
1671 If value is array-like, array of insertion points.
1672 If value is scalar, a single integer.
1674 See Also
1675 --------
1676 numpy.searchsorted : Similar method from NumPy.
1678 Examples
1679 --------
1680 >>> arr = pd.array([1, 2, 3, 5])
1681 >>> arr.searchsorted([4])
1682 array([3])
1683 """
1684 # Note: the base tests provided by pandas only test the basics.
1685 # We do not test
1686 # 1. Values outside the range of the `data_for_sorting` fixture
1687 # 2. Values between the values in the `data_for_sorting` fixture
1688 # 3. Missing values.
1689 arr = self.astype(object)
1690 if isinstance(value, ExtensionArray):
1691 value = value.astype(object)
1692 return arr.searchsorted(value, side=side, sorter=sorter)
1694 def equals(self, other: object) -> bool:
1695 """
1696 Return if another array is equivalent to this array.
1698 Equivalent means that both arrays have the same shape and dtype, and
1699 all values compare equal. Missing values in the same location are
1700 considered equal (in contrast with normal equality).
1702 Parameters
1703 ----------
1704 other : ExtensionArray
1705 Array to compare to this Array.
1707 Returns
1708 -------
1709 boolean
1710 Whether the arrays are equivalent.
1712 See Also
1713 --------
1714 numpy.array_equal : Equivalent method for numpy array.
1715 Series.equals : Equivalent method for Series.
1716 DataFrame.equals : Equivalent method for DataFrame.
1718 Examples
1719 --------
1720 >>> arr1 = pd.array([1, 2, np.nan])
1721 >>> arr2 = pd.array([1, 2, np.nan])
1722 >>> arr1.equals(arr2)
1723 True
1725 >>> arr1 = pd.array([1, 3, np.nan])
1726 >>> arr2 = pd.array([1, 2, np.nan])
1727 >>> arr1.equals(arr2)
1728 False
1729 """
1730 if type(self) != type(other):
1731 return False
1732 if self.dtype != other.dtype:
1733 return False
1734 elif len(self) != len(other):
1735 return False
1736 else:
1737 equal_values = self == other
1738 if isinstance(equal_values, ExtensionArray):
1739 # boolean array with NA -> fill with False
1740 equal_values = equal_values.fillna(False)
1741 # error: Unsupported left operand type for & ("ExtensionArray")
1742 equal_na = self.isna() & other.isna() # type: ignore[operator]
1743 return bool((equal_values | equal_na).all())
1745 def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
1746 """
1747 Pointwise comparison for set containment in the given values.
1749 Roughly equivalent to `np.array([x in values for x in self])`
1751 Parameters
1752 ----------
1753 values : np.ndarray or ExtensionArray
1754 Values to compare every element in the array against.
1756 Returns
1757 -------
1758 np.ndarray[bool]
1759 With true at indices where value is in `values`.
1761 See Also
1762 --------
1763 DataFrame.isin: Whether each element in the DataFrame is contained in values.
1764 Index.isin: Return a boolean array where the index values are in values.
1765 Series.isin: Whether elements in Series are contained in values.
1767 Examples
1768 --------
1769 >>> arr = pd.array([1, 2, 3])
1770 >>> arr.isin([1])
1771 <BooleanArray>
1772 [True, False, False]
1773 Length: 3, dtype: boolean
1774 """
1775 return isin(np.asarray(self), values)
1777 def _values_for_factorize(self) -> tuple[np.ndarray, Any]:
1778 """
1779 Return an array and missing value suitable for factorization.
1781 This method provides the values and NA sentinel to use in the
1782 factorization process. Subclasses may override this to customize
1783 the factorization behavior.
1785 Returns
1786 -------
1787 values : ndarray
1788 An array suitable for factorization. This should maintain order
1789 and be a supported dtype (Float64, Int64, UInt64, String, Object).
1790 By default, the extension array is cast to object dtype.
1791 na_value : object
1792 The value in `values` to consider missing. This will be treated
1793 as NA in the factorization routines, so it will be coded as
1794 `-1` and not included in `uniques`. By default,
1795 ``np.nan`` is used.
1797 See Also
1798 --------
1799 util.hash_pandas_object : Hash the pandas object.
1801 Notes
1802 -----
1803 The values returned by this method are also used in
1804 :func:`pandas.util.hash_pandas_object`. If needed, this can be
1805 overridden in the ``self._hash_pandas_object()`` method.
1807 Examples
1808 --------
1809 >>> pd.array([1, 2, 3])._values_for_factorize()
1810 (array([1, 2, 3], dtype=object), nan)
1811 """
1812 return self.astype(object), np.nan
1814 def factorize(
1815 self,
1816 use_na_sentinel: bool = True,
1817 ) -> tuple[np.ndarray, ExtensionArray]:
1818 """
1819 Encode the extension array as an enumerated type.
1821 This method encodes the array values as integer codes that index
1822 into an array of unique values. Missing values can be encoded
1823 with a sentinel value of -1.
1825 Parameters
1826 ----------
1827 use_na_sentinel : bool, default True
1828 If True, the sentinel -1 will be used for NaN values. If False,
1829 NaN values will be encoded as non-negative integers and will not drop the
1830 NaN from the uniques of the values.
1832 Returns
1833 -------
1834 codes : ndarray
1835 An integer NumPy array that's an indexer into the original
1836 ExtensionArray.
1837 uniques : ExtensionArray
1838 An ExtensionArray containing the unique values of `self`.
1840 .. note::
1842 uniques will *not* contain an entry for the NA value of
1843 the ExtensionArray if there are any missing values present
1844 in `self`.
1846 See Also
1847 --------
1848 factorize : Top-level factorize method that dispatches here.
1850 Notes
1851 -----
1852 :meth:`pandas.factorize` offers a `sort` keyword as well.
1854 Examples
1855 --------
1856 >>> idx1 = pd.PeriodIndex(
1857 ... ["2014-01", "2014-01", "2014-02", "2014-02", "2014-03", "2014-03"],
1858 ... freq="M",
1859 ... )
1860 >>> arr, idx = idx1.factorize()
1861 >>> arr
1862 array([0, 0, 1, 1, 2, 2])
1863 >>> idx
1864 PeriodIndex(['2014-01', '2014-02', '2014-03'], dtype='period[M]')
1865 """
1866 # Implementer note: There are two ways to override the behavior of
1867 # pandas.factorize
1868 # 1. _values_for_factorize and _from_factorize.
1869 # Specify the values passed to pandas' internal factorization
1870 # routines, and how to convert from those values back to the
1871 # original ExtensionArray.
1872 # 2. ExtensionArray.factorize.
1873 # Complete control over factorization.
1874 arr, na_value = self._values_for_factorize()
1876 codes, uniques = factorize_array(
1877 arr, use_na_sentinel=use_na_sentinel, na_value=na_value
1878 )
1880 uniques_ea = self._from_factorized(uniques, self)
1881 return codes, uniques_ea
1883 def repeat(self, repeats: int | Sequence[int], axis: AxisInt | None = None) -> Self:
1884 """
1885 Repeat elements of an ExtensionArray.
1887 Returns a new ExtensionArray where each element of the current ExtensionArray
1888 is repeated consecutively a given number of times.
1890 Parameters
1891 ----------
1892 repeats : int or array of ints
1893 The number of repetitions for each element. This should be a
1894 non-negative integer. Repeating 0 times will return an empty
1895 ExtensionArray.
1896 axis : None
1897 Must be ``None``. Has no effect but is accepted for compatibility
1898 with numpy.
1900 Returns
1901 -------
1902 ExtensionArray
1903 Newly created ExtensionArray with repeated elements.
1905 See Also
1906 --------
1907 Series.repeat : Equivalent function for Series.
1908 Index.repeat : Equivalent function for Index.
1909 numpy.repeat : Similar method for :class:`numpy.ndarray`.
1910 ExtensionArray.take : Take arbitrary positions.
1912 Examples
1913 --------
1914 >>> cat = pd.Categorical(["a", "b", "c"])
1915 >>> cat
1916 ['a', 'b', 'c']
1917 Categories (3, str): ['a', 'b', 'c']
1918 >>> cat.repeat(2)
1919 ['a', 'a', 'b', 'b', 'c', 'c']
1920 Categories (3, str): ['a', 'b', 'c']
1921 >>> cat.repeat([1, 2, 3])
1922 ['a', 'b', 'b', 'c', 'c', 'c']
1923 Categories (3, str): ['a', 'b', 'c']
1924 """
1925 nv.validate_repeat((), {"axis": axis})
1926 ind = np.arange(len(self)).repeat(repeats)
1927 return self.take(ind)
1929 def value_counts(self, dropna: bool = True) -> Series:
1930 """
1931 Return a Series containing counts of unique values.
1933 This method returns a Series with unique values as the index and their
1934 counts as the values.
1936 Parameters
1937 ----------
1938 dropna : bool, default True
1939 Don't include counts of NA values.
1941 Returns
1942 -------
1943 Series
1944 A Series with unique values as index and counts as values.
1946 See Also
1947 --------
1948 Series.value_counts : Equivalent method on Series.
1949 DataFrame.value_counts : Equivalent method on DataFrame.
1950 Index.value_counts : Equivalent method on Index.
1952 Examples
1953 --------
1954 >>> from pandas.core.arrays import IntegerArray
1955 >>> arr = IntegerArray._from_sequence([3, 3, 3, 1, 2, 2])
1956 >>> arr.value_counts()
1957 3 3
1958 1 1
1959 2 2
1960 Name: count, dtype: Int64
1961 """
1962 from pandas.core.algorithms import value_counts_internal as value_counts
1964 result = value_counts(self.to_numpy(copy=False), sort=False, dropna=dropna)
1965 result.index = result.index.astype(self.dtype)
1966 return result
1968 # ------------------------------------------------------------------------
1969 # Indexing methods
1970 # ------------------------------------------------------------------------
1972 def take(
1973 self,
1974 indices: TakeIndexer,
1975 *,
1976 allow_fill: bool = False,
1977 fill_value: Any = None,
1978 ) -> Self:
1979 """
1980 Take elements from an array.
1982 This method selects elements from the array at the specified
1983 indices, optionally handling negative indices as missing values
1984 when ``allow_fill=True``.
1986 Parameters
1987 ----------
1988 indices : sequence of int or one-dimensional np.ndarray of int
1989 Indices to be taken.
1990 allow_fill : bool, default False
1991 How to handle negative values in `indices`.
1993 * False: negative values in `indices` indicate positional indices
1994 from the right (the default). This is similar to
1995 :func:`numpy.take`.
1997 * True: negative values in `indices` indicate
1998 missing values. These values are set to `fill_value`. Any other
1999 other negative values raise a ``ValueError``.
2001 fill_value : any, optional
2002 Fill value to use for NA-indices when `allow_fill` is True.
2003 This may be ``None``, in which case the default NA value for
2004 the type, ``self.dtype.na_value``, is used.
2006 For many ExtensionArrays, there will be two representations of
2007 `fill_value`: a user-facing "boxed" scalar, and a low-level
2008 physical NA value. `fill_value` should be the user-facing version,
2009 and the implementation should handle translating that to the
2010 physical version for processing the take if necessary.
2012 Returns
2013 -------
2014 ExtensionArray
2015 An array formed with selected `indices`.
2017 Raises
2018 ------
2019 IndexError
2020 When the indices are out of bounds for the array.
2021 ValueError
2022 When `indices` contains negative values other than ``-1``
2023 and `allow_fill` is True.
2025 See Also
2026 --------
2027 numpy.take : Take elements from an array along an axis.
2028 api.extensions.take : Take elements from an array.
2030 Notes
2031 -----
2032 ExtensionArray.take is called by ``Series.__getitem__``, ``.loc``,
2033 ``iloc``, when `indices` is a sequence of values. Additionally,
2034 it's called by :meth:`Series.reindex`, or any other method
2035 that causes realignment, with a `fill_value`.
2037 Scalar values for `indices` are not supported. If needed,
2038 use a list-like wrapping a single element (e.g. ``[index]``).
2040 Examples
2041 --------
2042 Here's an example implementation, which relies on casting the
2043 extension array to object dtype. This uses the helper method
2044 :func:`pandas.api.extensions.take`.
2046 .. code-block:: python
2048 def take(self, indices, allow_fill=False, fill_value=None):
2049 from pandas.api.extensions import take
2051 # If the ExtensionArray is backed by an ndarray, then
2052 # just pass that here instead of coercing to object.
2053 data = self.astype(object)
2055 if allow_fill and fill_value is None:
2056 fill_value = self.dtype.na_value
2058 # fill value should always be translated from the scalar
2059 # type for the array, to the physical storage type for
2060 # the data, before passing to take.
2062 result = take(
2063 data, indices, fill_value=fill_value, allow_fill=allow_fill
2064 )
2065 return self._from_sequence(result, dtype=self.dtype)
2066 """
2067 # Implementer note: The `fill_value` parameter should be a user-facing
2068 # value, an instance of self.dtype.type. When passed `fill_value=None`,
2069 # the default of `self.dtype.na_value` should be used.
2070 # This may differ from the physical storage type your ExtensionArray
2071 # uses. In this case, your implementation is responsible for casting
2072 # the user-facing type to the storage type, before using
2073 # pandas.api.extensions.take
2074 raise AbstractMethodError(self)
2076 def copy(self) -> Self:
2077 """
2078 Return a copy of the array.
2080 This method creates a copy of the `ExtensionArray` where modifying the
2081 data in the copy will not affect the original array. This is useful when
2082 you want to manipulate data without altering the original dataset.
2084 Returns
2085 -------
2086 ExtensionArray
2087 A new `ExtensionArray` object that is a copy of the current instance.
2089 See Also
2090 --------
2091 DataFrame.copy : Return a copy of the DataFrame.
2092 Series.copy : Return a copy of the Series.
2094 Examples
2095 --------
2096 >>> arr = pd.array([1, 2, 3])
2097 >>> arr2 = arr.copy()
2098 >>> arr[0] = 2
2099 >>> arr2
2100 <IntegerArray>
2101 [1, 2, 3]
2102 Length: 3, dtype: Int64
2103 """
2104 raise AbstractMethodError(self)
2106 @overload
2107 def view(self, dtype: None = ...) -> Self: ...
2109 @overload
2110 def view(self, dtype: Dtype | None = ...) -> ArrayLike: ...
2112 def view(self, dtype: Dtype | None = None) -> ArrayLike:
2113 """
2114 Return a view on the array.
2116 This method returns a new object referencing the same underlying
2117 data, without copying. Modifications to either the view or the
2118 original array will be reflected in both.
2120 Parameters
2121 ----------
2122 dtype : str, np.dtype, or ExtensionDtype, optional
2123 Default None.
2125 Returns
2126 -------
2127 ExtensionArray or np.ndarray
2128 A view on the :class:`ExtensionArray`'s data.
2130 See Also
2131 --------
2132 api.extensions.ExtensionArray.ravel: Return a flattened view on input array.
2133 Index.view: Equivalent function for Index.
2134 ndarray.view: New view of array with the same data.
2136 Examples
2137 --------
2138 This gives view on the underlying data of an ``ExtensionArray`` and is not a
2139 copy. Modifications on either the view or the original ``ExtensionArray``
2140 will be reflected on the underlying data:
2142 >>> arr = pd.array([1, 2, 3])
2143 >>> arr2 = arr.view()
2144 >>> arr[0] = 2
2145 >>> arr2
2146 <IntegerArray>
2147 [2, 2, 3]
2148 Length: 3, dtype: Int64
2149 """
2150 # NB:
2151 # - This must return a *new* object referencing the same data, not self.
2152 # - The only case that *must* be implemented is with dtype=None,
2153 # giving a view with the same dtype as self.
2154 if dtype is not None:
2155 raise NotImplementedError(dtype)
2156 return self[:]
2158 # ------------------------------------------------------------------------
2159 # Printing
2160 # ------------------------------------------------------------------------
2162 def __repr__(self) -> str:
2163 if self.ndim > 1:
2164 return self._repr_2d()
2166 from pandas.io.formats.printing import format_object_summary
2168 # the short repr has no trailing newline, while the truncated
2169 # repr does. So we include a newline in our template, and strip
2170 # any trailing newlines from format_object_summary
2171 data = format_object_summary(
2172 self, self._formatter(), indent_for_name=False
2173 ).rstrip(", \n")
2174 class_name = f"<{type(self).__name__}>\n"
2175 footer = self._get_repr_footer()
2176 return f"{class_name}{data}\n{footer}"
2178 def _get_repr_footer(self) -> str:
2179 # GH#24278
2180 if self.ndim > 1:
2181 return f"Shape: {self.shape}, dtype: {self.dtype}"
2182 return f"Length: {len(self)}, dtype: {self.dtype}"
2184 def _repr_2d(self) -> str:
2185 from pandas.io.formats.printing import format_object_summary
2187 # the short repr has no trailing newline, while the truncated
2188 # repr does. So we include a newline in our template, and strip
2189 # any trailing newlines from format_object_summary
2190 lines = [
2191 format_object_summary(x, self._formatter(), indent_for_name=False).rstrip(
2192 ", \n"
2193 )
2194 for x in self
2195 ]
2196 data = ",\n".join(lines)
2197 class_name = f"<{type(self).__name__}>"
2198 footer = self._get_repr_footer()
2199 return f"{class_name}\n[\n{data}\n]\n{footer}"
2201 def _formatter(self, boxed: bool = False) -> Callable[[Any], str | None]:
2202 """
2203 Formatting function for scalar values.
2205 This is used in the default '__repr__'. The returned formatting
2206 function receives instances of your scalar type.
2208 Parameters
2209 ----------
2210 boxed : bool, default False
2211 An indicator for whether or not your array is being printed
2212 within a Series, DataFrame, or Index (True), or just by
2213 itself (False). This may be useful if you want scalar values
2214 to appear differently within a Series versus on its own (e.g.
2215 quoted or not).
2217 Returns
2218 -------
2219 Callable[[Any], str]
2220 A callable that gets instances of the scalar type and
2221 returns a string. By default, :func:`repr` is used
2222 when ``boxed=False`` and :func:`str` is used when
2223 ``boxed=True``.
2225 See Also
2226 --------
2227 api.extensions.ExtensionArray._concat_same_type : Concatenate multiple
2228 array of this dtype.
2229 api.extensions.ExtensionArray._explode : Transform each element of
2230 list-like to a row.
2231 api.extensions.ExtensionArray._from_factorized : Reconstruct an
2232 ExtensionArray after factorization.
2233 api.extensions.ExtensionArray._from_sequence : Construct a new
2234 ExtensionArray from a sequence of scalars.
2236 Examples
2237 --------
2238 >>> class MyExtensionArray(pd.arrays.NumpyExtensionArray):
2239 ... def _formatter(self, boxed=False):
2240 ... return lambda x: "*" + str(x) + "*"
2241 >>> MyExtensionArray(np.array([1, 2, 3, 4]))
2242 <MyExtensionArray>
2243 [*1*, *2*, *3*, *4*]
2244 Length: 4, dtype: int64
2245 """
2246 if boxed:
2247 return str
2248 return repr
2250 # ------------------------------------------------------------------------
2251 # Reshaping
2252 # ------------------------------------------------------------------------
2254 def transpose(self, *axes: int) -> Self:
2255 """
2256 Return a transposed view on this array.
2258 Because ExtensionArrays are always 1D, this is a no-op. It is included
2259 for compatibility with np.ndarray.
2261 Returns
2262 -------
2263 ExtensionArray
2265 Examples
2266 --------
2267 >>> pd.array([1, 2, 3]).transpose()
2268 <IntegerArray>
2269 [1, 2, 3]
2270 Length: 3, dtype: Int64
2271 """
2272 return self[:]
2274 @property
2275 def T(self) -> Self:
2276 return self.transpose()
2278 def ravel(self, order: Literal["C", "F", "A", "K"] | None = "C") -> Self:
2279 """
2280 Return a flattened view on this array.
2282 Since ExtensionArrays are 1-dimensional, this method returns the
2283 array itself. The ``order`` parameter is accepted for compatibility
2284 with NumPy but is ignored.
2286 Parameters
2287 ----------
2288 order : {None, 'C', 'F', 'A', 'K'}, default 'C'
2290 Returns
2291 -------
2292 ExtensionArray
2293 A flattened view on the array.
2295 See Also
2296 --------
2297 ExtensionArray.tolist: Return a list of the values.
2299 Notes
2300 -----
2301 - Because ExtensionArrays are 1D-only, this is a no-op.
2302 - The "order" argument is ignored, is for compatibility with NumPy.
2304 Examples
2305 --------
2306 >>> pd.array([1, 2, 3]).ravel()
2307 <IntegerArray>
2308 [1, 2, 3]
2309 Length: 3, dtype: Int64
2310 """
2311 return self
2313 @classmethod
2314 def _concat_same_type(cls, to_concat: Sequence[Self]) -> Self:
2315 """
2316 Concatenate multiple arrays of this dtype.
2318 This method joins a sequence of ExtensionArrays of the same dtype
2319 into a single ExtensionArray. All arrays in the sequence must
2320 have the same dtype.
2322 Parameters
2323 ----------
2324 to_concat : sequence of this type
2325 An array of the same dtype to concatenate.
2327 Returns
2328 -------
2329 ExtensionArray
2331 See Also
2332 --------
2333 api.extensions.ExtensionArray._explode : Transform each element of
2334 list-like to a row.
2335 api.extensions.ExtensionArray._formatter : Formatting function for
2336 scalar values.
2337 api.extensions.ExtensionArray._from_factorized : Reconstruct an
2338 ExtensionArray after factorization.
2340 Examples
2341 --------
2342 >>> arr1 = pd.array([1, 2, 3])
2343 >>> arr2 = pd.array([4, 5, 6])
2344 >>> pd.arrays.IntegerArray._concat_same_type([arr1, arr2])
2345 <IntegerArray>
2346 [1, 2, 3, 4, 5, 6]
2347 Length: 6, dtype: Int64
2348 """
2349 # Implementer note: this method will only be called with a sequence of
2350 # ExtensionArrays of this class and with the same dtype as self. This
2351 # should allow "easy" concatenation (no upcasting needed), and result
2352 # in a new ExtensionArray of the same dtype.
2353 # Note: this strict behaviour is only guaranteed starting with pandas 1.1
2354 raise AbstractMethodError(cls)
2356 # The _can_hold_na attribute is set to True so that pandas internals
2357 # will use the ExtensionDtype.na_value as the NA value in operations
2358 # such as take(), reindex(), shift(), etc. In addition, those results
2359 # will then be of the ExtensionArray subclass rather than an array
2360 # of objects
2361 @cache_readonly
2362 def _can_hold_na(self) -> bool:
2363 return self.dtype._can_hold_na
2365 def _accumulate(
2366 self, name: str, *, skipna: bool = True, **kwargs
2367 ) -> ExtensionArray:
2368 """
2369 Return an ExtensionArray performing an accumulation operation.
2371 The underlying data type might change.
2373 Parameters
2374 ----------
2375 name : str
2376 Name of the function, supported values are:
2377 - cummin
2378 - cummax
2379 - cumsum
2380 - cumprod
2381 skipna : bool, default True
2382 If True, skip NA values.
2383 **kwargs
2384 Additional keyword arguments passed to the accumulation function.
2385 Currently, there is no supported kwarg.
2387 Returns
2388 -------
2389 array
2390 An array performing the accumulation operation.
2392 Raises
2393 ------
2394 NotImplementedError : subclass does not define accumulations
2396 See Also
2397 --------
2398 api.extensions.ExtensionArray._concat_same_type : Concatenate multiple
2399 array of this dtype.
2400 api.extensions.ExtensionArray.view : Return a view on the array.
2401 api.extensions.ExtensionArray._explode : Transform each element of
2402 list-like to a row.
2404 Examples
2405 --------
2406 >>> arr = pd.array([1, 2, 3])
2407 >>> arr._accumulate(name="cumsum")
2408 <IntegerArray>
2409 [1, 3, 6]
2410 Length: 3, dtype: Int64
2411 """
2412 raise NotImplementedError(f"cannot perform {name} with type {self.dtype}")
2414 def _reduce(
2415 self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
2416 ):
2417 """
2418 Return a scalar result of performing the reduction operation.
2420 This method dispatches to the appropriate reduction method (e.g.,
2421 sum, mean, min, max) based on the `name` parameter and returns
2422 the result.
2424 Parameters
2425 ----------
2426 name : str
2427 Name of the function, supported values are:
2428 { any, all, min, max, sum, mean, median, prod,
2429 std, var, sem, kurt, skew }.
2430 skipna : bool, default True
2431 If True, skip NaN values.
2432 keepdims : bool, default False
2433 If False, a scalar is returned.
2434 If True, the result has dimension with size one along the reduced axis.
2435 **kwargs
2436 Additional keyword arguments passed to the reduction function.
2437 Currently, `ddof` is the only supported kwarg.
2439 Returns
2440 -------
2441 scalar or ndarray:
2442 The result of the reduction operation. The type of the result
2443 depends on `keepdims`:
2444 - If `keepdims` is `False`, a scalar value is returned.
2445 - If `keepdims` is `True`, the result is wrapped in a numpy array with
2446 a single element.
2448 Raises
2449 ------
2450 TypeError : subclass does not define operations
2452 See Also
2453 --------
2454 Series.min : Return the minimum value.
2455 Series.max : Return the maximum value.
2456 Series.sum : Return the sum of values.
2457 Series.mean : Return the mean of values.
2458 Series.median : Return the median of values.
2459 Series.std : Return the standard deviation.
2460 Series.var : Return the variance.
2461 Series.prod : Return the product of values.
2462 Series.sem : Return the standard error of the mean.
2463 Series.kurt : Return the kurtosis.
2464 Series.skew : Return the skewness.
2466 Examples
2467 --------
2468 >>> pd.array([1, 2, 3])._reduce("min")
2469 np.int64(1)
2470 >>> pd.array([1, 2, 3])._reduce("max")
2471 np.int64(3)
2472 >>> pd.array([1, 2, 3])._reduce("sum")
2473 np.int64(6)
2474 >>> pd.array([1, 2, 3])._reduce("mean")
2475 np.float64(2.0)
2476 >>> pd.array([1, 2, 3])._reduce("median")
2477 np.float64(2.0)
2478 """
2479 meth = getattr(self, name, None)
2480 if meth is None:
2481 raise TypeError(
2482 f"'{type(self).__name__}' with dtype {self.dtype} "
2483 f"does not support operation '{name}'"
2484 )
2485 if name != "count":
2486 kwargs["skipna"] = skipna
2487 result = meth(**kwargs)
2488 if keepdims:
2489 if name in ["min", "max"]:
2490 result = self._from_sequence([result], dtype=self.dtype)
2491 else:
2492 result = np.array([result])
2494 return result
2496 def count(self):
2497 """
2498 Count the number of non-NA values in the array.
2500 This method returns the number of elements in the
2501 :class:`ExtensionArray` that are not missing (i.e., not NA/null).
2503 Returns
2504 -------
2505 scalar
2506 Number of non-NA values in the Series.
2508 See Also
2509 --------
2510 Series.count : Count the number of non-NA values in a Series.
2512 Examples
2513 --------
2514 >>> s = pd.array([1, pd.NA, 3])
2515 >>> s.count()
2516 np.int64(2)
2517 """
2518 return (~self.isna()).sum()
2520 # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
2521 # Incompatible types in assignment (expression has type "None", base class
2522 # "object" defined the type as "Callable[[object], int]")
2523 __hash__: ClassVar[None] # type: ignore[assignment]
2525 # ------------------------------------------------------------------------
2526 # Non-Optimized Default Methods; in the case of the private methods here,
2527 # these are not guaranteed to be stable across pandas versions.
2529 def _values_for_json(self) -> np.ndarray:
2530 """
2531 Specify how to render our entries in to_json.
2533 Notes
2534 -----
2535 The dtype on the returned ndarray is not restricted, but for non-native
2536 types that are not specifically handled in objToJSON.c, to_json is
2537 liable to raise. In these cases, it may be safer to return an ndarray
2538 of strings.
2539 """
2540 return np.asarray(self)
2542 def _hash_pandas_object(
2543 self, *, encoding: str, hash_key: str, categorize: bool
2544 ) -> npt.NDArray[np.uint64]:
2545 """
2546 Hook for hash_pandas_object.
2548 Default is to use the values returned by _values_for_factorize.
2550 Parameters
2551 ----------
2552 encoding : str
2553 Encoding for data & key when strings.
2554 hash_key : str
2555 Hash_key for string key to encode.
2556 categorize : bool
2557 Whether to first categorize object arrays before hashing. This is more
2558 efficient when the array contains duplicate values.
2560 Returns
2561 -------
2562 np.ndarray[uint64]
2563 An array of hashed values.
2565 See Also
2566 --------
2567 api.extensions.ExtensionArray._values_for_factorize : Return an array and
2568 missing value suitable for factorization.
2569 util.hash_array : Given a 1d array, return an array of hashed values.
2571 Examples
2572 --------
2573 >>> pd.array([1, 2])._hash_pandas_object(
2574 ... encoding="utf-8", hash_key="1000000000000000", categorize=False
2575 ... )
2576 array([ 6238072747940578789, 15839785061582574730], dtype=uint64)
2577 """
2578 from pandas.core.util.hashing import hash_array
2580 values, _ = self._values_for_factorize()
2581 return hash_array(
2582 values, encoding=encoding, hash_key=hash_key, categorize=categorize
2583 )
2585 def _explode(self) -> tuple[Self, npt.NDArray[np.uint64]]:
2586 """
2587 Transform each element of list-like to a row.
2589 For arrays that do not contain list-like elements the default
2590 implementation of this method just returns a copy and an array
2591 of ones (unchanged index).
2593 Returns
2594 -------
2595 ExtensionArray
2596 Array with the exploded values.
2597 np.ndarray[uint64]
2598 The original lengths of each list-like for determining the
2599 resulting index.
2601 See Also
2602 --------
2603 Series.explode : The method on the ``Series`` object that this
2604 extension array method is meant to support.
2606 Examples
2607 --------
2608 >>> import pyarrow as pa
2609 >>> a = pd.array(
2610 ... [[1, 2, 3], [4], [5, 6]], dtype=pd.ArrowDtype(pa.list_(pa.int64()))
2611 ... )
2612 >>> a._explode()
2613 (<ArrowExtensionArray>
2614 [1, 2, 3, 4, 5, 6]
2615 Length: 6, dtype: int64[pyarrow], array([3, 1, 2], dtype=int32))
2616 """
2617 values = self.copy()
2618 counts = np.ones(shape=(len(self),), dtype=np.uint64)
2619 return values, counts
2621 def tolist(self) -> list[Any]:
2622 """
2623 Return a list of the values.
2625 These are each a scalar type, which is a Python scalar
2626 (for str, int, float) or a pandas scalar
2627 (for Timestamp/Timedelta/Interval/Period)
2629 Returns
2630 -------
2631 list
2632 Python list of values in array.
2634 See Also
2635 --------
2636 Index.to_list: Return a list of the values in the Index.
2637 Series.to_list: Return a list of the values in the Series.
2639 Examples
2640 --------
2641 >>> arr = pd.array([1, 2, 3])
2642 >>> arr.tolist()
2643 [1, 2, 3]
2644 """
2645 if self.ndim > 1:
2646 return [x.tolist() for x in self]
2647 return list(self)
2649 def delete(self, loc: ScalarIndexer | slice | list[int] | np.ndarray) -> Self:
2650 indexer = np.delete(np.arange(len(self)), loc)
2651 return self.take(indexer)
2653 def insert(self, loc: int, item) -> Self:
2654 """
2655 Insert an item at the given position.
2657 This method creates a new ExtensionArray with the specified item
2658 inserted at the given index. The original array is not modified.
2660 Parameters
2661 ----------
2662 loc : int
2663 Index where the `item` needs to be inserted.
2664 item : scalar-like
2665 Value to be inserted.
2667 Returns
2668 -------
2669 ExtensionArray
2670 With `item` inserted at `loc`.
2672 See Also
2673 --------
2674 Index.insert: Make new Index inserting new item at location.
2676 Notes
2677 -----
2678 This method should be both type and dtype-preserving. If the item
2679 cannot be held in an array of this type/dtype, either ValueError or
2680 TypeError should be raised.
2682 The default implementation relies on _from_sequence to raise on invalid
2683 items.
2685 Examples
2686 --------
2687 >>> arr = pd.array([1, 2, 3])
2688 >>> arr.insert(2, -1)
2689 <IntegerArray>
2690 [1, 2, -1, 3]
2691 Length: 4, dtype: Int64
2692 """
2693 loc = validate_insert_loc(loc, len(self))
2695 item_arr = type(self)._from_sequence([item], dtype=self.dtype)
2697 return type(self)._concat_same_type([self[:loc], item_arr, self[loc:]])
2699 def _putmask(self, mask: npt.NDArray[np.bool_], value) -> None:
2700 """
2701 Analogue to np.putmask(self, mask, value)
2703 Parameters
2704 ----------
2705 mask : np.ndarray[bool]
2706 value : scalar or listlike
2707 If listlike, must be arraylike with same length as self.
2709 Returns
2710 -------
2711 None
2713 Notes
2714 -----
2715 Unlike np.putmask, we do not repeat listlike values with mismatched length.
2716 'value' should either be a scalar or an arraylike with the same length
2717 as self.
2718 """
2719 if is_list_like(value):
2720 val = value[mask]
2721 else:
2722 val = value
2724 self[mask] = val
2726 def _where(self, mask: npt.NDArray[np.bool_], value) -> Self:
2727 """
2728 Analogue to np.where(mask, self, value)
2730 Parameters
2731 ----------
2732 mask : np.ndarray[bool]
2733 value : scalar or listlike
2735 Returns
2736 -------
2737 same type as self
2738 """
2739 result = self.copy()
2741 if is_list_like(value):
2742 val = value[~mask]
2743 else:
2744 val = value
2746 result[~mask] = val
2747 return result
2749 @property
2750 def _is_monotonic_increasing(self) -> bool:
2751 """
2752 Return True if the array values are monotonically increasing.
2754 Subclasses can override this for performance. The default
2755 implementation falls back to computing ranks via ``_rank``.
2757 Returns
2758 -------
2759 bool
2760 Whether the array values are monotonically increasing.
2762 See Also
2763 --------
2764 ExtensionArray._is_monotonic_decreasing : Check for monotonically
2765 decreasing values.
2767 Examples
2768 --------
2769 >>> arr = pd.array([1, 2, 3])
2770 >>> arr._is_monotonic_increasing
2771 True
2772 """
2773 return self._monotonic_check()[0]
2775 @property
2776 def _is_monotonic_decreasing(self) -> bool:
2777 """
2778 Return True if the array values are monotonically decreasing.
2780 Subclasses can override this for performance. The default
2781 implementation falls back to computing ranks via ``_rank``.
2783 Returns
2784 -------
2785 bool
2786 Whether the array values are monotonically decreasing.
2788 See Also
2789 --------
2790 ExtensionArray._is_monotonic_increasing : Check for monotonically
2791 increasing values.
2793 Examples
2794 --------
2795 >>> arr = pd.array([3, 2, 1])
2796 >>> arr._is_monotonic_decreasing
2797 True
2798 """
2799 return self._monotonic_check()[1]
2801 def _monotonic_check(self) -> tuple[bool, bool]:
2802 """
2803 Return (is_monotonic_increasing, is_monotonic_decreasing).
2805 Default implementation using ``_values_for_argsort``. Subclasses
2806 should override ``_is_monotonic_increasing`` and
2807 ``_is_monotonic_decreasing`` instead of this method.
2808 """
2809 if self._hasna:
2810 return False, False
2811 try:
2812 values = self._values_for_argsort()
2813 inc, dec, _ = is_monotonic(values)
2814 except TypeError:
2815 # Either ``_values_for_argsort`` is not implemented or the dtype is
2816 # not orderable by ``is_monotonic`` (e.g. complex).
2817 return False, False
2818 return inc, dec
2820 def _rank(
2821 self,
2822 *,
2823 axis: AxisInt = 0,
2824 method: RankMethod = "average",
2825 na_option: RankNaOption = "keep",
2826 ascending: bool = True,
2827 pct: bool = False,
2828 ):
2829 """
2830 See Series.rank.__doc__.
2831 """
2832 if axis != 0:
2833 raise NotImplementedError
2835 return rank(
2836 self._values_for_argsort(),
2837 axis=axis,
2838 method=method,
2839 na_option=na_option,
2840 ascending=ascending,
2841 pct=pct,
2842 mask=np.asarray(self.isna(), dtype="bool") if self._hasna else None,
2843 )
2845 @classmethod
2846 def _empty(cls, shape: Shape, dtype: ExtensionDtype):
2847 """
2848 Create an ExtensionArray with the given shape and dtype.
2850 See also
2851 --------
2852 ExtensionDtype.empty
2853 ExtensionDtype.empty is the 'official' public version of this API.
2854 """
2855 # Implementer note: while ExtensionDtype.empty is the public way to
2856 # call this method, it is still required to implement this `_empty`
2857 # method as well (it is called internally in pandas)
2858 obj = cls._from_sequence([], dtype=dtype)
2860 taker = np.broadcast_to(np.intp(-1), shape)
2861 result = obj.take(taker, allow_fill=True)
2862 if not isinstance(result, cls) or dtype != result.dtype:
2863 raise NotImplementedError(
2864 f"Default 'empty' implementation is invalid for dtype='{dtype}'"
2865 )
2866 return result
2868 def _quantile(self, qs: npt.NDArray[np.float64], interpolation: str) -> Self:
2869 """
2870 Compute the quantiles of self for each quantile in `qs`.
2872 Parameters
2873 ----------
2874 qs : np.ndarray[float64]
2875 interpolation: str
2877 Returns
2878 -------
2879 same type as self
2880 """
2881 mask = np.asarray(self.isna())
2882 arr = np.asarray(self)
2883 fill_value = np.nan
2885 res_values = quantile_with_mask(arr, mask, fill_value, qs, interpolation)
2886 return type(self)._from_sequence(res_values)
2888 def _mode(self, dropna: bool = True) -> Self:
2889 """
2890 Returns the mode(s) of the ExtensionArray.
2892 Always returns `ExtensionArray` even if only one value.
2894 Parameters
2895 ----------
2896 dropna : bool, default True
2897 Don't consider counts of NA values.
2899 Returns
2900 -------
2901 same type as self
2902 Sorted, if possible.
2903 """
2904 # error: Incompatible return value type (got "Union[ExtensionArray,
2905 # Tuple[np.ndarray, npt.NDArray[np.bool_]]", expected "Self")
2906 result, _ = mode(self, dropna=dropna)
2907 return result # type: ignore[return-value]
2909 def round(self, decimals: int = 0) -> Self:
2910 """
2911 Round each value in the array to the given number of decimals.
2913 Parameters
2914 ----------
2915 decimals : int, default 0
2916 Number of decimal places to round to. If decimals is negative,
2917 it specifies the number of positions to the left of the decimal
2918 point.
2920 Returns
2921 -------
2922 ExtensionArray
2923 Rounded values of the ExtensionArray.
2925 See Also
2926 --------
2927 DataFrame.round : Round values of a DataFrame.
2928 Series.round : Round values of a Series.
2930 Notes
2931 -----
2932 This is a non-performant default implementation. Subclasses are
2933 encouraged to override it to avoid the elementwise loop.
2935 Examples
2936 --------
2937 >>> arr = pd.array([1.234, 5.678, pd.NA], dtype="Float64")
2938 >>> arr.round(1)
2939 <FloatingArray>
2940 [1.2, 5.7, <NA>]
2941 Length: 3, dtype: Float64
2942 """
2943 if self.dtype._is_boolean:
2944 return self.copy()
2945 if not self.dtype._is_numeric:
2946 raise TypeError(f"Cannot round dtype {self.dtype} as it is non-numeric")
2947 # Python's builtin round on complex emits DeprecationWarning (and
2948 # raises TypeError in a future Python release); use np.round there.
2949 round_fn = np.round if self.dtype.kind == "c" else round
2950 rounded = [
2951 round_fn(item, decimals) if not item_isna else item
2952 for item, item_isna in zip(self, self.isna(), strict=True)
2953 ]
2954 return type(self)._from_sequence(rounded, dtype=self.dtype)
2956 def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
2957 # the fallback at the end of this method np.asarray()s the values: M8/m8
2958 # survive to be truth-tested, the rest flatten to object
2959 ops.disallow_datetimelike_logical_ufunc(ufunc, inputs)
2961 if any(
2962 isinstance(other, (ABCSeries, ABCIndex, ABCDataFrame)) for other in inputs
2963 ):
2964 return NotImplemented
2966 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(
2967 self, ufunc, method, *inputs, **kwargs
2968 )
2969 if result is not NotImplemented:
2970 return result
2972 if "out" in kwargs:
2973 return arraylike.dispatch_ufunc_with_out(
2974 self, ufunc, method, *inputs, **kwargs
2975 )
2977 if method == "reduce":
2978 result = arraylike.dispatch_reduction_ufunc(
2979 self, ufunc, method, *inputs, **kwargs
2980 )
2981 if result is not NotImplemented:
2982 return result
2984 return arraylike.default_array_ufunc(self, ufunc, method, *inputs, **kwargs)
2986 def map(self, mapper, na_action: Literal["ignore"] | None = None):
2987 """
2988 Map values using an input mapping or function.
2990 Parameters
2991 ----------
2992 mapper : function, dict, or Series
2993 Mapping correspondence.
2994 na_action : {None, 'ignore'}, default None
2995 If 'ignore', propagate NA values, without passing them to the
2996 mapping correspondence. If 'ignore' is not supported, a
2997 ``NotImplementedError`` should be raised.
2999 Returns
3000 -------
3001 Union[ndarray, Index, ExtensionArray]
3002 The output of the mapping function applied to the array.
3003 If the function returns a tuple with more than one element
3004 a MultiIndex will be returned.
3005 """
3006 result = map_array(self, mapper, na_action=na_action)
3007 if isinstance(result, np.ndarray):
3008 return self._cast_pointwise_result(result)
3009 return result
3011 # ------------------------------------------------------------------------
3012 # GroupBy Methods
3014 def _groupby_op(
3015 self,
3016 *,
3017 how: str,
3018 has_dropped_na: bool,
3019 min_count: int,
3020 ngroups: int,
3021 ids: npt.NDArray[np.intp],
3022 **kwargs,
3023 ) -> ArrayLike:
3024 """
3025 Dispatch GroupBy reduction or transformation operation.
3027 This is an *experimental* API to allow ExtensionArray authors to implement
3028 reductions and transformations. The API is subject to change.
3030 Parameters
3031 ----------
3032 how : {'any', 'all', 'sum', 'prod', 'min', 'max', 'mean', 'median',
3033 'median', 'var', 'std', 'sem', 'nth', 'last', 'ohlc',
3034 'cumprod', 'cumsum', 'cummin', 'cummax', 'rank'}
3035 has_dropped_na : bool
3036 min_count : int
3037 ngroups : int
3038 ids : np.ndarray[np.intp]
3039 ids[i] gives the integer label for the group that self[i] belongs to.
3040 **kwargs : operation-specific
3041 'any', 'all' -> ['skipna']
3042 'var', 'std', 'sem' -> ['ddof']
3043 'cumprod', 'cumsum', 'cummin', 'cummax' -> ['skipna']
3044 'rank' -> ['ties_method', 'ascending', 'na_option', 'pct']
3046 Returns
3047 -------
3048 np.ndarray or ExtensionArray
3049 """
3050 if how in ["first", "last"]:
3051 return self._groupby_first_last(
3052 how=how,
3053 min_count=min_count,
3054 ngroups=ngroups,
3055 ids=ids,
3056 skipna=kwargs.get("skipna", True),
3057 )
3059 from pandas.core.arrays.string_ import StringDtype
3060 from pandas.core.groupby.ops import WrappedCythonOp
3062 op = WrappedCythonOp(how=how, has_dropped_na=has_dropped_na)
3064 initial: Any = 0
3065 # GH#43682
3066 if isinstance(self.dtype, StringDtype):
3067 # StringArray
3068 if op.how in [
3069 "prod",
3070 "mean",
3071 "median",
3072 "cumsum",
3073 "cumprod",
3074 "std",
3075 "sem",
3076 "var",
3077 "skew",
3078 "kurt",
3079 ]:
3080 raise TypeError(
3081 f"dtype '{self.dtype}' does not support operation '{how}'"
3082 )
3083 if op.how not in ["any", "all"]:
3084 # Fail early to avoid conversion to object
3085 op._get_cython_function(op.kind, op.how, np.dtype(object), False)
3087 arr = self
3088 if op.how == "sum":
3089 initial = ""
3090 # https://github.com/pandas-dev/pandas/issues/60229
3091 # All NA should result in the empty string.
3092 assert "skipna" in kwargs
3093 if kwargs["skipna"] and min_count == 0:
3094 arr = arr.fillna("")
3095 npvalues = arr.to_numpy(object, na_value=np.nan)
3096 else:
3097 raise NotImplementedError(
3098 f"function is not implemented for this dtype: {self.dtype}"
3099 )
3101 res_values = op._cython_op_ndim_compat(
3102 npvalues,
3103 min_count=min_count,
3104 ngroups=ngroups,
3105 comp_ids=ids,
3106 mask=None,
3107 initial=initial,
3108 **kwargs,
3109 )
3111 if op.how in op.cast_blocklist:
3112 # i.e. how in ["rank"], since other cast_blocklist methods don't go
3113 # through cython_operation
3114 return res_values
3116 if isinstance(self.dtype, StringDtype):
3117 dtype = self.dtype
3118 string_array_cls = dtype.construct_array_type()
3119 return string_array_cls._from_sequence(res_values, dtype=dtype)
3121 else:
3122 raise NotImplementedError
3124 def _groupby_quantile(
3125 self,
3126 *,
3127 qs: npt.NDArray[np.float64],
3128 interpolation: Literal["linear", "lower", "higher", "nearest", "midpoint"],
3129 ids: npt.NDArray[np.intp],
3130 ngroups: int,
3131 starts: npt.NDArray[np.int64],
3132 ends: npt.NDArray[np.int64],
3133 ) -> ArrayLike:
3134 """
3135 Dispatch GroupBy quantile operation.
3137 Parameters
3138 ----------
3139 qs : np.ndarray[float64]
3140 Quantile(s) to compute.
3141 interpolation : {'linear', 'lower', 'higher', 'nearest', 'midpoint'}
3142 ids : np.ndarray[np.intp]
3143 Group labels.
3144 ngroups : int
3145 starts : np.ndarray[int64]
3146 ends : np.ndarray[int64]
3148 Returns
3149 -------
3150 np.ndarray or ExtensionArray
3151 """
3152 from pandas.core.arrays.string_ import StringDtype
3154 if isinstance(self.dtype, StringDtype) or is_object_dtype(self.dtype):
3155 raise TypeError(
3156 f"dtype '{self.dtype}' does not support operation 'quantile'"
3157 )
3158 if is_bool_dtype(self.dtype):
3159 raise TypeError("Cannot use quantile with bool dtype")
3161 mask = np.asarray(isna(self))
3162 nqs = len(qs)
3164 if is_integer_dtype(self.dtype) or is_float_dtype(self.dtype):
3165 vals = self.to_numpy(dtype=float, na_value=np.nan)
3166 else:
3167 vals = np.asarray(self)
3169 inference: np.dtype | None = (
3170 np.dtype(np.int64) if is_integer_dtype(self.dtype) else None
3171 )
3173 out = np.empty((ngroups, nqs), dtype=np.float64)
3174 libgroupby.group_quantile(
3175 out,
3176 values=vals,
3177 mask=mask, # type: ignore[arg-type]
3178 labels=ids,
3179 qs=qs,
3180 interpolation=interpolation,
3181 starts=starts,
3182 ends=ends,
3183 result_mask=None,
3184 is_datetimelike=False,
3185 )
3186 out = out.ravel("K") # type: ignore[assignment]
3188 if inference is not None and not (
3189 is_integer_dtype(inference) and interpolation in {"linear", "midpoint"}
3190 ):
3191 out = out.astype(inference)
3193 return out
3195 def _groupby_first_last(
3196 self,
3197 *,
3198 how: str,
3199 min_count: int,
3200 ngroups: int,
3201 ids: npt.NDArray[np.intp],
3202 skipna: bool = True,
3203 ) -> Self:
3204 """
3205 Optimized implementation of groupby first/last for ExtensionArrays.
3207 Uses an index-based approach: computes the index of the first/last
3208 non-NA element per group, then gathers results via take(). This avoids
3209 any dtype conversion and works for all EA types.
3210 """
3211 if self.ndim != 1:
3212 raise NotImplementedError(
3213 "groupby first/last only supports 1D ExtensionArrays"
3214 )
3215 isna_mask = np.asarray(self.isna(), dtype=np.uint8)
3217 result_indices, result_mask = libgroupby.group_first_last_indexer(
3218 labels=ids,
3219 mask=isna_mask,
3220 ngroups=ngroups,
3221 skipna=skipna,
3222 is_last=(how == "last"),
3223 )
3225 # Apply min_count: require at least min_count non-NA observations.
3226 # For first/last, the natural minimum is 1 (need at least one value).
3227 if min_count > 1:
3228 nobs = np.zeros(ngroups, dtype=np.int64)
3229 non_na_indices = np.where((~isna_mask.view(bool)) & (ids >= 0))[0]
3230 np.add.at(nobs, ids[non_na_indices], 1)
3231 below = nobs < min_count
3232 result_mask[below] = 1
3233 result_indices[below] = -1
3235 # take() with allow_fill=True treats -1 as "fill with NA"
3236 return self.take(result_indices, allow_fill=True)
3239class ExtensionArrayNaResult(ExtensionArray):
3240 @overload
3241 def any(self, *, skipna: Literal[True] = ...) -> bool: ...
3243 @overload
3244 def any(self, *, skipna: bool) -> bool | NAType: ...
3246 def any(self, *, skipna: bool = True) -> bool | NAType:
3247 raise AbstractMethodError(self)
3249 @overload
3250 def all(self, *, skipna: Literal[True] = ...) -> bool: ...
3252 @overload
3253 def all(self, *, skipna: bool) -> bool | NAType: ...
3255 def all(self, *, skipna: bool = True) -> bool | NAType:
3256 raise AbstractMethodError(self)
3258 def sum(
3259 self,
3260 *,
3261 axis: AxisInt | None = None,
3262 skipna: bool = True,
3263 min_count: int = 0,
3264 **kwargs,
3265 ):
3266 raise AbstractMethodError(self)
3268 def __invert__(self) -> Self:
3269 raise AbstractMethodError(self)
3272class ExtensionOpsMixin:
3273 """
3274 A base class for linking the operators to their dunder names.
3276 .. note::
3278 You may want to set ``__array_priority__`` if you want your
3279 implementation to be called when involved in binary operations
3280 with NumPy arrays.
3281 """
3283 @classmethod
3284 def _create_arithmetic_method(cls, op):
3285 raise AbstractMethodError(cls)
3287 @classmethod
3288 def _add_arithmetic_ops(cls) -> None:
3289 setattr(cls, "__add__", cls._create_arithmetic_method(operator.add))
3290 setattr(cls, "__radd__", cls._create_arithmetic_method(roperator.radd))
3291 setattr(cls, "__sub__", cls._create_arithmetic_method(operator.sub))
3292 setattr(cls, "__rsub__", cls._create_arithmetic_method(roperator.rsub))
3293 setattr(cls, "__mul__", cls._create_arithmetic_method(operator.mul))
3294 setattr(cls, "__rmul__", cls._create_arithmetic_method(roperator.rmul))
3295 setattr(cls, "__pow__", cls._create_arithmetic_method(operator.pow))
3296 setattr(cls, "__rpow__", cls._create_arithmetic_method(roperator.rpow))
3297 setattr(cls, "__mod__", cls._create_arithmetic_method(operator.mod))
3298 setattr(cls, "__rmod__", cls._create_arithmetic_method(roperator.rmod))
3299 setattr(cls, "__floordiv__", cls._create_arithmetic_method(operator.floordiv))
3300 setattr(
3301 cls, "__rfloordiv__", cls._create_arithmetic_method(roperator.rfloordiv)
3302 )
3303 setattr(cls, "__truediv__", cls._create_arithmetic_method(operator.truediv))
3304 setattr(cls, "__rtruediv__", cls._create_arithmetic_method(roperator.rtruediv))
3305 setattr(cls, "__divmod__", cls._create_arithmetic_method(divmod))
3306 setattr(cls, "__rdivmod__", cls._create_arithmetic_method(roperator.rdivmod))
3308 @classmethod
3309 def _create_comparison_method(cls, op):
3310 raise AbstractMethodError(cls)
3312 @classmethod
3313 def _add_comparison_ops(cls) -> None:
3314 setattr(cls, "__eq__", cls._create_comparison_method(operator.eq))
3315 setattr(cls, "__ne__", cls._create_comparison_method(operator.ne))
3316 setattr(cls, "__lt__", cls._create_comparison_method(operator.lt))
3317 setattr(cls, "__gt__", cls._create_comparison_method(operator.gt))
3318 setattr(cls, "__le__", cls._create_comparison_method(operator.le))
3319 setattr(cls, "__ge__", cls._create_comparison_method(operator.ge))
3321 @classmethod
3322 def _create_logical_method(cls, op):
3323 raise AbstractMethodError(cls)
3325 @classmethod
3326 def _add_logical_ops(cls) -> None:
3327 setattr(cls, "__and__", cls._create_logical_method(operator.and_))
3328 setattr(cls, "__rand__", cls._create_logical_method(roperator.rand_))
3329 setattr(cls, "__or__", cls._create_logical_method(operator.or_))
3330 setattr(cls, "__ror__", cls._create_logical_method(roperator.ror_))
3331 setattr(cls, "__xor__", cls._create_logical_method(operator.xor))
3332 setattr(cls, "__rxor__", cls._create_logical_method(roperator.rxor))
3335@set_module("pandas.api.extensions")
3336class ExtensionScalarOpsMixin(ExtensionOpsMixin):
3337 """
3338 A mixin for defining ops on an ExtensionArray.
3340 It is assumed that the underlying scalar objects have the operators
3341 already defined.
3343 Notes
3344 -----
3345 If you have defined a subclass MyExtensionArray(ExtensionArray), then
3346 use MyExtensionArray(ExtensionArray, ExtensionScalarOpsMixin) to
3347 get the arithmetic operators. After the definition of MyExtensionArray,
3348 insert the lines
3350 MyExtensionArray._add_arithmetic_ops()
3351 MyExtensionArray._add_comparison_ops()
3353 to link the operators to your class.
3355 .. note::
3357 You may want to set ``__array_priority__`` if you want your
3358 implementation to be called when involved in binary operations
3359 with NumPy arrays.
3360 """
3362 @classmethod
3363 def _create_method(cls, op, coerce_to_dtype: bool = True, result_dtype=None):
3364 """
3365 A class method that returns a method that will correspond to an
3366 operator for an ExtensionArray subclass, by dispatching to the
3367 relevant operator defined on the individual elements of the
3368 ExtensionArray.
3370 Parameters
3371 ----------
3372 op : function
3373 An operator that takes arguments op(a, b)
3374 coerce_to_dtype : bool, default True
3375 boolean indicating whether to attempt to convert
3376 the result to the underlying ExtensionArray dtype.
3377 If it's not possible to create a new ExtensionArray with the
3378 values, an ndarray is returned instead.
3380 Returns
3381 -------
3382 Callable[[Any, Any], Union[ndarray, ExtensionArray]]
3383 A method that can be bound to a class. When used, the method
3384 receives the two arguments, one of which is the instance of
3385 this class, and should return an ExtensionArray or an ndarray.
3387 Returning an ndarray may be necessary when the result of the
3388 `op` cannot be stored in the ExtensionArray. The dtype of the
3389 ndarray uses NumPy's normal inference rules.
3391 Examples
3392 --------
3393 Given an ExtensionArray subclass called MyExtensionArray, use
3395 __add__ = cls._create_method(operator.add)
3397 in the class definition of MyExtensionArray to create the operator
3398 for addition, that will be based on the operator implementation
3399 of the underlying elements of the ExtensionArray
3400 """
3402 def _binop(self, other):
3403 def convert_values(param):
3404 if isinstance(param, ExtensionArray) or is_list_like(param):
3405 ovalues = param
3406 else: # Assume its an object
3407 ovalues = [param] * len(self)
3408 return ovalues
3410 if isinstance(other, (ABCSeries, ABCIndex, ABCDataFrame)):
3411 # rely on pandas to unbox and dispatch to us
3412 return NotImplemented
3414 lvalues = self
3415 rvalues = convert_values(other)
3417 # If the operator is not defined for the underlying objects,
3418 # a TypeError should be raised
3419 res = [op(a, b) for (a, b) in zip(lvalues, rvalues, strict=True)]
3421 def _maybe_convert(arr):
3422 if coerce_to_dtype:
3423 # https://github.com/pandas-dev/pandas/issues/22850
3424 # We catch all regular exceptions here, and fall back
3425 # to an ndarray.
3426 res = self._cast_pointwise_result(arr)
3427 if not isinstance(res, type(self)):
3428 # exception raised in _from_sequence; ensure we have ndarray
3429 res = np.asarray(arr)
3430 else:
3431 res = np.asarray(arr, dtype=result_dtype)
3432 return res
3434 if op.__name__ in {"divmod", "rdivmod"}:
3435 a, b = zip(*res, strict=True)
3436 return _maybe_convert(a), _maybe_convert(b)
3438 return _maybe_convert(res)
3440 op_name = f"__{op.__name__}__"
3441 return set_function_name(_binop, op_name, cls)
3443 @classmethod
3444 def _create_arithmetic_method(cls, op):
3445 return cls._create_method(op)
3447 @classmethod
3448 def _create_comparison_method(cls, op):
3449 return cls._create_method(op, coerce_to_dtype=False, result_dtype=bool)