Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2An interface for extending pandas with custom arrays.
4.. warning::
6 This is an experimental API and subject to breaking changes
7 without warning.
8"""
10from __future__ import annotations
12import operator
13from typing import (
14 TYPE_CHECKING,
15 Any,
16 ClassVar,
17 Literal,
18 Self,
19 cast,
20 overload,
21)
22import warnings
24import numpy as np
26from pandas._libs import (
27 algos as libalgos,
28 lib,
29)
30from pandas.compat import set_function_name
31from pandas.compat.numpy import function as nv
32from pandas.errors import AbstractMethodError
33from pandas.util._decorators import (
34 cache_readonly,
35 set_module,
36)
37from pandas.util._exceptions import find_stack_level
38from pandas.util._validators import (
39 validate_bool_kwarg,
40 validate_insert_loc,
41)
43from pandas.core.dtypes.astype import astype_is_view
44from pandas.core.dtypes.common import (
45 is_integer,
46 is_list_like,
47 is_scalar,
48 pandas_dtype,
49)
50from pandas.core.dtypes.dtypes import ExtensionDtype
51from pandas.core.dtypes.generic import (
52 ABCDataFrame,
53 ABCIndex,
54 ABCSeries,
55)
56from pandas.core.dtypes.missing import isna
58from pandas.core import (
59 arraylike,
60 missing,
61 roperator,
62)
63from pandas.core.algorithms import (
64 duplicated,
65 factorize_array,
66 isin,
67 map_array,
68 mode,
69 rank,
70 unique,
71)
72from pandas.core.array_algos.quantile import quantile_with_mask
73from pandas.core.missing import _fill_limit_area_1d
74from pandas.core.sorting import (
75 nargminmax,
76 nargsort,
77)
79if TYPE_CHECKING:
80 from collections.abc import (
81 Callable,
82 Iterator,
83 Sequence,
84 )
86 from pandas._libs.missing import NAType
87 from pandas._typing import (
88 ArrayLike,
89 AstypeArg,
90 AxisInt,
91 Dtype,
92 DtypeObj,
93 FillnaOptions,
94 InterpolateOptions,
95 NumpySorter,
96 NumpyValueArrayLike,
97 PositionalIndexer,
98 ScalarIndexer,
99 SequenceIndexer,
100 Shape,
101 SortKind,
102 TakeIndexer,
103 npt,
104 )
106 from pandas import (
107 Index,
108 Series,
109 )
111_extension_array_shared_docs: dict[str, str] = {}
114@set_module("pandas.api.extensions")
115class ExtensionArray:
116 """
117 Abstract base class for custom 1-D array types.
119 pandas will recognize instances of this class as proper arrays
120 with a custom type and will not attempt to coerce them to objects. They
121 may be stored directly inside a :class:`DataFrame` or :class:`Series`.
123 Attributes
124 ----------
125 dtype
126 nbytes
127 ndim
128 shape
130 Methods
131 -------
132 argsort
133 astype
134 copy
135 dropna
136 duplicated
137 factorize
138 fillna
139 equals
140 insert
141 interpolate
142 isin
143 isna
144 item
145 ravel
146 repeat
147 searchsorted
148 shift
149 take
150 tolist
151 unique
152 view
153 _accumulate
154 _concat_same_type
155 _explode
156 _formatter
157 _from_factorized
158 _from_sequence
159 _from_sequence_of_strings
160 _hash_pandas_object
161 _pad_or_backfill
162 _reduce
163 _values_for_argsort
164 _values_for_factorize
166 See Also
167 --------
168 api.extensions.ExtensionDtype : A custom data type, to be paired with an
169 ExtensionArray.
170 api.extensions.ExtensionArray.dtype : An instance of ExtensionDtype.
172 Notes
173 -----
174 The interface includes the following abstract methods that must be
175 implemented by subclasses:
177 * _from_sequence
178 * _from_factorized
179 * __getitem__
180 * __len__
181 * __eq__
182 * dtype
183 * nbytes
184 * isna
185 * take
186 * copy
187 * _concat_same_type
188 * interpolate
190 A default repr displaying the type, (truncated) data, length,
191 and dtype is provided. It can be customized or replaced by
192 by overriding:
194 * __repr__ : A default repr for the ExtensionArray.
195 * _formatter : Print scalars inside a Series or DataFrame.
197 Some methods require casting the ExtensionArray to an ndarray of Python
198 objects with ``self.astype(object)``, which may be expensive. When
199 performance is a concern, we highly recommend overriding the following
200 methods:
202 * fillna
203 * _pad_or_backfill
204 * dropna
205 * unique
206 * factorize / _values_for_factorize
207 * argsort, argmax, argmin / _values_for_argsort
208 * searchsorted
209 * map
211 The remaining methods implemented on this class should be performant,
212 as they only compose abstract methods. Still, a more efficient
213 implementation may be available, and these methods can be overridden.
215 One can implement methods to handle array accumulations or reductions.
217 * _accumulate
218 * _reduce
220 One can implement methods to handle parsing from strings that will be used
221 in methods such as ``pandas.io.parsers.read_csv``.
223 * _from_sequence_of_strings
225 This class does not inherit from 'abc.ABCMeta' for performance reasons.
226 Methods and properties required by the interface raise
227 ``pandas.errors.AbstractMethodError`` and no ``register`` method is
228 provided for registering virtual subclasses.
230 ExtensionArrays are limited to 1 dimension.
232 They may be backed by none, one, or many NumPy arrays. For example,
233 ``pandas.Categorical`` is an extension array backed by two arrays,
234 one for codes and one for categories. An array of IPv6 address may
235 be backed by a NumPy structured array with two fields, one for the
236 lower 64 bits and one for the upper 64 bits. Or they may be backed
237 by some other storage type, like Python lists. Pandas makes no
238 assumptions on how the data are stored, just that it can be converted
239 to a NumPy array.
240 The ExtensionArray interface does not impose any rules on how this data
241 is stored. However, currently, the backing data cannot be stored in
242 attributes called ``.values`` or ``._values`` to ensure full compatibility
243 with pandas internals. But other names as ``.data``, ``._data``,
244 ``._items``, ... can be freely used.
246 If implementing NumPy's ``__array_ufunc__`` interface, pandas expects
247 that
249 1. You defer by returning ``NotImplemented`` when any Series are present
250 in `inputs`. Pandas will extract the arrays and call the ufunc again.
251 2. You define a ``_HANDLED_TYPES`` tuple as an attribute on the class.
252 Pandas inspect this to determine whether the ufunc is valid for the
253 types present.
255 See :ref:`extending.extension.ufunc` for more.
257 By default, ExtensionArrays are not hashable. Immutable subclasses may
258 override this behavior.
260 Examples
261 --------
262 Please see the following:
264 https://github.com/pandas-dev/pandas/blob/main/pandas/tests/extension/list/array.py
265 """
267 # '_typ' is for pandas.core.dtypes.generic.ABCExtensionArray.
268 # Don't override this.
269 _typ = "extension"
271 # similar to __array_priority__, positions ExtensionArray after Index,
272 # Series, and DataFrame. EA subclasses may override to choose which EA
273 # subclass takes priority. If overriding, the value should always be
274 # strictly less than 2000 to be below Index.__pandas_priority__.
275 __pandas_priority__ = 1000
277 _readonly = False
279 # ------------------------------------------------------------------------
280 # Constructors
281 # ------------------------------------------------------------------------
283 @classmethod
284 def _from_sequence(
285 cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
286 ) -> Self:
287 """
288 Construct a new ExtensionArray from a sequence of scalars.
290 Parameters
291 ----------
292 scalars : Sequence
293 Each element will be an instance of the scalar type for this
294 array, ``cls.dtype.type`` or be converted into this type in this method.
295 dtype : dtype, optional
296 Construct for this particular dtype. This should be a Dtype
297 compatible with the ExtensionArray.
298 copy : bool, default False
299 If True, copy the underlying data.
301 Returns
302 -------
303 ExtensionArray
305 See Also
306 --------
307 api.extensions.ExtensionArray._from_sequence_of_strings : Construct a new
308 ExtensionArray from a sequence of strings.
309 api.extensions.ExtensionArray._hash_pandas_object : Hook for
310 hash_pandas_object.
312 Examples
313 --------
314 >>> pd.arrays.IntegerArray._from_sequence([4, 5])
315 <IntegerArray>
316 [4, 5]
317 Length: 2, dtype: Int64
318 """
319 raise AbstractMethodError(cls)
321 @classmethod
322 def _from_sequence_of_strings(
323 cls, strings, *, dtype: ExtensionDtype, copy: bool = False
324 ) -> Self:
325 """
326 Construct a new ExtensionArray from a sequence of strings.
328 Parameters
329 ----------
330 strings : Sequence
331 Each element will be an instance of the scalar type for this
332 array, ``cls.dtype.type``.
333 dtype : ExtensionDtype
334 Construct for this particular dtype. This should be a Dtype
335 compatible with the ExtensionArray.
336 copy : bool, default False
337 If True, copy the underlying data.
339 Returns
340 -------
341 ExtensionArray
343 See Also
344 --------
345 api.extensions.ExtensionArray._from_sequence : Construct a new ExtensionArray
346 from a sequence of scalars.
347 api.extensions.ExtensionArray._from_factorized : Reconstruct an ExtensionArray
348 after factorization.
350 Examples
351 --------
352 >>> pd.arrays.IntegerArray._from_sequence_of_strings(
353 ... ["1", "2", "3"], dtype=pd.Int64Dtype()
354 ... )
355 <IntegerArray>
356 [1, 2, 3]
357 Length: 3, dtype: Int64
358 """
359 raise AbstractMethodError(cls)
361 @classmethod
362 def _from_factorized(cls, values, original):
363 """
364 Reconstruct an ExtensionArray after factorization.
366 Parameters
367 ----------
368 values : ndarray
369 An integer ndarray with the factorized values.
370 original : ExtensionArray
371 The original ExtensionArray that factorize was called on.
373 See Also
374 --------
375 factorize : Top-level factorize method that dispatches here.
376 ExtensionArray.factorize : Encode the extension array as an enumerated type.
378 Examples
379 --------
380 >>> interv_arr = pd.arrays.IntervalArray(
381 ... [pd.Interval(0, 1), pd.Interval(1, 5), pd.Interval(1, 5)]
382 ... )
383 >>> codes, uniques = pd.factorize(interv_arr)
384 >>> pd.arrays.IntervalArray._from_factorized(uniques, interv_arr)
385 <IntervalArray>
386 [(0, 1], (1, 5]]
387 Length: 2, dtype: interval[int64, right]
388 """
389 raise AbstractMethodError(cls)
391 @classmethod
392 def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
393 """
394 Strict analogue to _from_sequence, allowing only sequences of scalars
395 that should be specifically inferred to the given dtype.
397 Parameters
398 ----------
399 scalars : sequence
400 dtype : ExtensionDtype
402 Raises
403 ------
404 TypeError or ValueError
406 Notes
407 -----
408 This is called in a try/except block when casting the result of a
409 pointwise operation in ExtensionArray._cast_pointwise_result.
410 """
411 try:
412 return cls._from_sequence(scalars, dtype=dtype, copy=False)
413 except (ValueError, TypeError):
414 raise
415 except Exception:
416 warnings.warn(
417 "_from_scalars should only raise ValueError or TypeError. "
418 "Consider overriding _from_scalars where appropriate.",
419 stacklevel=find_stack_level(),
420 )
421 raise
423 def _cast_pointwise_result(self, values) -> ArrayLike:
424 """
425 Construct an ExtensionArray after a pointwise operation.
427 Cast the result of a pointwise operation (e.g. Series.map) to an
428 array. This is not required to return an ExtensionArray of the same
429 type as self or of the same dtype. It can also return another
430 ExtensionArray of the same "family" if you implement multiple
431 ExtensionArrays/Dtypes that are interoperable (e.g. if you have float
432 array with units, this method can return an int array with units).
434 If converting to your own ExtensionArray is not possible, this method
435 falls back to returning an array with the default type inference.
436 If you only need to cast to `self.dtype`, it is recommended to override
437 `_from_scalars` instead of this method.
439 Parameters
440 ----------
441 values : sequence
443 Returns
444 -------
445 ExtensionArray or ndarray
446 """
447 try:
448 return type(self)._from_scalars(values, dtype=self.dtype)
449 except (ValueError, TypeError):
450 values = np.asarray(values, dtype=object)
451 return lib.maybe_convert_objects(values, convert_non_numeric=True)
453 # ------------------------------------------------------------------------
454 # Must be a Sequence
455 # ------------------------------------------------------------------------
456 @overload
457 def __getitem__(self, item: ScalarIndexer) -> Any: ...
459 @overload
460 def __getitem__(self, item: SequenceIndexer) -> Self: ...
462 def __getitem__(self, item: PositionalIndexer) -> Self | Any:
463 """
464 Select a subset of self.
466 Parameters
467 ----------
468 item : int, slice, or ndarray
469 * int: The position in 'self' to get.
471 * slice: A slice object, where 'start', 'stop', and 'step' are
472 integers or None
474 * ndarray: A 1-d boolean NumPy ndarray the same length as 'self'
476 * list[int]: A list of int
478 Returns
479 -------
480 item : scalar or ExtensionArray
482 Notes
483 -----
484 For scalar ``item``, return a scalar value suitable for the array's
485 type. This should be an instance of ``self.dtype.type``.
487 For slice ``key``, return an instance of ``ExtensionArray``, even
488 if the slice is length 0 or 1.
490 For a boolean mask, return an instance of ``ExtensionArray``, filtered
491 to the values where ``item`` is True.
492 """
493 raise AbstractMethodError(self)
495 def __setitem__(self, key, value) -> None:
496 """
497 Set one or more values inplace.
499 This method is not required to satisfy the pandas extension array
500 interface.
502 Parameters
503 ----------
504 key : int, ndarray, or slice
505 When called from, e.g. ``Series.__setitem__``, ``key`` will be
506 one of
508 * scalar int
509 * ndarray of integers.
510 * boolean ndarray
511 * slice object
513 value : ExtensionDtype.type, Sequence[ExtensionDtype.type], or object
514 value or values to be set of ``key``.
516 Returns
517 -------
518 None
520 Raises
521 ------
522 ValueError
523 If the array is readonly and modification is attempted.
524 """
525 # Some notes to the ExtensionArray implementer who may have ended up
526 # here. While this method is not required for the interface, if you
527 # *do* choose to implement __setitem__, then some semantics should be
528 # observed:
529 #
530 # * Setting multiple values : ExtensionArrays should support setting
531 # multiple values at once, 'key' will be a sequence of integers and
532 # 'value' will be a same-length sequence.
533 #
534 # * Broadcasting : For a sequence 'key' and a scalar 'value',
535 # each position in 'key' should be set to 'value'.
536 #
537 # * Coercion : Most users will expect basic coercion to work. For
538 # example, a string like '2018-01-01' is coerced to a datetime
539 # when setting on a datetime64ns array. In general, if the
540 # __init__ method coerces that value, then so should __setitem__
541 # Note, also, that Series/DataFrame.where internally use __setitem__
542 # on a copy of the data.
543 # Check if the array is readonly
544 if self._readonly:
545 raise ValueError("Cannot modify read-only array")
547 raise NotImplementedError(f"{type(self)} does not implement __setitem__.")
549 def __len__(self) -> int:
550 """
551 Length of this array
553 Returns
554 -------
555 length : int
556 """
557 raise AbstractMethodError(self)
559 def __iter__(self) -> Iterator[Any]:
560 """
561 Iterate over elements of the array.
562 """
563 # This needs to be implemented so that pandas recognizes extension
564 # arrays as list-like. The default implementation makes successive
565 # calls to ``__getitem__``, which may be slower than necessary.
566 for i in range(len(self)):
567 yield self[i]
569 def __contains__(self, item: object) -> bool | np.bool_:
570 """
571 Return for `item in self`.
572 """
573 # GH37867
574 # comparisons of any item to pd.NA always return pd.NA, so e.g. "a" in [pd.NA]
575 # would raise a TypeError. The implementation below works around that.
576 if is_scalar(item) and isna(item):
577 if not self._can_hold_na:
578 return False
579 elif item is self.dtype.na_value or isinstance(item, self.dtype.type):
580 return self._hasna
581 else:
582 return False
583 else:
584 # error: Item "ExtensionArray" of "Union[ExtensionArray, ndarray]" has no
585 # attribute "any"
586 return (item == self).any() # type: ignore[union-attr]
588 # error: Signature of "__eq__" incompatible with supertype "object"
589 def __eq__(self, other: object) -> ArrayLike: # type: ignore[override]
590 """
591 Return for `self == other` (element-wise equality).
592 """
593 # Implementer note: this should return a boolean numpy ndarray or
594 # a boolean ExtensionArray.
595 # When `other` is one of Series, Index, or DataFrame, this method should
596 # return NotImplemented (to ensure that those objects are responsible for
597 # first unpacking the arrays, and then dispatch the operation to the
598 # underlying arrays)
599 raise AbstractMethodError(self)
601 # error: Signature of "__ne__" incompatible with supertype "object"
602 def __ne__(self, other: object) -> ArrayLike: # type: ignore[override]
603 """
604 Return for `self != other` (element-wise in-equality).
605 """
606 # error: Unsupported operand type for ~ ("ExtensionArray")
607 return ~(self == other) # type: ignore[operator]
609 def item(self, index: int | None = None):
610 """
611 Return the array element at the specified position as a Python scalar.
613 Parameters
614 ----------
615 index : int, optional
616 Position of the element. If not provided, the array must contain
617 exactly one element.
619 Returns
620 -------
621 scalar
622 The element at the specified position.
624 Raises
625 ------
626 ValueError
627 If no index is provided and the array does not have exactly
628 one element.
629 IndexError
630 If the specified position is out of bounds.
632 See Also
633 --------
634 numpy.ndarray.item : Return the item of an array as a scalar.
636 Examples
637 --------
638 >>> arr = pd.array([1], dtype="Int64")
639 >>> arr.item()
640 np.int64(1)
642 >>> arr = pd.array([1, 2, 3], dtype="Int64")
643 >>> arr.item(0)
644 np.int64(1)
645 >>> arr.item(2)
646 np.int64(3)
647 """
648 if index is None:
649 if len(self) != 1:
650 raise ValueError(
651 "can only convert an array of size 1 to a Python scalar"
652 )
653 return self[0]
654 else:
655 if not is_integer(index):
656 raise TypeError(f"index must be an integer, got {type(index)}")
657 return self[index]
659 def to_numpy(
660 self,
661 dtype: npt.DTypeLike | None = None,
662 copy: bool = False,
663 na_value: object = lib.no_default,
664 ) -> np.ndarray:
665 """
666 Convert to a NumPy ndarray.
668 This is similar to :meth:`numpy.asarray`, but may provide additional control
669 over how the conversion is done.
671 Parameters
672 ----------
673 dtype : str or numpy.dtype, optional
674 The dtype to pass to :meth:`numpy.asarray`.
675 copy : bool, default False
676 Whether to ensure that the returned value is a not a view on
677 another array. Note that ``copy=False`` does not *ensure* that
678 ``to_numpy()`` is no-copy. Rather, ``copy=True`` ensure that
679 a copy is made, even if not strictly necessary.
680 na_value : Any, optional
681 The value to use for missing values. The default value depends
682 on `dtype` and the type of the array.
684 Returns
685 -------
686 numpy.ndarray
687 """
688 result = np.asarray(self, dtype=dtype)
689 if copy or na_value is not lib.no_default:
690 result = result.copy()
691 elif self._readonly and astype_is_view(self.dtype, result.dtype):
692 # If the ExtensionArray is readonly, make the numpy array readonly too
693 result = result.view()
694 result.flags.writeable = False
696 if na_value is not lib.no_default:
697 result[self.isna()] = na_value # type: ignore[index]
699 return result
701 # ------------------------------------------------------------------------
702 # Required attributes
703 # ------------------------------------------------------------------------
705 @property
706 def dtype(self) -> ExtensionDtype:
707 """
708 An instance of ExtensionDtype.
710 See Also
711 --------
712 api.extensions.ExtensionDtype : Base class for extension dtypes.
713 api.extensions.ExtensionArray : Base class for extension array types.
714 api.extensions.ExtensionArray.dtype : The dtype of an ExtensionArray.
715 Series.dtype : The dtype of a Series.
716 DataFrame.dtype : The dtype of a DataFrame.
718 Examples
719 --------
720 >>> pd.array([1, 2, 3]).dtype
721 Int64Dtype()
722 """
723 raise AbstractMethodError(self)
725 @property
726 def shape(self) -> Shape:
727 """
728 Return a tuple of the array dimensions.
730 See Also
731 --------
732 numpy.ndarray.shape : Similar attribute which returns the shape of an array.
733 DataFrame.shape : Return a tuple representing the dimensionality of the
734 DataFrame.
735 Series.shape : Return a tuple representing the dimensionality of the Series.
737 Examples
738 --------
739 >>> arr = pd.array([1, 2, 3])
740 >>> arr.shape
741 (3,)
742 """
743 return (len(self),)
745 @property
746 def size(self) -> int:
747 """
748 The number of elements in the array.
749 """
750 # error: Incompatible return value type (got "signedinteger[_64Bit]",
751 # expected "int") [return-value]
752 return np.prod(self.shape) # type: ignore[return-value]
754 @property
755 def ndim(self) -> int:
756 """
757 Extension Arrays are only allowed to be 1-dimensional.
759 See Also
760 --------
761 ExtensionArray.shape: Return a tuple of the array dimensions.
762 ExtensionArray.size: The number of elements in the array.
764 Examples
765 --------
766 >>> arr = pd.array([1, 2, 3])
767 >>> arr.ndim
768 1
769 """
770 return 1
772 @property
773 def nbytes(self) -> int:
774 """
775 The number of bytes needed to store this object in memory.
777 See Also
778 --------
779 ExtensionArray.shape: Return a tuple of the array dimensions.
780 ExtensionArray.size: The number of elements in the array.
782 Examples
783 --------
784 >>> pd.array([1, 2, 3]).nbytes
785 27
786 """
787 # If this is expensive to compute, return an approximate lower bound
788 # on the number of bytes needed.
789 raise AbstractMethodError(self)
791 # ------------------------------------------------------------------------
792 # Additional Methods
793 # ------------------------------------------------------------------------
795 @overload
796 def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
798 @overload
799 def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
801 @overload
802 def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
804 def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
805 """
806 Cast to a NumPy array or ExtensionArray with 'dtype'.
808 Parameters
809 ----------
810 dtype : str or dtype
811 Typecode or data-type to which the array is cast.
812 copy : bool, default True
813 Whether to copy the data, even if not necessary. If False,
814 a copy is made only if the old dtype does not match the
815 new dtype.
817 Returns
818 -------
819 np.ndarray or pandas.api.extensions.ExtensionArray
820 An ``ExtensionArray`` if ``dtype`` is ``ExtensionDtype``,
821 otherwise a Numpy ndarray with ``dtype`` for its dtype.
823 See Also
824 --------
825 Series.astype : Cast a Series to a different dtype.
826 DataFrame.astype : Cast a DataFrame to a different dtype.
827 api.extensions.ExtensionArray : Base class for ExtensionArray objects.
828 core.arrays.DatetimeArray._from_sequence : Create a DatetimeArray from a
829 sequence.
830 core.arrays.TimedeltaArray._from_sequence : Create a TimedeltaArray from
831 a sequence.
833 Examples
834 --------
835 >>> arr = pd.array([1, 2, 3])
836 >>> arr
837 <IntegerArray>
838 [1, 2, 3]
839 Length: 3, dtype: Int64
841 Casting to another ``ExtensionDtype`` returns an ``ExtensionArray``:
843 >>> arr1 = arr.astype("Float64")
844 >>> arr1
845 <FloatingArray>
846 [1.0, 2.0, 3.0]
847 Length: 3, dtype: Float64
848 >>> arr1.dtype
849 Float64Dtype()
851 Otherwise, we will get a Numpy ndarray:
853 >>> arr2 = arr.astype("float64")
854 >>> arr2
855 array([1., 2., 3.])
856 >>> arr2.dtype
857 dtype('float64')
858 """
859 dtype = pandas_dtype(dtype)
860 if dtype == self.dtype:
861 if not copy:
862 return self
863 else:
864 return self.copy()
866 if isinstance(dtype, ExtensionDtype):
867 cls = dtype.construct_array_type()
868 return cls._from_sequence(self, dtype=dtype, copy=copy)
870 elif lib.is_np_dtype(dtype, "M"):
871 from pandas.core.arrays import DatetimeArray
873 return DatetimeArray._from_sequence(self, dtype=dtype, copy=copy)
875 elif lib.is_np_dtype(dtype, "m"):
876 from pandas.core.arrays import TimedeltaArray
878 return TimedeltaArray._from_sequence(self, dtype=dtype, copy=copy)
880 if not copy:
881 return np.asarray(self, dtype=dtype)
882 else:
883 return np.array(self, dtype=dtype, copy=copy)
885 def isna(self) -> np.ndarray | ExtensionArraySupportsAnyAll:
886 """
887 A 1-D array indicating if each value is missing.
889 Returns
890 -------
891 numpy.ndarray or pandas.api.extensions.ExtensionArray
892 In most cases, this should return a NumPy ndarray. For
893 exceptional cases like ``SparseArray``, where returning
894 an ndarray would be expensive, an ExtensionArray may be
895 returned.
897 See Also
898 --------
899 ExtensionArray.dropna: Return ExtensionArray without NA values.
900 ExtensionArray.fillna: Fill NA/NaN values using the specified method.
902 Notes
903 -----
904 If returning an ExtensionArray, then
906 * ``na_values._is_boolean`` should be True
907 * ``na_values`` should implement :func:`ExtensionArray._reduce`
908 * ``na_values`` should implement :func:`ExtensionArray._accumulate`
909 * ``na_values.any`` and ``na_values.all`` should be implemented
911 Examples
912 --------
913 >>> arr = pd.array([1, 2, np.nan, np.nan])
914 >>> arr.isna()
915 array([False, False, True, True])
916 """
917 raise AbstractMethodError(self)
919 @property
920 def _hasna(self) -> bool:
921 # GH#22680
922 """
923 Equivalent to `self.isna().any()`.
925 Some ExtensionArray subclasses may be able to optimize this check.
926 """
927 return bool(self.isna().any())
929 def _values_for_argsort(self) -> np.ndarray:
930 """
931 Return values for sorting.
933 Returns
934 -------
935 ndarray
936 The transformed values should maintain the ordering between values
937 within the array.
939 See Also
940 --------
941 ExtensionArray.argsort : Return the indices that would sort this array.
943 Notes
944 -----
945 The caller is responsible for *not* modifying these values in-place, so
946 it is safe for implementers to give views on ``self``.
948 Functions that use this (e.g. ``ExtensionArray.argsort``) should ignore
949 entries with missing values in the original array (according to
950 ``self.isna()``). This means that the corresponding entries in the returned
951 array don't need to be modified to sort correctly.
953 Examples
954 --------
955 In most cases, this is the underlying Numpy array of the ``ExtensionArray``:
957 >>> arr = pd.array([1, 2, 3])
958 >>> arr._values_for_argsort()
959 array([1, 2, 3])
960 """
961 # Note: this is used in `ExtensionArray.argsort/argmin/argmax`.
962 return np.array(self)
964 def argsort(
965 self,
966 *,
967 ascending: bool = True,
968 kind: SortKind = "quicksort",
969 na_position: str = "last",
970 **kwargs,
971 ) -> np.ndarray:
972 """
973 Return the indices that would sort this array.
975 Parameters
976 ----------
977 ascending : bool, default True
978 Whether the indices should result in an ascending
979 or descending sort.
980 kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional
981 Sorting algorithm.
982 na_position : {'first', 'last'}, default 'last'
983 If ``'first'``, put ``NaN`` values at the beginning.
984 If ``'last'``, put ``NaN`` values at the end.
985 **kwargs
986 Passed through to :func:`numpy.argsort`.
988 Returns
989 -------
990 np.ndarray[np.intp]
991 Array of indices that sort ``self``. If NaN values are contained,
992 NaN values are placed at the end.
994 See Also
995 --------
996 numpy.argsort : Sorting implementation used internally.
998 Examples
999 --------
1000 >>> arr = pd.array([3, 1, 2, 5, 4])
1001 >>> arr.argsort()
1002 array([1, 2, 0, 4, 3])
1003 """
1004 # Implementer note: You have two places to override the behavior of
1005 # argsort.
1006 # 1. _values_for_argsort : construct the values passed to np.argsort
1007 # 2. argsort : total control over sorting. In case of overriding this,
1008 # it is recommended to also override argmax/argmin
1009 ascending = nv.validate_argsort_with_ascending(ascending, (), kwargs)
1011 values = self._values_for_argsort()
1012 return nargsort(
1013 values,
1014 kind=kind,
1015 ascending=ascending,
1016 na_position=na_position,
1017 mask=np.asarray(self.isna()),
1018 )
1020 def argmin(self, skipna: bool = True) -> int:
1021 """
1022 Return the index of minimum value.
1024 In case of multiple occurrences of the minimum value, the index
1025 corresponding to the first occurrence is returned.
1027 Parameters
1028 ----------
1029 skipna : bool, default True
1031 Returns
1032 -------
1033 int
1035 See Also
1036 --------
1037 ExtensionArray.argmax : Return the index of the maximum value.
1039 Examples
1040 --------
1041 >>> arr = pd.array([3, 1, 2, 5, 4])
1042 >>> arr.argmin()
1043 np.int64(1)
1044 """
1045 # Implementer note: You have two places to override the behavior of
1046 # argmin.
1047 # 1. _values_for_argsort : construct the values used in nargminmax
1048 # 2. argmin itself : total control over sorting.
1049 validate_bool_kwarg(skipna, "skipna")
1050 if not skipna and self._hasna:
1051 raise ValueError("Encountered an NA value with skipna=False")
1052 return nargminmax(self, "argmin")
1054 def argmax(self, skipna: bool = True) -> int:
1055 """
1056 Return the index of maximum value.
1058 In case of multiple occurrences of the maximum value, the index
1059 corresponding to the first occurrence is returned.
1061 Parameters
1062 ----------
1063 skipna : bool, default True
1065 Returns
1066 -------
1067 int
1069 See Also
1070 --------
1071 ExtensionArray.argmin : Return the index of the minimum value.
1073 Examples
1074 --------
1075 >>> arr = pd.array([3, 1, 2, 5, 4])
1076 >>> arr.argmax()
1077 np.int64(3)
1078 """
1079 # Implementer note: You have two places to override the behavior of
1080 # argmax.
1081 # 1. _values_for_argsort : construct the values used in nargminmax
1082 # 2. argmax itself : total control over sorting.
1083 validate_bool_kwarg(skipna, "skipna")
1084 if not skipna and self._hasna:
1085 raise ValueError("Encountered an NA value with skipna=False")
1086 return nargminmax(self, "argmax")
1088 def interpolate(
1089 self,
1090 *,
1091 method: InterpolateOptions,
1092 axis: int,
1093 index: Index,
1094 limit,
1095 limit_direction,
1096 limit_area,
1097 copy: bool,
1098 **kwargs,
1099 ) -> Self:
1100 """
1101 Fill NaN values using an interpolation method.
1103 Parameters
1104 ----------
1105 method : str, default 'linear'
1106 Interpolation technique to use. One of:
1107 * 'linear': Ignore the index and treat the values as equally spaced.
1108 This is the only method supported on MultiIndexes.
1109 * 'time': Works on daily and higher resolution data to interpolate
1110 given length of interval.
1111 * 'index', 'values': use the actual numerical values of the index.
1112 * 'nearest', 'zero', 'slinear', 'quadratic', 'cubic', 'barycentric',
1113 'polynomial': Passed to scipy.interpolate.interp1d, whereas 'spline'
1114 is passed to scipy.interpolate.UnivariateSpline. These methods use
1115 the numerical values of the index.
1116 Both 'polynomial' and 'spline' require that you also specify an
1117 order (int), e.g. arr.interpolate(method='polynomial', order=5).
1118 * 'krogh', 'piecewise_polynomial', 'spline', 'pchip', 'akima',
1119 'cubicspline': Wrappers around the SciPy interpolation methods
1120 of similar names. See Notes.
1121 * 'from_derivatives': Refers to scipy.interpolate.BPoly.from_derivatives.
1122 axis : int
1123 Axis to interpolate along. For 1-dimensional data, use 0.
1124 index : Index
1125 Index to use for interpolation.
1126 limit : int or None
1127 Maximum number of consecutive NaNs to fill. Must be greater than 0.
1128 limit_direction : {'forward', 'backward', 'both'}
1129 Consecutive NaNs will be filled in this direction.
1130 limit_area : {'inside', 'outside'} or None
1131 If limit is specified, consecutive NaNs will be filled with this
1132 restriction.
1133 * None: No fill restriction.
1134 * 'inside': Only fill NaNs surrounded by valid values (interpolate).
1135 * 'outside': Only fill NaNs outside valid values (extrapolate).
1136 copy : bool
1137 If True, a copy of the object is returned with interpolated values.
1138 **kwargs : optional
1139 Keyword arguments to pass on to the interpolating function.
1141 Returns
1142 -------
1143 ExtensionArray
1144 An ExtensionArray with interpolated values.
1146 See Also
1147 --------
1148 Series.interpolate : Interpolate values in a Series.
1149 DataFrame.interpolate : Interpolate values in a DataFrame.
1151 Notes
1152 -----
1153 - All parameters must be specified as keyword arguments.
1154 - The 'krogh', 'piecewise_polynomial', 'spline', 'pchip' and 'akima'
1155 methods are wrappers around the respective SciPy implementations of
1156 similar names. These use the actual numerical values of the index.
1158 Examples
1159 --------
1160 Interpolating values in a NumPy array:
1162 >>> arr = pd.arrays.NumpyExtensionArray(np.array([0, 1, np.nan, 3]))
1163 >>> arr.interpolate(
1164 ... method="linear",
1165 ... limit=3,
1166 ... limit_direction="forward",
1167 ... index=pd.Index(range(len(arr))),
1168 ... fill_value=1,
1169 ... copy=False,
1170 ... axis=0,
1171 ... limit_area="inside",
1172 ... )
1173 <NumpyExtensionArray>
1174 [0.0, 1.0, 2.0, 3.0]
1175 Length: 4, dtype: float64
1177 Interpolating values in a FloatingArray:
1179 >>> arr = pd.array([1.0, pd.NA, 3.0, 4.0, pd.NA, 6.0], dtype="Float64")
1180 >>> arr.interpolate(
1181 ... method="linear",
1182 ... axis=0,
1183 ... index=pd.Index(range(len(arr))),
1184 ... limit=None,
1185 ... limit_direction="both",
1186 ... limit_area=None,
1187 ... copy=True,
1188 ... )
1189 <FloatingArray>
1190 [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
1191 Length: 6, dtype: Float64
1192 """
1193 # NB: we return type(self) even if copy=False
1194 raise NotImplementedError(
1195 f"{type(self).__name__} does not implement interpolate"
1196 )
1198 def _pad_or_backfill(
1199 self,
1200 *,
1201 method: FillnaOptions,
1202 limit: int | None = None,
1203 limit_area: Literal["inside", "outside"] | None = None,
1204 copy: bool = True,
1205 ) -> Self:
1206 """
1207 Pad or backfill values, used by Series/DataFrame ffill and bfill.
1209 Parameters
1210 ----------
1211 method : {'backfill', 'bfill', 'pad', 'ffill'}
1212 Method to use for filling holes in reindexed Series:
1214 * pad / ffill: propagate last valid observation forward to next valid.
1215 * backfill / bfill: use NEXT valid observation to fill gap.
1217 limit : int, default None
1218 This is the maximum number of consecutive
1219 NaN values to forward/backward fill. In other words, if there is
1220 a gap with more than this number of consecutive NaNs, it will only
1221 be partially filled. If method is not specified, this is the
1222 maximum number of entries along the entire axis where NaNs will be
1223 filled.
1225 limit_area : {'inside', 'outside'} or None, default None
1226 Specifies which area to limit filling.
1227 - 'inside': Limit the filling to the area within the gaps.
1228 - 'outside': Limit the filling to the area outside the gaps.
1229 If `None`, no limitation is applied.
1231 copy : bool, default True
1232 Whether to make a copy of the data before filling. If False, then
1233 the original should be modified and no new memory should be allocated.
1234 For ExtensionArray subclasses that cannot do this, it is at the
1235 author's discretion whether to ignore "copy=False" or to raise.
1236 The base class implementation ignores the keyword if any NAs are
1237 present.
1239 Returns
1240 -------
1241 Same type as self
1242 The filled array with the same type as the original.
1244 See Also
1245 --------
1246 Series.ffill : Forward fill missing values.
1247 Series.bfill : Backward fill missing values.
1248 DataFrame.ffill : Forward fill missing values in DataFrame.
1249 DataFrame.bfill : Backward fill missing values in DataFrame.
1250 api.types.isna : Check for missing values.
1251 api.types.isnull : Check for missing values.
1253 Examples
1254 --------
1255 >>> arr = pd.array([np.nan, np.nan, 2, 3, np.nan, np.nan])
1256 >>> arr._pad_or_backfill(method="backfill", limit=1)
1257 <IntegerArray>
1258 [<NA>, 2, 2, 3, <NA>, <NA>]
1259 Length: 6, dtype: Int64
1260 """
1261 mask = self.isna()
1263 if mask.any():
1264 # NB: the base class does not respect the "copy" keyword
1265 meth = missing.clean_fill_method(method)
1267 npmask = np.asarray(mask)
1268 if limit_area is not None and not npmask.all():
1269 _fill_limit_area_1d(npmask, limit_area)
1270 if meth == "pad":
1271 indexer = libalgos.get_fill_indexer(npmask, limit=limit)
1272 return self.take(indexer, allow_fill=True)
1273 else:
1274 # i.e. meth == "backfill"
1275 indexer = libalgos.get_fill_indexer(npmask[::-1], limit=limit)[::-1]
1276 return self[::-1].take(indexer, allow_fill=True)
1278 else:
1279 if not copy:
1280 return self
1281 new_values = self.copy()
1282 return new_values
1284 def fillna(
1285 self,
1286 value: object | ArrayLike,
1287 limit: int | None = None,
1288 copy: bool = True,
1289 ) -> Self:
1290 """
1291 Fill NA/NaN values using the specified method.
1293 Parameters
1294 ----------
1295 value : scalar, array-like
1296 If a scalar value is passed it is used to fill all missing values.
1297 Alternatively, an array-like "value" can be given. It's expected
1298 that the array-like have the same length as 'self'.
1299 limit : int, default None
1300 The maximum number of entries where NA values will be filled.
1301 copy : bool, default True
1302 Whether to make a copy of the data before filling. If False, then
1303 the original should be modified and no new memory should be allocated.
1304 For ExtensionArray subclasses that cannot do this, it is at the
1305 author's discretion whether to ignore "copy=False" or to raise.
1307 Returns
1308 -------
1309 ExtensionArray
1310 With NA/NaN filled.
1312 See Also
1313 --------
1314 api.extensions.ExtensionArray.dropna : Return ExtensionArray without
1315 NA values.
1316 api.extensions.ExtensionArray.isna : A 1-D array indicating if
1317 each value is missing.
1319 Examples
1320 --------
1321 >>> arr = pd.array([np.nan, np.nan, 2, 3, np.nan, np.nan])
1322 >>> arr.fillna(0)
1323 <IntegerArray>
1324 [0, 0, 2, 3, 0, 0]
1325 Length: 6, dtype: Int64
1326 """
1327 mask = self.isna()
1328 if limit is not None and limit < len(self):
1329 # isna can return an ExtensionArray, we're assuming that comparisons
1330 # are implemented.
1331 # mypy doesn't like that mask can be an EA which need not have `cumsum`
1332 modify = mask.cumsum() > limit # type: ignore[union-attr]
1333 if modify.any():
1334 # Only copy mask if necessary
1335 mask = mask.copy()
1336 mask[modify] = False
1337 # error: Argument 2 to "check_value_size" has incompatible type
1338 # "ExtensionArray"; expected "ndarray"
1339 value = missing.check_value_size(
1340 value,
1341 mask, # type: ignore[arg-type]
1342 len(self),
1343 )
1345 if mask.any():
1346 # fill with value
1347 if not copy:
1348 new_values = self[:]
1349 else:
1350 new_values = self.copy()
1351 new_values[mask] = value
1352 elif not copy:
1353 new_values = self[:]
1354 else:
1355 new_values = self.copy()
1356 return new_values
1358 def dropna(self) -> Self:
1359 """
1360 Return ExtensionArray without NA values.
1362 Returns
1363 -------
1364 Self
1365 An ExtensionArray of the same type as the original but with all
1366 NA values removed.
1368 See Also
1369 --------
1370 Series.dropna : Remove missing values from a Series.
1371 DataFrame.dropna : Remove missing values from a DataFrame.
1372 api.extensions.ExtensionArray.isna : Check for missing values in
1373 an ExtensionArray.
1375 Examples
1376 --------
1377 >>> pd.array([1, 2, np.nan]).dropna()
1378 <IntegerArray>
1379 [1, 2]
1380 Length: 2, dtype: Int64
1381 """
1382 # error: Unsupported operand type for ~ ("ExtensionArray")
1383 return self[~self.isna()] # type: ignore[operator]
1385 def duplicated(
1386 self, keep: Literal["first", "last", False] = "first"
1387 ) -> npt.NDArray[np.bool_]:
1388 """
1389 Return boolean ndarray denoting duplicate values.
1391 Parameters
1392 ----------
1393 keep : {'first', 'last', False}, default 'first'
1394 - ``first`` : Mark duplicates as ``True`` except for the first occurrence.
1395 - ``last`` : Mark duplicates as ``True`` except for the last occurrence.
1396 - False : Mark all duplicates as ``True``.
1398 Returns
1399 -------
1400 ndarray[bool]
1401 With true in indices where elements are duplicated and false otherwise.
1403 See Also
1404 --------
1405 DataFrame.duplicated : Return boolean Series denoting
1406 duplicate rows.
1407 Series.duplicated : Indicate duplicate Series values.
1408 api.extensions.ExtensionArray.unique : Compute the ExtensionArray
1409 of unique values.
1411 Examples
1412 --------
1413 >>> pd.array([1, 1, 2, 3, 3], dtype="Int64").duplicated()
1414 array([False, True, False, False, True])
1415 """
1416 mask = self.isna().astype(np.bool_, copy=False)
1417 return duplicated(values=self, keep=keep, mask=mask)
1419 def shift(self, periods: int = 1, fill_value: object = None) -> ExtensionArray:
1420 """
1421 Shift values by desired number.
1423 Newly introduced missing values are filled with
1424 ``self.dtype.na_value``.
1426 Parameters
1427 ----------
1428 periods : int, default 1
1429 The number of periods to shift. Negative values are allowed
1430 for shifting backwards.
1432 fill_value : object, optional
1433 The scalar value to use for newly introduced missing values.
1434 The default is ``self.dtype.na_value``.
1436 Returns
1437 -------
1438 ExtensionArray
1439 Shifted.
1441 See Also
1442 --------
1443 api.extensions.ExtensionArray.transpose : Return a transposed view on
1444 this array.
1445 api.extensions.ExtensionArray.factorize : Encode the extension array as an
1446 enumerated type.
1448 Notes
1449 -----
1450 If ``self`` is empty or ``periods`` is 0, a copy of ``self`` is
1451 returned.
1453 If ``periods > len(self)``, then an array of size
1454 len(self) is returned, with all values filled with
1455 ``self.dtype.na_value``.
1457 For 2-dimensional ExtensionArrays, we are always shifting along axis=0.
1459 Examples
1460 --------
1461 >>> arr = pd.array([1, 2, 3])
1462 >>> arr.shift(2)
1463 <IntegerArray>
1464 [<NA>, <NA>, 1]
1465 Length: 3, dtype: Int64
1466 """
1467 # Note: this implementation assumes that `self.dtype.na_value` can be
1468 # stored in an instance of your ExtensionArray with `self.dtype`.
1469 if not len(self) or periods == 0:
1470 return self.copy()
1472 if isna(fill_value):
1473 fill_value = self.dtype.na_value
1475 empty = self._from_sequence(
1476 [fill_value] * min(abs(periods), len(self)), dtype=self.dtype
1477 )
1478 if periods > 0:
1479 a = empty
1480 b = self[:-periods]
1481 else:
1482 a = self[abs(periods) :]
1483 b = empty
1484 return self._concat_same_type([a, b])
1486 def unique(self) -> Self:
1487 """
1488 Compute the ExtensionArray of unique values.
1490 Returns
1491 -------
1492 pandas.api.extensions.ExtensionArray
1493 With unique values from the input array.
1495 See Also
1496 --------
1497 Index.unique: Return unique values in the index.
1498 Series.unique: Return unique values of Series object.
1499 unique: Return unique values based on a hash table.
1501 Examples
1502 --------
1503 >>> arr = pd.array([1, 2, 3, 1, 2, 3])
1504 >>> arr.unique()
1505 <IntegerArray>
1506 [1, 2, 3]
1507 Length: 3, dtype: Int64
1508 """
1509 uniques = unique(self.astype(object))
1510 return self._from_sequence(uniques, dtype=self.dtype)
1512 def searchsorted(
1513 self,
1514 value: NumpyValueArrayLike | ExtensionArray,
1515 side: Literal["left", "right"] = "left",
1516 sorter: NumpySorter | None = None,
1517 ) -> npt.NDArray[np.intp] | np.intp:
1518 """
1519 Find indices where elements should be inserted to maintain order.
1521 Find the indices into a sorted array `self` (a) such that, if the
1522 corresponding elements in `value` were inserted before the indices,
1523 the order of `self` would be preserved.
1525 Assuming that `self` is sorted:
1527 ====== ================================
1528 `side` returned index `i` satisfies
1529 ====== ================================
1530 left ``self[i-1] < value <= self[i]``
1531 right ``self[i-1] <= value < self[i]``
1532 ====== ================================
1534 Parameters
1535 ----------
1536 value : array-like, list or scalar
1537 Value(s) to insert into `self`.
1538 side : {'left', 'right'}, optional
1539 If 'left', the index of the first suitable location found is given.
1540 If 'right', return the last such index. If there is no suitable
1541 index, return either 0 or N (where N is the length of `self`).
1542 sorter : 1-D array-like, optional
1543 Optional array of integer indices that sort array a into ascending
1544 order. They are typically the result of argsort.
1546 Returns
1547 -------
1548 array of ints or int
1549 If value is array-like, array of insertion points.
1550 If value is scalar, a single integer.
1552 See Also
1553 --------
1554 numpy.searchsorted : Similar method from NumPy.
1556 Examples
1557 --------
1558 >>> arr = pd.array([1, 2, 3, 5])
1559 >>> arr.searchsorted([4])
1560 array([3])
1561 """
1562 # Note: the base tests provided by pandas only test the basics.
1563 # We do not test
1564 # 1. Values outside the range of the `data_for_sorting` fixture
1565 # 2. Values between the values in the `data_for_sorting` fixture
1566 # 3. Missing values.
1567 arr = self.astype(object)
1568 if isinstance(value, ExtensionArray):
1569 value = value.astype(object)
1570 return arr.searchsorted(value, side=side, sorter=sorter)
1572 def equals(self, other: object) -> bool:
1573 """
1574 Return if another array is equivalent to this array.
1576 Equivalent means that both arrays have the same shape and dtype, and
1577 all values compare equal. Missing values in the same location are
1578 considered equal (in contrast with normal equality).
1580 Parameters
1581 ----------
1582 other : ExtensionArray
1583 Array to compare to this Array.
1585 Returns
1586 -------
1587 boolean
1588 Whether the arrays are equivalent.
1590 See Also
1591 --------
1592 numpy.array_equal : Equivalent method for numpy array.
1593 Series.equals : Equivalent method for Series.
1594 DataFrame.equals : Equivalent method for DataFrame.
1596 Examples
1597 --------
1598 >>> arr1 = pd.array([1, 2, np.nan])
1599 >>> arr2 = pd.array([1, 2, np.nan])
1600 >>> arr1.equals(arr2)
1601 True
1603 >>> arr1 = pd.array([1, 3, np.nan])
1604 >>> arr2 = pd.array([1, 2, np.nan])
1605 >>> arr1.equals(arr2)
1606 False
1607 """
1608 if type(self) != type(other):
1609 return False
1610 other = cast(ExtensionArray, other)
1611 if self.dtype != other.dtype:
1612 return False
1613 elif len(self) != len(other):
1614 return False
1615 else:
1616 equal_values = self == other
1617 if isinstance(equal_values, ExtensionArray):
1618 # boolean array with NA -> fill with False
1619 equal_values = equal_values.fillna(False)
1620 # error: Unsupported left operand type for & ("ExtensionArray")
1621 equal_na = self.isna() & other.isna() # type: ignore[operator]
1622 return bool((equal_values | equal_na).all())
1624 def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
1625 """
1626 Pointwise comparison for set containment in the given values.
1628 Roughly equivalent to `np.array([x in values for x in self])`
1630 Parameters
1631 ----------
1632 values : np.ndarray or ExtensionArray
1633 Values to compare every element in the array against.
1635 Returns
1636 -------
1637 np.ndarray[bool]
1638 With true at indices where value is in `values`.
1640 See Also
1641 --------
1642 DataFrame.isin: Whether each element in the DataFrame is contained in values.
1643 Index.isin: Return a boolean array where the index values are in values.
1644 Series.isin: Whether elements in Series are contained in values.
1646 Examples
1647 --------
1648 >>> arr = pd.array([1, 2, 3])
1649 >>> arr.isin([1])
1650 <BooleanArray>
1651 [True, False, False]
1652 Length: 3, dtype: boolean
1653 """
1654 return isin(np.asarray(self), values)
1656 def _values_for_factorize(self) -> tuple[np.ndarray, Any]:
1657 """
1658 Return an array and missing value suitable for factorization.
1660 Returns
1661 -------
1662 values : ndarray
1663 An array suitable for factorization. This should maintain order
1664 and be a supported dtype (Float64, Int64, UInt64, String, Object).
1665 By default, the extension array is cast to object dtype.
1666 na_value : object
1667 The value in `values` to consider missing. This will be treated
1668 as NA in the factorization routines, so it will be coded as
1669 `-1` and not included in `uniques`. By default,
1670 ``np.nan`` is used.
1672 See Also
1673 --------
1674 util.hash_pandas_object : Hash the pandas object.
1676 Notes
1677 -----
1678 The values returned by this method are also used in
1679 :func:`pandas.util.hash_pandas_object`. If needed, this can be
1680 overridden in the ``self._hash_pandas_object()`` method.
1682 Examples
1683 --------
1684 >>> pd.array([1, 2, 3])._values_for_factorize()
1685 (array([1, 2, 3], dtype=object), nan)
1686 """
1687 return self.astype(object), np.nan
1689 def factorize(
1690 self,
1691 use_na_sentinel: bool = True,
1692 ) -> tuple[np.ndarray, ExtensionArray]:
1693 """
1694 Encode the extension array as an enumerated type.
1696 Parameters
1697 ----------
1698 use_na_sentinel : bool, default True
1699 If True, the sentinel -1 will be used for NaN values. If False,
1700 NaN values will be encoded as non-negative integers and will not drop the
1701 NaN from the uniques of the values.
1703 Returns
1704 -------
1705 codes : ndarray
1706 An integer NumPy array that's an indexer into the original
1707 ExtensionArray.
1708 uniques : ExtensionArray
1709 An ExtensionArray containing the unique values of `self`.
1711 .. note::
1713 uniques will *not* contain an entry for the NA value of
1714 the ExtensionArray if there are any missing values present
1715 in `self`.
1717 See Also
1718 --------
1719 factorize : Top-level factorize method that dispatches here.
1721 Notes
1722 -----
1723 :meth:`pandas.factorize` offers a `sort` keyword as well.
1725 Examples
1726 --------
1727 >>> idx1 = pd.PeriodIndex(
1728 ... ["2014-01", "2014-01", "2014-02", "2014-02", "2014-03", "2014-03"],
1729 ... freq="M",
1730 ... )
1731 >>> arr, idx = idx1.factorize()
1732 >>> arr
1733 array([0, 0, 1, 1, 2, 2])
1734 >>> idx
1735 PeriodIndex(['2014-01', '2014-02', '2014-03'], dtype='period[M]')
1736 """
1737 # Implementer note: There are two ways to override the behavior of
1738 # pandas.factorize
1739 # 1. _values_for_factorize and _from_factorize.
1740 # Specify the values passed to pandas' internal factorization
1741 # routines, and how to convert from those values back to the
1742 # original ExtensionArray.
1743 # 2. ExtensionArray.factorize.
1744 # Complete control over factorization.
1745 arr, na_value = self._values_for_factorize()
1747 codes, uniques = factorize_array(
1748 arr, use_na_sentinel=use_na_sentinel, na_value=na_value
1749 )
1751 uniques_ea = self._from_factorized(uniques, self)
1752 return codes, uniques_ea
1754 _extension_array_shared_docs["repeat"] = """
1755 Repeat elements of a %(klass)s.
1757 Returns a new %(klass)s where each element of the current %(klass)s
1758 is repeated consecutively a given number of times.
1760 Parameters
1761 ----------
1762 repeats : int or array of ints
1763 The number of repetitions for each element. This should be a
1764 non-negative integer. Repeating 0 times will return an empty
1765 %(klass)s.
1766 axis : None
1767 Must be ``None``. Has no effect but is accepted for compatibility
1768 with numpy.
1770 Returns
1771 -------
1772 %(klass)s
1773 Newly created %(klass)s with repeated elements.
1775 See Also
1776 --------
1777 Series.repeat : Equivalent function for Series.
1778 Index.repeat : Equivalent function for Index.
1779 numpy.repeat : Similar method for :class:`numpy.ndarray`.
1780 ExtensionArray.take : Take arbitrary positions.
1782 Examples
1783 --------
1784 >>> cat = pd.Categorical(['a', 'b', 'c'])
1785 >>> cat
1786 ['a', 'b', 'c']
1787 Categories (3, str): ['a', 'b', 'c']
1788 >>> cat.repeat(2)
1789 ['a', 'a', 'b', 'b', 'c', 'c']
1790 Categories (3, str): ['a', 'b', 'c']
1791 >>> cat.repeat([1, 2, 3])
1792 ['a', 'b', 'b', 'c', 'c', 'c']
1793 Categories (3, str): ['a', 'b', 'c']
1794 """
1796 def repeat(self, repeats: int | Sequence[int], axis: AxisInt | None = None) -> Self:
1797 """
1798 Repeat elements of an ExtensionArray.
1800 Returns a new ExtensionArray where each element of the current ExtensionArray
1801 is repeated consecutively a given number of times.
1803 Parameters
1804 ----------
1805 repeats : int or array of ints
1806 The number of repetitions for each element. This should be a
1807 non-negative integer. Repeating 0 times will return an empty
1808 ExtensionArray.
1809 axis : None
1810 Must be ``None``. Has no effect but is accepted for compatibility
1811 with numpy.
1813 Returns
1814 -------
1815 ExtensionArray
1816 Newly created ExtensionArray with repeated elements.
1818 See Also
1819 --------
1820 Series.repeat : Equivalent function for Series.
1821 Index.repeat : Equivalent function for Index.
1822 numpy.repeat : Similar method for :class:`numpy.ndarray`.
1823 ExtensionArray.take : Take arbitrary positions.
1825 Examples
1826 --------
1827 >>> cat = pd.Categorical(["a", "b", "c"])
1828 >>> cat
1829 ['a', 'b', 'c']
1830 Categories (3, str): ['a', 'b', 'c']
1831 >>> cat.repeat(2)
1832 ['a', 'a', 'b', 'b', 'c', 'c']
1833 Categories (3, str): ['a', 'b', 'c']
1834 >>> cat.repeat([1, 2, 3])
1835 ['a', 'b', 'b', 'c', 'c', 'c']
1836 Categories (3, str): ['a', 'b', 'c']
1837 """
1838 nv.validate_repeat((), {"axis": axis})
1839 ind = np.arange(len(self)).repeat(repeats)
1840 return self.take(ind)
1842 def value_counts(self, dropna: bool = True) -> Series:
1843 """
1844 Return a Series containing counts of unique values.
1846 This method returns a Series with unique values as the index and their
1847 counts as the values.
1849 Parameters
1850 ----------
1851 dropna : bool, default True
1852 Don't include counts of NA values.
1854 Returns
1855 -------
1856 Series
1857 A Series with unique values as index and counts as values.
1859 See Also
1860 --------
1861 Series.value_counts : Equivalent method on Series.
1862 DataFrame.value_counts : Equivalent method on DataFrame.
1863 Index.value_counts : Equivalent method on Index.
1865 Examples
1866 --------
1867 >>> from pandas.core.arrays import IntegerArray
1868 >>> arr = IntegerArray._from_sequence([3, 3, 3, 1, 2, 2])
1869 >>> arr.value_counts()
1870 3 3
1871 1 1
1872 2 2
1873 Name: count, dtype: Int64
1874 """
1875 from pandas.core.algorithms import value_counts_internal as value_counts
1877 result = value_counts(self.to_numpy(copy=False), sort=False, dropna=dropna)
1878 result.index = result.index.astype(self.dtype)
1879 return result
1881 # ------------------------------------------------------------------------
1882 # Indexing methods
1883 # ------------------------------------------------------------------------
1885 def take(
1886 self,
1887 indices: TakeIndexer,
1888 *,
1889 allow_fill: bool = False,
1890 fill_value: Any = None,
1891 ) -> Self:
1892 """
1893 Take elements from an array.
1895 Parameters
1896 ----------
1897 indices : sequence of int or one-dimensional np.ndarray of int
1898 Indices to be taken.
1899 allow_fill : bool, default False
1900 How to handle negative values in `indices`.
1902 * False: negative values in `indices` indicate positional indices
1903 from the right (the default). This is similar to
1904 :func:`numpy.take`.
1906 * True: negative values in `indices` indicate
1907 missing values. These values are set to `fill_value`. Any other
1908 other negative values raise a ``ValueError``.
1910 fill_value : any, optional
1911 Fill value to use for NA-indices when `allow_fill` is True.
1912 This may be ``None``, in which case the default NA value for
1913 the type, ``self.dtype.na_value``, is used.
1915 For many ExtensionArrays, there will be two representations of
1916 `fill_value`: a user-facing "boxed" scalar, and a low-level
1917 physical NA value. `fill_value` should be the user-facing version,
1918 and the implementation should handle translating that to the
1919 physical version for processing the take if necessary.
1921 Returns
1922 -------
1923 ExtensionArray
1924 An array formed with selected `indices`.
1926 Raises
1927 ------
1928 IndexError
1929 When the indices are out of bounds for the array.
1930 ValueError
1931 When `indices` contains negative values other than ``-1``
1932 and `allow_fill` is True.
1934 See Also
1935 --------
1936 numpy.take : Take elements from an array along an axis.
1937 api.extensions.take : Take elements from an array.
1939 Notes
1940 -----
1941 ExtensionArray.take is called by ``Series.__getitem__``, ``.loc``,
1942 ``iloc``, when `indices` is a sequence of values. Additionally,
1943 it's called by :meth:`Series.reindex`, or any other method
1944 that causes realignment, with a `fill_value`.
1946 Examples
1947 --------
1948 Here's an example implementation, which relies on casting the
1949 extension array to object dtype. This uses the helper method
1950 :func:`pandas.api.extensions.take`.
1952 .. code-block:: python
1954 def take(self, indices, allow_fill=False, fill_value=None):
1955 from pandas.api.extensions import take
1957 # If the ExtensionArray is backed by an ndarray, then
1958 # just pass that here instead of coercing to object.
1959 data = self.astype(object)
1961 if allow_fill and fill_value is None:
1962 fill_value = self.dtype.na_value
1964 # fill value should always be translated from the scalar
1965 # type for the array, to the physical storage type for
1966 # the data, before passing to take.
1968 result = take(
1969 data, indices, fill_value=fill_value, allow_fill=allow_fill
1970 )
1971 return self._from_sequence(result, dtype=self.dtype)
1972 """
1973 # Implementer note: The `fill_value` parameter should be a user-facing
1974 # value, an instance of self.dtype.type. When passed `fill_value=None`,
1975 # the default of `self.dtype.na_value` should be used.
1976 # This may differ from the physical storage type your ExtensionArray
1977 # uses. In this case, your implementation is responsible for casting
1978 # the user-facing type to the storage type, before using
1979 # pandas.api.extensions.take
1980 raise AbstractMethodError(self)
1982 def copy(self) -> Self:
1983 """
1984 Return a copy of the array.
1986 This method creates a copy of the `ExtensionArray` where modifying the
1987 data in the copy will not affect the original array. This is useful when
1988 you want to manipulate data without altering the original dataset.
1990 Returns
1991 -------
1992 ExtensionArray
1993 A new `ExtensionArray` object that is a copy of the current instance.
1995 See Also
1996 --------
1997 DataFrame.copy : Return a copy of the DataFrame.
1998 Series.copy : Return a copy of the Series.
2000 Examples
2001 --------
2002 >>> arr = pd.array([1, 2, 3])
2003 >>> arr2 = arr.copy()
2004 >>> arr[0] = 2
2005 >>> arr2
2006 <IntegerArray>
2007 [1, 2, 3]
2008 Length: 3, dtype: Int64
2009 """
2010 raise AbstractMethodError(self)
2012 @overload
2013 def view(self) -> Self: ...
2015 @overload
2016 def view(self, dtype: Dtype | None = ...) -> ArrayLike: ...
2018 def view(self, dtype: Dtype | None = None) -> ArrayLike:
2019 """
2020 Return a view on the array.
2022 Parameters
2023 ----------
2024 dtype : str, np.dtype, or ExtensionDtype, optional
2025 Default None.
2027 Returns
2028 -------
2029 ExtensionArray or np.ndarray
2030 A view on the :class:`ExtensionArray`'s data.
2032 See Also
2033 --------
2034 api.extensions.ExtensionArray.ravel: Return a flattened view on input array.
2035 Index.view: Equivalent function for Index.
2036 ndarray.view: New view of array with the same data.
2038 Examples
2039 --------
2040 This gives view on the underlying data of an ``ExtensionArray`` and is not a
2041 copy. Modifications on either the view or the original ``ExtensionArray``
2042 will be reflected on the underlying data:
2044 >>> arr = pd.array([1, 2, 3])
2045 >>> arr2 = arr.view()
2046 >>> arr[0] = 2
2047 >>> arr2
2048 <IntegerArray>
2049 [2, 2, 3]
2050 Length: 3, dtype: Int64
2051 """
2052 # NB:
2053 # - This must return a *new* object referencing the same data, not self.
2054 # - The only case that *must* be implemented is with dtype=None,
2055 # giving a view with the same dtype as self.
2056 if dtype is not None:
2057 raise NotImplementedError(dtype)
2058 return self[:]
2060 # ------------------------------------------------------------------------
2061 # Printing
2062 # ------------------------------------------------------------------------
2064 def __repr__(self) -> str:
2065 if self.ndim > 1:
2066 return self._repr_2d()
2068 from pandas.io.formats.printing import format_object_summary
2070 # the short repr has no trailing newline, while the truncated
2071 # repr does. So we include a newline in our template, and strip
2072 # any trailing newlines from format_object_summary
2073 data = format_object_summary(
2074 self, self._formatter(), indent_for_name=False
2075 ).rstrip(", \n")
2076 class_name = f"<{type(self).__name__}>\n"
2077 footer = self._get_repr_footer()
2078 return f"{class_name}{data}\n{footer}"
2080 def _get_repr_footer(self) -> str:
2081 # GH#24278
2082 if self.ndim > 1:
2083 return f"Shape: {self.shape}, dtype: {self.dtype}"
2084 return f"Length: {len(self)}, dtype: {self.dtype}"
2086 def _repr_2d(self) -> str:
2087 from pandas.io.formats.printing import format_object_summary
2089 # the short repr has no trailing newline, while the truncated
2090 # repr does. So we include a newline in our template, and strip
2091 # any trailing newlines from format_object_summary
2092 lines = [
2093 format_object_summary(x, self._formatter(), indent_for_name=False).rstrip(
2094 ", \n"
2095 )
2096 for x in self
2097 ]
2098 data = ",\n".join(lines)
2099 class_name = f"<{type(self).__name__}>"
2100 footer = self._get_repr_footer()
2101 return f"{class_name}\n[\n{data}\n]\n{footer}"
2103 def _formatter(self, boxed: bool = False) -> Callable[[Any], str | None]:
2104 """
2105 Formatting function for scalar values.
2107 This is used in the default '__repr__'. The returned formatting
2108 function receives instances of your scalar type.
2110 Parameters
2111 ----------
2112 boxed : bool, default False
2113 An indicated for whether or not your array is being printed
2114 within a Series, DataFrame, or Index (True), or just by
2115 itself (False). This may be useful if you want scalar values
2116 to appear differently within a Series versus on its own (e.g.
2117 quoted or not).
2119 Returns
2120 -------
2121 Callable[[Any], str]
2122 A callable that gets instances of the scalar type and
2123 returns a string. By default, :func:`repr` is used
2124 when ``boxed=False`` and :func:`str` is used when
2125 ``boxed=True``.
2127 See Also
2128 --------
2129 api.extensions.ExtensionArray._concat_same_type : Concatenate multiple
2130 array of this dtype.
2131 api.extensions.ExtensionArray._explode : Transform each element of
2132 list-like to a row.
2133 api.extensions.ExtensionArray._from_factorized : Reconstruct an
2134 ExtensionArray after factorization.
2135 api.extensions.ExtensionArray._from_sequence : Construct a new
2136 ExtensionArray from a sequence of scalars.
2138 Examples
2139 --------
2140 >>> class MyExtensionArray(pd.arrays.NumpyExtensionArray):
2141 ... def _formatter(self, boxed=False):
2142 ... return lambda x: "*" + str(x) + "*"
2143 >>> MyExtensionArray(np.array([1, 2, 3, 4]))
2144 <MyExtensionArray>
2145 [*1*, *2*, *3*, *4*]
2146 Length: 4, dtype: int64
2147 """
2148 if boxed:
2149 return str
2150 return repr
2152 # ------------------------------------------------------------------------
2153 # Reshaping
2154 # ------------------------------------------------------------------------
2156 def transpose(self, *axes: int) -> Self:
2157 """
2158 Return a transposed view on this array.
2160 Because ExtensionArrays are always 1D, this is a no-op. It is included
2161 for compatibility with np.ndarray.
2163 Returns
2164 -------
2165 ExtensionArray
2167 Examples
2168 --------
2169 >>> pd.array([1, 2, 3]).transpose()
2170 <IntegerArray>
2171 [1, 2, 3]
2172 Length: 3, dtype: Int64
2173 """
2174 return self[:]
2176 @property
2177 def T(self) -> Self:
2178 return self.transpose()
2180 def ravel(self, order: Literal["C", "F", "A", "K"] | None = "C") -> Self:
2181 """
2182 Return a flattened view on this array.
2184 Parameters
2185 ----------
2186 order : {None, 'C', 'F', 'A', 'K'}, default 'C'
2188 Returns
2189 -------
2190 ExtensionArray
2191 A flattened view on the array.
2193 See Also
2194 --------
2195 ExtensionArray.tolist: Return a list of the values.
2197 Notes
2198 -----
2199 - Because ExtensionArrays are 1D-only, this is a no-op.
2200 - The "order" argument is ignored, is for compatibility with NumPy.
2202 Examples
2203 --------
2204 >>> pd.array([1, 2, 3]).ravel()
2205 <IntegerArray>
2206 [1, 2, 3]
2207 Length: 3, dtype: Int64
2208 """
2209 return self
2211 @classmethod
2212 def _concat_same_type(cls, to_concat: Sequence[Self]) -> Self:
2213 """
2214 Concatenate multiple array of this dtype.
2216 Parameters
2217 ----------
2218 to_concat : sequence of this type
2219 An array of the same dtype to concatenate.
2221 Returns
2222 -------
2223 ExtensionArray
2225 See Also
2226 --------
2227 api.extensions.ExtensionArray._explode : Transform each element of
2228 list-like to a row.
2229 api.extensions.ExtensionArray._formatter : Formatting function for
2230 scalar values.
2231 api.extensions.ExtensionArray._from_factorized : Reconstruct an
2232 ExtensionArray after factorization.
2234 Examples
2235 --------
2236 >>> arr1 = pd.array([1, 2, 3])
2237 >>> arr2 = pd.array([4, 5, 6])
2238 >>> pd.arrays.IntegerArray._concat_same_type([arr1, arr2])
2239 <IntegerArray>
2240 [1, 2, 3, 4, 5, 6]
2241 Length: 6, dtype: Int64
2242 """
2243 # Implementer note: this method will only be called with a sequence of
2244 # ExtensionArrays of this class and with the same dtype as self. This
2245 # should allow "easy" concatenation (no upcasting needed), and result
2246 # in a new ExtensionArray of the same dtype.
2247 # Note: this strict behaviour is only guaranteed starting with pandas 1.1
2248 raise AbstractMethodError(cls)
2250 # The _can_hold_na attribute is set to True so that pandas internals
2251 # will use the ExtensionDtype.na_value as the NA value in operations
2252 # such as take(), reindex(), shift(), etc. In addition, those results
2253 # will then be of the ExtensionArray subclass rather than an array
2254 # of objects
2255 @cache_readonly
2256 def _can_hold_na(self) -> bool:
2257 return self.dtype._can_hold_na
2259 def _accumulate(
2260 self, name: str, *, skipna: bool = True, **kwargs
2261 ) -> ExtensionArray:
2262 """
2263 Return an ExtensionArray performing an accumulation operation.
2265 The underlying data type might change.
2267 Parameters
2268 ----------
2269 name : str
2270 Name of the function, supported values are:
2271 - cummin
2272 - cummax
2273 - cumsum
2274 - cumprod
2275 skipna : bool, default True
2276 If True, skip NA values.
2277 **kwargs
2278 Additional keyword arguments passed to the accumulation function.
2279 Currently, there is no supported kwarg.
2281 Returns
2282 -------
2283 array
2284 An array performing the accumulation operation.
2286 Raises
2287 ------
2288 NotImplementedError : subclass does not define accumulations
2290 See Also
2291 --------
2292 api.extensions.ExtensionArray._concat_same_type : Concatenate multiple
2293 array of this dtype.
2294 api.extensions.ExtensionArray.view : Return a view on the array.
2295 api.extensions.ExtensionArray._explode : Transform each element of
2296 list-like to a row.
2298 Examples
2299 --------
2300 >>> arr = pd.array([1, 2, 3])
2301 >>> arr._accumulate(name="cumsum")
2302 <IntegerArray>
2303 [1, 3, 6]
2304 Length: 3, dtype: Int64
2305 """
2306 raise NotImplementedError(f"cannot perform {name} with type {self.dtype}")
2308 def _reduce(
2309 self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
2310 ):
2311 """
2312 Return a scalar result of performing the reduction operation.
2314 Parameters
2315 ----------
2316 name : str
2317 Name of the function, supported values are:
2318 { any, all, min, max, sum, mean, median, prod,
2319 std, var, sem, kurt, skew }.
2320 skipna : bool, default True
2321 If True, skip NaN values.
2322 keepdims : bool, default False
2323 If False, a scalar is returned.
2324 If True, the result has dimension with size one along the reduced axis.
2325 **kwargs
2326 Additional keyword arguments passed to the reduction function.
2327 Currently, `ddof` is the only supported kwarg.
2329 Returns
2330 -------
2331 scalar or ndarray:
2332 The result of the reduction operation. The type of the result
2333 depends on `keepdims`:
2334 - If `keepdims` is `False`, a scalar value is returned.
2335 - If `keepdims` is `True`, the result is wrapped in a numpy array with
2336 a single element.
2338 Raises
2339 ------
2340 TypeError : subclass does not define operations
2342 See Also
2343 --------
2344 Series.min : Return the minimum value.
2345 Series.max : Return the maximum value.
2346 Series.sum : Return the sum of values.
2347 Series.mean : Return the mean of values.
2348 Series.median : Return the median of values.
2349 Series.std : Return the standard deviation.
2350 Series.var : Return the variance.
2351 Series.prod : Return the product of values.
2352 Series.sem : Return the standard error of the mean.
2353 Series.kurt : Return the kurtosis.
2354 Series.skew : Return the skewness.
2356 Examples
2357 --------
2358 >>> pd.array([1, 2, 3])._reduce("min")
2359 np.int64(1)
2360 >>> pd.array([1, 2, 3])._reduce("max")
2361 np.int64(3)
2362 >>> pd.array([1, 2, 3])._reduce("sum")
2363 np.int64(6)
2364 >>> pd.array([1, 2, 3])._reduce("mean")
2365 np.float64(2.0)
2366 >>> pd.array([1, 2, 3])._reduce("median")
2367 np.float64(2.0)
2368 """
2369 meth = getattr(self, name, None)
2370 if meth is None:
2371 raise TypeError(
2372 f"'{type(self).__name__}' with dtype {self.dtype} "
2373 f"does not support operation '{name}'"
2374 )
2375 result = meth(skipna=skipna, **kwargs)
2376 if keepdims:
2377 if name in ["min", "max"]:
2378 result = self._from_sequence([result], dtype=self.dtype)
2379 else:
2380 result = np.array([result])
2382 return result
2384 # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
2385 # Incompatible types in assignment (expression has type "None", base class
2386 # "object" defined the type as "Callable[[object], int]")
2387 __hash__: ClassVar[None] # type: ignore[assignment]
2389 # ------------------------------------------------------------------------
2390 # Non-Optimized Default Methods; in the case of the private methods here,
2391 # these are not guaranteed to be stable across pandas versions.
2393 def _values_for_json(self) -> np.ndarray:
2394 """
2395 Specify how to render our entries in to_json.
2397 Notes
2398 -----
2399 The dtype on the returned ndarray is not restricted, but for non-native
2400 types that are not specifically handled in objToJSON.c, to_json is
2401 liable to raise. In these cases, it may be safer to return an ndarray
2402 of strings.
2403 """
2404 return np.asarray(self)
2406 def _hash_pandas_object(
2407 self, *, encoding: str, hash_key: str, categorize: bool
2408 ) -> npt.NDArray[np.uint64]:
2409 """
2410 Hook for hash_pandas_object.
2412 Default is to use the values returned by _values_for_factorize.
2414 Parameters
2415 ----------
2416 encoding : str
2417 Encoding for data & key when strings.
2418 hash_key : str
2419 Hash_key for string key to encode.
2420 categorize : bool
2421 Whether to first categorize object arrays before hashing. This is more
2422 efficient when the array contains duplicate values.
2424 Returns
2425 -------
2426 np.ndarray[uint64]
2427 An array of hashed values.
2429 See Also
2430 --------
2431 api.extensions.ExtensionArray._values_for_factorize : Return an array and
2432 missing value suitable for factorization.
2433 util.hash_array : Given a 1d array, return an array of hashed values.
2435 Examples
2436 --------
2437 >>> pd.array([1, 2])._hash_pandas_object(
2438 ... encoding="utf-8", hash_key="1000000000000000", categorize=False
2439 ... )
2440 array([ 6238072747940578789, 15839785061582574730], dtype=uint64)
2441 """
2442 from pandas.core.util.hashing import hash_array
2444 values, _ = self._values_for_factorize()
2445 return hash_array(
2446 values, encoding=encoding, hash_key=hash_key, categorize=categorize
2447 )
2449 def _explode(self) -> tuple[Self, npt.NDArray[np.uint64]]:
2450 """
2451 Transform each element of list-like to a row.
2453 For arrays that do not contain list-like elements the default
2454 implementation of this method just returns a copy and an array
2455 of ones (unchanged index).
2457 Returns
2458 -------
2459 ExtensionArray
2460 Array with the exploded values.
2461 np.ndarray[uint64]
2462 The original lengths of each list-like for determining the
2463 resulting index.
2465 See Also
2466 --------
2467 Series.explode : The method on the ``Series`` object that this
2468 extension array method is meant to support.
2470 Examples
2471 --------
2472 >>> import pyarrow as pa
2473 >>> a = pd.array(
2474 ... [[1, 2, 3], [4], [5, 6]], dtype=pd.ArrowDtype(pa.list_(pa.int64()))
2475 ... )
2476 >>> a._explode()
2477 (<ArrowExtensionArray>
2478 [1, 2, 3, 4, 5, 6]
2479 Length: 6, dtype: int64[pyarrow], array([3, 1, 2], dtype=int32))
2480 """
2481 values = self.copy()
2482 counts = np.ones(shape=(len(self),), dtype=np.uint64)
2483 return values, counts
2485 def tolist(self) -> list:
2486 """
2487 Return a list of the values.
2489 These are each a scalar type, which is a Python scalar
2490 (for str, int, float) or a pandas scalar
2491 (for Timestamp/Timedelta/Interval/Period)
2493 Returns
2494 -------
2495 list
2496 Python list of values in array.
2498 See Also
2499 --------
2500 Index.to_list: Return a list of the values in the Index.
2501 Series.to_list: Return a list of the values in the Series.
2503 Examples
2504 --------
2505 >>> arr = pd.array([1, 2, 3])
2506 >>> arr.tolist()
2507 [1, 2, 3]
2508 """
2509 if self.ndim > 1:
2510 return [x.tolist() for x in self]
2511 return list(self)
2513 def delete(self, loc: PositionalIndexer) -> Self:
2514 indexer = np.delete(np.arange(len(self)), loc)
2515 return self.take(indexer)
2517 def insert(self, loc: int, item) -> Self:
2518 """
2519 Insert an item at the given position.
2521 Parameters
2522 ----------
2523 loc : int
2524 Index where the `item` needs to be inserted.
2525 item : scalar-like
2526 Value to be inserted.
2528 Returns
2529 -------
2530 ExtensionArray
2531 With `item` inserted at `loc`.
2533 See Also
2534 --------
2535 Index.insert: Make new Index inserting new item at location.
2537 Notes
2538 -----
2539 This method should be both type and dtype-preserving. If the item
2540 cannot be held in an array of this type/dtype, either ValueError or
2541 TypeError should be raised.
2543 The default implementation relies on _from_sequence to raise on invalid
2544 items.
2546 Examples
2547 --------
2548 >>> arr = pd.array([1, 2, 3])
2549 >>> arr.insert(2, -1)
2550 <IntegerArray>
2551 [1, 2, -1, 3]
2552 Length: 4, dtype: Int64
2553 """
2554 loc = validate_insert_loc(loc, len(self))
2556 item_arr = type(self)._from_sequence([item], dtype=self.dtype)
2558 return type(self)._concat_same_type([self[:loc], item_arr, self[loc:]])
2560 def _putmask(self, mask: npt.NDArray[np.bool_], value) -> None:
2561 """
2562 Analogue to np.putmask(self, mask, value)
2564 Parameters
2565 ----------
2566 mask : np.ndarray[bool]
2567 value : scalar or listlike
2568 If listlike, must be arraylike with same length as self.
2570 Returns
2571 -------
2572 None
2574 Notes
2575 -----
2576 Unlike np.putmask, we do not repeat listlike values with mismatched length.
2577 'value' should either be a scalar or an arraylike with the same length
2578 as self.
2579 """
2580 if is_list_like(value):
2581 val = value[mask]
2582 else:
2583 val = value
2585 self[mask] = val
2587 def _where(self, mask: npt.NDArray[np.bool_], value) -> Self:
2588 """
2589 Analogue to np.where(mask, self, value)
2591 Parameters
2592 ----------
2593 mask : np.ndarray[bool]
2594 value : scalar or listlike
2596 Returns
2597 -------
2598 same type as self
2599 """
2600 result = self.copy()
2602 if is_list_like(value):
2603 val = value[~mask]
2604 else:
2605 val = value
2607 result[~mask] = val
2608 return result
2610 def _rank(
2611 self,
2612 *,
2613 axis: AxisInt = 0,
2614 method: str = "average",
2615 na_option: str = "keep",
2616 ascending: bool = True,
2617 pct: bool = False,
2618 ):
2619 """
2620 See Series.rank.__doc__.
2621 """
2622 if axis != 0:
2623 raise NotImplementedError
2625 return rank(
2626 self._values_for_argsort(),
2627 axis=axis,
2628 method=method,
2629 na_option=na_option,
2630 ascending=ascending,
2631 pct=pct,
2632 mask=np.asarray(self.isna(), dtype="bool") if self._hasna else None,
2633 )
2635 @classmethod
2636 def _empty(cls, shape: Shape, dtype: ExtensionDtype):
2637 """
2638 Create an ExtensionArray with the given shape and dtype.
2640 See also
2641 --------
2642 ExtensionDtype.empty
2643 ExtensionDtype.empty is the 'official' public version of this API.
2644 """
2645 # Implementer note: while ExtensionDtype.empty is the public way to
2646 # call this method, it is still required to implement this `_empty`
2647 # method as well (it is called internally in pandas)
2648 obj = cls._from_sequence([], dtype=dtype)
2650 taker = np.broadcast_to(np.intp(-1), shape)
2651 result = obj.take(taker, allow_fill=True)
2652 if not isinstance(result, cls) or dtype != result.dtype:
2653 raise NotImplementedError(
2654 f"Default 'empty' implementation is invalid for dtype='{dtype}'"
2655 )
2656 return result
2658 def _quantile(self, qs: npt.NDArray[np.float64], interpolation: str) -> Self:
2659 """
2660 Compute the quantiles of self for each quantile in `qs`.
2662 Parameters
2663 ----------
2664 qs : np.ndarray[float64]
2665 interpolation: str
2667 Returns
2668 -------
2669 same type as self
2670 """
2671 mask = np.asarray(self.isna())
2672 arr = np.asarray(self)
2673 fill_value = np.nan
2675 res_values = quantile_with_mask(arr, mask, fill_value, qs, interpolation)
2676 return type(self)._from_sequence(res_values)
2678 def _mode(self, dropna: bool = True) -> Self:
2679 """
2680 Returns the mode(s) of the ExtensionArray.
2682 Always returns `ExtensionArray` even if only one value.
2684 Parameters
2685 ----------
2686 dropna : bool, default True
2687 Don't consider counts of NA values.
2689 Returns
2690 -------
2691 same type as self
2692 Sorted, if possible.
2693 """
2694 # error: Incompatible return value type (got "Union[ExtensionArray,
2695 # Tuple[np.ndarray, npt.NDArray[np.bool_]]", expected "Self")
2696 result, _ = mode(self, dropna=dropna)
2697 return result # type: ignore[return-value]
2699 def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
2700 if any(
2701 isinstance(other, (ABCSeries, ABCIndex, ABCDataFrame)) for other in inputs
2702 ):
2703 return NotImplemented
2705 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(
2706 self, ufunc, method, *inputs, **kwargs
2707 )
2708 if result is not NotImplemented:
2709 return result
2711 if "out" in kwargs:
2712 return arraylike.dispatch_ufunc_with_out(
2713 self, ufunc, method, *inputs, **kwargs
2714 )
2716 if method == "reduce":
2717 result = arraylike.dispatch_reduction_ufunc(
2718 self, ufunc, method, *inputs, **kwargs
2719 )
2720 if result is not NotImplemented:
2721 return result
2723 return arraylike.default_array_ufunc(self, ufunc, method, *inputs, **kwargs)
2725 def map(self, mapper, na_action: Literal["ignore"] | None = None):
2726 """
2727 Map values using an input mapping or function.
2729 Parameters
2730 ----------
2731 mapper : function, dict, or Series
2732 Mapping correspondence.
2733 na_action : {None, 'ignore'}, default None
2734 If 'ignore', propagate NA values, without passing them to the
2735 mapping correspondence. If 'ignore' is not supported, a
2736 ``NotImplementedError`` should be raised.
2738 Returns
2739 -------
2740 Union[ndarray, Index, ExtensionArray]
2741 The output of the mapping function applied to the array.
2742 If the function returns a tuple with more than one element
2743 a MultiIndex will be returned.
2744 """
2745 return map_array(self, mapper, na_action=na_action)
2747 # ------------------------------------------------------------------------
2748 # GroupBy Methods
2750 def _groupby_op(
2751 self,
2752 *,
2753 how: str,
2754 has_dropped_na: bool,
2755 min_count: int,
2756 ngroups: int,
2757 ids: npt.NDArray[np.intp],
2758 **kwargs,
2759 ) -> ArrayLike:
2760 """
2761 Dispatch GroupBy reduction or transformation operation.
2763 This is an *experimental* API to allow ExtensionArray authors to implement
2764 reductions and transformations. The API is subject to change.
2766 Parameters
2767 ----------
2768 how : {'any', 'all', 'sum', 'prod', 'min', 'max', 'mean', 'median',
2769 'median', 'var', 'std', 'sem', 'nth', 'last', 'ohlc',
2770 'cumprod', 'cumsum', 'cummin', 'cummax', 'rank'}
2771 has_dropped_na : bool
2772 min_count : int
2773 ngroups : int
2774 ids : np.ndarray[np.intp]
2775 ids[i] gives the integer label for the group that self[i] belongs to.
2776 **kwargs : operation-specific
2777 'any', 'all' -> ['skipna']
2778 'var', 'std', 'sem' -> ['ddof']
2779 'cumprod', 'cumsum', 'cummin', 'cummax' -> ['skipna']
2780 'rank' -> ['ties_method', 'ascending', 'na_option', 'pct']
2782 Returns
2783 -------
2784 np.ndarray or ExtensionArray
2785 """
2786 from pandas.core.arrays.string_ import StringDtype
2787 from pandas.core.groupby.ops import WrappedCythonOp
2789 kind = WrappedCythonOp.get_kind_from_how(how)
2790 op = WrappedCythonOp(how=how, kind=kind, has_dropped_na=has_dropped_na)
2792 initial: Any = 0
2793 # GH#43682
2794 if isinstance(self.dtype, StringDtype):
2795 # StringArray
2796 if op.how in [
2797 "prod",
2798 "mean",
2799 "median",
2800 "cumsum",
2801 "cumprod",
2802 "std",
2803 "sem",
2804 "var",
2805 "skew",
2806 "kurt",
2807 ]:
2808 raise TypeError(
2809 f"dtype '{self.dtype}' does not support operation '{how}'"
2810 )
2811 if op.how not in ["any", "all"]:
2812 # Fail early to avoid conversion to object
2813 op._get_cython_function(op.kind, op.how, np.dtype(object), False)
2815 arr = self
2816 if op.how == "sum":
2817 initial = ""
2818 # https://github.com/pandas-dev/pandas/issues/60229
2819 # All NA should result in the empty string.
2820 assert "skipna" in kwargs
2821 if kwargs["skipna"] and min_count == 0:
2822 arr = arr.fillna("")
2823 npvalues = arr.to_numpy(object, na_value=np.nan)
2824 else:
2825 raise NotImplementedError(
2826 f"function is not implemented for this dtype: {self.dtype}"
2827 )
2829 res_values = op._cython_op_ndim_compat(
2830 npvalues,
2831 min_count=min_count,
2832 ngroups=ngroups,
2833 comp_ids=ids,
2834 mask=None,
2835 initial=initial,
2836 **kwargs,
2837 )
2839 if op.how in op.cast_blocklist:
2840 # i.e. how in ["rank"], since other cast_blocklist methods don't go
2841 # through cython_operation
2842 return res_values
2844 if isinstance(self.dtype, StringDtype):
2845 dtype = self.dtype
2846 string_array_cls = dtype.construct_array_type()
2847 return string_array_cls._from_sequence(res_values, dtype=dtype)
2849 else:
2850 raise NotImplementedError
2853class ExtensionArraySupportsAnyAll(ExtensionArray):
2854 @overload
2855 def any(self, *, skipna: Literal[True] = ...) -> bool: ...
2857 @overload
2858 def any(self, *, skipna: bool) -> bool | NAType: ...
2860 def any(self, *, skipna: bool = True) -> bool | NAType:
2861 raise AbstractMethodError(self)
2863 @overload
2864 def all(self, *, skipna: Literal[True] = ...) -> bool: ...
2866 @overload
2867 def all(self, *, skipna: bool) -> bool | NAType: ...
2869 def all(self, *, skipna: bool = True) -> bool | NAType:
2870 raise AbstractMethodError(self)
2873class ExtensionOpsMixin:
2874 """
2875 A base class for linking the operators to their dunder names.
2877 .. note::
2879 You may want to set ``__array_priority__`` if you want your
2880 implementation to be called when involved in binary operations
2881 with NumPy arrays.
2882 """
2884 @classmethod
2885 def _create_arithmetic_method(cls, op):
2886 raise AbstractMethodError(cls)
2888 @classmethod
2889 def _add_arithmetic_ops(cls) -> None:
2890 setattr(cls, "__add__", cls._create_arithmetic_method(operator.add))
2891 setattr(cls, "__radd__", cls._create_arithmetic_method(roperator.radd))
2892 setattr(cls, "__sub__", cls._create_arithmetic_method(operator.sub))
2893 setattr(cls, "__rsub__", cls._create_arithmetic_method(roperator.rsub))
2894 setattr(cls, "__mul__", cls._create_arithmetic_method(operator.mul))
2895 setattr(cls, "__rmul__", cls._create_arithmetic_method(roperator.rmul))
2896 setattr(cls, "__pow__", cls._create_arithmetic_method(operator.pow))
2897 setattr(cls, "__rpow__", cls._create_arithmetic_method(roperator.rpow))
2898 setattr(cls, "__mod__", cls._create_arithmetic_method(operator.mod))
2899 setattr(cls, "__rmod__", cls._create_arithmetic_method(roperator.rmod))
2900 setattr(cls, "__floordiv__", cls._create_arithmetic_method(operator.floordiv))
2901 setattr(
2902 cls, "__rfloordiv__", cls._create_arithmetic_method(roperator.rfloordiv)
2903 )
2904 setattr(cls, "__truediv__", cls._create_arithmetic_method(operator.truediv))
2905 setattr(cls, "__rtruediv__", cls._create_arithmetic_method(roperator.rtruediv))
2906 setattr(cls, "__divmod__", cls._create_arithmetic_method(divmod))
2907 setattr(cls, "__rdivmod__", cls._create_arithmetic_method(roperator.rdivmod))
2909 @classmethod
2910 def _create_comparison_method(cls, op):
2911 raise AbstractMethodError(cls)
2913 @classmethod
2914 def _add_comparison_ops(cls) -> None:
2915 setattr(cls, "__eq__", cls._create_comparison_method(operator.eq))
2916 setattr(cls, "__ne__", cls._create_comparison_method(operator.ne))
2917 setattr(cls, "__lt__", cls._create_comparison_method(operator.lt))
2918 setattr(cls, "__gt__", cls._create_comparison_method(operator.gt))
2919 setattr(cls, "__le__", cls._create_comparison_method(operator.le))
2920 setattr(cls, "__ge__", cls._create_comparison_method(operator.ge))
2922 @classmethod
2923 def _create_logical_method(cls, op):
2924 raise AbstractMethodError(cls)
2926 @classmethod
2927 def _add_logical_ops(cls) -> None:
2928 setattr(cls, "__and__", cls._create_logical_method(operator.and_))
2929 setattr(cls, "__rand__", cls._create_logical_method(roperator.rand_))
2930 setattr(cls, "__or__", cls._create_logical_method(operator.or_))
2931 setattr(cls, "__ror__", cls._create_logical_method(roperator.ror_))
2932 setattr(cls, "__xor__", cls._create_logical_method(operator.xor))
2933 setattr(cls, "__rxor__", cls._create_logical_method(roperator.rxor))
2936@set_module("pandas.api.extensions")
2937class ExtensionScalarOpsMixin(ExtensionOpsMixin):
2938 """
2939 A mixin for defining ops on an ExtensionArray.
2941 It is assumed that the underlying scalar objects have the operators
2942 already defined.
2944 Notes
2945 -----
2946 If you have defined a subclass MyExtensionArray(ExtensionArray), then
2947 use MyExtensionArray(ExtensionArray, ExtensionScalarOpsMixin) to
2948 get the arithmetic operators. After the definition of MyExtensionArray,
2949 insert the lines
2951 MyExtensionArray._add_arithmetic_ops()
2952 MyExtensionArray._add_comparison_ops()
2954 to link the operators to your class.
2956 .. note::
2958 You may want to set ``__array_priority__`` if you want your
2959 implementation to be called when involved in binary operations
2960 with NumPy arrays.
2961 """
2963 @classmethod
2964 def _create_method(cls, op, coerce_to_dtype: bool = True, result_dtype=None):
2965 """
2966 A class method that returns a method that will correspond to an
2967 operator for an ExtensionArray subclass, by dispatching to the
2968 relevant operator defined on the individual elements of the
2969 ExtensionArray.
2971 Parameters
2972 ----------
2973 op : function
2974 An operator that takes arguments op(a, b)
2975 coerce_to_dtype : bool, default True
2976 boolean indicating whether to attempt to convert
2977 the result to the underlying ExtensionArray dtype.
2978 If it's not possible to create a new ExtensionArray with the
2979 values, an ndarray is returned instead.
2981 Returns
2982 -------
2983 Callable[[Any, Any], Union[ndarray, ExtensionArray]]
2984 A method that can be bound to a class. When used, the method
2985 receives the two arguments, one of which is the instance of
2986 this class, and should return an ExtensionArray or an ndarray.
2988 Returning an ndarray may be necessary when the result of the
2989 `op` cannot be stored in the ExtensionArray. The dtype of the
2990 ndarray uses NumPy's normal inference rules.
2992 Examples
2993 --------
2994 Given an ExtensionArray subclass called MyExtensionArray, use
2996 __add__ = cls._create_method(operator.add)
2998 in the class definition of MyExtensionArray to create the operator
2999 for addition, that will be based on the operator implementation
3000 of the underlying elements of the ExtensionArray
3001 """
3003 def _binop(self, other):
3004 def convert_values(param):
3005 if isinstance(param, ExtensionArray) or is_list_like(param):
3006 ovalues = param
3007 else: # Assume its an object
3008 ovalues = [param] * len(self)
3009 return ovalues
3011 if isinstance(other, (ABCSeries, ABCIndex, ABCDataFrame)):
3012 # rely on pandas to unbox and dispatch to us
3013 return NotImplemented
3015 lvalues = self
3016 rvalues = convert_values(other)
3018 # If the operator is not defined for the underlying objects,
3019 # a TypeError should be raised
3020 res = [op(a, b) for (a, b) in zip(lvalues, rvalues, strict=True)]
3022 def _maybe_convert(arr):
3023 if coerce_to_dtype:
3024 # https://github.com/pandas-dev/pandas/issues/22850
3025 # We catch all regular exceptions here, and fall back
3026 # to an ndarray.
3027 res = self._cast_pointwise_result(arr)
3028 if not isinstance(res, type(self)):
3029 # exception raised in _from_sequence; ensure we have ndarray
3030 res = np.asarray(arr)
3031 else:
3032 res = np.asarray(arr, dtype=result_dtype)
3033 return res
3035 if op.__name__ in {"divmod", "rdivmod"}:
3036 a, b = zip(*res, strict=True)
3037 return _maybe_convert(a), _maybe_convert(b)
3039 return _maybe_convert(res)
3041 op_name = f"__{op.__name__}__"
3042 return set_function_name(_binop, op_name, cls)
3044 @classmethod
3045 def _create_arithmetic_method(cls, op):
3046 return cls._create_method(op)
3048 @classmethod
3049 def _create_comparison_method(cls, op):
3050 return cls._create_method(op, coerce_to_dtype=False, result_dtype=bool)