Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from collections import abc
4from datetime import (
5 datetime,
6 tzinfo,
7)
8import functools
9from itertools import zip_longest
10import operator
11from typing import (
12 TYPE_CHECKING,
13 Any,
14 ClassVar,
15 Literal,
16 NoReturn,
17 Self,
18 SupportsIndex,
19 cast,
20 final,
21 overload,
22)
23import warnings
25import numpy as np
27from pandas._config import (
28 is_nan_na,
29 using_string_dtype,
30)
31from pandas._config.config import _global_config as config
33from pandas._libs import (
34 NaT,
35 algos as libalgos,
36 index as libindex,
37 lib,
38 writers,
39)
40from pandas._libs.internals import BlockValuesRefs
41import pandas._libs.join as libjoin
42from pandas._libs.lib import (
43 is_datetime_array,
44 no_default,
45)
46from pandas._libs.missing import is_matching_na
47from pandas._libs.tslibs import (
48 Period,
49 Timestamp,
50 tz_compare,
51)
52from pandas._libs.tslibs.parsing import parse_datetime_string_with_reso
53from pandas.compat.numpy import function as nv
54from pandas.errors import (
55 DuplicateLabelError,
56 InvalidIndexError,
57 Pandas4Warning,
58)
59from pandas.util._decorators import (
60 cache_readonly,
61 set_module,
62)
63from pandas.util._exceptions import (
64 find_stack_level,
65 rewrite_exception,
66)
68from pandas.core.dtypes.astype import (
69 astype_array,
70 astype_is_view,
71)
72from pandas.core.dtypes.cast import (
73 LossySetitemError,
74 can_hold_element,
75 common_dtype_categorical_compat,
76 find_result_type,
77 infer_dtype_from,
78 maybe_unbox_numpy_scalar,
79 np_can_hold_element,
80)
81from pandas.core.dtypes.common import (
82 ensure_int64,
83 ensure_object,
84 ensure_platform_int,
85 is_any_real_numeric_dtype,
86 is_bool_dtype,
87 is_ea_or_datetimelike_dtype,
88 is_float,
89 is_hashable,
90 is_integer,
91 is_iterator,
92 is_list_like,
93 is_numeric_dtype,
94 is_object_dtype,
95 is_scalar,
96 is_signed_integer_dtype,
97 is_string_dtype,
98 needs_i8_conversion,
99 pandas_dtype,
100 validate_all_hashable,
101)
102from pandas.core.dtypes.concat import concat_compat
103from pandas.core.dtypes.dtypes import (
104 ArrowDtype,
105 CategoricalDtype,
106 DatetimeTZDtype,
107 ExtensionDtype,
108 IntervalDtype,
109 PeriodDtype,
110 SparseDtype,
111)
112from pandas.core.dtypes.generic import (
113 ABCCategoricalIndex,
114 ABCDataFrame,
115 ABCDatetimeIndex,
116 ABCIntervalIndex,
117 ABCMultiIndex,
118 ABCPeriodIndex,
119 ABCRangeIndex,
120 ABCSeries,
121 ABCTimedeltaIndex,
122)
123from pandas.core.dtypes.inference import is_dict_like
124from pandas.core.dtypes.missing import (
125 array_equivalent,
126 is_valid_na_for_dtype,
127 isna,
128)
130from pandas.core import (
131 arraylike,
132 nanops,
133 ops,
134)
135from pandas.core.accessor import Accessor
136import pandas.core.algorithms as algos
137from pandas.core.array_algos.putmask import validate_putmask
138from pandas.core.arrays import (
139 ArrowExtensionArray,
140 BaseMaskedArray,
141 Categorical,
142 DatetimeArray,
143 ExtensionArray,
144 PeriodArray,
145 TimedeltaArray,
146)
147from pandas.core.arrays.floating import FloatingDtype
148from pandas.core.arrays.string_ import (
149 StringArray,
150 StringDtype,
151)
152from pandas.core.base import (
153 IndexOpsMixin,
154 PandasObject,
155)
156import pandas.core.common as com
157from pandas.core.construction import (
158 ensure_wrapped_if_datetimelike,
159 extract_array,
160 sanitize_array,
161)
162from pandas.core.indexers import (
163 disallow_ndim_indexing,
164 is_valid_positional_slice,
165)
166from pandas.core.indexes.frozen import FrozenList
167from pandas.core.missing import clean_reindex_fill_method
168from pandas.core.ops import get_op_result_name
169from pandas.core.sorting import (
170 ensure_key_mapped,
171 get_group_index_sorter,
172 nargsort,
173)
174from pandas.core.strings.accessor import StringMethods
176from pandas.io.formats.printing import (
177 PrettyDict,
178 default_pprint,
179 format_object_summary,
180 get_adjustment,
181)
183if TYPE_CHECKING:
184 from collections.abc import (
185 Callable,
186 Hashable,
187 Iterable,
188 Sequence,
189 )
191 from pandas._typing import (
192 AnyAll,
193 ArrayLike,
194 Axes,
195 Axis,
196 AxisInt,
197 DropKeep,
198 Dtype,
199 DtypeObj,
200 F,
201 Frequency,
202 IgnoreRaise,
203 IndexLabel,
204 IndexT,
205 JoinHow,
206 Level,
207 NaPosition,
208 ReindexMethod,
209 Shape,
210 SliceType,
211 npt,
212 )
214 from pandas import (
215 CategoricalIndex,
216 DataFrame,
217 MultiIndex,
218 Series,
219 )
220 from pandas.core.arrays import IntervalArray
222__all__ = ["Index"]
224str_t = str
226_dtype_obj = np.dtype("object")
228_masked_engines = {
229 "Complex128": libindex.MaskedComplex128Engine,
230 "Complex64": libindex.MaskedComplex64Engine,
231 "Float64": libindex.MaskedFloat64Engine,
232 "Float32": libindex.MaskedFloat32Engine,
233 "UInt64": libindex.MaskedUInt64Engine,
234 "UInt32": libindex.MaskedUInt32Engine,
235 "UInt16": libindex.MaskedUInt16Engine,
236 "UInt8": libindex.MaskedUInt8Engine,
237 "Int64": libindex.MaskedInt64Engine,
238 "Int32": libindex.MaskedInt32Engine,
239 "Int16": libindex.MaskedInt16Engine,
240 "Int8": libindex.MaskedInt8Engine,
241 "boolean": libindex.MaskedBoolEngine,
242 "double[pyarrow]": libindex.MaskedFloat64Engine,
243 "float64[pyarrow]": libindex.MaskedFloat64Engine,
244 "float32[pyarrow]": libindex.MaskedFloat32Engine,
245 "float[pyarrow]": libindex.MaskedFloat32Engine,
246 "uint64[pyarrow]": libindex.MaskedUInt64Engine,
247 "uint32[pyarrow]": libindex.MaskedUInt32Engine,
248 "uint16[pyarrow]": libindex.MaskedUInt16Engine,
249 "uint8[pyarrow]": libindex.MaskedUInt8Engine,
250 "int64[pyarrow]": libindex.MaskedInt64Engine,
251 "int32[pyarrow]": libindex.MaskedInt32Engine,
252 "int16[pyarrow]": libindex.MaskedInt16Engine,
253 "int8[pyarrow]": libindex.MaskedInt8Engine,
254 "bool[pyarrow]": libindex.MaskedBoolEngine,
255}
258def _maybe_return_indexers(meth: F) -> F:
259 """
260 Decorator to simplify 'return_indexers' checks in Index.join.
261 """
263 @functools.wraps(meth)
264 def join(
265 self: Index,
266 other: Index,
267 *,
268 how: JoinHow = "left",
269 level: Level | None = None,
270 return_indexers: bool = False,
271 sort: bool = False,
272 ) -> Index | tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
273 join_index, lidx, ridx = meth(self, other, how=how, level=level, sort=sort)
274 if not return_indexers:
275 return join_index
277 if lidx is not None:
278 lidx = ensure_platform_int(lidx)
279 if ridx is not None:
280 ridx = ensure_platform_int(ridx)
281 return join_index, lidx, ridx
283 return cast("F", join)
286def _new_Index(cls: type[Index], d: dict) -> Index:
287 """
288 This is called upon unpickling, rather than the default which doesn't
289 have arguments and breaks __new__.
290 """
291 # required for backward compat, because PI can't be instantiated with
292 # ordinals through __new__ GH #13277
293 d["copy"] = False
294 if issubclass(cls, ABCPeriodIndex):
295 from pandas.core.indexes.period import _new_PeriodIndex
297 return _new_PeriodIndex(cls, **d) # type: ignore[no-untyped-call]
299 if issubclass(cls, ABCMultiIndex):
300 if "labels" in d and "codes" not in d:
301 # GH#23752 "labels" kwarg has been replaced with "codes"
302 d["codes"] = d.pop("labels")
304 # Since this was a valid MultiIndex at pickle-time, we don't need to
305 # check validty at un-pickle time.
306 d["verify_integrity"] = False
308 elif "dtype" not in d and "data" in d:
309 # Prevent Index.__new__ from conducting inference;
310 # "data" key not in RangeIndex
311 d["dtype"] = d["data"].dtype
312 return cls.__new__(cls, **d) # pyright: ignore[reportArgumentType]
315@set_module("pandas")
316class Index(IndexOpsMixin, PandasObject):
317 """
318 Immutable sequence used for indexing and alignment.
320 The basic object storing axis labels for all pandas objects.
322 .. versionchanged:: 2.0.0
324 Index can hold all numpy numeric dtypes (except float16). Previously only
325 int64/uint64/float64 dtypes were accepted.
327 Parameters
328 ----------
329 data : array-like (1-dimensional)
330 An array-like structure containing the data for the index. This could be a
331 Python list, a NumPy array, or a pandas Series.
332 dtype : str, numpy.dtype, or ExtensionDtype, optional
333 Data type for the output Index. If not specified, this will be
334 inferred from `data`.
335 See the :ref:`user guide <basics.dtypes>` for more usages.
336 copy : bool, default None
337 Whether to copy input data, only relevant for array, Series, and Index
338 inputs (for other input, e.g. a list, a new array is created anyway).
339 Defaults to True for array input and False for Index/Series.
340 Set to False to avoid copying array input at your own risk (if you
341 know the input data won't be modified elsewhere).
342 Set to True to force copying Series/Index input up front.
343 name : object
344 Name to be stored in the index.
345 tupleize_cols : bool (default: True)
346 When True, attempt to create a MultiIndex if possible.
348 See Also
349 --------
350 RangeIndex : Index implementing a monotonic integer range.
351 CategoricalIndex : Index of :class:`Categorical` s.
352 MultiIndex : A multi-level, or hierarchical Index.
353 IntervalIndex : An Index of :class:`Interval` s.
354 DatetimeIndex : Index of datetime64 data.
355 TimedeltaIndex : Index of timedelta64 data.
356 PeriodIndex : Index of Period data.
358 Notes
359 -----
360 An Index instance can **only** contain hashable objects.
361 An Index instance *can not* hold numpy float16 dtype.
363 NumPy arrays with ``dtype=object`` are handled like other list-like inputs:
364 pandas may infer a more specific dtype (for example ``str`` for an array of
365 Python strings) instead of preserving NumPy object dtype. Values that
366 NumPy already stores in a dedicated dtype (such as ``int64`` integers) are
367 not altered in the same way. To force object dtype, pass ``dtype=object``.
369 Examples
370 --------
371 >>> pd.Index([1, 2, 3])
372 Index([1, 2, 3], dtype='int64')
374 >>> pd.Index(list("abc"))
375 Index(['a', 'b', 'c'], dtype='str')
377 >>> pd.Index([1, 2, 3], dtype="uint8")
378 Index([1, 2, 3], dtype='uint8')
379 """
381 # similar to __array_priority__, positions Index after Series and DataFrame
382 # but before ExtensionArray. Should NOT be overridden by subclasses.
383 __pandas_priority__ = 2000
385 # Cython methods; see github.com/cython/cython/issues/2647
386 # for why we need to wrap these instead of making them class attributes
387 # Moreover, cython will choose the appropriate-dtyped sub-function
388 # given the dtypes of the passed arguments
390 @final
391 def _left_indexer_unique(self, other: Self) -> npt.NDArray[np.intp]:
392 # Caller is responsible for ensuring other.dtype == self.dtype
393 sv = self._get_join_target()
394 ov = other._get_join_target()
395 # similar but not identical to ov.searchsorted(sv)
396 return libjoin.left_join_indexer_unique(sv, ov)
398 @final
399 def _left_indexer(
400 self, other: Self
401 ) -> tuple[ArrayLike, npt.NDArray[np.intp], npt.NDArray[np.intp]]:
402 # Caller is responsible for ensuring other.dtype == self.dtype
403 sv = self._get_join_target()
404 ov = other._get_join_target()
405 joined_ndarray, lidx, ridx = libjoin.left_join_indexer(sv, ov)
406 joined = self._from_join_target(joined_ndarray)
407 return joined, lidx, ridx
409 @final
410 def _inner_indexer(
411 self, other: Self
412 ) -> tuple[ArrayLike, npt.NDArray[np.intp], npt.NDArray[np.intp]]:
413 # Caller is responsible for ensuring other.dtype == self.dtype
414 sv = self._get_join_target()
415 ov = other._get_join_target()
416 joined_ndarray, lidx, ridx = libjoin.inner_join_indexer(sv, ov)
417 joined = self._from_join_target(joined_ndarray)
418 return joined, lidx, ridx
420 @final
421 def _outer_indexer(
422 self, other: Self
423 ) -> tuple[ArrayLike, npt.NDArray[np.intp], npt.NDArray[np.intp]]:
424 # Caller is responsible for ensuring other.dtype == self.dtype
425 sv = self._get_join_target()
426 ov = other._get_join_target()
427 joined_ndarray, lidx, ridx = libjoin.outer_join_indexer(sv, ov)
428 joined = self._from_join_target(joined_ndarray)
429 return joined, lidx, ridx
431 _typ: str = "index"
432 _data: ExtensionArray | np.ndarray
433 _data_cls: type[ExtensionArray] | tuple[type[np.ndarray], type[ExtensionArray]] = (
434 np.ndarray,
435 ExtensionArray,
436 )
437 _id: object | None = None
438 _name: Hashable = None
439 # MultiIndex.levels previously allowed setting the index name. We
440 # don't allow this anymore, and raise if it happens rather than
441 # failing silently.
442 _no_setting_name: bool = False
443 _comparables: list[str] = ["name"]
444 _attributes: list[str] = ["name"]
446 @cache_readonly
447 def _can_hold_strings(self) -> bool:
448 return not is_numeric_dtype(self.dtype)
450 _engine_types: dict[np.dtype | ExtensionDtype, type[libindex.IndexEngine]] = {
451 np.dtype(np.int8): libindex.Int8Engine,
452 np.dtype(np.int16): libindex.Int16Engine,
453 np.dtype(np.int32): libindex.Int32Engine,
454 np.dtype(np.int64): libindex.Int64Engine,
455 np.dtype(np.uint8): libindex.UInt8Engine,
456 np.dtype(np.uint16): libindex.UInt16Engine,
457 np.dtype(np.uint32): libindex.UInt32Engine,
458 np.dtype(np.uint64): libindex.UInt64Engine,
459 np.dtype(np.float32): libindex.Float32Engine,
460 np.dtype(np.float64): libindex.Float64Engine,
461 np.dtype(np.complex64): libindex.Complex64Engine,
462 np.dtype(np.complex128): libindex.Complex128Engine,
463 }
465 @property
466 def _engine_type(
467 self,
468 ) -> type[libindex.IndexEngine | libindex.ExtensionEngine]:
469 return self._engine_types.get(self.dtype, libindex.ObjectEngine)
471 # whether we support partial string indexing. Overridden
472 # in DatetimeIndex and PeriodIndex
473 _supports_partial_string_indexing = False
475 _accessors = {"str"}
477 str = Accessor("str", StringMethods)
479 _references: BlockValuesRefs | None = None
481 # --------------------------------------------------------------------
482 # Constructors
484 def __new__(
485 cls,
486 data: Any = None,
487 dtype: Dtype | None = None,
488 copy: bool | None = None,
489 name: Hashable = None,
490 tupleize_cols: bool = True,
491 ) -> Self:
492 from pandas.core.indexes.range import RangeIndex
494 name = maybe_extract_name(name, data, cls)
496 if dtype is not None:
497 dtype = pandas_dtype(dtype)
499 data_dtype = getattr(data, "dtype", None)
501 refs = None
502 if not copy and isinstance(data, (ABCSeries, Index)):
503 refs = data._references
505 # GH 63306, GH 63388
506 data, copy = cls._maybe_copy_array_input(data, copy, dtype)
508 # range
509 if isinstance(data, (range, RangeIndex)):
510 result = RangeIndex(start=data, copy=bool(copy), name=name)
511 if dtype is not None:
512 return result.astype(dtype, copy=False) # type: ignore[return-value] # pyright: ignore[reportReturnType]
513 # error: Incompatible return value type (got "MultiIndex",
514 # expected "Self")
515 return result # type: ignore[return-value]
517 elif is_ea_or_datetimelike_dtype(dtype):
518 # non-EA dtype indexes have special casting logic, so we punt here
519 if isinstance(data, (set, frozenset)):
520 data = list(data)
522 elif is_ea_or_datetimelike_dtype(data_dtype):
523 pass
525 elif isinstance(data, (np.ndarray, ABCMultiIndex)):
526 if isinstance(data, ABCMultiIndex):
527 data = data._values
529 if data.dtype.kind not in "iufcbmM":
530 # GH#11836 we need to avoid having numpy coerce
531 # things that look like ints/floats to ints unless
532 # they are actually ints, e.g. '0' and 0.0
533 # should not be coerced
534 data = com.asarray_tuplesafe(data, dtype=_dtype_obj)
535 elif isinstance(data, (ABCSeries, Index)):
536 # GH 56244: Avoid potential inference on object types
537 pass
538 elif is_scalar(data):
539 raise cls._raise_scalar_data_error(data)
540 elif hasattr(data, "__array__"):
541 return cls(np.asarray(data), dtype=dtype, copy=copy, name=name)
542 elif not is_list_like(data) and not isinstance(data, memoryview):
543 # 2022-11-16 the memoryview check is only necessary on some CI
544 # builds, not clear why
545 raise cls._raise_scalar_data_error(data)
547 else:
548 if tupleize_cols:
549 # GH21470: convert iterable to list before determining if empty
550 if is_iterator(data):
551 data = list(data)
553 if data and all(isinstance(e, tuple) for e in data):
554 # we must be all tuples, otherwise don't construct
555 # 10697
556 from pandas.core.indexes.multi import MultiIndex
558 # error: Incompatible return value type (got "MultiIndex",
559 # expected "Self")
560 return MultiIndex.from_tuples( # type: ignore[return-value]
561 data, names=name
562 )
563 # other iterable of some kind
565 if not isinstance(data, (list, tuple)):
566 # we allow set/frozenset, which Series/sanitize_array does not, so
567 # cast to list here
568 data = list(data)
569 if len(data) == 0:
570 # unlike Series, we default to object dtype:
571 data = np.array(data, dtype=object)
573 if len(data) and isinstance(data[0], tuple):
574 # Ensure we get 1-D array of tuples instead of 2D array.
575 data = com.asarray_tuplesafe(data, dtype=_dtype_obj)
577 try:
578 arr = sanitize_array(data, None, dtype=dtype, copy=bool(copy))
579 except ValueError as err:
580 if "index must be specified when data is not list-like" in str(err):
581 raise cls._raise_scalar_data_error(data) from err
582 if "Data must be 1-dimensional" in str(err):
583 raise ValueError("Index data must be 1-dimensional") from err
584 raise
585 arr = ensure_wrapped_if_datetimelike(arr) # type: ignore[no-untyped-call]
587 klass = cls._dtype_to_subclass(arr.dtype)
589 arr = klass._ensure_array(arr, arr.dtype, copy=False)
590 out = klass._simple_new(arr, name, refs=refs)
591 # Preserve freq when wrapping a DatetimeIndex/TimedeltaIndex input,
592 # since _simple_new doesn't copy Index-level attributes like _freq.
593 if isinstance(data, (ABCDatetimeIndex, ABCTimedeltaIndex)) and isinstance(
594 out, (ABCDatetimeIndex, ABCTimedeltaIndex)
595 ):
596 out._freq = data._freq
597 return out # type: ignore[return-value] # pyright: ignore[reportReturnType]
599 @classmethod
600 def _ensure_array(cls, data: ArrayLike, dtype: DtypeObj, copy: bool) -> ArrayLike:
601 """
602 Ensure we have a valid array to pass to _simple_new.
603 """
604 if data.ndim > 1:
605 # GH#13601, GH#20285, GH#27125
606 raise ValueError("Index data must be 1-dimensional")
607 elif dtype == np.float16:
608 # float16 not supported (no indexing engine)
609 raise NotImplementedError("float16 indexes are not supported")
611 if copy:
612 # asarray_tuplesafe does not always copy underlying data,
613 # so need to make sure that this happens
614 data = data.copy()
615 return data
617 @final
618 @classmethod
619 def _dtype_to_subclass(cls, dtype: DtypeObj) -> type[Index]:
620 # Delay import for perf. https://github.com/pandas-dev/pandas/pull/31423
622 if isinstance(dtype, ExtensionDtype):
623 return dtype.index_class
625 if dtype.kind == "M":
626 from pandas import DatetimeIndex
628 return DatetimeIndex
630 elif dtype.kind == "m":
631 from pandas import TimedeltaIndex
633 return TimedeltaIndex
635 elif dtype.kind == "O":
636 # NB: assuming away MultiIndex
637 return Index
639 elif issubclass(dtype.type, str) or is_numeric_dtype(dtype):
640 return Index
642 raise NotImplementedError(dtype)
644 # NOTE for new Index creation:
646 # - _simple_new: It returns new Index with the same type as the caller.
647 # All metadata (such as name) must be provided by caller's responsibility.
648 # Using _shallow_copy is recommended because it fills these metadata
649 # otherwise specified.
651 # - _shallow_copy: It returns new Index with the same type (using
652 # _simple_new), but fills caller's metadata otherwise specified. Passed
653 # kwargs will overwrite corresponding metadata.
655 # See each method's docstring.
657 @classmethod
658 def _simple_new(
659 cls,
660 values: ArrayLike,
661 name: Hashable | None = None,
662 refs: BlockValuesRefs | None = None,
663 ) -> Self:
664 """
665 We require that we have a dtype compat for the values. If we are passed
666 a non-dtype compat, then coerce using the constructor.
668 Must be careful not to recurse.
669 """
670 assert isinstance(values, cls._data_cls), type(values)
672 result = object.__new__(cls)
673 result._data = values
674 result._name = name
675 result._cache = {}
676 result._reset_identity()
677 if refs is not None:
678 result._references = refs
679 else:
680 result._references = BlockValuesRefs()
681 result._references.add_index_reference(result)
683 return result
685 @classmethod
686 def _with_infer(cls, *args: Any, **kwargs: Any) -> Self:
687 """
688 Constructor that uses the 1.0.x behavior inferring numeric dtypes
689 for ndarray[object] inputs.
690 """
691 result = cls(*args, **kwargs)
693 if result.dtype == _dtype_obj and not result._is_multi:
694 # error: Argument 1 to "maybe_convert_objects" has incompatible type
695 # "Union[ExtensionArray, ndarray[Any, Any]]"; expected
696 # "ndarray[Any, Any]"
697 values = lib.maybe_convert_objects(result._values) # type: ignore[arg-type]
698 if values.dtype.kind in "iufb":
699 return Index(values, name=result.name, copy=False) # type: ignore[return-value]
701 return result
703 @cache_readonly
704 def _constructor(self) -> type[Self]:
705 return type(self)
707 @final
708 def _maybe_check_unique(self) -> None:
709 """
710 Check that an Index has no duplicates.
712 This is typically only called via
713 `NDFrame.flags.allows_duplicate_labels.setter` when it's set to
714 True (duplicates aren't allowed).
716 Raises
717 ------
718 DuplicateLabelError
719 When the index is not unique.
720 """
721 if not self.is_unique:
722 msg = """Index has duplicates."""
723 duplicates = self._format_duplicate_message()
724 msg += f"\n{duplicates}"
726 raise DuplicateLabelError(msg)
728 @final
729 def _format_duplicate_message(self) -> DataFrame:
730 """
731 Construct the DataFrame for a DuplicateLabelError.
733 This returns a DataFrame indicating the labels and positions
734 of duplicates in an index. This should only be called when it's
735 already known that duplicates are present.
737 Examples
738 --------
739 >>> idx = pd.Index(["a", "b", "a"])
740 >>> idx._format_duplicate_message()
741 positions
742 label
743 a [0, 2]
744 """
745 from pandas import Series
747 duplicates = self[self.duplicated(keep="first")].unique()
748 assert len(duplicates)
750 out = (
751 Series(np.arange(len(self)), copy=False)
752 .groupby(self, observed=False)
753 .agg(list)[duplicates]
754 )
755 if self._is_multi:
756 # test_format_duplicate_labels_message_multi
757 # error: "Type[Index]" has no attribute "from_tuples" [attr-defined]
758 out.index = type(self).from_tuples(out.index) # type: ignore[attr-defined]
760 if self.nlevels == 1:
761 out = out.rename_axis("label")
762 return out.to_frame(name="positions")
764 # --------------------------------------------------------------------
765 # Index Internals Methods
767 def _shallow_copy(self, values: ArrayLike, name: Hashable = no_default) -> Self:
768 """
769 Create a new Index with the same class as the caller, don't copy the
770 data, use the same object attributes with passed in attributes taking
771 precedence.
773 *this is an internal non-public method*
775 Parameters
776 ----------
777 values : the values to create the new Index, optional
778 name : Label, defaults to self.name
779 """
780 name = self._name if name is no_default else name
782 return self._simple_new(values, name=name, refs=self._references)
784 def _view(self) -> Self:
785 """
786 fastpath to make a shallow copy, i.e. new object with same data.
787 """
788 result = self._simple_new(self._values, name=self._name, refs=self._references)
790 result._cache = self._cache
791 return result
793 @final
794 def _rename(self, name: Hashable) -> Self:
795 """
796 fastpath for rename if new name is already validated.
797 """
798 result = self._view()
799 result._name = name
800 return result
802 @final
803 def is_(self, other: Index) -> bool:
804 """
805 More flexible, faster check like ``is`` but that works through views.
807 Note: this is *not* the same as ``Index.identical()``, which checks
808 that metadata is also the same.
810 Parameters
811 ----------
812 other : Index
813 Other Index to compare against.
815 Returns
816 -------
817 bool
818 True if both have same underlying data, False otherwise.
820 See Also
821 --------
822 Index.identical : Works like ``Index.is_`` but also checks metadata.
824 Examples
825 --------
826 >>> idx1 = pd.Index(["1", "2", "3"])
827 >>> idx1.is_(idx1.view())
828 True
830 >>> idx1.is_(idx1.copy())
831 False
832 """
833 if self is other:
834 return True
835 elif not hasattr(other, "_id"):
836 return False
837 elif self._id is None or other._id is None: # pyright: ignore[reportAttributeAccessIssue]
838 return False
839 else:
840 return self._id is other._id # pyright: ignore[reportAttributeAccessIssue]
842 @final
843 def _reset_identity(self) -> None:
844 """
845 Initializes or resets ``_id`` attribute with new object.
846 """
847 self._id = object()
849 @final
850 def _cleanup(self) -> None:
851 if "_engine" in self._cache:
852 self._engine.clear_mapping() # type: ignore[no-untyped-call]
854 @cache_readonly
855 def _engine(
856 self,
857 ) -> libindex.IndexEngine | libindex.ExtensionEngine | libindex.MaskedIndexEngine:
858 # For base class (object dtype) we get ObjectEngine
859 dtype = self.dtype
860 target_values = self._get_engine_target()
862 if isinstance(dtype, ArrowDtype) and dtype.kind in "Mm":
863 import pyarrow as pa
865 values = self._values
866 pa_type = values._pa_array.type # type: ignore[union-attr]
867 if pa.types.is_timestamp(pa_type):
868 target_values = values._to_datetimearray() # type: ignore[union-attr]
869 return libindex.DatetimeEngine(target_values._ndarray)
870 elif pa.types.is_duration(pa_type):
871 target_values = values._to_timedeltaarray() # type: ignore[union-attr]
872 return libindex.TimedeltaEngine(target_values._ndarray)
874 if isinstance(target_values, ExtensionArray):
875 if isinstance(target_values, (BaseMaskedArray, ArrowExtensionArray)):
876 try:
877 return _masked_engines[target_values.dtype.name](target_values)
878 except KeyError:
879 # Not supported yet e.g. decimal
880 pass
881 elif self._engine_type is libindex.ObjectEngine:
882 return libindex.ExtensionEngine(target_values)
884 # to avoid a reference cycle, bind `target_values` to a local variable, so
885 # `self` is not passed into the lambda.
886 if target_values.dtype == bool:
887 return libindex.BoolEngine(target_values) # type: ignore[arg-type]
888 elif target_values.dtype == np.complex64:
889 return libindex.Complex64Engine(target_values) # type: ignore[arg-type]
890 elif target_values.dtype == np.complex128:
891 return libindex.Complex128Engine(target_values) # type: ignore[arg-type]
892 elif needs_i8_conversion(dtype):
893 # We need to keep M8/m8 dtype when initializing the Engine,
894 # but don't want to change _get_engine_target bc it is used
895 # elsewhere
896 # error: Item "ExtensionArray" of "Union[ExtensionArray,
897 # ndarray[Any, Any]]" has no attribute "_ndarray" [union-attr]
898 target_values = self._data._ndarray # type: ignore[union-attr]
899 elif isinstance(dtype, StringDtype):
900 return libindex.StringObjectEngine(target_values, dtype.na_value)
902 # error: Argument 1 to "ExtensionEngine" has incompatible type
903 # "ndarray[Any, Any]"; expected "ExtensionArray"
904 return self._engine_type(target_values) # type: ignore[arg-type]
906 @final
907 @cache_readonly
908 def _dir_additions_for_owner(self) -> set[str_t]:
909 """
910 Add the string-like labels to the owner dataframe/series dir output.
912 If this is a MultiIndex, it's first level values are used.
913 """
914 max_items = config["display"]["max_dir_items"]
915 # GH#18587 avoid hashing the entire Index in the common case where
916 # only the first ``max_items`` unique values are needed. Examining a
917 # bounded prefix suffices when it already contains enough uniques;
918 # otherwise fall back to the full unique() call.
919 if max_items is not None and len(self) > max_items * 10:
920 prefix_uniq = self[: max_items * 10].unique(level=0)
921 if len(prefix_uniq) >= max_items:
922 return {
923 c
924 for c in prefix_uniq[:max_items]
925 if isinstance(c, str) and c.isidentifier()
926 }
927 return {
928 c
929 for c in self.unique(level=0)[:max_items]
930 if isinstance(c, str) and c.isidentifier()
931 }
933 # --------------------------------------------------------------------
934 # Array-Like Methods
936 # ndarray compat
937 def __len__(self) -> int:
938 """
939 Return the length of the Index.
940 """
941 return len(self._data)
943 def __array__(
944 self, dtype: npt.DTypeLike | None = None, copy: bool | None = None
945 ) -> np.ndarray:
946 """
947 The array interface, return my values.
948 """
949 if copy is None:
950 # Note, that the if branch exists for NumPy 1.x support
951 return np.asarray(self._data, dtype=dtype)
953 return np.array(self._data, dtype=dtype, copy=copy)
955 def __array_ufunc__(
956 self, ufunc: np.ufunc, method: str_t, *inputs: Any, **kwargs: Any
957 ) -> Any:
958 if any(isinstance(other, (ABCSeries, ABCDataFrame)) for other in inputs):
959 return NotImplemented
961 # self._values only reaches the ExtensionArray guard when it is an EA, so a
962 # bool Index against datetimelike data needs this here
963 ops.disallow_datetimelike_logical_ufunc(ufunc, inputs)
965 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(
966 self, ufunc, method, *inputs, **kwargs
967 )
968 if result is not NotImplemented:
969 return result
971 if "out" in kwargs:
972 # e.g. test_dti_isub_tdi
973 return arraylike.dispatch_ufunc_with_out(
974 self, ufunc, method, *inputs, **kwargs
975 )
977 if method == "reduce":
978 result = arraylike.dispatch_reduction_ufunc(
979 self, ufunc, method, *inputs, **kwargs
980 )
981 if result is not NotImplemented:
982 return result
984 new_inputs = [x if x is not self else x._values for x in inputs]
985 result = getattr(ufunc, method)(*new_inputs, **kwargs)
986 if ufunc.nout == 2:
987 # i.e. np.divmod, np.modf, np.frexp
988 return tuple(self.__array_wrap__(x) for x in result)
989 elif method == "reduce":
990 result = lib.item_from_zerodim(result)
991 return maybe_unbox_numpy_scalar(result, object_with_dtype=self)
992 elif is_scalar(result):
993 # e.g. matmul
994 return maybe_unbox_numpy_scalar(result, object_with_dtype=self)
996 if result.dtype == np.float16:
997 result = result.astype(np.float32)
999 return self.__array_wrap__(result)
1001 @final
1002 def __array_wrap__(
1003 self,
1004 result: np.ndarray,
1005 context: tuple | None = None,
1006 return_scalar: bool = False,
1007 ) -> Any:
1008 """
1009 Gets called after a ufunc and other functions e.g. np.split.
1010 """
1011 result = lib.item_from_zerodim(result) # type: ignore[assignment]
1012 if np.ndim(result) > 1:
1013 # Reached in plotting tests with e.g. np.nonzero(index)
1014 return result
1016 return Index(result, name=self.name)
1018 @cache_readonly
1019 def dtype(self) -> DtypeObj:
1020 """
1021 Return the dtype object of the underlying data.
1023 The dtype describes the type of elements stored in the Index, such as
1024 ``int64``, ``float64``, ``object``, or an extension dtype.
1026 See Also
1027 --------
1028 Index.inferred_type: Return a string of the type inferred from the values.
1030 Examples
1031 --------
1032 >>> idx = pd.Index([1, 2, 3])
1033 >>> idx
1034 Index([1, 2, 3], dtype='int64')
1035 >>> idx.dtype
1036 dtype('int64')
1037 """
1038 return self._data.dtype
1040 @final
1041 def ravel(self, order: str_t = "C") -> Self:
1042 """
1043 Return a view on self.
1045 Since Index objects are one-dimensional, this simply returns a view
1046 of the original Index unchanged.
1048 Parameters
1049 ----------
1050 order : {'K', 'A', 'C', 'F'}, default 'C'
1051 Specify the memory layout of the view. This parameter is not
1052 implemented currently.
1054 Returns
1055 -------
1056 Index
1057 A view on self.
1059 See Also
1060 --------
1061 numpy.ndarray.ravel : Return a flattened array.
1063 Examples
1064 --------
1065 >>> s = pd.Series([1, 2, 3], index=["a", "b", "c"])
1066 >>> s.index.ravel()
1067 Index(['a', 'b', 'c'], dtype='str')
1068 """
1069 return self[:] # type: ignore[return-value]
1071 def view(self, cls: npt.DTypeLike | None = None) -> Index:
1072 """
1073 Return a view of the Index with the specified dtype or a new Index instance.
1075 This method returns a view of the calling Index object if no arguments are
1076 provided. If a dtype is specified through the `cls` argument, it attempts
1077 to return a view of the Index with the specified dtype. Note that viewing
1078 the Index as a different dtype reinterprets the underlying data, which can
1079 lead to unexpected results for non-numeric or incompatible dtype conversions.
1081 Parameters
1082 ----------
1083 cls : data-type or ndarray sub-class, optional
1084 Data-type descriptor of the returned view, e.g., float32 or int16.
1085 Omitting it results in the view having the same data-type as `self`.
1086 This argument can also be specified as an ndarray sub-class,
1087 e.g., np.int64 or np.float32 which then specifies the type of
1088 the returned object.
1090 Returns
1091 -------
1092 Index or ndarray
1093 A view of the Index. If `cls` is None, the returned object is an Index
1094 view with the same dtype as the calling object. If a numeric `cls` is
1095 specified an ndarray view with the new dtype is returned.
1097 Raises
1098 ------
1099 ValueError
1100 If attempting to change to a dtype in a way that is not compatible with
1101 the original dtype's memory layout, for example, viewing an 'int64' Index
1102 as 'str'.
1104 See Also
1105 --------
1106 Index.copy : Returns a copy of the Index.
1107 numpy.ndarray.view : Returns a new view of array with the same data.
1109 Examples
1110 --------
1111 >>> idx = pd.Index([-1, 0, 1])
1112 >>> idx.view()
1113 Index([-1, 0, 1], dtype='int64')
1115 >>> idx.view(np.uint64)
1116 array([18446744073709551615, 0, 1],
1117 dtype=uint64)
1119 Viewing as 'int32' or 'float32' reinterprets the memory, which may lead to
1120 unexpected behavior:
1122 >>> idx.view("float32")
1123 array([ nan, nan, 0.e+00, 0.e+00, 1.e-45, 0.e+00], dtype=float32)
1124 """
1125 # we need to see if we are subclassing an
1126 # index type here
1127 if cls is not None:
1128 dtype = cls
1129 if isinstance(cls, str):
1130 dtype = pandas_dtype(cls) # type: ignore[assignment]
1132 if needs_i8_conversion(dtype): # type: ignore[arg-type]
1133 idx_cls = self._dtype_to_subclass(dtype) # type: ignore[arg-type]
1134 arr = self.array.view(dtype) # type: ignore[arg-type]
1135 if isinstance(arr, ExtensionArray):
1136 # here we exclude non-supported dt64/td64 dtypes
1137 return idx_cls._simple_new(
1138 arr, name=self.name, refs=self._references
1139 )
1140 return arr # type: ignore[return-value]
1142 result = self._data.view(cls) # type: ignore[arg-type]
1143 else:
1144 result = self._view() # type: ignore[assignment]
1145 if isinstance(result, Index):
1146 result._id = self._id
1147 return result # type: ignore[return-value]
1149 def astype(self, dtype: Dtype, copy: bool = True) -> Index:
1150 """
1151 Create an Index with values cast to dtypes.
1153 The class of a new Index is determined by dtype. When conversion is
1154 impossible, a TypeError exception is raised.
1156 Parameters
1157 ----------
1158 dtype : numpy dtype or pandas type
1159 Dtype for the result Index.
1160 copy : bool, default True
1161 By default, astype always returns a newly allocated object.
1162 If copy is set to False and internal requirements on dtype are
1163 satisfied, the original data is used to create a new Index
1164 or the original Index is returned.
1166 Returns
1167 -------
1168 Index
1169 Index with values cast to specified dtype.
1171 See Also
1172 --------
1173 Index.dtype: Return the dtype object of the underlying data.
1174 Index.dtypes: Return the dtype object of the underlying data.
1175 Index.convert_dtypes: Convert columns to the best possible dtypes.
1177 Examples
1178 --------
1179 >>> idx = pd.Index([1, 2, 3])
1180 >>> idx
1181 Index([1, 2, 3], dtype='int64')
1182 >>> idx.astype("float")
1183 Index([1.0, 2.0, 3.0], dtype='float64')
1184 """
1185 if dtype is not None:
1186 dtype = pandas_dtype(dtype)
1188 if self.dtype == dtype:
1189 # Ensure that self.astype(self.dtype) is self
1190 return self.copy() if copy else self
1192 values = self._data
1193 if isinstance(values, ExtensionArray):
1194 with rewrite_exception(type(values).__name__, type(self).__name__):
1195 new_values = values.astype(dtype, copy=copy)
1197 elif isinstance(dtype, ExtensionDtype):
1198 cls = dtype.construct_array_type()
1199 # Note: for RangeIndex and CategoricalDtype self vs self._values
1200 # behaves differently here.
1201 new_values = cls._from_sequence(self, dtype=dtype, copy=copy)
1203 else:
1204 # GH#13149 specifically use astype_array instead of astype
1205 new_values = astype_array(values, dtype=dtype, copy=copy)
1207 # pass copy=False because any copying will be done in the astype above
1208 result = Index(new_values, name=self.name, dtype=new_values.dtype, copy=False)
1209 if (
1210 not copy
1211 and self._references is not None
1212 and astype_is_view(self.dtype, dtype)
1213 ):
1214 result._references = self._references
1215 result._references.add_index_reference(result)
1216 return result
1218 def take(
1219 self,
1220 indices: Sequence[int] | np.ndarray,
1221 axis: Axis = 0,
1222 allow_fill: bool | lib.NoDefault = no_default,
1223 fill_value: object = no_default,
1224 **kwargs: Any,
1225 ) -> Self:
1226 """
1227 Return a new Index of the values selected by the indices.
1229 For internal compatibility with numpy arrays.
1231 Parameters
1232 ----------
1233 indices : array-like
1234 Indices to be taken.
1235 axis : {0 or 'index'}, optional
1236 The axis over which to select values, always 0 or 'index'.
1237 allow_fill : bool, optional
1238 How to handle negative values in `indices`.
1240 * False: negative values in `indices` indicate positional indices
1241 from the right, matching :func:`numpy.take`.
1242 * True: negative values in `indices` indicate missing values. ``-1``
1243 entries are set to ``fill_value`` (using the default NA value of
1244 the caller's dtype if not supplied). Any other negative values
1245 raise a ``ValueError``.
1246 * Not supplied: defaults to ``allow_fill=False`` unless ``fill_value``
1247 is explicitly provided, in which case fill semantics apply
1248 (``allow_fill=True``).
1250 fill_value : scalar, optional
1251 If fill semantics apply (see ``allow_fill``), indices specified by
1252 ``-1`` are filled with ``fill_value``. Not specifying ``fill_value``
1253 or passing ``fill_value=None`` will fill using the default NA value
1254 of the caller's dtype. If the Index's dtype cannot hold
1255 ``fill_value``, the result is promoted to a common dtype (e.g. an
1256 integer Index + ``np.nan`` fill is cast to float, while an integer
1257 Index + a string fill is cast to object).
1258 **kwargs
1259 Required for compatibility with numpy.
1261 Returns
1262 -------
1263 Index
1264 An index formed of elements at the given indices. Will be the same
1265 type as self, except for RangeIndex.
1267 See Also
1268 --------
1269 numpy.ndarray.take: Return an array formed from the
1270 elements of a at the given indices.
1272 Examples
1273 --------
1274 >>> idx = pd.Index(["a", "b", "c"])
1275 >>> idx.take([2, 2, 1, 2])
1276 Index(['c', 'c', 'b', 'c'], dtype='str')
1277 """
1278 if kwargs:
1279 nv.validate_take((), kwargs)
1280 if is_scalar(indices):
1281 raise TypeError("Expected indices to be array-like")
1282 indices = ensure_platform_int(indices)
1284 if allow_fill is no_default:
1285 if fill_value is None:
1286 # GH#65210: preserve pre-3.1 wrap behavior for this one case,
1287 # but warn since the sentinel-based default would otherwise
1288 # flip it into fill-with-NA semantics.
1289 warnings.warn(
1290 "Passing fill_value=None without allow_fill previously "
1291 "used numpy-style wrapping of negative indices. In a "
1292 "future version this will trigger fill semantics "
1293 "(filling -1 entries with the dtype's NA value). Pass "
1294 "allow_fill=False to keep wrapping behavior, or "
1295 "allow_fill=True to opt into fill semantics.",
1296 Pandas4Warning,
1297 stacklevel=find_stack_level(),
1298 )
1299 allow_fill = False
1300 else:
1301 # Default: opt into fill semantics only if fill_value is
1302 # explicit.
1303 allow_fill = fill_value is not no_default
1305 if allow_fill:
1306 # Per the ExtensionArray convention, fill_value=None (or unsupplied)
1307 # falls back to self._na_value.
1308 if fill_value is no_default or fill_value is None:
1309 fill_value = self._na_value
1310 if (indices < -1).any():
1311 raise ValueError("When allow_fill=True, all indices must be >= -1")
1312 elif fill_value is no_default:
1313 # algos.take does not understand the no_default sentinel; it will
1314 # ignore fill_value anyway since allow_fill is False.
1315 fill_value = None
1317 if indices.ndim == 1 and lib.is_range_indexer(indices, len(self)):
1318 return self.copy()
1320 values = self._values
1321 if isinstance(values, np.ndarray):
1322 taken = algos.take(
1323 values, indices, allow_fill=allow_fill, fill_value=fill_value
1324 )
1325 else:
1326 # algos.take passes 'axis' keyword which not all EAs accept
1327 taken = values.take(indices, allow_fill=allow_fill, fill_value=fill_value)
1328 return self._constructor._simple_new(taken, name=self.name) # pyright: ignore[reportArgumentType]
1330 def repeat(self, repeats: int | Sequence[int], axis: None = None) -> Self:
1331 """
1332 Repeat elements of an Index.
1334 Returns a new Index where each element of the current Index
1335 is repeated consecutively a given number of times.
1337 Parameters
1338 ----------
1339 repeats : int or array of ints
1340 The number of repetitions for each element. This should be a
1341 non-negative integer. Repeating 0 times will return an empty
1342 Index.
1343 axis : None
1344 Must be ``None``. Has no effect but is accepted for compatibility
1345 with numpy.
1347 Returns
1348 -------
1349 Index
1350 Newly created Index with repeated elements.
1352 See Also
1353 --------
1354 Series.repeat : Equivalent function for Series.
1355 numpy.repeat : Similar method for :class:`numpy.ndarray`.
1357 Examples
1358 --------
1359 >>> idx = pd.Index(["a", "b", "c"])
1360 >>> idx
1361 Index(['a', 'b', 'c'], dtype='str')
1362 >>> idx.repeat(2)
1363 Index(['a', 'a', 'b', 'b', 'c', 'c'], dtype='str')
1364 >>> idx.repeat([1, 2, 3])
1365 Index(['a', 'b', 'b', 'c', 'c', 'c'], dtype='str')
1366 """
1367 repeats = ensure_platform_int(repeats) # type: ignore[assignment]
1368 nv.validate_repeat((), {"axis": axis})
1369 res_values = self._values.repeat(repeats) # pyright: ignore[reportArgumentType]
1371 # _constructor so RangeIndex-> Index with an int64 dtype
1372 return self._constructor._simple_new(res_values, name=self.name)
1374 # --------------------------------------------------------------------
1375 # Copying Methods
1377 def copy(
1378 self,
1379 name: Hashable | None = None,
1380 deep: bool = False,
1381 ) -> Self:
1382 """
1383 Make a copy of this object.
1385 Name is set on the new object.
1387 Parameters
1388 ----------
1389 name : Label, optional
1390 Set name for new object.
1391 deep : bool, default False
1392 If True attempts to make a deep copy of the Index.
1393 Else makes a shallow copy.
1395 Returns
1396 -------
1397 Index
1398 Index refer to new object which is a copy of this object.
1400 See Also
1401 --------
1402 Index.delete: Make new Index with passed location(-s) deleted.
1403 Index.drop: Make new Index with passed list of labels deleted.
1405 Notes
1406 -----
1407 In most cases, there should be no functional difference from using
1408 ``deep``, but if ``deep`` is passed it will attempt to deepcopy.
1410 Examples
1411 --------
1412 >>> idx = pd.Index(["a", "b", "c"])
1413 >>> new_idx = idx.copy()
1414 >>> idx is new_idx
1415 False
1416 """
1418 name = self._validate_names(name=name, deep=deep)[0]
1419 if deep:
1420 new_data = self._data.copy()
1421 new_index = type(self)._simple_new(new_data, name=name)
1422 else:
1423 new_index = self._rename(name=name)
1424 return new_index
1426 @final
1427 def __copy__(self) -> Self:
1428 return self.copy(deep=False)
1430 @final
1431 def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self:
1432 """
1433 Parameters
1434 ----------
1435 memo, default None
1436 Standard signature. Unused
1437 """
1438 return self.copy(deep=True)
1440 # --------------------------------------------------------------------
1441 # Rendering Methods
1443 @final
1444 def __repr__(self) -> str_t:
1445 """
1446 Return a string representation for this object.
1447 """
1448 klass_name = type(self).__name__
1449 data = self._format_data()
1450 attrs = self._format_attrs()
1452 display_width = config["display"]["width"] or 80
1453 indent = len(klass_name) + 1 # length of "ClassName("
1454 indent_str = " " * indent
1456 # Determine the length of the line where attrs start
1457 if "\n" in data:
1458 last_line = data.rsplit("\n", 1)[-1]
1459 line_len = len(last_line)
1460 else:
1461 line_len = indent + len(data)
1463 adj = get_adjustment()
1465 # Build the attrs portion with width-aware wrapping.
1466 # Reserve 1 char for trailing comma when not the last attr.
1467 parts: list[str_t] = []
1468 num_attrs = len(attrs)
1469 for attr_idx, (key, val) in enumerate(attrs):
1470 attr = f"{key}={val}"
1471 attr_len = adj.len(attr)
1472 comma = 1 if attr_idx < num_attrs - 1 else 0
1473 if not parts:
1474 # First attr: wrap onto new line if it won't fit on the
1475 # current line (which already contains data)
1476 if line_len + attr_len + comma > display_width:
1477 parts.append("\n" + indent_str + attr)
1478 line_len = indent + attr_len
1479 else:
1480 parts.append(attr)
1481 line_len += attr_len
1482 else:
1483 sep = ", "
1484 if line_len + len(sep) + attr_len + comma > display_width:
1485 parts.append(",\n" + indent_str + attr)
1486 line_len = indent + attr_len
1487 else:
1488 parts.append(sep + attr)
1489 line_len += len(sep) + attr_len
1491 prepr = "".join(parts)
1492 if prepr.startswith("\n"):
1493 # First attr wrapped to a new line; strip trailing whitespace
1494 # from the data portion (e.g. trailing ", " separator)
1495 data = data.rstrip()
1496 return f"{klass_name}({data}{prepr})"
1498 def _formatter_func(self, val: object) -> str_t:
1499 """
1500 Return the formatted value.
1501 """
1502 return default_pprint(val)
1504 @final
1505 def _format_data(self, name: str_t | None = None) -> str_t:
1506 """
1507 Return the formatted data as a unicode string.
1508 """
1509 # do we want to justify (only do so for non-objects)
1510 is_justify = True
1512 if self.inferred_type == "string":
1513 is_justify = False
1514 elif isinstance(self.dtype, CategoricalDtype):
1515 self = cast("CategoricalIndex", self)
1516 if is_string_dtype(self.categories.dtype):
1517 is_justify = False
1518 elif isinstance(self, ABCRangeIndex):
1519 # We will do the relevant formatting via attrs
1520 return ""
1522 return format_object_summary(
1523 self,
1524 self._formatter_func,
1525 is_justify=is_justify,
1526 name=name,
1527 line_break_each_value=self._is_multi,
1528 )
1530 def _format_attrs(self) -> list[tuple[str_t, str_t | int | bool | None]]:
1531 """
1532 Return a list of tuples of the (attr,formatted_value).
1533 """
1534 attrs: list[tuple[str_t, str_t | int | bool | None]] = []
1536 if not self._is_multi:
1537 attrs.append(("dtype", f"'{self.dtype}'"))
1539 if self.name is not None:
1540 attrs.append(("name", default_pprint(self.name)))
1541 elif self._is_multi and any(x is not None for x in self.names):
1542 attrs.append(("names", default_pprint(self.names)))
1544 max_seq_items = config["display"]["max_seq_items"] or len(self)
1545 if len(self) > max_seq_items:
1546 attrs.append(("length", len(self)))
1547 return attrs
1549 @final
1550 def _get_level_names(self) -> range | Sequence[Hashable]:
1551 """
1552 Return a name or list of names with None replaced by the level number.
1553 """
1554 if self._is_multi:
1555 return maybe_sequence_to_range( # type: ignore[return-value]
1556 [
1557 level if name is None else name
1558 for level, name in enumerate(self.names)
1559 ]
1560 )
1561 else:
1562 return range(1) if self.name is None else [self.name]
1564 def _mpl_repr(self) -> np.ndarray:
1565 # how to represent ourselves to matplotlib
1566 if isinstance(self.dtype, np.dtype) and self.dtype.kind == "M":
1567 return self.astype(object, copy=False)._values # type: ignore[return-value] # pyright: ignore[reportReturnType]
1568 elif isinstance(self.dtype, ExtensionDtype):
1569 values = cast("ExtensionArray", self._values)
1570 if self.dtype.kind in "mM":
1571 # e.g. ArrowDtype - relying on default of NaT for those dtypes
1572 # (explicitly specifying NaT raises an error)
1573 return values.to_numpy()
1574 if self.dtype.kind == "O":
1575 return values.to_numpy(na_value=None)
1576 if self.hasnans:
1577 return values.to_numpy(na_value=np.nan)
1578 else:
1579 return values.to_numpy()
1580 return self._values # type: ignore[return-value] # pyright: ignore[reportReturnType]
1582 _default_na_rep = "NaN"
1584 @final
1585 def _format_flat(
1586 self,
1587 *,
1588 include_name: bool,
1589 formatter: Callable | None = None,
1590 ) -> list[str_t]:
1591 """
1592 Render a string representation of the Index.
1593 """
1594 from pandas.io.formats.format import format_name
1596 header = []
1597 if include_name:
1598 header.append(format_name(self.name))
1600 if formatter is not None:
1601 return header + list(self.map(formatter))
1603 return self._format_with_header(header=header, na_rep=self._default_na_rep)
1605 def _format_with_header(self, *, header: list[str_t], na_rep: str_t) -> list[str_t]:
1606 from pandas.io.formats.format import format_array
1608 values = self._values
1610 if (
1611 is_object_dtype(values.dtype)
1612 or is_string_dtype(values.dtype)
1613 or isinstance(self.dtype, (IntervalDtype, CategoricalDtype))
1614 ):
1615 # TODO: why do we need different justify for these cases?
1616 justify = "all"
1617 else:
1618 justify = "left"
1619 # passing leading_space=False breaks test_format_missing,
1620 # test_index_repr_in_frame_with_nan, but would otherwise make
1621 # trim_front unnecessary
1622 formatted = format_array(values, None, justify=justify)
1623 result = trim_front(formatted)
1624 return header + result
1626 def _get_values_for_csv(
1627 self,
1628 *,
1629 na_rep: str_t = "",
1630 decimal: str_t = ".",
1631 float_format: str_t | None = None,
1632 date_format: str_t | None = None,
1633 quoting: int | None = None,
1634 ) -> npt.NDArray[np.object_]:
1635 return get_values_for_csv(
1636 self._values,
1637 na_rep=na_rep,
1638 decimal=decimal,
1639 float_format=float_format,
1640 date_format=date_format,
1641 quoting=quoting,
1642 )
1644 def _summary(self, name: str_t | None = None) -> str_t:
1645 """
1646 Return a summarized representation.
1648 Parameters
1649 ----------
1650 name : str
1651 name to use in the summary representation
1653 Returns
1654 -------
1655 String with a summarized representation of the index
1656 """
1657 if len(self) > 0:
1658 head = self[0]
1659 if hasattr(head, "format") and not isinstance(head, str):
1660 head = head.format() # pyright: ignore[reportAttributeAccessIssue]
1661 elif needs_i8_conversion(self.dtype):
1662 # e.g. Timedelta, display as values, not quoted
1663 head = self._formatter_func(head).replace("'", "")
1664 tail = self[-1]
1665 if hasattr(tail, "format") and not isinstance(tail, str):
1666 tail = tail.format() # pyright: ignore[reportAttributeAccessIssue]
1667 elif needs_i8_conversion(self.dtype):
1668 # e.g. Timedelta, display as values, not quoted
1669 tail = self._formatter_func(tail).replace("'", "")
1671 index_summary = f", {head} to {tail}"
1672 else:
1673 index_summary = ""
1675 if name is None:
1676 name = type(self).__name__
1677 return f"{name}: {len(self)} entries{index_summary}"
1679 # --------------------------------------------------------------------
1680 # Conversion Methods
1682 def to_flat_index(self) -> Self:
1683 """
1684 Identity method.
1686 This is implemented for compatibility with subclass implementations
1687 when chaining.
1689 Returns
1690 -------
1691 pd.Index
1692 Caller.
1694 See Also
1695 --------
1696 MultiIndex.to_flat_index : Subclass implementation.
1697 """
1698 return self
1700 @final
1701 def to_series(
1702 self, index: Axes | None = None, name: Hashable | None = None
1703 ) -> Series:
1704 """
1705 Create a Series with both index and values equal to the index keys.
1707 Useful with map for returning an indexer based on an index.
1709 Parameters
1710 ----------
1711 index : Index or array-like, optional
1712 Index of resulting Series. If None, defaults to original index.
1713 name : str, optional
1714 Name of resulting Series. If None, defaults to name of original
1715 index.
1717 Returns
1718 -------
1719 Series
1720 The dtype will be based on the type of the Index values.
1722 See Also
1723 --------
1724 Index.to_frame : Convert an Index to a DataFrame.
1725 Series.to_frame : Convert Series to DataFrame.
1727 Examples
1728 --------
1729 >>> idx = pd.Index(["Ant", "Bear", "Cow"], name="animal")
1731 By default, the original index and original name is reused.
1733 >>> idx.to_series()
1734 animal
1735 Ant Ant
1736 Bear Bear
1737 Cow Cow
1738 Name: animal, dtype: str
1740 To enforce a new index, specify new labels to ``index``:
1742 >>> idx.to_series(index=[0, 1, 2])
1743 0 Ant
1744 1 Bear
1745 2 Cow
1746 Name: animal, dtype: str
1748 To override the name of the resulting column, specify ``name``:
1750 >>> idx.to_series(name="zoo")
1751 animal
1752 Ant Ant
1753 Bear Bear
1754 Cow Cow
1755 Name: zoo, dtype: str
1756 """
1757 from pandas import Series
1759 if index is None:
1760 index = self._view()
1761 if name is None:
1762 name = self.name
1764 return Series(self._values.copy(), index=index, name=name)
1766 def to_frame(
1767 self, index: bool = True, name: Hashable = lib.no_default
1768 ) -> DataFrame:
1769 """
1770 Create a DataFrame with a column containing the Index.
1772 This method wraps the Index values into a single-column DataFrame,
1773 optionally preserving the original Index as the new DataFrame's index.
1775 Parameters
1776 ----------
1777 index : bool, default True
1778 Set the index of the returned DataFrame as the original Index.
1780 name : Hashable, defaults to index.name
1781 The passed name should substitute for the index name (if it has
1782 one).
1784 Returns
1785 -------
1786 DataFrame
1787 DataFrame containing the original Index data.
1789 See Also
1790 --------
1791 Index.to_series : Convert an Index to a Series.
1792 Series.to_frame : Convert Series to DataFrame.
1794 Examples
1795 --------
1796 >>> idx = pd.Index(["Ant", "Bear", "Cow"], name="animal")
1797 >>> idx.to_frame()
1798 animal
1799 animal
1800 Ant Ant
1801 Bear Bear
1802 Cow Cow
1804 By default, the original Index is reused. To enforce a new Index:
1806 >>> idx.to_frame(index=False)
1807 animal
1808 0 Ant
1809 1 Bear
1810 2 Cow
1812 To override the name of the resulting column, specify `name`:
1814 >>> idx.to_frame(index=False, name="zoo")
1815 zoo
1816 0 Ant
1817 1 Bear
1818 2 Cow
1819 """
1820 from pandas import DataFrame
1822 if name is lib.no_default:
1823 result_name = self._get_level_names()
1824 else:
1825 result_name = Index([name]) # type: ignore[assignment]
1826 result = DataFrame(self, copy=False)
1827 result.columns = result_name
1829 if index:
1830 result.index = self
1831 return result
1833 # --------------------------------------------------------------------
1834 # Name-Centric Methods
1836 @property
1837 def name(self) -> Hashable:
1838 """
1839 Return Index or MultiIndex name.
1841 The name (or label) of an Index provides metadata that can be used
1842 to identify the axis in DataFrames and Series.
1844 Returns
1845 -------
1846 label (hashable object)
1847 The name of the Index.
1849 See Also
1850 --------
1851 Index.set_names: Able to set new names partially and by level.
1852 Index.rename: Able to set new names partially and by level.
1853 Series.name: Corresponding Series property.
1855 Examples
1856 --------
1857 >>> idx = pd.Index([1, 2, 3], name="x")
1858 >>> idx
1859 Index([1, 2, 3], dtype='int64', name='x')
1860 >>> idx.name
1861 'x'
1862 """
1863 return self._name
1865 @name.setter
1866 def name(self, value: Hashable) -> None:
1867 if self._no_setting_name:
1868 # Used in MultiIndex.levels to avoid silently ignoring name updates.
1869 raise RuntimeError(
1870 "Cannot set name on a level of a MultiIndex. Use "
1871 "'MultiIndex.set_names' instead."
1872 )
1873 maybe_extract_name(value, None, type(self))
1874 self._name = value
1876 @final
1877 def _validate_names(
1878 self,
1879 name: Hashable = None,
1880 names: Sequence[Hashable] | None = None,
1881 deep: bool = False,
1882 ) -> list[Hashable]:
1883 """
1884 Handles the quirks of having a singular 'name' parameter for general
1885 Index and plural 'names' parameter for MultiIndex.
1886 """
1887 from copy import deepcopy
1889 if names is not None and name is not None:
1890 raise TypeError("Can only provide one of `names` and `name`")
1891 if names is None and name is None:
1892 new_names = deepcopy(self.names) if deep else self.names
1893 elif names is not None:
1894 if not is_list_like(names):
1895 raise TypeError("Must pass list-like as `names`.")
1896 new_names = names
1897 elif not is_list_like(name):
1898 new_names = [name]
1899 else:
1900 new_names = name
1902 if len(new_names) != len(self.names): # pyright: ignore[reportArgumentType]
1903 raise ValueError(
1904 f"Length of new names must be {len(self.names)}, got {len(new_names)}" # pyright: ignore[reportArgumentType]
1905 )
1907 # All items in 'new_names' need to be hashable
1908 validate_all_hashable(*new_names, error_name=f"{type(self).__name__}.name") # pyright: ignore[reportOptionalIterable, reportGeneralTypeIssues]
1910 return new_names # pyright: ignore[reportReturnType]
1912 def _get_default_index_names(
1913 self,
1914 names: Sequence[Hashable] | int | str_t | None = None,
1915 default: Hashable = None,
1916 ) -> list[Hashable]:
1917 """
1918 Get names of index.
1920 Parameters
1921 ----------
1922 names : int, str or 1-dimensional list, default None
1923 Index names to set.
1924 default : str
1925 Default name of index.
1927 Raises
1928 ------
1929 TypeError
1930 if names not str or list-like
1931 """
1932 from pandas.core.indexes.multi import MultiIndex
1934 if names is not None:
1935 if isinstance(names, (int, str)):
1936 names = [names]
1938 if not isinstance(names, list) and names is not None:
1939 raise ValueError("Index names must be str or 1-dimensional list")
1941 if not names:
1942 if isinstance(self, MultiIndex):
1943 names = com.fill_missing_names(self.names)
1944 else:
1945 names = [default] if self.name is None else [self.name]
1947 return names # pyright: ignore[reportReturnType]
1949 def _get_names(self) -> FrozenList:
1950 """
1951 Get names on index.
1953 This method returns a FrozenList containing the names of the object.
1954 It's primarily intended for internal use.
1956 Returns
1957 -------
1958 FrozenList
1959 A FrozenList containing the object's names, contains None if the object
1960 does not have a name.
1962 See Also
1963 --------
1964 Index.name : Index name as a string, or None for MultiIndex.
1966 Examples
1967 --------
1968 >>> idx = pd.Index([1, 2, 3], name="x")
1969 >>> idx.names
1970 FrozenList(['x'])
1972 >>> idx = pd.Index([1, 2, 3], name=("x", "y"))
1973 >>> idx.names
1974 FrozenList([('x', 'y')])
1976 If the index does not have a name set:
1978 >>> idx = pd.Index([1, 2, 3])
1979 >>> idx.names
1980 FrozenList([None])
1981 """
1982 return FrozenList((self.name,))
1984 def _set_names(
1985 self, values: Sequence[Hashable], *, level: int | Sequence[int] | None = None
1986 ) -> None:
1987 """
1988 Set new names on index. Each name has to be a hashable type.
1990 Parameters
1991 ----------
1992 values : str or sequence
1993 name(s) to set
1994 level : int, level name, or sequence of int/level names (default None)
1995 If the index is a MultiIndex (hierarchical), level(s) to set (None
1996 for all levels). Otherwise level must be None
1998 Raises
1999 ------
2000 TypeError if each name is not hashable.
2001 """
2002 if not is_list_like(values):
2003 raise ValueError("Names must be a list-like")
2004 if len(values) != 1:
2005 raise ValueError(f"Length of new names must be 1, got {len(values)}")
2007 # GH 20527
2008 # All items in 'name' need to be hashable:
2009 validate_all_hashable(*values, error_name=f"{type(self).__name__}.name")
2011 self._name = values[0]
2013 names = property(fset=_set_names, fget=_get_names)
2015 @overload
2016 def set_names(
2017 self,
2018 names: Hashable | Sequence[Hashable] | dict[Hashable, Hashable],
2019 *,
2020 level: Hashable | Sequence[Hashable] | None = ...,
2021 inplace: Literal[False] = ...,
2022 ) -> Self: ...
2024 @overload
2025 def set_names(
2026 self,
2027 names: Hashable | Sequence[Hashable] | dict[Hashable, Hashable],
2028 *,
2029 level: Hashable | Sequence[Hashable] | None = ...,
2030 inplace: Literal[True],
2031 ) -> None: ...
2033 @overload
2034 def set_names(
2035 self,
2036 names: Hashable | Sequence[Hashable] | dict[Hashable, Hashable],
2037 *,
2038 level: Hashable | Sequence[Hashable] | None = ...,
2039 inplace: bool = ...,
2040 ) -> Self | None: ...
2042 def set_names(
2043 self,
2044 names: Hashable | Sequence[Hashable] | dict[Hashable, Hashable],
2045 *,
2046 level: Hashable | Sequence[Hashable] | None = None,
2047 inplace: bool = False,
2048 ) -> Self | None:
2049 """
2050 Set Index or MultiIndex name.
2052 Able to set new names partially and by level.
2054 Parameters
2055 ----------
2056 names : Hashable or a sequence of the previous or dict-like for MultiIndex
2057 Name(s) to set.
2059 level : int, Hashable or a sequence of the previous, optional
2060 If the index is a MultiIndex and names is not dict-like, level(s) to set
2061 (None for all levels). Otherwise level must be None.
2063 inplace : bool, default False
2064 Modifies the object directly, instead of creating a new Index or
2065 MultiIndex.
2067 Returns
2068 -------
2069 Index or None
2070 The same type as the caller or None if ``inplace=True``.
2072 See Also
2073 --------
2074 Index.rename : Able to set new names without level.
2076 Examples
2077 --------
2078 >>> idx = pd.Index([1, 2, 3, 4])
2079 >>> idx
2080 Index([1, 2, 3, 4], dtype='int64')
2081 >>> idx.set_names("quarter")
2082 Index([1, 2, 3, 4], dtype='int64', name='quarter')
2084 >>> idx = pd.MultiIndex.from_product([["python", "cobra"], [2018, 2019]])
2085 >>> idx
2086 MultiIndex([('python', 2018),
2087 ('python', 2019),
2088 ( 'cobra', 2018),
2089 ( 'cobra', 2019)],
2090 )
2091 >>> idx = idx.set_names(["kind", "year"])
2092 >>> idx.set_names("species", level=0)
2093 MultiIndex([('python', 2018),
2094 ('python', 2019),
2095 ( 'cobra', 2018),
2096 ( 'cobra', 2019)],
2097 names=['species', 'year'])
2099 When renaming levels with a dict, levels can not be passed.
2101 >>> idx.set_names({"kind": "snake"})
2102 MultiIndex([('python', 2018),
2103 ('python', 2019),
2104 ( 'cobra', 2018),
2105 ( 'cobra', 2019)],
2106 names=['snake', 'year'])
2107 """
2108 if level is not None and not isinstance(self, ABCMultiIndex):
2109 raise ValueError("Level must be None for non-MultiIndex")
2111 if level is not None and not is_list_like(level) and is_list_like(names):
2112 raise TypeError("Names must be a string when a single level is provided.")
2114 if not is_list_like(names) and level is None and self.nlevels > 1:
2115 raise TypeError("Must pass list-like as `names`.")
2117 if is_dict_like(names) and not isinstance(self, ABCMultiIndex):
2118 raise TypeError("Can only pass dict-like as `names` for MultiIndex.")
2120 if is_dict_like(names) and level is not None:
2121 raise TypeError("Can not pass level for dictlike `names`.")
2123 if isinstance(self, ABCMultiIndex) and is_dict_like(names) and level is None:
2124 # Transform dict to list of new names and corresponding levels
2125 level_list: list[int] = []
2126 names_adjusted: list[Hashable] = []
2127 for i, name in enumerate(self.names):
2128 if name in names.keys(): # type: ignore[union-attr]
2129 level_list.append(i)
2130 names_adjusted.append(names[name]) # type: ignore[index]
2131 level = level_list
2132 names = names_adjusted
2134 if not is_list_like(names):
2135 names = [names] # type: ignore[assignment]
2136 if level is not None and not is_list_like(level):
2137 level = [level]
2139 if inplace:
2140 idx = self
2141 else:
2142 idx = self._view()
2144 idx._set_names(names, level=level) # type: ignore[arg-type]
2145 if not inplace:
2146 return idx # pyright: ignore[reportReturnType]
2147 return None
2149 @overload
2150 def rename(
2151 self, name: Hashable | Sequence[Hashable], *, inplace: Literal[False] = ...
2152 ) -> Self: ...
2154 @overload
2155 def rename(
2156 self, name: Hashable | Sequence[Hashable], *, inplace: Literal[True]
2157 ) -> None: ...
2159 def rename(
2160 self, name: Hashable | Sequence[Hashable], *, inplace: bool = False
2161 ) -> Self | None:
2162 """
2163 Alter Index or MultiIndex name.
2165 Able to set new names without level. Defaults to returning new index.
2166 Length of names must match number of levels in MultiIndex.
2168 Parameters
2169 ----------
2170 name : Hashable or a sequence of the previous
2171 Name(s) to set.
2172 inplace : bool, default False
2173 Modifies the object directly, instead of creating a new Index or
2174 MultiIndex.
2176 Returns
2177 -------
2178 Index or None
2179 The same type as the caller or None if ``inplace=True``.
2181 See Also
2182 --------
2183 Index.set_names : Able to set new names partially and by level.
2185 Examples
2186 --------
2187 >>> idx = pd.Index(["A", "C", "A", "B"], name="score")
2188 >>> idx.rename("grade")
2189 Index(['A', 'C', 'A', 'B'], dtype='str', name='grade')
2191 >>> idx = pd.MultiIndex.from_product(
2192 ... [["python", "cobra"], [2018, 2019]], names=["kind", "year"]
2193 ... )
2194 >>> idx
2195 MultiIndex([('python', 2018),
2196 ('python', 2019),
2197 ( 'cobra', 2018),
2198 ( 'cobra', 2019)],
2199 names=['kind', 'year'])
2200 >>> idx.rename(["species", "year"])
2201 MultiIndex([('python', 2018),
2202 ('python', 2019),
2203 ( 'cobra', 2018),
2204 ( 'cobra', 2019)],
2205 names=['species', 'year'])
2206 >>> idx.rename("species")
2207 Traceback (most recent call last):
2208 TypeError: Must pass list-like as `names`.
2209 """
2210 return self.set_names([name], inplace=inplace)
2212 # --------------------------------------------------------------------
2213 # Level-Centric Methods
2215 @property
2216 def nlevels(self) -> int:
2217 """
2218 Number of levels.
2219 """
2220 return 1
2222 def _sort_levels_monotonic(self) -> Self:
2223 """
2224 Compat with MultiIndex.
2225 """
2226 return self
2228 @final
2229 def _validate_index_level(self, level: Level) -> None:
2230 """
2231 Validate index level.
2233 For single-level Index getting level number is a no-op, but some
2234 verification must be done like in MultiIndex.
2236 """
2237 is_level_na = cast("bool", isna(level))
2238 is_name_na = cast("bool", isna(self.name))
2239 if is_level_na and is_name_na:
2240 if is_matching_na(level, self.name):
2241 return
2242 raise KeyError(
2243 f"Requested level ({level}) does not match index name ({self.name})"
2244 )
2245 elif is_level_na:
2246 raise KeyError(
2247 f"Requested level ({level}) does not match index name ({self.name})"
2248 )
2250 if is_integer(level):
2251 if is_integer(self.name) and level == self.name:
2252 return
2253 if level < 0 and level != -1:
2254 raise IndexError(
2255 "Too many levels: Index has only 1 level, "
2256 f"{level} is not a valid level number"
2257 )
2258 if level > 0:
2259 raise IndexError(
2260 f"Too many levels: Index has only 1 level, not {level + 1}"
2261 )
2262 return
2263 elif level != self.name:
2264 raise KeyError(
2265 f"Requested level ({level}) does not match index name ({self.name})"
2266 )
2268 def _get_level_number(self, level: Level) -> int:
2269 self._validate_index_level(level)
2270 return 0
2272 def sortlevel(
2273 self,
2274 level: Level | None = None,
2275 ascending: bool | list[bool] = True,
2276 sort_remaining: bool | None = None,
2277 na_position: NaPosition = "first",
2278 ) -> tuple[Self, np.ndarray]:
2279 """
2280 For internal compatibility with the Index API.
2282 Sort the Index. This is for compat with MultiIndex
2284 Parameters
2285 ----------
2286 ascending : bool, default True
2287 False to sort in descending order
2288 na_position : {'first' or 'last'}, default 'first'
2289 Argument 'first' puts NaNs at the beginning, 'last' puts NaNs at
2290 the end.
2292 .. versionadded:: 2.1.0
2294 level, sort_remaining are compat parameters
2296 Returns
2297 -------
2298 Index
2299 """
2300 if not isinstance(ascending, (list, bool)):
2301 raise TypeError(
2302 "ascending must be a single bool value or"
2303 "a list of bool values of length 1"
2304 )
2306 if isinstance(ascending, list):
2307 if len(ascending) != 1:
2308 raise TypeError("ascending must be a list of bool values of length 1")
2309 ascending = ascending[0]
2311 if not isinstance(ascending, bool):
2312 raise TypeError("ascending must be a bool value")
2314 return self.sort_values(
2315 return_indexer=True, ascending=ascending, na_position=na_position
2316 )
2318 def _get_level_values(self, level: Level) -> Index:
2319 """
2320 Return an Index of values for requested level.
2322 This is primarily useful to get an individual level of values from a
2323 MultiIndex, but is provided on Index as well for compatibility.
2325 Parameters
2326 ----------
2327 level : int or str
2328 It is either the integer position or the name of the level.
2330 Returns
2331 -------
2332 Index
2333 Calling object, as there is only one level in the Index.
2335 See Also
2336 --------
2337 MultiIndex.get_level_values : Get values for a level of a MultiIndex.
2339 Notes
2340 -----
2341 For Index, level should be 0, since there are no multiple levels.
2343 Examples
2344 --------
2345 >>> idx = pd.Index(list("abc"))
2346 >>> idx
2347 Index(['a', 'b', 'c'], dtype='str')
2349 Get level values by supplying `level` as integer:
2351 >>> idx.get_level_values(0)
2352 Index(['a', 'b', 'c'], dtype='str')
2353 """
2354 self._validate_index_level(level)
2355 return self
2357 get_level_values = _get_level_values
2359 @final
2360 def droplevel(self, level: IndexLabel = 0) -> Index:
2361 """
2362 Return index with requested level(s) removed.
2364 If resulting index has only 1 level left, the result will be
2365 of Index type, not MultiIndex. The original index is not modified inplace.
2367 Parameters
2368 ----------
2369 level : int, str, or list-like, default 0
2370 If a string is given, must be the name of a level
2371 If list-like, elements must be names or indexes of levels.
2373 Returns
2374 -------
2375 Index or MultiIndex
2376 Returns an Index or MultiIndex object, depending on the resulting index
2377 after removing the requested level(s).
2379 See Also
2380 --------
2381 Index.dropna : Return Index without NA/NaN values.
2383 Examples
2384 --------
2385 >>> mi = pd.MultiIndex.from_arrays(
2386 ... [[1, 2], [3, 4], [5, 6]], names=["x", "y", "z"]
2387 ... )
2388 >>> mi
2389 MultiIndex([(1, 3, 5),
2390 (2, 4, 6)],
2391 names=['x', 'y', 'z'])
2393 >>> mi.droplevel()
2394 MultiIndex([(3, 5),
2395 (4, 6)],
2396 names=['y', 'z'])
2398 >>> mi.droplevel(2)
2399 MultiIndex([(1, 3),
2400 (2, 4)],
2401 names=['x', 'y'])
2403 >>> mi.droplevel("z")
2404 MultiIndex([(1, 3),
2405 (2, 4)],
2406 names=['x', 'y'])
2408 >>> mi.droplevel(["x", "y"])
2409 Index([5, 6], dtype='int64', name='z')
2410 """
2411 if not isinstance(level, (tuple, list)):
2412 level = [level]
2414 levnums = sorted((self._get_level_number(lev) for lev in level), reverse=True)
2416 return self._drop_level_numbers(levnums)
2418 @final
2419 def _drop_level_numbers(self, levnums: list[int]) -> Index:
2420 """
2421 Drop MultiIndex levels by level _number_, not name.
2422 """
2424 if not levnums and not isinstance(self, ABCMultiIndex):
2425 return self
2426 if len(levnums) >= self.nlevels:
2427 raise ValueError(
2428 f"Cannot remove {len(levnums)} levels from an index with "
2429 f"{self.nlevels} levels: at least one level must be left."
2430 )
2431 # The two checks above guarantee that here self is a MultiIndex
2432 self = cast("MultiIndex", self)
2434 new_levels = list(self.levels)
2435 new_codes = list(self.codes)
2436 new_names = list(self.names)
2438 for i in levnums:
2439 new_levels.pop(i)
2440 new_codes.pop(i)
2441 new_names.pop(i)
2443 return droplevel_result(new_levels, new_codes, new_names)
2445 # --------------------------------------------------------------------
2446 # Introspection Methods
2448 @cache_readonly
2449 @final
2450 def _can_hold_na(self) -> bool:
2451 if isinstance(self.dtype, ExtensionDtype):
2452 return self.dtype._can_hold_na
2453 if self.dtype.kind in "iub":
2454 return False
2455 return True
2457 @property
2458 def is_monotonic_increasing(self) -> bool:
2459 """
2460 Return a boolean if the values are equal or increasing.
2462 A monotonically increasing index has values that are equal to or
2463 greater than the preceding value. This is useful for checking if
2464 an index is sorted in non-decreasing order.
2466 Returns
2467 -------
2468 bool
2470 See Also
2471 --------
2472 Index.is_monotonic_decreasing : Check if the values are equal or decreasing.
2474 Examples
2475 --------
2476 >>> pd.Index([1, 2, 3]).is_monotonic_increasing
2477 True
2478 >>> pd.Index([1, 2, 2]).is_monotonic_increasing
2479 True
2480 >>> pd.Index([1, 3, 2]).is_monotonic_increasing
2481 False
2482 """
2483 if isinstance(self._values, ExtensionArray) and (
2484 type(self._engine) is libindex.ObjectEngine
2485 ):
2486 # For custom EAs we use ObjectEngine by default, which gets the EA as
2487 # object ndarray -> use the EA's implementation directly instead of engine.
2488 # Subclasses of ObjectEngine (e.g. StringObjectEngine) are excluded.
2489 return self._values._is_monotonic_increasing
2490 return self._engine.is_monotonic_increasing
2492 @property
2493 def is_monotonic_decreasing(self) -> bool:
2494 """
2495 Return a boolean if the values are equal or decreasing.
2497 A monotonically decreasing index has values that are equal to or
2498 less than the preceding value. This is useful for checking if
2499 an index is sorted in non-increasing order.
2501 Returns
2502 -------
2503 bool
2505 See Also
2506 --------
2507 Index.is_monotonic_increasing : Check if the values are equal or increasing.
2509 Examples
2510 --------
2511 >>> pd.Index([3, 2, 1]).is_monotonic_decreasing
2512 True
2513 >>> pd.Index([3, 2, 2]).is_monotonic_decreasing
2514 True
2515 >>> pd.Index([3, 1, 2]).is_monotonic_decreasing
2516 False
2517 """
2518 if isinstance(self._values, ExtensionArray) and (
2519 type(self._engine) is libindex.ObjectEngine
2520 ):
2521 # For custom EAs we use ObjectEngine by default, which gets the EA as
2522 # object ndarray -> use the EA's implementation directly instead of engine.
2523 # Subclasses of ObjectEngine (e.g. StringObjectEngine) are excluded.
2524 return self._values._is_monotonic_decreasing
2525 return self._engine.is_monotonic_decreasing
2527 @final
2528 @property
2529 def _is_strictly_monotonic_increasing(self) -> bool:
2530 """
2531 Return if the index is strictly monotonic increasing
2532 (only increasing) values.
2534 Examples
2535 --------
2536 >>> Index([1, 2, 3])._is_strictly_monotonic_increasing
2537 True
2538 >>> Index([1, 2, 2])._is_strictly_monotonic_increasing
2539 False
2540 >>> Index([1, 3, 2])._is_strictly_monotonic_increasing
2541 False
2542 """
2543 return self.is_unique and self.is_monotonic_increasing
2545 @final
2546 @property
2547 def _is_strictly_monotonic_decreasing(self) -> bool:
2548 """
2549 Return if the index is strictly monotonic decreasing
2550 (only decreasing) values.
2552 Examples
2553 --------
2554 >>> Index([3, 2, 1])._is_strictly_monotonic_decreasing
2555 True
2556 >>> Index([3, 2, 2])._is_strictly_monotonic_decreasing
2557 False
2558 >>> Index([3, 1, 2])._is_strictly_monotonic_decreasing
2559 False
2560 """
2561 return self.is_unique and self.is_monotonic_decreasing
2563 @cache_readonly
2564 def is_unique(self) -> bool:
2565 """
2566 Return if the index has unique values.
2568 The uniqueness check is based on exact equality of values. An index
2569 with no repeated values returns ``True``, otherwise ``False``.
2571 Returns
2572 -------
2573 bool
2575 See Also
2576 --------
2577 Index.has_duplicates : Inverse method that checks if it has duplicate values.
2579 Examples
2580 --------
2581 >>> idx = pd.Index([1, 5, 7, 7])
2582 >>> idx.is_unique
2583 False
2585 >>> idx = pd.Index([1, 5, 7])
2586 >>> idx.is_unique
2587 True
2589 >>> idx = pd.Index(["Watermelon", "Orange", "Apple", "Watermelon"]).astype(
2590 ... "category"
2591 ... )
2592 >>> idx.is_unique
2593 False
2595 >>> idx = pd.Index(["Orange", "Apple", "Watermelon"]).astype("category")
2596 >>> idx.is_unique
2597 True
2598 """
2599 return self._engine.is_unique
2601 @final
2602 @property
2603 def has_duplicates(self) -> bool:
2604 """
2605 Check if the Index has duplicate values.
2607 This is the inverse of :attr:`~Index.is_unique`. It returns ``True``
2608 if any value appears more than once in the index, ``False`` otherwise.
2610 Returns
2611 -------
2612 bool
2613 Whether or not the Index has duplicate values.
2615 See Also
2616 --------
2617 Index.is_unique : Inverse method that checks if it has unique values.
2619 Examples
2620 --------
2621 >>> idx = pd.Index([1, 5, 7, 7])
2622 >>> idx.has_duplicates
2623 True
2625 >>> idx = pd.Index([1, 5, 7])
2626 >>> idx.has_duplicates
2627 False
2629 >>> idx = pd.Index(["Watermelon", "Orange", "Apple", "Watermelon"]).astype(
2630 ... "category"
2631 ... )
2632 >>> idx.has_duplicates
2633 True
2635 >>> idx = pd.Index(["Orange", "Apple", "Watermelon"]).astype("category")
2636 >>> idx.has_duplicates
2637 False
2638 """
2639 return not self.is_unique
2641 @cache_readonly
2642 def inferred_type(self) -> str_t:
2643 """
2644 Return a string of the type inferred from the values.
2646 This inspects the actual values in the Index to determine their type,
2647 which may differ from the stored :attr:`~Index.dtype`. For example,
2648 an object-dtype Index containing only integers returns ``'integer'``.
2650 See Also
2651 --------
2652 Index.dtype : Return the dtype object of the underlying data.
2654 Examples
2655 --------
2656 >>> idx = pd.Index([1, 2, 3])
2657 >>> idx
2658 Index([1, 2, 3], dtype='int64')
2659 >>> idx.inferred_type
2660 'integer'
2661 """
2662 return lib.infer_dtype(self._values, skipna=False)
2664 @cache_readonly
2665 @final
2666 def _is_all_dates(self) -> bool:
2667 """
2668 Whether or not the index values only consist of dates.
2669 """
2670 if needs_i8_conversion(self.dtype):
2671 return True
2672 elif self.dtype != _dtype_obj:
2673 # TODO(ExtensionIndex): 3rd party EA might override?
2674 # Note: this includes IntervalIndex, even when the left/right
2675 # contain datetime-like objects.
2676 return False
2677 elif self._is_multi:
2678 return False
2679 return is_datetime_array(ensure_object(self._values))
2681 @final
2682 @cache_readonly
2683 def _is_multi(self) -> bool:
2684 """
2685 Cached check equivalent to isinstance(self, MultiIndex)
2686 """
2687 return isinstance(self, ABCMultiIndex)
2689 # --------------------------------------------------------------------
2690 # Pickle Methods
2692 def __reduce__(self) -> tuple:
2693 d = {"data": self._data, "name": self.name}
2694 return _new_Index, (type(self), d), None
2696 # --------------------------------------------------------------------
2697 # Null Handling Methods
2699 @cache_readonly
2700 def _na_value(self) -> Hashable:
2701 """The expected NA value to use with this index."""
2702 dtype = self.dtype
2703 if isinstance(dtype, np.dtype):
2704 if dtype.kind in "mM":
2705 return NaT
2706 return np.nan
2707 return dtype.na_value
2709 @cache_readonly
2710 def _isnan(self) -> npt.NDArray[np.bool_]:
2711 """
2712 Return if each value is NaN.
2713 """
2714 if self._can_hold_na:
2715 return isna(self)
2716 else:
2717 # shouldn't reach to this condition by checking hasnans beforehand
2718 values = np.empty(len(self), dtype=np.bool_)
2719 values.fill(False)
2720 return values
2722 @cache_readonly
2723 def hasnans(self) -> bool:
2724 """
2725 Return True if there are any NaNs.
2727 Enables various performance speedups.
2729 Returns
2730 -------
2731 bool
2733 See Also
2734 --------
2735 Index.isna : Detect missing values.
2736 Index.dropna : Return Index without NA/NaN values.
2737 Index.fillna : Fill NA/NaN values with the specified value.
2739 Examples
2740 --------
2741 >>> s = pd.Series([1, 2, 3], index=["a", "b", None])
2742 >>> s
2743 a 1
2744 b 2
2745 None 3
2746 dtype: int64
2747 >>> s.index.hasnans
2748 True
2749 """
2750 if self._can_hold_na:
2751 return bool(self._isnan.any())
2752 else:
2753 return False
2755 @final
2756 def isna(self) -> npt.NDArray[np.bool_]:
2757 """
2758 Detect missing values.
2760 Return a boolean same-sized object indicating if the values are NA.
2761 NA values, such as ``None``, :attr:`numpy.NaN` or :attr:`pd.NaT`, get
2762 mapped to ``True`` values.
2763 Everything else get mapped to ``False`` values. Characters such as
2764 empty strings `''` or :attr:`numpy.inf` are not considered NA values.
2766 Returns
2767 -------
2768 numpy.ndarray[bool]
2769 A boolean array of whether my values are NA.
2771 See Also
2772 --------
2773 Index.notna : Boolean inverse of isna.
2774 Index.dropna : Omit entries with missing values.
2775 isna : Top-level isna.
2776 Series.isna : Detect missing values in Series object.
2778 Examples
2779 --------
2780 Show which entries in a pandas.Index are NA. The result is an
2781 array.
2783 >>> idx = pd.Index([5.2, 6.0, np.nan])
2784 >>> idx
2785 Index([5.2, 6.0, nan], dtype='float64')
2786 >>> idx.isna()
2787 array([False, False, True])
2789 Empty strings are not considered NA values. None is considered an NA
2790 value.
2792 >>> idx = pd.Index(["black", "", "red", None])
2793 >>> idx
2794 Index(['black', '', 'red', nan], dtype='str')
2795 >>> idx.isna()
2796 array([False, False, False, True])
2798 For datetimes, `NaT` (Not a Time) is considered as an NA value.
2800 >>> idx = pd.DatetimeIndex(
2801 ... [pd.Timestamp("1940-04-25"), pd.Timestamp(""), None, pd.NaT]
2802 ... )
2803 >>> idx
2804 DatetimeIndex(['1940-04-25', 'NaT', 'NaT', 'NaT'],
2805 dtype='datetime64[us]', freq=None)
2806 >>> idx.isna()
2807 array([False, True, True, True])
2808 """
2809 return self._isnan
2811 isnull = isna
2813 @final
2814 def notna(self) -> npt.NDArray[np.bool_]:
2815 """
2816 Detect existing (non-missing) values.
2818 Return a boolean same-sized object indicating if the values are not NA.
2819 Non-missing values get mapped to ``True``. Characters such as empty
2820 strings ``''`` or :attr:`numpy.inf` are not considered NA values.
2821 NA values, such as None or :attr:`numpy.NaN`, get mapped to ``False``
2822 values.
2824 Returns
2825 -------
2826 numpy.ndarray[bool]
2827 Boolean array to indicate which entries are not NA.
2829 See Also
2830 --------
2831 Index.notnull : Alias of notna.
2832 Index.isna: Inverse of notna.
2833 notna : Top-level notna.
2835 Examples
2836 --------
2837 Show which entries in an Index are not NA. The result is an
2838 array.
2840 >>> idx = pd.Index([5.2, 6.0, np.nan])
2841 >>> idx
2842 Index([5.2, 6.0, nan], dtype='float64')
2843 >>> idx.notna()
2844 array([ True, True, False])
2846 Empty strings are not considered NA values. None is considered a NA
2847 value.
2849 >>> idx = pd.Index(["black", "", "red", None])
2850 >>> idx
2851 Index(['black', '', 'red', nan], dtype='str')
2852 >>> idx.notna()
2853 array([ True, True, True, False])
2854 """
2855 return ~self.isna()
2857 notnull = notna
2859 def fillna(self, value: object) -> Index:
2860 """
2861 Fill NA/NaN values with the specified value.
2863 Returns a new Index with all NA/NaN entries replaced by the given
2864 scalar value.
2866 Parameters
2867 ----------
2868 value : scalar
2869 Scalar value to use to fill holes (e.g. 0).
2870 This value cannot be a list-likes.
2872 Returns
2873 -------
2874 Index
2875 NA/NaN values replaced with `value`.
2877 See Also
2878 --------
2879 DataFrame.fillna : Fill NaN values of a DataFrame.
2880 Series.fillna : Fill NaN Values of a Series.
2882 Examples
2883 --------
2884 >>> idx = pd.Index([np.nan, np.nan, 3])
2885 >>> idx.fillna(0)
2886 Index([0.0, 0.0, 3.0], dtype='float64')
2887 """
2888 if is_list_like(value, allow_sets=False) and not isinstance(value, tuple):
2889 raise TypeError(f"'value' must be a scalar, passed: {type(value).__name__}")
2891 if self.hasnans:
2892 values = self._values
2893 validate = getattr(values, "_validate_setitem_value", None)
2894 if validate is not None and not isinstance(self.dtype, np.dtype):
2895 # GH#25288 can_hold_element is permissive for most ExtensionArrays;
2896 # the array's own setitem validation is authoritative.
2897 try:
2898 validate(value)
2899 can_hold = True
2900 except (ValueError, TypeError):
2901 can_hold = False
2902 else:
2903 can_hold = can_hold_element(values, value)
2905 if not can_hold and not is_valid_na_for_dtype(value, self.dtype):
2906 # GH#45153 fillna with incompatible value requiring any
2907 # dtype casting is deprecated.
2908 warnings.warn(
2909 f"'{type(value).__name__}' is not supported as a fill "
2910 f"value for {self.dtype} dtype. In a future version, "
2911 f"calling fillna with an incompatible value will raise. "
2912 f"Explicitly cast to a common dtype before filling.",
2913 Pandas4Warning,
2914 stacklevel=find_stack_level(),
2915 )
2916 result = self.putmask(self._isnan, value)
2917 # no need to care metadata other than name
2918 # because it can't have freq if it has NaTs
2919 # _with_infer needed for test_fillna_categorical
2920 return Index._with_infer(result, name=self.name, copy=False)
2921 return self._view()
2923 def dropna(self, how: AnyAll = "any") -> Self:
2924 """
2925 Return Index without NA/NaN values.
2927 Removes missing values from the Index, returning a shorter Index
2928 containing only non-NA entries.
2930 Parameters
2931 ----------
2932 how : {'any', 'all'}, default 'any'
2933 If the Index is a MultiIndex, drop the value when any or all levels
2934 are NaN.
2936 Returns
2937 -------
2938 Index
2939 Returns an Index object after removing NA/NaN values.
2941 See Also
2942 --------
2943 Index.fillna : Fill NA/NaN values with the specified value.
2944 Index.isna : Detect missing values.
2946 Examples
2947 --------
2948 >>> idx = pd.Index([1, np.nan, 3])
2949 >>> idx.dropna()
2950 Index([1.0, 3.0], dtype='float64')
2951 """
2952 if how not in ("any", "all"):
2953 raise ValueError(f"invalid how option: {how}")
2955 if self.hasnans:
2956 res_values = self._values[~self._isnan]
2957 return type(self)._simple_new(res_values, name=self.name)
2958 return self._view()
2960 # --------------------------------------------------------------------
2961 # Uniqueness Methods
2963 def unique(self, level: Hashable | None = None) -> Self:
2964 """
2965 Return unique values in the index.
2967 Unique values are returned in order of appearance, this does NOT sort.
2969 Parameters
2970 ----------
2971 level : int or hashable, optional
2972 Only return values from specified level (for MultiIndex).
2973 If int, gets the level by integer position, else by level name.
2975 Returns
2976 -------
2977 Index
2978 Unique values in the index.
2980 See Also
2981 --------
2982 unique : Numpy array of unique values in that column.
2983 Series.unique : Return unique values of Series object.
2985 Examples
2986 --------
2987 >>> idx = pd.Index([1, 1, 2, 3, 3])
2988 >>> idx.unique()
2989 Index([1, 2, 3], dtype='int64')
2990 """
2991 if level is not None:
2992 self._validate_index_level(level)
2994 if self.is_unique:
2995 return self._view()
2997 result = super().unique() # type: ignore[no-untyped-call]
2998 return self._shallow_copy(result)
3000 def drop_duplicates(self, *, keep: DropKeep = "first") -> Self:
3001 """
3002 Return Index with duplicate values removed.
3004 The ``keep`` parameter controls which duplicate values are removed.
3005 The original Index is not modified.
3007 Parameters
3008 ----------
3009 keep : {'first', 'last', ``False``}, default 'first'
3010 - 'first' : Drop duplicates except for the first occurrence.
3011 - 'last' : Drop duplicates except for the last occurrence.
3012 - ``False`` : Drop all duplicates.
3014 Returns
3015 -------
3016 Index
3017 A new Index object with the duplicate values removed.
3019 See Also
3020 --------
3021 Series.drop_duplicates : Equivalent method on Series.
3022 DataFrame.drop_duplicates : Equivalent method on DataFrame.
3023 Index.duplicated : Related method on Index, indicating duplicate
3024 Index values.
3026 Examples
3027 --------
3028 Generate a pandas.Index with duplicate values.
3030 >>> idx = pd.Index(["llama", "cow", "llama", "beetle", "llama", "hippo"])
3032 The `keep` parameter controls which duplicate values are removed.
3033 The value 'first' keeps the first occurrence for each
3034 set of duplicated entries. The default value of keep is 'first'.
3036 >>> idx.drop_duplicates(keep="first")
3037 Index(['llama', 'cow', 'beetle', 'hippo'], dtype='str')
3039 The value 'last' keeps the last occurrence for each set of duplicated
3040 entries.
3042 >>> idx.drop_duplicates(keep="last")
3043 Index(['cow', 'beetle', 'llama', 'hippo'], dtype='str')
3045 The value ``False`` discards all sets of duplicated entries.
3047 >>> idx.drop_duplicates(keep=False)
3048 Index(['cow', 'beetle', 'hippo'], dtype='str')
3049 """
3050 if self.is_unique:
3051 return self._view()
3053 return super().drop_duplicates(keep=keep)
3055 def duplicated(self, keep: DropKeep = "first") -> npt.NDArray[np.bool_]:
3056 """
3057 Indicate duplicate index values.
3059 Duplicated values are indicated as ``True`` values in the resulting
3060 array. Either all duplicates, all except the first, or all except the
3061 last occurrence of duplicates can be indicated.
3063 Parameters
3064 ----------
3065 keep : {'first', 'last', False}, default 'first'
3066 The value or values in a set of duplicates to mark as missing.
3068 - 'first' : Mark duplicates as ``True`` except for the first
3069 occurrence.
3070 - 'last' : Mark duplicates as ``True`` except for the last
3071 occurrence.
3072 - ``False`` : Mark all duplicates as ``True``.
3074 Returns
3075 -------
3076 np.ndarray[bool]
3077 A numpy array of boolean values indicating duplicate index values.
3079 See Also
3080 --------
3081 Series.duplicated : Equivalent method on pandas.Series.
3082 DataFrame.duplicated : Equivalent method on pandas.DataFrame.
3083 Index.drop_duplicates : Remove duplicate values from Index.
3085 Examples
3086 --------
3087 By default, for each set of duplicated values, the first occurrence is
3088 set to False and all others to True:
3090 >>> idx = pd.Index(["llama", "cow", "llama", "beetle", "llama"])
3091 >>> idx.duplicated()
3092 array([False, False, True, False, True])
3094 which is equivalent to
3096 >>> idx.duplicated(keep="first")
3097 array([False, False, True, False, True])
3099 By using 'last', the last occurrence of each set of duplicated values
3100 is set on False and all others on True:
3102 >>> idx.duplicated(keep="last")
3103 array([ True, False, True, False, False])
3105 By setting keep on ``False``, all duplicates are True:
3107 >>> idx.duplicated(keep=False)
3108 array([ True, False, True, False, True])
3109 """
3110 if self.is_unique:
3111 # fastpath available bc we are immutable
3112 return np.zeros(len(self), dtype=bool)
3113 return self._duplicated(keep=keep)
3115 # --------------------------------------------------------------------
3116 # Arithmetic & Logical Methods
3118 def __iadd__(self, other: object) -> Self: # type: ignore[misc]
3119 # alias for __add__
3120 return self + other
3122 @final
3123 def __bool__(self) -> NoReturn:
3124 raise ValueError(
3125 f"The truth value of a {type(self).__name__} is ambiguous. "
3126 "Use a.empty, a.item(), a.any() or a.all()."
3127 )
3129 # --------------------------------------------------------------------
3130 # Set Operation Methods
3132 def _get_reconciled_name_object(self, other: Index) -> Self:
3133 """
3134 If the result of a set operation will be self,
3135 return a shallow copy of self.
3136 """
3137 name = get_op_result_name(self, other) # type: ignore[no-untyped-call]
3138 if self.name is not name:
3139 return self.rename(name)
3140 return self.copy(deep=False)
3142 @final
3143 def _validate_sort_keyword(self, sort: bool | None) -> None:
3144 if sort not in [None, False, True]:
3145 raise ValueError(
3146 "The 'sort' keyword only takes the values of "
3147 f"None, True, or False; {sort} was passed."
3148 )
3150 @final
3151 def _dti_setop_align_tzs(self, other: Index, setop: str_t) -> tuple[Index, Index]:
3152 """
3153 With mismatched timezones, cast both to UTC.
3154 """
3155 # Caller is responsible for checking
3156 # `self.dtype != other.dtype`
3157 if (
3158 isinstance(self, ABCDatetimeIndex)
3159 and isinstance(other, ABCDatetimeIndex)
3160 and self.tz is not None
3161 and other.tz is not None
3162 ):
3163 # GH#39328, GH#45357, GH#60080
3164 # If both timezones are the same, no need to convert to UTC
3165 if self.tz == other.tz:
3166 return self, other
3167 else:
3168 left = self.tz_convert("UTC")
3169 right = other.tz_convert("UTC")
3170 return left, right
3171 return self, other
3173 @final
3174 def union(self, other: Axes, sort: bool | None = None) -> Index:
3175 """
3176 Form the union of two Index objects.
3178 If the Index objects are incompatible, both Index objects will be
3179 cast to dtype('object') first.
3181 Parameters
3182 ----------
3183 other : Index or array-like
3184 Index or an array-like object containing elements to form the union
3185 with the original Index.
3186 sort : bool or None, default None
3187 Whether to sort the resulting Index.
3189 * None : Sort the result, except when
3191 1. `self` and `other` are equal.
3192 2. `self` or `other` has length 0.
3193 3. Some values in `self` or `other` cannot be compared.
3194 A RuntimeWarning is issued in this case.
3196 * False : do not sort the result.
3197 * True : Sort the result (which may raise TypeError).
3199 Returns
3200 -------
3201 Index
3202 Returns a new Index object with elements from both the original
3203 Index and the ``other`` Index.
3205 If either index contains duplicate entries, the result preserves
3206 duplicates up to the maximum number of occurrences in either index.
3208 See Also
3209 --------
3210 Index.unique : Return unique values in the index.
3211 Index.intersection : Form the intersection of two Index objects.
3212 Index.difference : Return a new Index with elements of index not in `other`.
3214 Examples
3215 --------
3216 Union matching dtypes
3218 >>> idx1 = pd.Index([1, 2, 3, 4])
3219 >>> idx2 = pd.Index([3, 4, 5, 6])
3220 >>> idx1.union(idx2)
3221 Index([1, 2, 3, 4, 5, 6], dtype='int64')
3223 Union mismatched dtypes
3225 >>> idx1 = pd.Index(["a", "b", "c", "d"])
3226 >>> idx2 = pd.Index([1, 2, 3, 4])
3227 >>> idx1.union(idx2)
3228 Index(['a', 'b', 'c', 'd', 1, 2, 3, 4], dtype='object')
3230 Union with duplicate values
3232 >>> idx1 = pd.Index([1, 2, 2, 3])
3233 >>> idx2 = pd.Index([3, 3, 4])
3234 >>> idx1.union(idx2)
3235 Index([1, 2, 2, 3, 3, 4], dtype='int64')
3237 MultiIndex case
3239 >>> idx1 = pd.MultiIndex.from_arrays(
3240 ... [[1, 1, 2, 2], ["Red", "Blue", "Red", "Blue"]]
3241 ... )
3242 >>> idx1
3243 MultiIndex([(1, 'Red'),
3244 (1, 'Blue'),
3245 (2, 'Red'),
3246 (2, 'Blue')],
3247 )
3248 >>> idx2 = pd.MultiIndex.from_arrays(
3249 ... [[3, 3, 2, 2], ["Red", "Green", "Red", "Green"]]
3250 ... )
3251 >>> idx2
3252 MultiIndex([(3, 'Red'),
3253 (3, 'Green'),
3254 (2, 'Red'),
3255 (2, 'Green')],
3256 )
3257 >>> idx1.union(idx2)
3258 MultiIndex([(1, 'Blue'),
3259 (1, 'Red'),
3260 (2, 'Blue'),
3261 (2, 'Green'),
3262 (2, 'Red'),
3263 (3, 'Green'),
3264 (3, 'Red')],
3265 )
3266 >>> idx1.union(idx2, sort=False)
3267 MultiIndex([(1, 'Red'),
3268 (1, 'Blue'),
3269 (2, 'Red'),
3270 (2, 'Blue'),
3271 (3, 'Red'),
3272 (3, 'Green'),
3273 (2, 'Green')],
3274 )
3275 """
3276 self._validate_sort_keyword(sort)
3277 self._assert_can_do_setop(other)
3278 other, result_name = self._convert_can_do_setop(other)
3280 if self.dtype != other.dtype:
3281 if (
3282 isinstance(self, ABCMultiIndex)
3283 and not is_object_dtype(_unpack_nested_dtype(other))
3284 and len(other) > 0
3285 ):
3286 raise NotImplementedError(
3287 "Can only union MultiIndex with MultiIndex or Index of tuples, "
3288 "try mi.to_flat_index().union(other) instead."
3289 )
3290 self, other = self._dti_setop_align_tzs(other, "union")
3292 dtype = self._find_common_type_compat(other)
3293 left = self.astype(dtype, copy=False)
3294 right = other.astype(dtype, copy=False)
3295 return left.union(right, sort=sort)
3297 elif not len(other) or self.equals(other):
3298 # NB: whether this (and the `if not len(self)` check below) come before
3299 # or after the dtype equality check above affects the returned dtype
3300 result = self._get_reconciled_name_object(other)
3301 if sort is True:
3302 return result.sort_values()
3303 return result
3305 elif not len(self):
3306 result = other._get_reconciled_name_object(self)
3307 if sort is True:
3308 return result.sort_values()
3309 return result
3311 union_result = self._union(other, sort=sort)
3313 return self._wrap_setop_result(other, union_result)
3315 def _union(self, other: Index, sort: bool | None) -> Index | ArrayLike | MultiIndex:
3316 """
3317 Specific union logic should go here. In subclasses, union behavior
3318 should be overwritten here rather than in `self.union`.
3320 Parameters
3321 ----------
3322 other : Index or array-like
3323 sort : False or None, default False
3324 Whether to sort the resulting index.
3326 * True : sort the result
3327 * False : do not sort the result.
3328 * None : sort the result, except when `self` and `other` are equal
3329 or when the values cannot be compared.
3331 Returns
3332 -------
3333 Index, MultiIndex, or ArrayLike
3334 """
3335 lvals = self._values
3336 rvals = other._values
3338 if (
3339 sort in (None, True)
3340 and (self.is_unique or other.is_unique)
3341 and self._can_use_libjoin
3342 and other._can_use_libjoin
3343 ):
3344 # Both are monotonic and at least one is unique, so can use outer join
3345 # (actually don't need either unique, but without this restriction
3346 # test_union_same_value_duplicated_in_both fails)
3347 try:
3348 return self._outer_indexer(other)[0] # pyright: ignore[reportArgumentType]
3349 except TypeError:
3350 # incomparable objects; should only be for object dtype
3351 value_list = list(lvals)
3353 # worth making this faster? a very unusual case
3354 value_set = set(lvals)
3355 value_list.extend(x for x in rvals if x not in value_set)
3356 # If objects are unorderable, we must have object dtype.
3357 return np.array(value_list, dtype=object)
3359 elif not other.is_unique:
3360 # other has duplicates
3361 result_dups = algos.union_with_duplicates(self, other)
3362 return _maybe_try_sort(result_dups, sort)
3364 # The rest of this method is analogous to Index._intersection_via_get_indexer
3366 # Self may have duplicates; other already checked as unique
3367 # find indexes of things in "other" that are not in "self"
3368 if self._index_as_unique:
3369 indexer = self.get_indexer(other)
3370 missing = (indexer == -1).nonzero()[0]
3371 else:
3372 missing = algos.unique1d(self.get_indexer_non_unique(other)[1])
3374 result: Index | MultiIndex | ArrayLike
3375 if self._is_multi:
3376 # Preserve MultiIndex to avoid losing dtypes
3377 result = self.append(other.take(missing))
3379 elif len(missing) > 0:
3380 other_diff = rvals.take(missing)
3381 result = concat_compat((lvals, other_diff))
3382 else:
3383 result = lvals
3385 result = _maybe_try_sort(result, sort)
3387 return result
3389 @final
3390 def _wrap_setop_result(
3391 self, other: Index, result: Index | ArrayLike | MultiIndex
3392 ) -> Index:
3393 name = get_op_result_name(self, other) # type: ignore[no-untyped-call]
3394 if isinstance(result, Index):
3395 if result.name != name:
3396 result = result.rename(name)
3397 else:
3398 result = self._shallow_copy(result, name=name)
3399 return result
3401 @final
3402 def intersection(self, other: Axes, sort: bool = False) -> Index:
3403 # default sort keyword is different here from other setops intentionally
3404 # done in GH#25063
3405 """
3406 Form the intersection of two Index objects.
3408 This returns a new Index with elements common to the index and `other`.
3410 Parameters
3411 ----------
3412 other : Index or array-like
3413 An Index or an array-like object containing elements to form the
3414 intersection with the original Index.
3415 sort : True, False or None, default False
3416 Whether to sort the resulting index.
3418 * None : sort the result, except when `self` and `other` are equal
3419 or when the values cannot be compared.
3420 * False : do not sort the result.
3421 * True : Sort the result (which may raise TypeError).
3423 Returns
3424 -------
3425 Index
3426 Returns a new Index object with elements common to both the original Index
3427 and the `other` Index.
3429 See Also
3430 --------
3431 Index.union : Form the union of two Index objects.
3432 Index.difference : Return a new Index with elements of index not in other.
3433 Index.isin : Return a boolean array where the index values are in values.
3435 Examples
3436 --------
3437 >>> idx1 = pd.Index([1, 2, 3, 4])
3438 >>> idx2 = pd.Index([3, 4, 5, 6])
3439 >>> idx1.intersection(idx2)
3440 Index([3, 4], dtype='int64')
3441 """
3442 self._validate_sort_keyword(sort)
3443 self._assert_can_do_setop(other)
3444 other, result_name = self._convert_can_do_setop(other)
3446 if self.dtype != other.dtype:
3447 self, other = self._dti_setop_align_tzs(other, "intersection")
3449 if self.equals(other):
3450 if not self.is_unique:
3451 result = self.unique()._get_reconciled_name_object(other)
3452 else:
3453 result = self._get_reconciled_name_object(other)
3454 if sort is True:
3455 result = result.sort_values()
3456 return result
3458 if len(self) == 0 or len(other) == 0:
3459 # fastpath; we need to be careful about having commutativity
3461 if self._is_multi or other._is_multi:
3462 # _convert_can_do_setop ensures that we have both or neither
3463 # We retain self.levels
3464 return self[:0].rename(result_name)
3466 dtype = self._find_common_type_compat(other)
3467 if self.dtype == dtype:
3468 # Slicing allows us to retain DTI/TDI.freq, RangeIndex
3470 # Note: self[:0] vs other[:0] affects
3471 # 1) which index's `freq` we get in DTI/TDI cases
3472 # This may be a historical artifact, i.e. no documented
3473 # reason for this choice.
3474 # 2) The `step` we get in RangeIndex cases
3475 if len(self) == 0:
3476 return self[:0].rename(result_name)
3477 else:
3478 return other[:0].rename(result_name)
3480 return Index([], dtype=dtype, name=result_name)
3482 elif not self._should_compare(other):
3483 # We can infer that the intersection is empty.
3484 if isinstance(self, ABCMultiIndex):
3485 return self[:0].rename(result_name)
3486 return Index([], name=result_name)
3488 elif self.dtype != other.dtype:
3489 dtype = self._find_common_type_compat(other)
3490 this = self.astype(dtype, copy=False)
3491 other = other.astype(dtype, copy=False)
3492 return this.intersection(other, sort=sort)
3494 result = self._intersection(other, sort=sort) # type: ignore[assignment]
3495 return self._wrap_intersection_result(other, result)
3497 def _intersection(
3498 self, other: Index, sort: bool = False
3499 ) -> Index | ArrayLike | MultiIndex:
3500 """
3501 intersection specialized to the case with matching dtypes.
3502 """
3503 if self._can_use_libjoin and other._can_use_libjoin:
3504 try:
3505 res_indexer, indexer, _ = self._inner_indexer(other) # pyright: ignore[reportArgumentType]
3506 except TypeError:
3507 # non-comparable; should only be for object dtype
3508 pass
3509 else:
3510 # TODO: algos.unique1d should preserve DTA/TDA
3511 if is_numeric_dtype(self.dtype):
3512 # This is faster, because Index.unique() checks for uniqueness
3513 # before calculating the unique values.
3514 res = algos.unique1d(res_indexer)
3515 else:
3516 result = self.take(indexer)
3517 res = result.drop_duplicates() # type: ignore[assignment]
3518 return ensure_wrapped_if_datetimelike(res) # type: ignore[no-untyped-call]
3520 res_values = self._intersection_via_get_indexer(other, sort=sort)
3521 res_values = _maybe_try_sort(res_values, sort) # type: ignore[assignment]
3522 return res_values
3524 def _wrap_intersection_result(
3525 self, other: Index, result: Index | ArrayLike | MultiIndex
3526 ) -> Index:
3527 # We will override for MultiIndex to handle empty results
3528 return self._wrap_setop_result(other, result)
3530 @final
3531 def _intersection_via_get_indexer(
3532 self, other: Index | MultiIndex, sort: bool | None
3533 ) -> ArrayLike | MultiIndex:
3534 """
3535 Find the intersection of two Indexes using get_indexer.
3537 Returns
3538 -------
3539 np.ndarray or ExtensionArray or MultiIndex
3540 The returned array will be unique.
3541 """
3542 left_unique = self.unique()
3543 right_unique = other.unique()
3545 # even though we are unique, we need get_indexer_for for IntervalIndex
3546 indexer = left_unique.get_indexer_for(right_unique)
3548 mask = indexer != -1
3550 taker = indexer.take(mask.nonzero()[0])
3551 if sort is False:
3552 # sort bc we want the elements in the same order they are in self
3553 # unnecessary in the case with sort=None bc we will sort later
3554 taker = np.sort(taker)
3556 result: MultiIndex | ExtensionArray | np.ndarray
3557 if isinstance(left_unique, ABCMultiIndex):
3558 result = left_unique.take(taker)
3559 else:
3560 result = left_unique.take(taker)._values
3561 return result
3563 @final
3564 def difference(self, other: Axes, sort: bool | None = None) -> Index:
3565 """
3566 Return a new Index with elements of index not in `other`.
3568 This is the set difference of two Index objects.
3570 Parameters
3571 ----------
3572 other : Index or array-like
3573 Index object or an array-like object containing elements to be compared
3574 with the elements of the original Index.
3575 sort : bool or None, default None
3576 Whether to sort the resulting index. By default, the
3577 values are attempted to be sorted, but any TypeError from
3578 incomparable elements is caught by pandas.
3580 * None : Attempt to sort the result, but catch any TypeErrors
3581 from comparing incomparable elements.
3582 * False : Do not sort the result.
3583 * True : Sort the result (which may raise TypeError).
3585 Returns
3586 -------
3587 Index
3588 Returns a new Index object containing elements that are in the original
3589 Index but not in the `other` Index.
3591 See Also
3592 --------
3593 Index.symmetric_difference : Compute the symmetric difference of two Index
3594 objects.
3595 Index.intersection : Form the intersection of two Index objects.
3597 Examples
3598 --------
3599 >>> idx1 = pd.Index([2, 1, 3, 4])
3600 >>> idx2 = pd.Index([3, 4, 5, 6])
3601 >>> idx1.difference(idx2)
3602 Index([1, 2], dtype='int64')
3603 >>> idx1.difference(idx2, sort=False)
3604 Index([2, 1], dtype='int64')
3605 """
3606 self._validate_sort_keyword(sort)
3607 self._assert_can_do_setop(other)
3608 other, result_name = self._convert_can_do_setop(other)
3610 # Note: we do NOT call _dti_setop_align_tzs here, as there
3611 # is no requirement that .difference be commutative, so it does
3612 # not cast to object.
3614 if self.equals(other):
3615 # Note: we do not (yet) sort even if sort=None GH#24959
3616 return self[:0].rename(result_name)
3618 if len(other) == 0:
3619 # Note: we do not (yet) sort even if sort=None GH#24959
3620 result = self.unique().rename(result_name)
3621 if sort is True:
3622 return result.sort_values()
3623 return result
3625 if not self._should_compare(other):
3626 # Nothing matches -> difference is everything
3627 result = self.unique().rename(result_name)
3628 if sort is True:
3629 return result.sort_values()
3630 return result
3632 result = self._difference(other, sort=sort)
3633 return self._wrap_difference_result(other, result)
3635 def _difference(self, other: Index, sort: bool | None) -> Index:
3636 # overridden by RangeIndex
3637 this = self
3638 if isinstance(self, ABCCategoricalIndex) and self.hasnans and other.hasnans:
3639 this = this.dropna()
3640 other = other.unique()
3641 the_diff = this[other.get_indexer_for(this) == -1]
3642 the_diff = the_diff if this.is_unique else the_diff.unique()
3643 the_diff = cast("Index", _maybe_try_sort(the_diff, sort))
3644 return the_diff
3646 def _wrap_difference_result(
3647 self, other: Index, result: Index | ArrayLike | MultiIndex
3648 ) -> Index:
3649 # We will override for MultiIndex to handle empty results
3650 return self._wrap_setop_result(other, result)
3652 def symmetric_difference(
3653 self,
3654 other: Axes,
3655 result_name: abc.Hashable | None = None,
3656 sort: bool | None = None,
3657 ) -> Index:
3658 """
3659 Compute the symmetric difference of two Index objects.
3661 The result contains elements that are in either of the two Index
3662 objects but not in their intersection. This is the set-theoretic
3663 symmetric difference, analogous to the ``^`` operator on Python sets.
3665 Parameters
3666 ----------
3667 other : Index or array-like
3668 Index or an array-like object with elements to compute the symmetric
3669 difference with the original Index.
3670 result_name : str
3671 A string representing the name of the resulting Index, if desired.
3672 sort : bool or None, default None
3673 Whether to sort the resulting index. By default, the
3674 values are attempted to be sorted, but any TypeError from
3675 incomparable elements is caught by pandas.
3677 * None : Attempt to sort the result, but catch any TypeErrors
3678 from comparing incomparable elements.
3679 * False : Do not sort the result.
3680 * True : Sort the result (which may raise TypeError).
3682 Returns
3683 -------
3684 Index
3685 Returns a new Index object containing elements that appear in either the
3686 original Index or the `other` Index, but not both.
3688 See Also
3689 --------
3690 Index.difference : Return a new Index with elements of index not in other.
3691 Index.union : Form the union of two Index objects.
3692 Index.intersection : Form the intersection of two Index objects.
3694 Notes
3695 -----
3696 ``symmetric_difference`` contains elements that appear in either
3697 ``idx1`` or ``idx2`` but not both. Equivalent to the Index created by
3698 ``idx1.difference(idx2) | idx2.difference(idx1)`` with duplicates
3699 dropped.
3701 Examples
3702 --------
3703 >>> idx1 = pd.Index([1, 2, 3, 4])
3704 >>> idx2 = pd.Index([2, 3, 4, 5])
3705 >>> idx1.symmetric_difference(idx2)
3706 Index([1, 5], dtype='int64')
3707 """
3708 self._validate_sort_keyword(sort)
3709 self._assert_can_do_setop(other)
3710 other, result_name_update = self._convert_can_do_setop(other)
3711 if result_name is None:
3712 result_name = result_name_update
3714 if self.dtype != other.dtype:
3715 self, other = self._dti_setop_align_tzs(other, "symmetric_difference")
3717 if not self._should_compare(other):
3718 return self.union(other, sort=sort).rename(result_name)
3720 elif self.dtype != other.dtype:
3721 dtype = self._find_common_type_compat(other)
3722 this = self.astype(dtype, copy=False)
3723 that = other.astype(dtype, copy=False)
3724 return this.symmetric_difference(that, sort=sort).rename(result_name)
3726 this = self.unique()
3727 other = other.unique()
3728 indexer = this.get_indexer_for(other)
3730 # {this} minus {other}
3731 common_indexer = indexer.take((indexer != -1).nonzero()[0])
3732 left_indexer = np.setdiff1d(
3733 np.arange(this.size), common_indexer, assume_unique=True
3734 )
3735 left_diff = this.take(left_indexer) # pyright: ignore[reportArgumentType]
3737 # {other} minus {this}
3738 right_indexer = (indexer == -1).nonzero()[0]
3739 right_diff = other.take(right_indexer)
3741 res_values = left_diff.append(right_diff)
3742 result = _maybe_try_sort(res_values, sort)
3744 if not self._is_multi:
3745 return Index(result, name=result_name, dtype=res_values.dtype)
3746 else:
3747 left_diff = cast("MultiIndex", left_diff)
3748 if len(result) == 0:
3749 # result might be an Index, if other was an Index
3750 return left_diff.remove_unused_levels().set_names(result_name)
3751 result = cast("Index", result)
3752 return result.set_names(result_name)
3754 @final
3755 def _assert_can_do_setop(self, other: Axes) -> bool:
3756 if not is_list_like(other):
3757 raise TypeError("Input must be Index or array-like")
3758 return True
3760 def _convert_can_do_setop(self, other: Axes) -> tuple[Index, Hashable]:
3761 if not isinstance(other, Index):
3762 other = Index(other, name=self.name)
3763 result_name = self.name
3764 else:
3765 result_name = get_op_result_name(self, other) # type: ignore[no-untyped-call]
3766 return other, result_name
3768 # --------------------------------------------------------------------
3769 # Indexing Methods
3771 def get_loc(self, key: Hashable) -> int | slice | npt.NDArray[np.bool_]:
3772 """
3773 Get integer location, slice or boolean mask for requested label.
3775 The return type depends on the characteristics of the index: an
3776 integer for a unique index, a slice for a monotonic index with
3777 duplicate entries, or a boolean mask for a non-monotonic index
3778 with duplicates.
3780 Parameters
3781 ----------
3782 key : label
3783 The key to check its location if it is present in the index.
3785 Returns
3786 -------
3787 int if unique index, slice if monotonic index, else mask
3788 Integer location, slice or boolean mask.
3790 See Also
3791 --------
3792 Index.get_slice_bound : Calculate slice bound that corresponds to
3793 given label.
3794 Index.get_indexer : Computes indexer and mask for new index given
3795 the current index.
3796 Index.get_non_unique : Returns indexer and masks for new index given
3797 the current index.
3798 Index.get_indexer_for : Returns an indexer even when non-unique.
3800 Examples
3801 --------
3802 >>> unique_index = pd.Index(list("abc"))
3803 >>> unique_index.get_loc("b")
3804 1
3806 >>> monotonic_index = pd.Index(list("abbc"))
3807 >>> monotonic_index.get_loc("b")
3808 slice(1, 3, None)
3810 >>> non_monotonic_index = pd.Index(list("abcb"))
3811 >>> non_monotonic_index.get_loc("b")
3812 array([False, True, False, True])
3813 """
3814 casted_key = self._maybe_cast_indexer(key)
3815 try:
3816 return self._engine.get_loc(casted_key)
3817 except KeyError as err:
3818 if isinstance(casted_key, slice) or (
3819 isinstance(casted_key, abc.Iterable)
3820 and any(isinstance(x, slice) for x in casted_key)
3821 ):
3822 raise InvalidIndexError(key) from err
3823 raise KeyError(key) from err
3824 except TypeError:
3825 # If we have a listlike key, _check_indexing_error will raise
3826 # InvalidIndexError. Otherwise we fall through and re-raise
3827 # the TypeError.
3828 self._check_indexing_error(key)
3829 raise
3831 @final
3832 def get_indexer(
3833 self,
3834 target: Axes,
3835 method: ReindexMethod | None = None,
3836 limit: int | None = None,
3837 tolerance: Any = None,
3838 ) -> npt.NDArray[np.intp]:
3839 """
3840 Compute indexer and mask for new index given the current index.
3842 The indexer should be then used as an input to ndarray.take to align the
3843 current data to the new index.
3845 Parameters
3846 ----------
3847 target : Index
3848 An iterable containing the values to be used for computing indexer.
3849 method : {None, 'pad'/'ffill', 'backfill'/'bfill', 'nearest'}, optional
3850 * default: exact matches only.
3851 * pad / ffill: find the PREVIOUS index value if no exact match.
3852 * backfill / bfill: use NEXT index value if no exact match
3853 * nearest: use the NEAREST index value if no exact match. Tied
3854 distances are broken by preferring the larger index value.
3855 limit : int, optional
3856 Maximum number of consecutive labels in ``target`` to match for
3857 inexact matches.
3858 tolerance : optional
3859 Maximum distance between original and new labels for inexact
3860 matches. The values of the index at the matching locations must
3861 satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
3863 Tolerance may be a scalar value, which applies the same tolerance
3864 to all values, or list-like, which applies variable tolerance per
3865 element. List-like includes list, tuple, array, Series, and must be
3866 the same size as the index and its dtype must exactly match the
3867 index's type.
3869 Returns
3870 -------
3871 np.ndarray[np.intp]
3872 Integers from 0 to n - 1 indicating that the index at these
3873 positions matches the corresponding target values. Missing values
3874 in the target are marked by -1.
3876 See Also
3877 --------
3878 Index.get_indexer_for : Returns an indexer even when non-unique.
3879 Index.get_non_unique : Returns indexer and masks for new index given
3880 the current index.
3882 Notes
3883 -----
3884 Returns -1 for unmatched values, for further explanation see the
3885 example below.
3887 Examples
3888 --------
3889 >>> index = pd.Index(["c", "a", "b"])
3890 >>> index.get_indexer(["a", "b", "x"])
3891 array([ 1, 2, -1])
3893 Notice that the return value is an array of locations in ``index``
3894 and ``x`` is marked by -1, as it is not in ``index``.
3895 """
3896 method = clean_reindex_fill_method(method)
3897 orig_target = target
3898 target = self._maybe_cast_listlike_indexer(target)
3900 self._check_indexing_method(method, limit, tolerance)
3902 if not self._index_as_unique:
3903 raise InvalidIndexError(self._requires_unique_msg)
3905 if len(target) == 0:
3906 return np.array([], dtype=np.intp)
3908 if not self._should_compare(target) and not self._should_partial_index(target):
3909 # IntervalIndex get special treatment bc numeric scalars can be
3910 # matched to Interval scalars
3911 return self._get_indexer_non_comparable(target, method=method, unique=True)
3913 if isinstance(self.dtype, CategoricalDtype):
3914 # _maybe_cast_listlike_indexer ensures target has our dtype
3915 # (could improve perf by doing _should_compare check earlier?)
3916 assert self.dtype == target.dtype
3917 target = cast("CategoricalIndex", target)
3919 indexer = self._engine.get_indexer(target.codes)
3920 if self.hasnans and target.hasnans:
3921 # After _maybe_cast_listlike_indexer, target elements which do not
3922 # belong to some category are changed to NaNs
3923 # Mask to track actual NaN values compared to inserted NaN values
3924 # GH#45361
3925 target_nans = isna(orig_target)
3926 loc = self.get_loc(np.nan)
3927 mask = target.isna()
3928 indexer[target_nans] = loc # type: ignore[assignment]
3929 indexer[mask & ~target_nans] = -1
3930 return indexer
3932 if isinstance(target.dtype, CategoricalDtype):
3933 # potential fastpath
3934 # get an indexer for unique categories then propagate to codes via take_nd
3935 # get_indexer instead of _get_indexer needed for MultiIndex cases
3936 # e.g. test_append_different_columns_types
3937 target = cast("CategoricalIndex", target)
3938 categories_indexer = self.get_indexer(target.categories)
3940 indexer = algos.take_nd(categories_indexer, target.codes, fill_value=-1)
3942 if (not self._is_multi and self.hasnans) and target.hasnans:
3943 # Exclude MultiIndex because hasnans raises NotImplementedError
3944 # we should only get here if we are unique, so loc is an integer
3945 # GH#41934
3946 loc = self.get_loc(np.nan)
3947 mask = target.isna()
3948 indexer[mask] = loc # type: ignore[assignment]
3950 return ensure_platform_int(indexer)
3952 pself, ptarget = self._maybe_downcast_for_indexing(target)
3953 if pself is not self or ptarget is not target:
3954 return pself.get_indexer(
3955 ptarget, method=method, limit=limit, tolerance=tolerance
3956 )
3958 if self.dtype == target.dtype and self.equals(target):
3959 # Only call equals if we have same dtype to avoid inference/casting
3960 return np.arange(len(target), dtype=np.intp)
3962 if self.dtype != target.dtype and not self._should_partial_index(target):
3963 # _should_partial_index e.g. IntervalIndex with numeric scalars
3964 # that can be matched to Interval scalars.
3965 dtype = self._find_common_type_compat(target)
3967 this = self.astype(dtype, copy=False)
3968 target = target.astype(dtype, copy=False)
3969 return this._get_indexer(
3970 target, method=method, limit=limit, tolerance=tolerance
3971 )
3973 return self._get_indexer(target, method, limit, tolerance)
3975 def _get_indexer(
3976 self,
3977 target: Index,
3978 method: str_t | None = None,
3979 limit: int | None = None,
3980 tolerance: Any = None,
3981 ) -> npt.NDArray[np.intp]:
3982 if tolerance is not None:
3983 tolerance = self._convert_tolerance(tolerance, target)
3985 if method in ["pad", "backfill"]:
3986 indexer = self._get_fill_indexer(target, method, limit, tolerance)
3987 elif method == "nearest":
3988 indexer = self._get_nearest_indexer(target, limit, tolerance)
3989 else:
3990 if target._is_multi and self._is_multi:
3991 engine = self._engine
3992 # error: Item "IndexEngine" of "Union[IndexEngine, ExtensionEngine]"
3993 # has no attribute "_extract_level_codes"
3994 tgt_values = engine._extract_level_codes( # type: ignore[union-attr]
3995 target
3996 )
3997 else:
3998 tgt_values = target._get_engine_target()
4000 indexer = self._engine.get_indexer(tgt_values) # pyright: ignore[reportArgumentType]
4002 if method is not None and target._can_hold_na and not target._is_multi:
4003 # GH#32572 NaT/NaN in the target should not be matched
4004 # by pad/backfill/nearest
4005 na_mask = target._isnan
4006 if na_mask.any():
4007 indexer[na_mask] = -1
4009 return ensure_platform_int(indexer)
4011 @final
4012 def _should_partial_index(self, target: Index) -> bool:
4013 """
4014 Should we attempt partial-matching indexing?
4015 """
4016 if isinstance(self.dtype, IntervalDtype):
4017 if isinstance(target.dtype, IntervalDtype):
4018 return False
4019 # "Index" has no attribute "left"
4020 return self.left._should_compare(target) # type: ignore[attr-defined]
4021 return False
4023 @final
4024 def _check_indexing_method(
4025 self,
4026 method: str_t | None,
4027 limit: int | None = None,
4028 tolerance: Any = None,
4029 ) -> None:
4030 """
4031 Raise if we have a get_indexer `method` that is not supported or valid.
4032 """
4033 if method not in [None, "bfill", "backfill", "pad", "ffill", "nearest"]:
4034 # in practice the clean_reindex_fill_method call would raise
4035 # before we get here
4036 raise ValueError("Invalid fill method") # pragma: no cover
4038 if self._is_multi:
4039 if method == "nearest":
4040 raise NotImplementedError(
4041 "method='nearest' not implemented yet "
4042 "for MultiIndex; see GitHub issue 9365"
4043 )
4044 if method in ("pad", "backfill"):
4045 if tolerance is not None:
4046 raise NotImplementedError(
4047 "tolerance not implemented yet for MultiIndex"
4048 )
4050 if isinstance(self.dtype, (IntervalDtype, CategoricalDtype)):
4051 # GH#37871 for now this is only for IntervalIndex and CategoricalIndex
4052 if method is not None:
4053 raise NotImplementedError(
4054 f"method {method} not yet implemented for {type(self).__name__}"
4055 )
4057 if method is None:
4058 if tolerance is not None:
4059 raise ValueError(
4060 "tolerance argument only valid if doing pad, "
4061 "backfill or nearest reindexing"
4062 )
4063 if limit is not None:
4064 raise ValueError(
4065 "limit argument only valid if doing pad, "
4066 "backfill or nearest reindexing"
4067 )
4069 def _convert_tolerance(
4070 self, tolerance: Any, target: np.ndarray | Index
4071 ) -> np.ndarray:
4072 # override this method on subclasses
4073 tolerance = np.asarray(tolerance)
4074 if target.size != tolerance.size and tolerance.size > 1:
4075 raise ValueError("list-like tolerance size must match target index size")
4076 elif is_numeric_dtype(self) and not np.issubdtype(tolerance.dtype, np.number):
4077 if tolerance.ndim > 0:
4078 raise ValueError(
4079 f"tolerance argument for {type(self).__name__} with dtype "
4080 f"{self.dtype} must contain numeric elements if it is list type"
4081 )
4083 raise ValueError(
4084 f"tolerance argument for {type(self).__name__} with dtype {self.dtype} "
4085 f"must be numeric if it is a scalar: {tolerance!r}"
4086 )
4087 return tolerance
4089 @final
4090 def _get_fill_indexer(
4091 self,
4092 target: Index,
4093 method: str_t,
4094 limit: int | None = None,
4095 tolerance: Any = None,
4096 ) -> npt.NDArray[np.intp]:
4097 if self._is_multi:
4098 if not (self.is_monotonic_increasing or self.is_monotonic_decreasing):
4099 raise ValueError("index must be monotonic increasing or decreasing")
4100 encoded = self.append(target)._engine.values # type: ignore[union-attr]
4101 self_encoded = Index(encoded[: len(self)], copy=False)
4102 target_encoded = Index(encoded[len(self) :], copy=False)
4103 return self_encoded._get_fill_indexer(
4104 target_encoded, method, limit, tolerance
4105 )
4107 if self.is_monotonic_increasing and target.is_monotonic_increasing:
4108 target_values = target._get_engine_target()
4109 own_values = self._get_engine_target()
4110 if not isinstance(target_values, np.ndarray) or not isinstance(
4111 own_values, np.ndarray
4112 ):
4113 raise NotImplementedError
4115 if method == "pad":
4116 indexer = libalgos.pad(own_values, target_values, limit=limit)
4117 else:
4118 # i.e. "backfill"
4119 indexer = libalgos.backfill(own_values, target_values, limit=limit)
4120 else:
4121 indexer = self._get_fill_indexer_searchsorted(target, method, limit)
4122 if tolerance is not None and len(self):
4123 indexer = self._filter_indexer_tolerance(target, indexer, tolerance)
4124 return indexer
4126 @final
4127 def _get_fill_indexer_searchsorted(
4128 self, target: Index, method: str_t, limit: int | None = None
4129 ) -> npt.NDArray[np.intp]:
4130 """
4131 Fallback pad/backfill get_indexer that works for monotonic decreasing
4132 indexes and non-monotonic targets.
4133 """
4134 if limit is not None:
4135 raise ValueError(
4136 f"limit argument for {method!r} method only well-defined "
4137 "if index and target are monotonic"
4138 )
4140 side: Literal["left", "right"] = "left" if method == "pad" else "right"
4142 # find exact matches first (this simplifies the algorithm)
4143 indexer = self.get_indexer(target)
4144 nonexact = indexer == -1
4145 indexer[nonexact] = self._searchsorted_monotonic(target[nonexact], side)
4146 if side == "left":
4147 # searchsorted returns "indices into a sorted array such that,
4148 # if the corresponding elements in v were inserted before the
4149 # indices, the order of a would be preserved".
4150 # Thus, we need to subtract 1 to find values to the left.
4151 indexer[nonexact] -= 1
4152 # This also mapped not found values (values of 0 from
4153 # np.searchsorted) to -1, which conveniently is also our
4154 # sentinel for missing values
4155 else:
4156 # Mark indices to the right of the largest value as not found
4157 indexer[indexer == len(self)] = -1
4158 return indexer
4160 @final
4161 def _get_nearest_indexer(
4162 self, target: Index, limit: int | None, tolerance: Any
4163 ) -> npt.NDArray[np.intp]:
4164 """
4165 Get the indexer for the nearest index labels; requires an index with
4166 values that can be subtracted from each other (e.g., not strings or
4167 tuples).
4168 """
4169 if not len(self):
4170 return self._get_fill_indexer(target, "pad")
4172 left_indexer = self.get_indexer(target, "pad", limit=limit)
4173 right_indexer = self.get_indexer(target, "backfill", limit=limit)
4175 left_distances = self._difference_compat(target, left_indexer)
4176 right_distances = self._difference_compat(target, right_indexer)
4178 op = operator.lt if self.is_monotonic_increasing else operator.le
4179 indexer = np.where(
4180 # error: Argument 1&2 has incompatible type "Union[ExtensionArray,
4181 # ndarray[Any, Any]]"; expected "Union[SupportsDunderLE,
4182 # SupportsDunderGE, SupportsDunderGT, SupportsDunderLT]"
4183 op(left_distances, right_distances) # type: ignore[arg-type]
4184 | (right_indexer == -1),
4185 left_indexer,
4186 right_indexer,
4187 )
4188 if tolerance is not None:
4189 indexer = self._filter_indexer_tolerance(target, indexer, tolerance)
4190 return indexer
4192 @final
4193 def _filter_indexer_tolerance(
4194 self,
4195 target: Index,
4196 indexer: npt.NDArray[np.intp],
4197 tolerance: Any,
4198 ) -> npt.NDArray[np.intp]:
4199 distance = self._difference_compat(target, indexer)
4201 return np.where(distance <= tolerance, indexer, -1)
4203 @final
4204 def _difference_compat(
4205 self, target: Index, indexer: npt.NDArray[np.intp]
4206 ) -> ArrayLike:
4207 # Compatibility for PeriodArray, for which __sub__ returns an ndarray[object]
4208 # of DateOffset objects, which do not support __abs__ (and would be slow
4209 # if they did)
4211 if isinstance(self.dtype, PeriodDtype):
4212 # Note: we only get here with matching dtypes
4213 own_values = cast("PeriodArray", self._data)._ndarray
4214 target_values = cast("PeriodArray", target._data)._ndarray
4215 diff = own_values[indexer] - target_values
4216 else:
4217 # error: Unsupported left operand type for - ("ExtensionArray")
4218 diff = self._values[indexer] - target._values # type: ignore[operator]
4219 return abs(diff)
4221 # --------------------------------------------------------------------
4222 # Indexer Conversion Methods
4224 @final
4225 def _validate_positional_slice(self, key: slice) -> None:
4226 """
4227 For positional indexing, a slice must have either int or None
4228 for each of start, stop, and step.
4229 """
4230 self._validate_indexer("positional", key.start, "iloc")
4231 self._validate_indexer("positional", key.stop, "iloc")
4232 self._validate_indexer("positional", key.step, "iloc")
4234 def _convert_slice_indexer(
4235 self, key: slice, kind: Literal["loc", "getitem"]
4236 ) -> slice:
4237 """
4238 Convert a slice indexer.
4240 By definition, these are labels unless 'iloc' is passed in.
4241 Floats are not allowed as the start, step, or stop of the slice.
4243 Parameters
4244 ----------
4245 key : label of the slice bound
4246 kind : {'loc', 'getitem'}
4247 """
4249 # potentially cast the bounds to integers
4250 start, stop, step = key.start, key.stop, key.step
4252 # figure out if this is a positional indexer
4253 is_index_slice = is_valid_positional_slice(key)
4255 # TODO(GH#50617): once Series.__[gs]etitem__ is removed we should be able
4256 # to simplify this.
4257 if kind == "getitem":
4258 # called from the getitem slicers, validate that we are in fact integers
4259 if is_index_slice:
4260 # In this case the _validate_indexer checks below are redundant
4261 return key
4262 elif self.dtype.kind in "iu":
4263 # Note: these checks are redundant if we know is_index_slice
4264 self._validate_indexer("slice", key.start, "getitem")
4265 self._validate_indexer("slice", key.stop, "getitem")
4266 self._validate_indexer("slice", key.step, "getitem")
4267 return key
4269 # convert the slice to an indexer here; checking that the user didn't
4270 # pass a positional slice to loc
4271 is_positional = is_index_slice and self._should_fallback_to_positional
4273 # if we are mixed and have integers
4274 if is_positional:
4275 try:
4276 # Validate start & stop
4277 if start is not None:
4278 self.get_loc(start)
4279 if stop is not None:
4280 self.get_loc(stop)
4281 is_positional = False
4282 except KeyError:
4283 pass
4285 if com.is_null_slice(key):
4286 # It doesn't matter if we are positional or label based
4287 indexer = key
4288 elif is_positional:
4289 if kind == "loc":
4290 # GH#16121, GH#24612, GH#31810
4291 raise TypeError(
4292 "Slicing a positional slice with .loc is not allowed, "
4293 "Use .loc with labels or .iloc with positions instead.",
4294 )
4295 indexer = key
4296 else:
4297 indexer = self.slice_indexer(start, stop, step)
4299 return indexer
4301 @final
4302 def _raise_invalid_indexer(
4303 self,
4304 form: Literal["slice", "positional"],
4305 key: object,
4306 reraise: lib.NoDefault | Exception | None = lib.no_default,
4307 ) -> None:
4308 """
4309 Raise consistent invalid indexer message.
4310 """
4311 msg = (
4312 f"cannot do {form} indexing on {type(self).__name__} with these "
4313 f"indexers [{key}] of type {type(key).__name__}"
4314 )
4315 if reraise is not lib.no_default:
4316 raise TypeError(msg) from reraise
4317 raise TypeError(msg)
4319 # --------------------------------------------------------------------
4320 # Reindex Methods
4322 @final
4323 def _validate_can_reindex(self, indexer: np.ndarray) -> None:
4324 """
4325 Check if we are allowing reindexing with this particular indexer.
4327 Parameters
4328 ----------
4329 indexer : an integer ndarray
4331 Raises
4332 ------
4333 ValueError if its a duplicate axis
4334 """
4335 # trying to reindex on an axis with duplicates
4336 if not self._index_as_unique and len(indexer):
4337 raise ValueError("cannot reindex on an axis with duplicate labels")
4339 def reindex(
4340 self,
4341 target: Axes,
4342 method: ReindexMethod | None = None,
4343 level: Level | None = None,
4344 limit: int | None = None,
4345 tolerance: float | None = None,
4346 ) -> tuple[Index, npt.NDArray[np.intp] | None]:
4347 """
4348 Create index with target's values.
4350 Optionally provides an indexer that maps the new labels back to the
4351 original positions, and supports filling methods for non-exact matches.
4353 Parameters
4354 ----------
4355 target : an iterable
4356 An iterable containing the values to be used for creating the new index.
4357 method : {None, 'pad'/'ffill', 'backfill'/'bfill', 'nearest'}, optional
4358 * default: exact matches only.
4359 * pad / ffill: find the PREVIOUS index value if no exact match.
4360 * backfill / bfill: use NEXT index value if no exact match
4361 * nearest: use the NEAREST index value if no exact match. Tied
4362 distances are broken by preferring the larger index value.
4363 level : int, optional
4364 Level of multiindex.
4365 limit : int, optional
4366 Maximum number of consecutive labels in ``target`` to match for
4367 inexact matches.
4368 tolerance : int, float, or list-like, optional
4369 Maximum distance between original and new labels for inexact
4370 matches. The values of the index at the matching locations must
4371 satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
4373 Tolerance may be a scalar value, which applies the same tolerance
4374 to all values, or list-like, which applies variable tolerance per
4375 element. List-like includes list, tuple, array, Series, and must be
4376 the same size as the index and its dtype must exactly match the
4377 index's type.
4379 Returns
4380 -------
4381 new_index : pd.Index
4382 Resulting index.
4383 indexer : np.ndarray[np.intp] or None
4384 Indices of output values in original index.
4386 Raises
4387 ------
4388 TypeError
4389 If ``method`` passed along with ``level``.
4390 ValueError
4391 If non-unique multi-index
4392 ValueError
4393 If non-unique index and ``method`` or ``limit`` passed.
4395 See Also
4396 --------
4397 Series.reindex : Conform Series to new index with optional filling logic.
4398 DataFrame.reindex : Conform DataFrame to new index with optional filling logic.
4400 Examples
4401 --------
4402 >>> idx = pd.Index(["car", "bike", "train", "tractor"])
4403 >>> idx
4404 Index(['car', 'bike', 'train', 'tractor'], dtype='str')
4405 >>> idx.reindex(["car", "bike"])
4406 (Index(['car', 'bike'], dtype='str'), array([0, 1]))
4407 """
4408 # GH6552: preserve names when reindexing to non-named target
4409 # (i.e. neither Index nor Series).
4410 preserve_names = not hasattr(target, "name")
4412 # GH7774: preserve dtype/tz if target is empty and not an Index.
4413 if is_iterator(target):
4414 target = list(target)
4416 if not isinstance(target, Index) and len(target) == 0:
4417 if level is not None and self._is_multi:
4418 idx = self.levels[level] # type: ignore[attr-defined]
4419 else:
4420 idx = self
4421 target = idx[:0]
4422 else:
4423 target = ensure_index(target)
4425 if level is not None and (
4426 isinstance(self, ABCMultiIndex) or isinstance(target, ABCMultiIndex)
4427 ):
4428 if method is not None:
4429 raise TypeError("Fill method not supported if level passed")
4431 # TODO: tests where passing `keep_order=not self._is_multi`
4432 # makes a difference for non-MultiIndex case
4433 target, indexer, _ = self._join_level(
4434 target, # type: ignore[arg-type]
4435 level,
4436 how="right",
4437 keep_order=not self._is_multi,
4438 )
4440 elif self.equals(target):
4441 indexer = None
4442 elif self._index_as_unique:
4443 indexer = self.get_indexer(
4444 target, method=method, limit=limit, tolerance=tolerance
4445 )
4446 elif self._is_multi:
4447 raise ValueError("cannot handle a non-unique multi-index!")
4448 elif not self.is_unique:
4449 # GH#42568
4450 raise ValueError("cannot reindex on an axis with duplicate labels")
4451 else:
4452 indexer, _ = self.get_indexer_non_unique(target)
4454 target = self._wrap_reindex_result(target, indexer, preserve_names) # type: ignore[arg-type]
4455 return target, indexer
4457 def _wrap_reindex_result(
4458 self, target: Index, indexer: npt.NDArray[np.intp] | None, preserve_names: bool
4459 ) -> Index:
4460 target = self._maybe_preserve_names(target, preserve_names)
4461 return target
4463 def _maybe_preserve_names(self, target: IndexT, preserve_names: bool) -> IndexT:
4464 if preserve_names and target.nlevels == 1 and target.name != self.name:
4465 target = target.copy(deep=False)
4466 target.name = self.name
4467 return target
4469 @final
4470 def _reindex_non_unique(
4471 self, target: Index
4472 ) -> tuple[Index, npt.NDArray[np.intp], npt.NDArray[np.intp] | None]:
4473 """
4474 Create a new index with target's values (move/add/delete values as
4475 necessary) use with non-unique Index and a possibly non-unique target.
4477 Parameters
4478 ----------
4479 target : an iterable
4481 Returns
4482 -------
4483 new_index : pd.Index
4484 Resulting index.
4485 indexer : np.ndarray[np.intp]
4486 Indices of output values in original index.
4487 new_indexer : np.ndarray[np.intp] or None
4489 """
4490 target = ensure_index(target)
4491 if len(target) == 0:
4492 # GH#13691
4493 return self[:0], np.array([], dtype=np.intp), None
4495 indexer, missing = self.get_indexer_non_unique(target)
4496 check = indexer != -1
4497 new_labels: Index | np.ndarray = self.take(indexer[check])
4498 new_indexer = None
4500 if len(missing):
4501 length = np.arange(len(indexer), dtype=np.intp)
4503 missing = ensure_platform_int(missing)
4504 missing_labels = target.take(missing)
4505 missing_indexer = length[~check]
4506 cur_labels = self.take(indexer[check]).values
4507 cur_indexer = length[check]
4509 # Index constructor below will do inference
4510 new_labels = np.empty((len(indexer),), dtype=object)
4511 new_labels[cur_indexer] = cur_labels
4512 new_labels[missing_indexer] = missing_labels
4514 # GH#38906
4515 if not len(self):
4516 new_indexer = np.arange(0, dtype=np.intp)
4518 # a unique indexer
4519 elif target.is_unique:
4520 # see GH5553, make sure we use the right indexer
4521 new_indexer = np.arange(len(indexer), dtype=np.intp)
4522 new_indexer[cur_indexer] = np.arange(len(cur_labels))
4523 new_indexer[missing_indexer] = -1
4525 # we have a non_unique selector, need to use the original
4526 # indexer here
4527 else:
4528 # need to retake to have the same size as the indexer
4529 indexer[~check] = -1
4531 # reset the new indexer to account for the new size
4532 new_indexer = np.arange(len(self.take(indexer)), dtype=np.intp)
4533 new_indexer[~check] = -1
4535 if not isinstance(self, ABCMultiIndex):
4536 new_index = Index(new_labels, name=self.name, copy=False)
4537 else:
4538 new_index = type(self).from_tuples(new_labels, names=self.names)
4539 return new_index, indexer, new_indexer
4541 # --------------------------------------------------------------------
4542 # Join Methods
4544 @overload
4545 def join(
4546 self,
4547 other: Index,
4548 *,
4549 how: JoinHow = ...,
4550 level: Level = ...,
4551 return_indexers: Literal[True],
4552 sort: bool = ...,
4553 ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]: ...
4555 @overload
4556 def join(
4557 self,
4558 other: Index,
4559 *,
4560 how: JoinHow = ...,
4561 level: Level = ...,
4562 return_indexers: Literal[False] = ...,
4563 sort: bool = ...,
4564 ) -> Index: ...
4566 @overload
4567 def join(
4568 self,
4569 other: Index,
4570 *,
4571 how: JoinHow = ...,
4572 level: Level = ...,
4573 return_indexers: bool = ...,
4574 sort: bool = ...,
4575 ) -> (
4576 Index | tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]
4577 ): ...
4579 @final
4580 @_maybe_return_indexers
4581 def join(
4582 self,
4583 other: Index,
4584 *,
4585 how: JoinHow = "left",
4586 level: Level | None = None,
4587 return_indexers: bool = False,
4588 sort: bool = False,
4589 ) -> Index | tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
4590 """
4591 Compute join_index and indexers to conform data structures to the new index.
4593 This method computes the result of a join between two Index objects
4594 using the specified join type, and optionally returns integer indexers
4595 that map each original index to the new joined index.
4597 Parameters
4598 ----------
4599 other : Index
4600 The other index on which join is performed.
4601 how : {'left', 'right', 'inner', 'outer'}
4602 level : int or level name, default None
4603 It is either the integer position or the name of the level.
4604 return_indexers : bool, default False
4605 Whether to return the indexers or not for both the index objects.
4606 sort : bool, default False
4607 Sort the join keys lexicographically in the result Index. If False,
4608 the order of the join keys depends on the join type (how keyword).
4610 Returns
4611 -------
4612 join_index, (left_indexer, right_indexer)
4613 The new index.
4615 See Also
4616 --------
4617 DataFrame.join : Join columns with `other` DataFrame either on index
4618 or on a key.
4619 DataFrame.merge : Merge DataFrame or named Series objects with a
4620 database-style join.
4622 Examples
4623 --------
4624 >>> idx1 = pd.Index([1, 2, 3])
4625 >>> idx2 = pd.Index([4, 5, 6])
4626 >>> idx1.join(idx2, how="outer")
4627 Index([1, 2, 3, 4, 5, 6], dtype='int64')
4628 >>> idx1.join(other=idx2, how="outer", return_indexers=True)
4629 (Index([1, 2, 3, 4, 5, 6], dtype='int64'),
4630 array([ 0, 1, 2, -1, -1, -1]), array([-1, -1, -1, 0, 1, 2]))
4631 """
4632 if not isinstance(other, Index):
4633 warnings.warn(
4634 f"Passing {type(other).__name__} to {type(self).__name__}.join "
4635 "is deprecated and will raise in a future version. "
4636 "Pass an Index instead.",
4637 Pandas4Warning,
4638 stacklevel=find_stack_level(),
4639 )
4641 other = ensure_index(other)
4642 sort = sort or how == "outer"
4644 if isinstance(self, ABCDatetimeIndex) and isinstance(other, ABCDatetimeIndex):
4645 if (self.tz is None) ^ (other.tz is None):
4646 # Raise instead of casting to object below.
4647 raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex")
4649 if not self._is_multi and not other._is_multi:
4650 # We have specific handling for MultiIndex below
4651 pself, pother = self._maybe_downcast_for_indexing(other)
4652 if pself is not self or pother is not other:
4653 return pself.join(
4654 pother, how=how, level=level, return_indexers=True, sort=sort
4655 )
4657 # try to figure out the join level
4658 # GH3662
4659 if level is None and (self._is_multi or other._is_multi):
4660 # have the same levels/names so a simple join
4661 if self.names == other.names:
4662 pass
4663 else:
4664 return self._join_multi(other, how=how)
4666 # join on the level
4667 if level is not None and (self._is_multi or other._is_multi):
4668 return self._join_level(other, level, how=how)
4670 if len(self) == 0 or len(other) == 0:
4671 try:
4672 return self._join_empty(other, how, sort)
4673 except TypeError:
4674 # object dtype; non-comparable objects
4675 pass
4677 if self.dtype != other.dtype:
4678 dtype = self._find_common_type_compat(other)
4679 this = self.astype(dtype, copy=False)
4680 that = other.astype(dtype, copy=False)
4681 join_index, lidx, ridx = this.join(that, how=how, return_indexers=True)
4682 # left/right joins keep the dtype of the side the values come from
4683 if how == "left":
4684 join_index = self if lidx is None else self.take(lidx)
4685 elif how == "right":
4686 join_index = other if ridx is None else other.take(ridx)
4687 return join_index, lidx, ridx
4688 elif (
4689 isinstance(self, ABCCategoricalIndex)
4690 and isinstance(other, ABCCategoricalIndex)
4691 and not self.ordered
4692 and not self.categories.equals(other.categories)
4693 ):
4694 # dtypes are "equal" but categories are in different order
4695 other = Index(other._values.reorder_categories(self.categories), copy=False)
4697 _validate_join_method(how)
4699 if (
4700 self.is_monotonic_increasing
4701 and other.is_monotonic_increasing
4702 and (self.is_unique or other.is_unique)
4703 ):
4704 if self._can_use_libjoin and other._can_use_libjoin:
4705 if self.equals(other):
4706 ret_index = other if how == "right" else self
4707 return ret_index, None, None
4708 try:
4709 return self._join_monotonic(other, how=how)
4710 except TypeError:
4711 # object dtype; non-comparable objects
4712 pass
4713 elif isinstance(self, ABCRangeIndex):
4714 # RangeIndex._join_monotonic doesn't use libjoin;
4715 # it uses pure-Python range operations.
4716 try:
4717 return self._join_monotonic(other, how=how)
4718 except (TypeError, NotImplementedError):
4719 pass
4721 if (
4722 self.equals(other)
4723 and (self.is_unique or other.is_unique)
4724 and (not sort or self.is_monotonic_increasing)
4725 ):
4726 ret_index = other if how == "right" else self
4727 return ret_index, None, None
4729 if not self.is_unique or not other.is_unique:
4730 return self._join_non_unique(other, how=how, sort=sort)
4732 return self._join_via_get_indexer(other, how, sort)
4734 def _join_empty(
4735 self, other: Index, how: JoinHow, sort: bool
4736 ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
4737 assert len(self) == 0 or len(other) == 0
4738 _validate_join_method(how)
4740 lidx: np.ndarray | None
4741 ridx: np.ndarray | None
4743 if len(other):
4744 how = cast("JoinHow", {"left": "right", "right": "left"}.get(how, how))
4745 join_index, ridx, lidx = other._join_empty(self, how, sort)
4746 elif how in ["left", "outer"]:
4747 if sort and not self.is_monotonic_increasing:
4748 lidx = self.argsort()
4749 join_index = self.take(lidx)
4750 else:
4751 lidx = None
4752 join_index = self._view()
4753 ridx = np.broadcast_to(np.intp(-1), len(join_index))
4754 else:
4755 join_index = other._view()
4756 lidx = np.array([], dtype=np.intp)
4757 ridx = None
4758 return join_index, lidx, ridx
4760 @final
4761 def _join_via_get_indexer(
4762 self, other: Index, how: JoinHow, sort: bool
4763 ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
4764 # Fallback if we do not have any fastpaths available based on
4765 # uniqueness/monotonicity
4767 # Note: at this point we have checked matching dtypes
4768 lindexer: npt.NDArray[np.intp] | None
4769 rindexer: npt.NDArray[np.intp] | None
4771 if how == "left":
4772 if sort:
4773 join_index, lindexer = self.sort_values(return_indexer=True)
4774 rindexer = other.get_indexer_for(join_index)
4775 return join_index, lindexer, rindexer
4776 else:
4777 join_index = self
4778 elif how == "right":
4779 if sort:
4780 join_index, rindexer = other.sort_values(return_indexer=True)
4781 lindexer = self.get_indexer_for(join_index)
4782 return join_index, lindexer, rindexer
4783 else:
4784 join_index = other
4785 elif how == "inner":
4786 join_index = self.intersection(other, sort=sort)
4787 elif how == "outer":
4788 try:
4789 join_index = self.union(other, sort=sort)
4790 except TypeError:
4791 join_index = self.union(other)
4792 try:
4793 join_index = _maybe_try_sort(join_index, sort) # type: ignore[assignment]
4794 except TypeError:
4795 pass
4797 names = other.names if how == "right" else self.names
4798 if join_index.names != names: # pyright: ignore[reportAttributeAccessIssue]
4799 join_index = join_index.set_names(names) # pyright: ignore[reportAttributeAccessIssue]
4801 if join_index is self:
4802 lindexer = None
4803 else:
4804 lindexer = self.get_indexer_for(join_index) # pyright: ignore[reportArgumentType]
4805 if join_index is other:
4806 rindexer = None
4807 else:
4808 rindexer = other.get_indexer_for(join_index) # pyright: ignore[reportArgumentType]
4809 return join_index, lindexer, rindexer # pyright: ignore[reportReturnType]
4811 @final
4812 def _join_multi(
4813 self, other: Index, how: JoinHow
4814 ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
4815 from pandas.core.indexes.multi import MultiIndex
4816 from pandas.core.reshape.merge import restore_dropped_levels_multijoin
4818 # figure out join names
4819 self_names_list = list(self.names)
4820 other_names_list = list(other.names)
4821 self_names_order = self_names_list.index
4822 other_names_order = other_names_list.index
4823 self_names = set(self_names_list)
4824 other_names = set(other_names_list)
4825 overlap = self_names & other_names
4827 # need at least 1 in common
4828 if not overlap:
4829 raise ValueError("cannot join with no overlapping index names")
4831 if isinstance(self, MultiIndex) and isinstance(other, MultiIndex):
4832 # Drop the non-matching levels from left and right respectively
4833 ldrop_names = sorted(self_names - overlap, key=self_names_order)
4834 rdrop_names = sorted(other_names - overlap, key=other_names_order)
4836 # if only the order differs
4837 if not len(ldrop_names + rdrop_names):
4838 self_jnlevels = self
4839 other_jnlevels = other.reorder_levels(self.names)
4840 else:
4841 self_jnlevels = self.droplevel(ldrop_names) # type: ignore[assignment]
4842 other_jnlevels = other.droplevel(rdrop_names) # type: ignore[assignment]
4844 # Join left and right
4845 # Join on same leveled multi-index frames is supported
4846 join_idx, lidx, ridx = self_jnlevels.join(
4847 other_jnlevels, how=how, return_indexers=True
4848 )
4850 # Restore the dropped levels
4851 # Returned index level order is
4852 # common levels, ldrop_names, rdrop_names
4853 dropped_names = ldrop_names + rdrop_names
4855 # error: Argument 5/6 to "restore_dropped_levels_multijoin" has
4856 # incompatible type "Optional[ndarray[Any, dtype[signedinteger[Any
4857 # ]]]]"; expected "ndarray[Any, dtype[signedinteger[Any]]]"
4858 levels, codes, names = restore_dropped_levels_multijoin(
4859 self,
4860 other,
4861 dropped_names,
4862 join_idx,
4863 lidx, # type: ignore[arg-type]
4864 ridx, # type: ignore[arg-type]
4865 )
4867 # Re-create the multi-index
4868 multi_join_idx = MultiIndex(
4869 levels=levels, codes=codes, names=names, verify_integrity=False
4870 )
4872 multi_join_idx = multi_join_idx.remove_unused_levels()
4874 # maintain the order of the index levels
4875 if how == "right":
4876 level_order = other_names_list + ldrop_names
4877 else:
4878 level_order = self_names_list + rdrop_names
4879 multi_join_idx = multi_join_idx.reorder_levels(level_order)
4881 return multi_join_idx, lidx, ridx
4883 jl = next(iter(overlap))
4885 # Case where only one index is multi
4886 # make the indices into mi's that match
4887 flip_order = False
4888 if isinstance(self, MultiIndex):
4889 self, other = other, self
4890 flip_order = True
4891 # flip if join method is right or left
4892 flip: dict[JoinHow, JoinHow] = {"right": "left", "left": "right"}
4893 how = flip.get(how, how)
4895 level = other.names.index(jl)
4896 result = self._join_level(other, level, how=how)
4898 if flip_order:
4899 return result[0], result[2], result[1]
4900 return result
4902 @final
4903 def _join_non_unique(
4904 self, other: Index, how: JoinHow = "left", sort: bool = False
4905 ) -> tuple[Index, npt.NDArray[np.intp], npt.NDArray[np.intp]]:
4906 from pandas.core.reshape.merge import get_join_indexers_non_unique
4908 # We only get here if dtypes match
4909 assert self.dtype == other.dtype
4911 left_idx, right_idx = get_join_indexers_non_unique(
4912 self._values, other._values, how=how, sort=sort
4913 )
4915 if how == "right":
4916 join_index = other.take(right_idx)
4917 else:
4918 join_index = self.take(left_idx)
4920 if how == "outer":
4921 mask = left_idx == -1
4922 if mask.any():
4923 right = other.take(right_idx)
4924 join_index = join_index.putmask(mask, right)
4926 if isinstance(join_index, ABCMultiIndex) and how == "outer":
4927 # test_join_index_levels
4928 join_index = join_index._sort_levels_monotonic()
4929 return join_index, left_idx, right_idx
4931 @final
4932 def _join_level(
4933 self, other: Index, level: Level, how: JoinHow = "left", keep_order: bool = True
4934 ) -> tuple[MultiIndex, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
4935 """
4936 The join method *only* affects the level of the resulting
4937 MultiIndex. Otherwise it just exactly aligns the Index data to the
4938 labels of the level in the MultiIndex.
4940 If ```keep_order == True```, the order of the data indexed by the
4941 MultiIndex will not be changed; otherwise, it will tie out
4942 with `other`.
4943 """
4944 from pandas.core.indexes.multi import MultiIndex
4946 def _get_leaf_sorter(labels: list[np.ndarray]) -> npt.NDArray[np.intp]:
4947 """
4948 Returns sorter for the inner most level while preserving the
4949 order of higher levels.
4951 Parameters
4952 ----------
4953 labels : list[np.ndarray]
4954 Each ndarray has signed integer dtype, not necessarily identical.
4956 Returns
4957 -------
4958 np.ndarray[np.intp]
4959 """
4960 if labels[0].size == 0:
4961 return np.empty(0, dtype=np.intp)
4963 if len(labels) == 1:
4964 return get_group_index_sorter(ensure_platform_int(labels[0]))
4966 # find indexers of beginning of each set of
4967 # same-key labels w.r.t all but last level
4968 tic = labels[0][:-1] != labels[0][1:]
4969 for lab in labels[1:-1]:
4970 tic |= lab[:-1] != lab[1:]
4972 starts = np.hstack(([True], tic, [True])).nonzero()[0]
4973 lab = ensure_int64(labels[-1])
4974 return lib.get_level_sorter(lab, ensure_platform_int(starts))
4976 if isinstance(self, MultiIndex) and isinstance(other, MultiIndex):
4977 raise TypeError("Join on level between two MultiIndex objects is ambiguous")
4979 left, right = self, other
4981 flip_order = not isinstance(self, MultiIndex)
4982 if flip_order:
4983 left, right = right, left
4984 flip: dict[JoinHow, JoinHow] = {"right": "left", "left": "right"}
4985 how = flip.get(how, how)
4987 assert isinstance(left, MultiIndex)
4989 level = left._get_level_number(level)
4990 old_level = left.levels[level]
4992 if not right.is_unique:
4993 raise NotImplementedError(
4994 "Index._join_level on non-unique index is not implemented"
4995 )
4997 new_level, left_lev_indexer, right_lev_indexer = old_level.join(
4998 right, how=how, return_indexers=True
4999 )
5001 if left_lev_indexer is None:
5002 if keep_order or len(left) == 0:
5003 left_indexer = None
5004 join_index = left
5005 else: # sort the leaves
5006 left_indexer = _get_leaf_sorter(left.codes[: level + 1])
5007 join_index = left[left_indexer]
5009 else:
5010 left_lev_indexer = ensure_platform_int(left_lev_indexer)
5011 rev_indexer = lib.get_reverse_indexer(left_lev_indexer, len(old_level))
5012 old_codes = left.codes[level]
5014 taker = old_codes[old_codes != -1]
5015 new_lev_codes = rev_indexer.take(taker)
5017 new_codes = list(left.codes)
5018 new_codes[level] = new_lev_codes
5020 new_levels = list(left.levels)
5021 new_levels[level] = new_level
5023 if keep_order: # just drop missing values. o.w. keep order
5024 left_indexer = np.arange(len(left), dtype=np.intp)
5025 left_indexer = cast("np.ndarray", left_indexer)
5026 mask = new_lev_codes != -1
5027 if not mask.all():
5028 new_codes = [lab[mask] for lab in new_codes]
5029 left_indexer = left_indexer[mask]
5031 elif level == 0: # outer most level, take the fast route
5032 max_new_lev = 0 if len(new_lev_codes) == 0 else new_lev_codes.max()
5033 ngroups = 1 + max_new_lev
5034 left_indexer, counts = libalgos.groupsort_indexer(
5035 new_lev_codes, ngroups
5036 )
5038 # missing values are placed first; drop them!
5039 left_indexer = left_indexer[counts[0] :]
5040 new_codes = [lab[left_indexer] for lab in new_codes]
5042 else: # sort the leaves
5043 mask = new_lev_codes != -1
5044 mask_all = mask.all()
5045 if not mask_all:
5046 new_codes = [lab[mask] for lab in new_codes]
5048 left_indexer = _get_leaf_sorter(new_codes[: level + 1])
5049 new_codes = [lab[left_indexer] for lab in new_codes]
5051 # left_indexers are w.r.t masked frame.
5052 # reverse to original frame!
5053 if not mask_all:
5054 left_indexer = mask.nonzero()[0][left_indexer]
5056 join_index = MultiIndex(
5057 levels=new_levels,
5058 codes=new_codes,
5059 names=left.names,
5060 verify_integrity=False,
5061 )
5063 if right_lev_indexer is not None:
5064 right_indexer = right_lev_indexer.take(join_index.codes[level])
5065 else:
5066 right_indexer = join_index.codes[level]
5068 if flip_order:
5069 left_indexer, right_indexer = right_indexer, left_indexer
5071 left_indexer = (
5072 None if left_indexer is None else ensure_platform_int(left_indexer)
5073 )
5074 right_indexer = (
5075 None if right_indexer is None else ensure_platform_int(right_indexer)
5076 )
5077 return join_index, left_indexer, right_indexer
5079 def _join_monotonic(
5080 self, other: Index, how: JoinHow = "left"
5081 ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
5082 # We only get here with (caller is responsible for ensuring):
5083 # 1) matching dtypes
5084 # 2) both monotonic increasing
5085 # 3) other.is_unique or self.is_unique
5086 assert other.dtype == self.dtype
5087 assert self._can_use_libjoin and other._can_use_libjoin
5089 ridx: npt.NDArray[np.intp] | None
5090 lidx: npt.NDArray[np.intp] | None
5092 if how == "left":
5093 if other.is_unique:
5094 # We can perform much better than the general case
5095 join_index = self
5096 lidx = None
5097 ridx = self._left_indexer_unique(other) # pyright: ignore[reportArgumentType]
5098 else:
5099 join_array, lidx, ridx = self._left_indexer(other) # pyright: ignore[reportArgumentType]
5100 join_index, lidx, ridx = self._wrap_join_result(
5101 join_array,
5102 other, # pyright: ignore[reportArgumentType]
5103 lidx,
5104 ridx,
5105 how, # pyright: ignore[reportArgumentType]
5106 )
5107 elif how == "right":
5108 if self.is_unique:
5109 # We can perform much better than the general case
5110 join_index = other
5111 lidx = other._left_indexer_unique(self)
5112 ridx = None
5113 else:
5114 join_array, ridx, lidx = other._left_indexer(self)
5115 join_index, lidx, ridx = self._wrap_join_result(
5116 join_array,
5117 other, # pyright: ignore[reportArgumentType]
5118 lidx,
5119 ridx,
5120 how, # pyright: ignore[reportArgumentType]
5121 )
5122 elif how == "inner":
5123 join_array, lidx, ridx = self._inner_indexer(other) # pyright: ignore[reportArgumentType]
5124 join_index, lidx, ridx = self._wrap_join_result(
5125 join_array,
5126 other, # pyright: ignore[reportArgumentType]
5127 lidx,
5128 ridx,
5129 how, # pyright: ignore[reportArgumentType]
5130 )
5131 elif how == "outer":
5132 join_array, lidx, ridx = self._outer_indexer(other) # pyright: ignore[reportArgumentType]
5133 join_index, lidx, ridx = self._wrap_join_result(
5134 join_array,
5135 other, # pyright: ignore[reportArgumentType]
5136 lidx,
5137 ridx,
5138 how, # pyright: ignore[reportArgumentType]
5139 )
5141 lidx = None if lidx is None else ensure_platform_int(lidx)
5142 ridx = None if ridx is None else ensure_platform_int(ridx)
5143 return join_index, lidx, ridx
5145 def _wrap_join_result(
5146 self,
5147 joined: ArrayLike,
5148 other: Self,
5149 lidx: npt.NDArray[np.intp] | None,
5150 ridx: npt.NDArray[np.intp] | None,
5151 how: JoinHow,
5152 ) -> tuple[Self, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
5153 assert other.dtype == self.dtype
5155 if lidx is not None and lib.is_range_indexer(lidx, len(self)):
5156 lidx = None
5157 if ridx is not None and lib.is_range_indexer(ridx, len(other)):
5158 ridx = None
5160 # return self or other if possible to maintain cached attributes
5161 if lidx is None:
5162 join_index = self
5163 elif ridx is None:
5164 join_index = other
5165 else:
5166 join_index = self._constructor._with_infer(
5167 joined, dtype=self.dtype, copy=False
5168 )
5170 names = other.names if how == "right" else self.names
5171 if join_index.names != names:
5172 join_index = join_index.set_names(names)
5174 return join_index, lidx, ridx
5176 @final
5177 @cache_readonly
5178 def _can_use_libjoin(self) -> bool:
5179 """
5180 Whether we can use the fastpaths implemented in _libs.join.
5182 This is driven by whether (in monotonic increasing cases that are
5183 guaranteed not to have NAs) we can convert to an np.ndarray without
5184 making a copy. If we cannot, this negates the performance benefit
5185 of using libjoin.
5186 """
5187 if not self.is_monotonic_increasing:
5188 # The libjoin functions all assume monotonicity.
5189 return False
5191 if type(self) is Index:
5192 # excludes EAs, but include masks, we get here with monotonic
5193 # values only, meaning no NA
5194 return (
5195 isinstance(self.dtype, np.dtype)
5196 or isinstance(self._values, (ArrowExtensionArray, BaseMaskedArray))
5197 or (
5198 isinstance(self.dtype, StringDtype)
5199 and self.dtype.storage == "python"
5200 )
5201 )
5202 # Exclude index types where the conversion to numpy converts to object dtype,
5203 # which negates the performance benefit of libjoin.
5204 # Also exclude RangeIndex which allocates memory in _get_join_target,
5205 # negating the performance benefit of the fastpath.
5206 return not isinstance(self, (ABCIntervalIndex, ABCMultiIndex, ABCRangeIndex))
5208 # --------------------------------------------------------------------
5209 # Uncategorized Methods
5211 @property
5212 def values(self) -> ArrayLike:
5213 """
5214 Return an array representing the data in the Index.
5216 .. warning::
5218 We recommend using :attr:`Index.array` or
5219 :meth:`Index.to_numpy`, depending on whether you need
5220 a reference to the underlying data or a NumPy array.
5221 The return type of ``.values`` depends on the dtype: it is a
5222 :class:`numpy.ndarray` for some dtypes (for example ``int64``
5223 or ``float64``) and an
5224 :class:`~pandas.api.extensions.ExtensionArray` for others (for
5225 example ``interval``, ``category``, or nullable ``Int64``),
5226 which makes it harder to write code that works across dtypes.
5227 :attr:`Index.array` always returns the underlying array as an
5228 :class:`~pandas.api.extensions.ExtensionArray`, and
5229 :meth:`Index.to_numpy` always returns a :class:`numpy.ndarray`
5230 and accepts ``dtype``, ``copy``, and ``na_value`` arguments to
5231 control the conversion.
5233 .. versionchanged:: 3.0.0
5235 The returned array is read-only.
5237 Returns
5238 -------
5239 array: numpy.ndarray or ExtensionArray
5241 See Also
5242 --------
5243 Index.array : Reference to the underlying data.
5244 Index.to_numpy : A NumPy array representing the underlying data.
5246 Examples
5247 --------
5248 For an :class:`Index` backed by a NumPy dtype such as ``int64``,
5249 ``.values`` returns a :class:`numpy.ndarray`:
5251 >>> idx = pd.Index([1, 2, 3])
5252 >>> idx
5253 Index([1, 2, 3], dtype='int64')
5254 >>> idx.values
5255 array([1, 2, 3])
5257 For an :class:`Index` backed by an extension dtype such as
5258 ``interval``, ``.values`` returns an
5259 :class:`~pandas.api.extensions.ExtensionArray` instead, while
5260 :meth:`Index.to_numpy` always returns a :class:`numpy.ndarray`:
5262 >>> idx = pd.interval_range(start=0, end=5)
5263 >>> idx.values
5264 <IntervalArray>
5265 [(0, 1], (1, 2], (2, 3], (3, 4], (4, 5]]
5266 Length: 5, dtype: interval[int64, right]
5267 >>> idx.to_numpy()
5268 array([Interval(0, 1, closed='right'), Interval(1, 2, closed='right'),
5269 Interval(2, 3, closed='right'), Interval(3, 4, closed='right'),
5270 Interval(4, 5, closed='right')], dtype=object)
5271 """
5272 data = self._data
5273 if isinstance(data, np.ndarray):
5274 data = data.view()
5275 data.flags.writeable = False
5276 else:
5277 # GH#38547 don't allow mutating the Index through its .values;
5278 # an Index is immutable, and in-place mutation here would
5279 # silently corrupt it (e.g. leave a stale lookup engine).
5280 data = data.view()
5281 data._readonly = True
5282 return data
5284 @cache_readonly
5285 def array(self) -> ExtensionArray:
5286 """
5287 The ExtensionArray of the data backing this Index.
5289 This property provides direct access to the underlying array data of
5290 an Index without requiring conversion to a NumPy array. It
5291 returns an ExtensionArray, which is the native storage format for
5292 pandas extension dtypes.
5294 .. versionchanged:: 3.1.0
5296 The returned array is read-only, since an Index is immutable.
5298 Returns
5299 -------
5300 ExtensionArray
5301 An ExtensionArray of the values stored within. For extension
5302 types, this is the actual array. For NumPy native types, this
5303 is a thin (no copy) wrapper around :class:`numpy.ndarray`.
5305 ``.array`` differs from ``.values``, which may require converting
5306 the data to a different form.
5308 See Also
5309 --------
5310 Index.to_numpy : Similar method that always returns a NumPy array.
5311 Series.to_numpy : Similar method that always returns a NumPy array.
5313 Notes
5314 -----
5315 This table lays out the different array types for each extension
5316 dtype within pandas.
5318 ================== =============================
5319 dtype array type
5320 ================== =============================
5321 category Categorical
5322 period PeriodArray
5323 interval IntervalArray
5324 IntegerNA IntegerArray
5325 string StringArray
5326 boolean BooleanArray
5327 datetime64[ns, tz] DatetimeArray
5328 ================== =============================
5330 For any 3rd-party extension types, the array type will be an
5331 ExtensionArray.
5333 For all remaining dtypes ``.array`` will be a
5334 :class:`arrays.NumpyExtensionArray` wrapping the actual ndarray
5335 stored within. If you absolutely need a NumPy array (possibly with
5336 copying / coercing data), then use :meth:`Series.to_numpy` instead.
5338 Examples
5339 --------
5340 For regular NumPy types like int, and float, a NumpyExtensionArray
5341 is returned.
5343 >>> pd.Index([1, 2, 3]).array
5344 <NumpyExtensionArray>
5345 [1, 2, 3]
5346 Length: 3, dtype: int64
5348 For extension types, like Categorical, the actual ExtensionArray
5349 is returned
5351 >>> idx = pd.Index(pd.Categorical(["a", "b", "a"]))
5352 >>> idx.array
5353 ['a', 'b', 'a']
5354 Categories (2, str): ['a', 'b']
5355 """
5356 array = self._data
5357 if isinstance(array, np.ndarray):
5358 from pandas.core.arrays.numpy_ import NumpyExtensionArray
5360 array = NumpyExtensionArray(array)
5361 # GH#38547 an Index is immutable; return a read-only view so the
5362 # Index cannot be silently corrupted by mutating the returned array
5363 # (which would also leave a stale lookup engine behind).
5364 array = array.view()
5365 array._readonly = True
5366 return array
5368 @property
5369 def _values(self) -> ExtensionArray | np.ndarray:
5370 """
5371 The best array representation.
5373 This is an ndarray or ExtensionArray.
5375 ``_values`` are consistent between ``Series`` and ``Index``.
5377 It may differ from the public '.values' method, though this
5378 difference is deprecated and will be removed in a future version.
5380 index | values | _values |
5381 ----------------- | --------------- | ------------- |
5382 Index | ndarray | ndarray |
5383 CategoricalIndex | Categorical | Categorical |
5384 DatetimeIndex | ndarray[M8ns] | DatetimeArray |
5385 DatetimeIndex[tz] | ndarray[M8ns]* | DatetimeArray |
5386 PeriodIndex | ndarray[object]*| PeriodArray |
5387 IntervalIndex | IntervalArray | IntervalArray |
5389 * Will return the ExtensionArray in a future version (deprecated).
5391 See Also
5392 --------
5393 values : Values
5394 """
5395 return self._data
5397 def _get_engine_target(self) -> ArrayLike:
5398 """
5399 Get the ndarray or ExtensionArray that we can pass to the IndexEngine
5400 constructor.
5401 """
5402 vals = self._values
5403 if isinstance(vals, StringArray):
5404 # GH#45652 much more performant than ExtensionEngine
5405 return vals._ndarray
5406 if isinstance(vals, ArrowExtensionArray) and self.dtype.kind in "Mm":
5407 import pyarrow as pa
5409 pa_type = vals._pa_array.type
5410 if pa.types.is_timestamp(pa_type):
5411 vals = vals._to_datetimearray()
5412 return vals._ndarray.view("i8")
5413 elif pa.types.is_duration(pa_type):
5414 vals = vals._to_timedeltaarray()
5415 return vals._ndarray.view("i8")
5416 if (
5417 type(self) is Index
5418 and isinstance(vals, ExtensionArray)
5419 and not isinstance(vals, BaseMaskedArray)
5420 and not (
5421 isinstance(vals, ArrowExtensionArray)
5422 and is_numeric_dtype(self.dtype)
5423 # Exclude decimal
5424 and self.dtype.kind != "O"
5425 )
5426 ):
5427 # TODO(ExtensionIndex): remove special-case, just use self._values
5428 return vals.astype(object)
5429 return vals
5431 @final
5432 def _get_join_target(self) -> np.ndarray:
5433 """
5434 Get the ndarray or ExtensionArray that we can pass to the join
5435 functions.
5436 """
5437 if isinstance(self._values, BaseMaskedArray):
5438 # This is only used if our array is monotonic, so no NAs present
5439 return self._values._data
5440 elif (
5441 isinstance(self._values, ArrowExtensionArray)
5442 and self.dtype.kind not in "mM"
5443 ):
5444 # This is only used if our array is monotonic, so no missing values
5445 # present
5446 # "mM" cases will go through _get_engine_target and cast to i8
5447 return self._values.to_numpy()
5449 target = self._get_engine_target()
5450 if not isinstance(target, np.ndarray):
5451 raise ValueError("_can_use_libjoin should return False.")
5452 return target
5454 def _from_join_target(self, result: np.ndarray) -> ArrayLike:
5455 """
5456 Cast the ndarray returned from one of the libjoin.foo_indexer functions
5457 back to type(self._data).
5458 """
5459 if isinstance(self.values, BaseMaskedArray):
5460 return type(self.values)(result, np.zeros(result.shape, dtype=np.bool_))
5461 elif isinstance(self.values, (ArrowExtensionArray, StringArray)):
5462 return type(self.values)._from_sequence(result, dtype=self.dtype)
5463 return result
5465 def memory_usage(self, deep: bool = False) -> int:
5466 """
5467 Memory usage of the values.
5469 Returns the number of bytes consumed by the Index. When ``deep=True``,
5470 the memory consumption of underlying objects referencing this Index
5471 (e.g., the characters of object-dtype values) is included.
5473 Parameters
5474 ----------
5475 deep : bool, default False
5476 Introspect the data deeply, interrogate
5477 `object` dtypes for system-level memory consumption.
5479 Returns
5480 -------
5481 bytes used
5482 Returns memory usage of the values in the Index in bytes.
5484 See Also
5485 --------
5486 numpy.ndarray.nbytes : Total bytes consumed by the elements of the
5487 array.
5489 Notes
5490 -----
5491 Memory usage does not include memory consumed by elements that
5492 are not components of the array if deep=False or if used on PyPy
5494 Examples
5495 --------
5496 >>> idx = pd.Index([1, 2, 3])
5497 >>> idx.memory_usage()
5498 24
5499 """
5500 result = self._memory_usage(deep=deep)
5502 # include our engine hashtable, only if it's already cached
5503 if "_engine" in self._cache:
5504 result += self._engine.sizeof(deep=deep)
5505 return result
5507 @final
5508 def where(self, cond: np.ndarray, other: object = None) -> Index:
5509 """
5510 Replace values where the condition is False.
5512 The replacement is taken from other.
5514 Parameters
5515 ----------
5516 cond : bool array-like with the same length as self
5517 Condition to select the values on.
5518 other : scalar, or array-like, default None
5519 Replacement if the condition is False.
5521 Returns
5522 -------
5523 pandas.Index
5524 A copy of self with values replaced from other
5525 where the condition is False.
5527 See Also
5528 --------
5529 Series.where : Same method for Series.
5530 DataFrame.where : Same method for DataFrame.
5532 Examples
5533 --------
5534 >>> idx = pd.Index(["car", "bike", "train", "tractor"])
5535 >>> idx
5536 Index(['car', 'bike', 'train', 'tractor'], dtype='str')
5537 >>> idx.where(idx.isin(["car", "train"]), "other")
5538 Index(['car', 'other', 'train', 'other'], dtype='str')
5539 """
5540 if isinstance(self, ABCMultiIndex):
5541 raise NotImplementedError(
5542 ".where is not supported for MultiIndex operations"
5543 )
5544 cond = np.asarray(cond, dtype=bool)
5546 from pandas import Series
5548 # Pass pandas objects (not their underlying arrays) so that CoW
5549 # references are tracked in the no-copy cases (GH#65265).
5550 ser = Series(self, copy=False)
5551 try:
5552 result_ser = ser.where(cond, other)
5553 except (TypeError, ValueError):
5554 # e.g. Categorical with a value not in categories;
5555 # find a common dtype and retry
5556 dtype = self._find_common_type_compat(other)
5557 if dtype == self.dtype:
5558 raise
5559 return self.astype(dtype).where(cond, other)
5560 return Index(result_ser, dtype=result_ser.dtype, name=self.name, copy=False)
5562 # construction helpers
5563 @final
5564 @classmethod
5565 def _raise_scalar_data_error(cls, data: object) -> NoReturn:
5566 # We return the TypeError so that we can raise it from the constructor
5567 # in order to keep mypy happy
5568 raise TypeError(
5569 f"{cls.__name__}(...) must be called with a collection of some "
5570 f"kind, {repr(data) if not isinstance(data, np.generic) else str(data)} "
5571 "was passed"
5572 )
5574 @classmethod
5575 def _maybe_copy_array_input(
5576 cls, data: object, copy: bool | None, dtype: Dtype | None
5577 ) -> tuple[Any, bool]:
5578 """
5579 Ensure that the input data is copied if necessary.
5580 GH#63388
5581 """
5582 if isinstance(data, (ExtensionArray, np.ndarray)):
5583 if copy is not False:
5584 if dtype is None or astype_is_view(data.dtype, pandas_dtype(dtype)):
5585 data = data.copy()
5586 copy = False
5587 return data, bool(copy)
5589 def _validate_fill_value(self, value: object) -> object:
5590 """
5591 Check if the value can be inserted into our array without casting,
5592 and convert it to an appropriate native type if necessary.
5594 Raises
5595 ------
5596 TypeError
5597 If the value cannot be inserted into an array of this dtype.
5598 """
5599 dtype = self.dtype
5600 if isinstance(dtype, np.dtype) and dtype.kind not in "mM":
5601 if isinstance(value, tuple) and dtype != object:
5602 # GH#54385
5603 raise TypeError
5604 try:
5605 return np_can_hold_element(dtype, value)
5606 except LossySetitemError as err:
5607 # re-raise as TypeError for consistency
5608 raise TypeError from err
5609 elif not can_hold_element(self._values, value):
5610 raise TypeError
5611 return value
5613 @cache_readonly
5614 def _is_memory_usage_qualified(self) -> bool:
5615 """
5616 Return a boolean if we need a qualified .info display.
5617 """
5618 return is_object_dtype(self.dtype) or (
5619 is_string_dtype(self.dtype) and self.dtype.storage == "python" # type: ignore[union-attr]
5620 )
5622 def __contains__(self, key: Any) -> bool:
5623 """
5624 Return a boolean indicating whether the provided key is in the index.
5626 Parameters
5627 ----------
5628 key : label
5629 The key to check if it is present in the index.
5631 Returns
5632 -------
5633 bool
5634 Whether the key search is in the index.
5636 Raises
5637 ------
5638 TypeError
5639 If the key is not hashable.
5641 See Also
5642 --------
5643 Index.isin : Returns an ndarray of boolean dtype indicating whether the
5644 list-like key is in the index.
5646 Examples
5647 --------
5648 >>> idx = pd.Index([1, 2, 3, 4])
5649 >>> idx
5650 Index([1, 2, 3, 4], dtype='int64')
5652 >>> 2 in idx
5653 True
5654 >>> 6 in idx
5655 False
5656 """
5657 hash(key)
5658 try:
5659 return key in self._engine
5660 except (OverflowError, TypeError, ValueError):
5661 return False
5663 # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
5664 # Incompatible types in assignment (expression has type "None", base class
5665 # "object" defined the type as "Callable[[object], int]")
5666 __hash__: ClassVar[None] # type: ignore[assignment]
5668 @final
5669 def __setitem__(self, key: object, value: object) -> None:
5670 raise TypeError("Index does not support mutable operations")
5672 @overload
5673 def __getitem__(self, key: SupportsIndex) -> Any: ...
5675 @overload
5676 def __getitem__(
5677 self, key: slice | np.ndarray | list[int] | list[bool] | Series
5678 ) -> Index: ...
5680 def __getitem__(self, key: Any) -> Any:
5681 """
5682 Override numpy.ndarray's __getitem__ method to work as desired.
5684 This function adds lists and Series as valid boolean indexers
5685 (ndarrays only supports ndarray with dtype=bool).
5687 If resulting ndim != 1, plain ndarray is returned instead of
5688 corresponding `Index` subclass.
5690 """
5691 getitem = self._data.__getitem__
5693 key = lib.item_from_zerodim(key)
5694 if is_integer(key) or is_float(key):
5695 # GH#44051 exclude bool, which would return a 2d ndarray
5696 key = com.cast_scalar_indexer(key)
5697 return maybe_unbox_numpy_scalar(getitem(key), object_with_dtype=self)
5699 if isinstance(key, slice):
5700 # This case is separated from the conditional above to avoid
5701 # pessimization com.is_bool_indexer and ndim checks.
5702 return self._getitem_slice(key)
5704 if com.is_bool_indexer(key):
5705 # if we have list[bools, length=1e5] then doing this check+convert
5706 # takes 166 µs + 2.1 ms and cuts the ndarray.__getitem__
5707 # time below from 3.8 ms to 496 µs
5708 # if we already have ndarray[bool], the overhead is 1.4 µs or .25%
5709 if isinstance(getattr(key, "dtype", None), ExtensionDtype):
5710 key = key.to_numpy(dtype=bool, na_value=False)
5711 else:
5712 key = np.asarray(key, dtype=bool)
5714 if not isinstance(self.dtype, ExtensionDtype):
5715 if len(key) == 0 and len(key) != len(self):
5716 raise ValueError(
5717 "The length of the boolean indexer cannot be 0 "
5718 "when the Index has length greater than 0."
5719 )
5721 result = getitem(key)
5722 # Because we ruled out integer above, we always get an arraylike here
5723 if result.ndim > 1:
5724 disallow_ndim_indexing(result)
5726 # NB: Using _constructor._simple_new would break if MultiIndex
5727 # didn't override __getitem__
5728 return self._constructor._simple_new(result, name=self._name)
5730 def _getitem_slice(self, slobj: slice) -> Self:
5731 """
5732 Fastpath for __getitem__ when we know we have a slice.
5733 """
5734 res = self._data[slobj]
5735 result = type(self)._simple_new(res, name=self._name, refs=self._references)
5736 if "_engine" in self._cache:
5737 reverse = slobj.step is not None and slobj.step < 0
5738 result._engine._update_from_sliced(self._engine, reverse=reverse) # type: ignore[union-attr]
5740 return result
5742 @final
5743 def _can_hold_identifiers_and_holds_name(self, name: Hashable) -> bool:
5744 """
5745 Faster check for ``name in self`` when we know `name` is a Python
5746 identifier (e.g. in NDFrame.__getattr__, which hits this to support
5747 . key lookup). For indexes that can't hold identifiers (everything
5748 but object & categorical) we just return False.
5750 https://github.com/pandas-dev/pandas/issues/19764
5751 """
5752 if (
5753 self.dtype == object
5754 or is_string_dtype(self.dtype)
5755 or isinstance(self.dtype, CategoricalDtype)
5756 ):
5757 return name in self
5758 return False
5760 def append(self, other: Index | Sequence[Index]) -> Index:
5761 """
5762 Append a collection of Index options together.
5764 This method concatenates one or more Index objects to the current
5765 index, returning a new Index containing all elements in order.
5767 Parameters
5768 ----------
5769 other : Index or list/tuple of indices
5770 Single Index or a collection of indices, which can be either a list or a
5771 tuple.
5773 Returns
5774 -------
5775 Index
5776 Returns a new Index object resulting from appending the provided other
5777 indices to the original Index.
5779 See Also
5780 --------
5781 Index.insert : Make new Index inserting new item at location.
5783 Examples
5784 --------
5785 >>> idx = pd.Index([1, 2, 3])
5786 >>> idx.append(pd.Index([4]))
5787 Index([1, 2, 3, 4], dtype='int64')
5788 """
5789 to_concat = [self]
5791 if isinstance(other, (list, tuple)):
5792 to_concat += list(other)
5793 else:
5794 # error: Argument 1 to "append" of "list" has incompatible type
5795 # "Union[Index, Sequence[Index]]"; expected "Index"
5796 to_concat.append(other) # type: ignore[arg-type]
5798 for obj in to_concat:
5799 if not isinstance(obj, Index):
5800 raise TypeError("all inputs must be Index")
5802 names = {obj.name for obj in to_concat}
5803 name = None if len(names) > 1 else self.name
5805 return self._concat(to_concat, name) # pyright: ignore[reportArgumentType]
5807 def _concat(self, to_concat: list[Index], name: Hashable) -> Index:
5808 """
5809 Concatenate multiple Index objects.
5810 """
5811 to_concat_vals = [x._values for x in to_concat]
5813 result = concat_compat(to_concat_vals)
5815 return Index._with_infer(result, name=name, copy=False)
5817 def putmask(self, mask: npt.NDArray[np.bool_], value: object) -> Index:
5818 """
5819 Return a new Index of the values set with the mask.
5821 Replaces elements of the Index where the corresponding mask entry is
5822 ``True``, returning a new Index rather than modifying in place.
5824 Parameters
5825 ----------
5826 mask : array-like of bool
5827 Array of booleans denoting where values should be replaced.
5828 value : scalar
5829 Scalar value to use to fill holes (e.g. 0).
5830 This value cannot be a list-likes.
5832 Returns
5833 -------
5834 Index
5835 A new Index of the values set with the mask.
5837 See Also
5838 --------
5839 numpy.putmask : Changes elements of an array
5840 based on conditional and input values.
5842 Examples
5843 --------
5844 >>> idx1 = pd.Index([1, 2, 3])
5845 >>> idx2 = pd.Index([5, 6, 7])
5846 >>> idx1.putmask([True, False, False], idx2)
5847 Index([5, 2, 3], dtype='int64')
5848 """
5849 mask, noop = validate_putmask(self._values, mask)
5850 if noop:
5851 return self.copy()
5852 return self.where(~mask, value)
5854 def equals(self, other: Any) -> bool:
5855 """
5856 Determine if two Index object are equal.
5858 The things that are being compared are:
5860 * The elements inside the Index object.
5861 * The order of the elements inside the Index object.
5863 Parameters
5864 ----------
5865 other : Any
5866 The other object to compare against.
5868 Returns
5869 -------
5870 bool
5871 True if "other" is an Index and it has the same elements and order
5872 as the calling index; False otherwise.
5874 See Also
5875 --------
5876 Index.identical: Checks that object attributes and types are also equal.
5877 Index.has_duplicates: Check if the Index has duplicate values.
5878 Index.is_unique: Return if the index has unique values.
5880 Examples
5881 --------
5882 >>> idx1 = pd.Index([1, 2, 3])
5883 >>> idx1
5884 Index([1, 2, 3], dtype='int64')
5885 >>> idx1.equals(pd.Index([1, 2, 3]))
5886 True
5888 The elements inside are compared
5890 >>> idx2 = pd.Index(["1", "2", "3"])
5891 >>> idx2
5892 Index(['1', '2', '3'], dtype='str')
5894 >>> idx1.equals(idx2)
5895 False
5897 The order is compared
5899 >>> ascending_idx = pd.Index([1, 2, 3])
5900 >>> ascending_idx
5901 Index([1, 2, 3], dtype='int64')
5902 >>> descending_idx = pd.Index([3, 2, 1])
5903 >>> descending_idx
5904 Index([3, 2, 1], dtype='int64')
5905 >>> ascending_idx.equals(descending_idx)
5906 False
5908 The dtype is *not* compared
5910 >>> int64_idx = pd.Index([1, 2, 3], dtype="int64")
5911 >>> int64_idx
5912 Index([1, 2, 3], dtype='int64')
5913 >>> uint64_idx = pd.Index([1, 2, 3], dtype="uint64")
5914 >>> uint64_idx
5915 Index([1, 2, 3], dtype='uint64')
5916 >>> int64_idx.equals(uint64_idx)
5917 True
5918 """
5919 if self.is_(other):
5920 return True
5922 if not isinstance(other, Index):
5923 return False
5925 if len(self) != len(other):
5926 # quickly return if the lengths are different
5927 return False
5929 if isinstance(self.dtype, StringDtype) and other.dtype != self.dtype:
5930 # TODO(infer_string) can we avoid this special case?
5931 # special case for object behavior
5932 return other.equals(self.astype(object))
5934 if is_object_dtype(self.dtype) and not is_object_dtype(other.dtype):
5935 # if other is not object, use other's logic for coercion
5936 return other.equals(self)
5938 if isinstance(other, ABCMultiIndex):
5939 # d-level MultiIndex can equal d-tuple Index
5940 return other.equals(self)
5942 if isinstance(self._values, ExtensionArray):
5943 # Dispatch to the ExtensionArray's .equals method.
5944 if not isinstance(other, type(self)):
5945 return False
5947 earr = cast("ExtensionArray", self._data)
5948 return earr.equals(other._data)
5950 if isinstance(other.dtype, ExtensionDtype):
5951 # All EA-backed Index subclasses override equals
5952 return other.equals(self)
5954 return array_equivalent(self._values, other._values)
5956 @final
5957 def identical(self, other: Index) -> bool:
5958 """
5959 Similar to equals, but checks that object attributes and types are also equal.
5961 While :meth:`Index.equals` only checks that the elements are the same,
5962 this method additionally verifies that the Index objects have matching
5963 types and attributes (e.g., ``name``).
5965 Parameters
5966 ----------
5967 other : Index
5968 The Index object you want to compare with the current Index object.
5970 Returns
5971 -------
5972 bool
5973 If two Index objects have equal elements and same type True,
5974 otherwise False.
5976 See Also
5977 --------
5978 Index.equals: Determine if two Index object are equal.
5979 Index.has_duplicates: Check if the Index has duplicate values.
5980 Index.is_unique: Return if the index has unique values.
5982 Examples
5983 --------
5984 >>> idx1 = pd.Index(["1", "2", "3"])
5985 >>> idx2 = pd.Index(["1", "2", "3"])
5986 >>> idx2.identical(idx1)
5987 True
5989 >>> idx1 = pd.Index(["1", "2", "3"], name="A")
5990 >>> idx2 = pd.Index(["1", "2", "3"], name="B")
5991 >>> idx2.identical(idx1)
5992 False
5993 """
5994 return (
5995 self.equals(other)
5996 and all(
5997 getattr(self, c, None) == getattr(other, c, None)
5998 for c in self._comparables
5999 )
6000 and type(self) == type(other)
6001 and self.dtype == other.dtype
6002 )
6004 @final
6005 def asof(self, label: Hashable) -> Hashable:
6006 """
6007 Return the label from the index, or, if not present, the previous one.
6009 Assuming that the index is sorted, return the passed index label if it
6010 is in the index, or return the previous index label if the passed one
6011 is not in the index.
6013 Parameters
6014 ----------
6015 label : object
6016 The label up to which the method returns the latest index label.
6018 Returns
6019 -------
6020 object
6021 The passed label if it is in the index. The previous label if the
6022 passed label is not in the sorted index or `NaN` if there is no
6023 such label.
6025 See Also
6026 --------
6027 Series.asof : Return the latest value in a Series up to the
6028 passed index.
6029 merge_asof : Perform an asof merge (similar to left join but it
6030 matches on nearest key rather than equal key).
6031 Index.get_loc : An `asof` is a thin wrapper around `get_loc`
6032 with method='pad'.
6034 Examples
6035 --------
6036 `Index.asof` returns the latest index label up to the passed label.
6038 >>> idx = pd.Index(["2013-12-31", "2014-01-02", "2014-01-03"])
6039 >>> idx.asof("2014-01-01")
6040 '2013-12-31'
6042 If the label is in the index, the method returns the passed label.
6044 >>> idx.asof("2014-01-02")
6045 '2014-01-02'
6047 If all of the labels in the index are later than the passed label,
6048 NaN is returned.
6050 >>> idx.asof("1999-01-02")
6051 nan
6053 If the index is not sorted, an error is raised.
6055 >>> idx_not_sorted = pd.Index(["2013-12-31", "2015-01-02", "2014-01-03"])
6056 >>> idx_not_sorted.asof("2013-12-31")
6057 Traceback (most recent call last):
6058 ValueError: index must be monotonic increasing or decreasing
6059 """
6060 self._searchsorted_monotonic(label) # validate sortedness
6061 try:
6062 loc = self.get_loc(label)
6063 except (KeyError, TypeError) as err:
6064 # KeyError -> No exact match, try for padded
6065 # TypeError -> passed e.g. non-hashable, fall through to get
6066 # the tested exception message
6067 indexer = self.get_indexer([label], method="pad")
6068 if indexer.ndim > 1 or indexer.size > 1:
6069 raise TypeError("asof requires scalar valued input") from err
6070 loc = indexer.item()
6071 if loc == -1:
6072 return self._na_value
6073 else:
6074 if isinstance(loc, slice):
6075 return self[loc][-1]
6077 return self[loc] # pyright: ignore[reportReturnType]
6079 def asof_locs(
6080 self, where: Index, mask: npt.NDArray[np.bool_]
6081 ) -> npt.NDArray[np.intp]:
6082 """
6083 Return the locations (indices) of labels in the index.
6085 As in the :meth:`pandas.Index.asof`, if the label (a particular entry in
6086 ``where``) is not in the index, the latest index label up to the
6087 passed label is chosen and its index returned.
6089 If all of the labels in the index are later than a label in ``where``,
6090 -1 is returned.
6092 ``mask`` is used to ignore ``NA`` values in the index during calculation.
6094 Parameters
6095 ----------
6096 where : Index
6097 An Index consisting of an array of timestamps.
6098 mask : np.ndarray[bool]
6099 Array of booleans denoting where values in the original
6100 data are not ``NA``.
6102 Returns
6103 -------
6104 np.ndarray[np.intp]
6105 An array of locations (indices) of the labels from the index
6106 which correspond to the return values of :meth:`pandas.Index.asof`
6107 for every element in ``where``.
6109 See Also
6110 --------
6111 Index.asof : Return the label from the index, or, if not present, the
6112 previous one.
6114 Examples
6115 --------
6116 >>> idx = pd.date_range("2023-06-01", periods=3, freq="D")
6117 >>> where = pd.DatetimeIndex(
6118 ... ["2023-05-30 00:12:00", "2023-06-01 00:00:00", "2023-06-02 23:59:59"]
6119 ... )
6120 >>> mask = np.ones(3, dtype=bool)
6121 >>> idx.asof_locs(where, mask)
6122 array([-1, 0, 1])
6124 We can use ``mask`` to ignore certain values in the index during calculation.
6126 >>> mask[1] = False
6127 >>> idx.asof_locs(where, mask)
6128 array([-1, 0, 0])
6129 """
6130 # error: No overload variant of "searchsorted" of "ndarray" matches argument
6131 # types "Union[ExtensionArray, ndarray[Any, Any]]", "str"
6132 # TODO: will be fixed when ExtensionArray.searchsorted() is fixed
6133 locs = self._values[mask].searchsorted(
6134 where._values, # pyright: ignore[reportArgumentType]
6135 side="right", # type: ignore[call-overload]
6136 )
6137 locs = np.where(locs > 0, locs - 1, 0)
6139 result = np.arange(len(self), dtype=np.intp)[mask].take(locs)
6141 first_value = self._values[mask.argmax()]
6142 result[(locs == 0) & (where._values < first_value)] = -1
6144 return result
6146 @overload
6147 def sort_values(
6148 self,
6149 *,
6150 return_indexer: Literal[False] = ...,
6151 ascending: bool = ...,
6152 na_position: NaPosition = ...,
6153 key: Callable | None = ...,
6154 ) -> Self: ...
6156 @overload
6157 def sort_values(
6158 self,
6159 *,
6160 return_indexer: Literal[True],
6161 ascending: bool = ...,
6162 na_position: NaPosition = ...,
6163 key: Callable | None = ...,
6164 ) -> tuple[Self, np.ndarray]: ...
6166 @overload
6167 def sort_values(
6168 self,
6169 *,
6170 return_indexer: bool = ...,
6171 ascending: bool = ...,
6172 na_position: NaPosition = ...,
6173 key: Callable | None = ...,
6174 ) -> Self | tuple[Self, np.ndarray]: ...
6176 def sort_values(
6177 self,
6178 *,
6179 return_indexer: bool = False,
6180 ascending: bool = True,
6181 na_position: NaPosition = "last",
6182 key: Callable | None = None,
6183 ) -> Self | tuple[Self, np.ndarray]:
6184 """
6185 Return a sorted copy of the index.
6187 Return a sorted copy of the index, and optionally return the indices
6188 that sorted the index itself.
6190 Parameters
6191 ----------
6192 return_indexer : bool, default False
6193 Should the indices that would sort the index be returned.
6194 ascending : bool, default True
6195 Should the index values be sorted in an ascending order.
6196 na_position : {'first' or 'last'}, default 'last'
6197 Argument 'first' puts NaNs at the beginning, 'last' puts NaNs at
6198 the end.
6199 key : callable, optional
6200 If not None, apply the key function to the index values
6201 before sorting. This is similar to the `key` argument in the
6202 builtin :meth:`sorted` function, with the notable difference that
6203 this `key` function should be *vectorized*. It should expect an
6204 ``Index`` and return an ``Index`` of the same shape.
6206 Returns
6207 -------
6208 sorted_index : pandas.Index
6209 Sorted copy of the index.
6210 indexer : numpy.ndarray, optional
6211 The indices that the index itself was sorted by.
6213 See Also
6214 --------
6215 Series.sort_values : Sort values of a Series.
6216 DataFrame.sort_values : Sort values in a DataFrame.
6218 Examples
6219 --------
6220 >>> idx = pd.Index([10, 100, 1, 1000])
6221 >>> idx
6222 Index([10, 100, 1, 1000], dtype='int64')
6224 Sort values in ascending order (default behavior).
6226 >>> idx.sort_values()
6227 Index([1, 10, 100, 1000], dtype='int64')
6229 Sort values in descending order, and also get the indices `idx` was
6230 sorted by.
6232 >>> idx.sort_values(ascending=False, return_indexer=True)
6233 (Index([1000, 100, 10, 1], dtype='int64'), array([3, 1, 0, 2]))
6234 """
6235 if key is None and (
6236 (ascending and self.is_monotonic_increasing)
6237 or (not ascending and self.is_monotonic_decreasing)
6238 ):
6239 if return_indexer:
6240 indexer = np.arange(len(self), dtype=np.intp)
6241 return self.copy(), indexer
6242 else:
6243 return self.copy()
6245 # GH 35584. Sort missing values according to na_position kwarg
6246 # ignore na_position for MultiIndex
6247 if not isinstance(self, ABCMultiIndex):
6248 _as = nargsort(
6249 items=self, ascending=ascending, na_position=na_position, key=key
6250 )
6251 else:
6252 idx = cast("Index", ensure_key_mapped(self, key))
6253 _as = idx.argsort(na_position=na_position)
6254 if not ascending:
6255 _as = _as[::-1]
6257 sorted_index = self.take(_as)
6259 if return_indexer:
6260 return sorted_index, _as # pyright: ignore[reportReturnType]
6261 else:
6262 return sorted_index # pyright: ignore[reportReturnType]
6264 def shift(self, periods: int = 1, freq: Frequency | None = None) -> Self:
6265 """
6266 Shift index by desired number of time frequency increments.
6268 This method is for shifting the values of datetime-like indexes
6269 by a specified time increment a given number of times.
6271 Parameters
6272 ----------
6273 periods : int, default 1
6274 Number of periods (or increments) to shift by,
6275 can be positive or negative.
6276 freq : pandas.DateOffset, pandas.Timedelta or str, optional
6277 Frequency increment to shift by.
6278 If None, the index is shifted by its own `freq` attribute.
6279 Offset aliases are valid strings, e.g., 'D', 'W', 'M' etc.
6281 Returns
6282 -------
6283 pandas.Index
6284 Shifted index.
6286 See Also
6287 --------
6288 Series.shift : Shift values of Series.
6290 Notes
6291 -----
6292 This method is only implemented for datetime-like index classes,
6293 i.e., DatetimeIndex, PeriodIndex and TimedeltaIndex.
6295 Examples
6296 --------
6297 Put the first 5 month starts of 2011 into an index.
6299 >>> month_starts = pd.date_range("1/1/2011", periods=5, freq="MS")
6300 >>> month_starts
6301 DatetimeIndex(['2011-01-01', '2011-02-01', '2011-03-01', '2011-04-01',
6302 '2011-05-01'],
6303 dtype='datetime64[us]', freq='MS')
6305 Shift the index by 10 days.
6307 >>> month_starts.shift(10, freq="D")
6308 DatetimeIndex(['2011-01-11', '2011-02-11', '2011-03-11', '2011-04-11',
6309 '2011-05-11'],
6310 dtype='datetime64[us]', freq=None)
6312 The default value of `freq` is the `freq` attribute of the index,
6313 which is 'MS' (month start) in this example.
6315 >>> month_starts.shift(10)
6316 DatetimeIndex(['2011-11-01', '2011-12-01', '2012-01-01', '2012-02-01',
6317 '2012-03-01'],
6318 dtype='datetime64[us]', freq='MS')
6319 """
6320 raise NotImplementedError(
6321 f"This method is only implemented for DatetimeIndex, PeriodIndex and "
6322 f"TimedeltaIndex; Got type {type(self).__name__}"
6323 )
6325 def argsort(self, *args: Any, **kwargs: Any) -> npt.NDArray[np.intp]:
6326 """
6327 Return the integer indices that would sort the index.
6329 This method returns an array of indices that indicate the order in
6330 which elements of the index should be arranged to produce a sorted
6331 index. It delegates to the underlying array's ``argsort`` method.
6333 Parameters
6334 ----------
6335 *args
6336 Passed to `numpy.ndarray.argsort`.
6337 **kwargs
6338 Passed to `numpy.ndarray.argsort`.
6340 Returns
6341 -------
6342 np.ndarray[np.intp]
6343 Integer indices that would sort the index if used as
6344 an indexer.
6346 See Also
6347 --------
6348 numpy.argsort : Similar method for NumPy arrays.
6349 Index.sort_values : Return sorted copy of Index.
6351 Examples
6352 --------
6353 >>> idx = pd.Index(["b", "a", "d", "c"])
6354 >>> idx
6355 Index(['b', 'a', 'd', 'c'], dtype='str')
6357 >>> order = idx.argsort()
6358 >>> order
6359 array([1, 0, 3, 2])
6361 >>> idx[order]
6362 Index(['a', 'b', 'c', 'd'], dtype='str')
6363 """
6364 # This works for either ndarray or EA, is overridden
6365 # by RangeIndex, MultIIndex
6366 return self._data.argsort(*args, **kwargs)
6368 def _check_indexing_error(self, key: object) -> None:
6369 if not is_scalar(key):
6370 # if key is not a scalar, directly raise an error (the code below
6371 # would convert to numpy arrays and raise later any way) - GH29926
6372 raise InvalidIndexError(key)
6374 @cache_readonly
6375 def _should_fallback_to_positional(self) -> bool:
6376 """
6377 Should an integer key be treated as positional?
6378 """
6379 return self.inferred_type not in {
6380 "integer",
6381 "mixed-integer",
6382 "floating",
6383 "complex",
6384 }
6386 def get_indexer_non_unique(
6387 self, target: Axes
6388 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]:
6389 """
6390 Compute indexer and mask for new index given the current index.
6392 The indexer should be then used as an input to ndarray.take to align the
6393 current data to the new index.
6395 Parameters
6396 ----------
6397 target : Index
6398 An iterable containing the values to be used for computing indexer.
6400 Returns
6401 -------
6402 indexer : np.ndarray[np.intp]
6403 Integers from 0 to n - 1 indicating that the index at these
6404 positions matches the corresponding target values. Missing values
6405 in the target are marked by -1.
6406 missing : np.ndarray[np.intp]
6407 An indexer into the target of the values not found.
6408 These correspond to the -1 in the indexer array.
6410 See Also
6411 --------
6412 Index.get_indexer : Computes indexer and mask for new index given
6413 the current index.
6414 Index.get_indexer_for : Returns an indexer even when non-unique.
6416 Examples
6417 --------
6418 >>> index = pd.Index(["c", "b", "a", "b", "b"])
6419 >>> index.get_indexer_non_unique(["b", "b"])
6420 (array([1, 3, 4, 1, 3, 4]), array([], dtype=int64))
6422 In the example below there are no matched values.
6424 >>> index = pd.Index(["c", "b", "a", "b", "b"])
6425 >>> index.get_indexer_non_unique(["q", "r", "t"])
6426 (array([-1, -1, -1]), array([0, 1, 2]))
6428 For this reason, the returned ``indexer`` contains only integers equal to -1.
6429 It demonstrates that there's no match between the index and the ``target``
6430 values at these positions. The mask [0, 1, 2] in the return value shows that
6431 the first, second, and third elements are missing.
6433 Notice that the return value is a tuple contains two items. In the example
6434 below the first item is an array of locations in ``index``. The second
6435 item is a mask shows that the first and third elements are missing.
6437 >>> index = pd.Index(["c", "b", "a", "b", "b"])
6438 >>> index.get_indexer_non_unique(["f", "b", "s"])
6439 (array([-1, 1, 3, 4, -1]), array([0, 2]))
6440 """
6441 target = self._maybe_cast_listlike_indexer(target)
6443 if not self._should_compare(target) and not self._should_partial_index(target):
6444 # _should_partial_index e.g. IntervalIndex with numeric scalars
6445 # that can be matched to Interval scalars.
6446 return self._get_indexer_non_comparable(target, method=None, unique=False)
6448 pself, ptarget = self._maybe_downcast_for_indexing(target)
6449 if pself is not self or ptarget is not target:
6450 return pself.get_indexer_non_unique(ptarget)
6452 if self.dtype != target.dtype:
6453 # TODO: if object, could use infer_dtype to preempt costly
6454 # conversion if still non-comparable?
6455 dtype = self._find_common_type_compat(target)
6457 this = self.astype(dtype, copy=False)
6458 that = target.astype(dtype, copy=False)
6459 return this.get_indexer_non_unique(that)
6461 # TODO: get_indexer has fastpaths for both Categorical-self and
6462 # Categorical-target. Can we do something similar here?
6464 # Note: _maybe_downcast_for_indexing ensures we never get here
6465 # with MultiIndex self and non-Multi target
6466 if self._is_multi and target._is_multi:
6467 engine = self._engine
6468 # Item "IndexEngine" of "Union[IndexEngine, ExtensionEngine]" has
6469 # no attribute "_extract_level_codes"
6470 tgt_values = engine._extract_level_codes(target) # type: ignore[union-attr]
6471 else:
6472 tgt_values = target._get_engine_target()
6474 indexer, missing = self._engine.get_indexer_non_unique(tgt_values) # pyright: ignore[reportArgumentType]
6475 return ensure_platform_int(indexer), ensure_platform_int(missing)
6477 @final
6478 def get_indexer_for(self, target: Axes) -> npt.NDArray[np.intp]:
6479 """
6480 Guaranteed return of an indexer even when non-unique.
6482 This dispatches to get_indexer or get_indexer_non_unique
6483 as appropriate.
6485 Parameters
6486 ----------
6487 target : Index
6488 An iterable containing the values to be used for computing indexer.
6490 Returns
6491 -------
6492 np.ndarray[np.intp]
6493 List of indices.
6495 See Also
6496 --------
6497 Index.get_indexer : Computes indexer and mask for new index given
6498 the current index.
6499 Index.get_non_unique : Returns indexer and masks for new index given
6500 the current index.
6502 Examples
6503 --------
6504 >>> idx = pd.Index([np.nan, "var1", np.nan])
6505 >>> idx.get_indexer_for([np.nan])
6506 array([0, 2])
6507 """
6508 if self._index_as_unique:
6509 return self.get_indexer(target)
6510 indexer, _ = self.get_indexer_non_unique(target)
6511 return indexer
6513 def _pairwise_indexer(self, target: Index) -> npt.NDArray[np.intp]:
6514 """
6515 Positions in self for each element of target, pairing the k-th
6516 occurrence of a label in target with its k-th occurrence in self.
6518 Unlike get_indexer, self may contain duplicates. target must contain
6519 every label of self. Returns -1 where self has no k-th occurrence.
6521 Parameters
6522 ----------
6523 target : Index
6525 Returns
6526 -------
6527 np.ndarray[np.intp]
6528 """
6529 labels = target.unique()
6530 self_codes = labels.get_indexer_for(self)
6531 target_codes = labels.get_indexer_for(target)
6532 self_rank = algos.occurrence_rank(self_codes)
6533 target_rank = algos.occurrence_rank(target_codes)
6534 stride = max(self_rank.max(initial=0), target_rank.max(initial=0)) + 1
6535 self_keys = Index(self_codes * stride + self_rank)
6536 target_keys = target_codes * stride + target_rank
6537 return self_keys.get_indexer(target_keys)
6539 def _get_indexer_strict(
6540 self, key: Axes, axis_name: str_t
6541 ) -> tuple[Index, np.ndarray]:
6542 """
6543 Analogue to get_indexer that raises if any elements are missing.
6544 """
6545 keyarr = key
6546 if not isinstance(keyarr, Index):
6547 keyarr = com.asarray_tuplesafe(keyarr)
6549 if self._index_as_unique:
6550 indexer = self.get_indexer_for(keyarr) # pyright: ignore[reportArgumentType]
6551 if not isinstance(keyarr, Index):
6552 keyarr = ensure_index(keyarr)
6553 keyarr.name = self.name
6554 else:
6555 keyarr, indexer, new_indexer = self._reindex_non_unique(keyarr) # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
6557 self._raise_if_missing(keyarr, indexer, axis_name)
6559 keyarr = self.take(indexer)
6560 if lib.is_np_dtype(keyarr.dtype, "mM") or isinstance(
6561 keyarr.dtype, DatetimeTZDtype
6562 ):
6563 # DTI/TDI.take can infer a freq in some cases when we dont want one
6564 if isinstance(key, list) or (
6565 isinstance(key, type(self))
6566 # "Index" has no attribute "freq"
6567 and key.freq is None # type: ignore[attr-defined]
6568 ):
6569 # error: "Index" has no attribute "_with_freq"; maybe "_with_infer"?
6570 keyarr = keyarr._with_freq(None) # type: ignore[attr-defined]
6572 return keyarr, indexer
6574 def _raise_if_missing(
6575 self, key: Index, indexer: npt.NDArray[np.intp], axis_name: str_t
6576 ) -> None:
6577 """
6578 Check that indexer can be used to return a result.
6580 e.g. at least one element was found,
6581 unless the list of keys was actually empty.
6583 Parameters
6584 ----------
6585 key : Index
6586 Targeted labels (only used to show correct error message).
6587 indexer: np.ndarray[np.intp]
6588 Indices corresponding to the key,
6589 (with -1 indicating not found).
6590 axis_name : str
6592 Raises
6593 ------
6594 KeyError
6595 If at least one key was requested but none was found.
6596 """
6597 if len(key) == 0:
6598 return
6600 # Count missing values
6601 missing_mask = indexer < 0
6602 nmissing = missing_mask.sum()
6604 if nmissing:
6605 if nmissing == len(indexer):
6606 raise KeyError(f"None of [{key}] are in the [{axis_name}]")
6608 not_found = list(ensure_index(key)[missing_mask.nonzero()[0]].unique())
6609 raise KeyError(f"{not_found} not in index")
6611 @overload
6612 def _get_indexer_non_comparable(
6613 self, target: Index, method: str_t | None, unique: Literal[True] = ...
6614 ) -> npt.NDArray[np.intp]: ...
6616 @overload
6617 def _get_indexer_non_comparable(
6618 self, target: Index, method: str_t | None, unique: Literal[False]
6619 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
6621 @overload
6622 def _get_indexer_non_comparable(
6623 self, target: Index, method: str_t | None, unique: bool = True
6624 ) -> npt.NDArray[np.intp] | tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
6626 @final
6627 def _get_indexer_non_comparable(
6628 self, target: Index, method: str_t | None, unique: bool = True
6629 ) -> npt.NDArray[np.intp] | tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]:
6630 """
6631 Called from get_indexer or get_indexer_non_unique when the target
6632 is of a non-comparable dtype.
6634 For get_indexer lookups with method=None, get_indexer is an _equality_
6635 check, so non-comparable dtypes mean we will always have no matches.
6637 For get_indexer lookups with a method, get_indexer is an _inequality_
6638 check, so non-comparable dtypes mean we will always raise TypeError.
6640 Parameters
6641 ----------
6642 target : Index
6643 method : str or None
6644 unique : bool, default True
6645 * True if called from get_indexer.
6646 * False if called from get_indexer_non_unique.
6648 Raises
6649 ------
6650 TypeError
6651 If doing an inequality check, i.e. method is not None.
6652 """
6653 if method is not None:
6654 other_dtype = _unpack_nested_dtype(target)
6655 raise TypeError(f"Cannot compare dtypes {self.dtype} and {other_dtype}")
6657 no_matches = -1 * np.ones(target.shape, dtype=np.intp)
6658 if unique:
6659 # This is for get_indexer
6660 return no_matches
6661 else:
6662 # This is for get_indexer_non_unique
6663 missing = np.arange(len(target), dtype=np.intp)
6664 return no_matches, missing
6666 @property
6667 def _index_as_unique(self) -> bool:
6668 """
6669 Whether we should treat this as unique for the sake of
6670 get_indexer vs get_indexer_non_unique.
6672 For IntervalIndex compat.
6673 """
6674 return self.is_unique
6676 _requires_unique_msg = "Reindexing only valid with uniquely valued Index objects"
6678 @final
6679 def _maybe_downcast_for_indexing(self, other: Index) -> tuple[Index, Index]:
6680 """
6681 When dealing with an object-dtype Index and a non-object Index, see
6682 if we can upcast the object-dtype one to improve performance.
6683 """
6685 if isinstance(self, ABCDatetimeIndex) and isinstance(other, ABCDatetimeIndex):
6686 if (
6687 self.tz is not None
6688 and other.tz is not None
6689 and not tz_compare(self.tz, other.tz)
6690 ):
6691 # standardize on UTC
6692 return self.tz_convert("UTC"), other.tz_convert("UTC")
6694 elif self.inferred_type == "date" and isinstance(other, ABCDatetimeIndex):
6695 try:
6696 result = type(other)(self)
6697 except ValueError:
6698 # e.g. out of bounds, or dates mixed with tz-aware Timestamps
6699 return self, other
6700 else:
6701 if self.is_unique and not result.is_unique:
6702 return self, other
6703 warnings.warn(
6704 # GH#62158 deprecate special-case treatment of date objects
6705 "Indexing a DatetimeIndex with a sequence of datetime.date "
6706 "objects is deprecated. Use Timestamp objects instead.",
6707 Pandas4Warning,
6708 stacklevel=find_stack_level(),
6709 )
6710 return result, other
6711 elif self.inferred_type == "timedelta" and isinstance(other, ABCTimedeltaIndex):
6712 # TODO: we dont have tests that get here
6713 return type(other)(self), other
6715 elif self.dtype.kind == "u" and other.dtype.kind == "i":
6716 # GH#41873
6717 if other.min() >= 0: # type: ignore[operator] # pyright: ignore[reportOperatorIssue]
6718 # lookup min as it may be cached
6719 # TODO: may need itemsize check if we have non-64-bit Indexes
6720 return self, other.astype(self.dtype)
6722 elif self._is_multi and not other._is_multi:
6723 try:
6724 # "Type[Index]" has no attribute "from_tuples"
6725 other = type(self).from_tuples(other) # type: ignore[attr-defined]
6726 except (TypeError, ValueError):
6727 # let's instead try with a straight Index
6728 self = Index(self._values, copy=False)
6730 if not is_object_dtype(self.dtype) and is_object_dtype(other.dtype):
6731 # Reverse op so we dont need to re-implement on the subclasses
6732 other, self = other._maybe_downcast_for_indexing(self)
6734 return self, other
6736 @final
6737 def _find_common_type_compat(self, target: Any) -> DtypeObj:
6738 """
6739 Implementation of find_common_type that adjusts for Index-specific
6740 special cases.
6741 """
6742 target_dtype, _ = infer_dtype_from(target)
6744 if isinstance(target, tuple):
6745 # GH#54385
6746 return np.dtype(object)
6748 if using_string_dtype():
6749 # special case: if left or right is a zero-length RangeIndex or
6750 # Index[object], those can be created by the default empty constructors
6751 # -> for that case ignore this dtype and always return the other
6752 # (https://github.com/pandas-dev/pandas/pull/60797)
6753 from pandas.core.indexes.range import RangeIndex
6755 if len(self) == 0 and (
6756 isinstance(self, RangeIndex) or self.dtype == np.object_
6757 ):
6758 return target_dtype
6759 if (
6760 isinstance(target, Index)
6761 and len(target) == 0
6762 and (isinstance(target, RangeIndex) or target_dtype == np.object_)
6763 ):
6764 return self.dtype
6766 # special case: if one dtype is uint64 and the other a signed int, return object
6767 # See https://github.com/pandas-dev/pandas/issues/26778 for discussion
6768 # Now it's:
6769 # * float | [u]int -> float
6770 # * uint64 | signed int -> object
6771 # We may change union(float | [u]int) to go to object.
6772 if self.dtype == "uint64" or target_dtype == "uint64":
6773 if is_signed_integer_dtype(self.dtype) or is_signed_integer_dtype(
6774 target_dtype
6775 ):
6776 return _dtype_obj
6778 dtype = find_result_type(self.dtype, target)
6779 dtype = common_dtype_categorical_compat([self, target], dtype)
6780 return dtype
6782 @final
6783 def _should_compare(self, other: Index) -> bool:
6784 """
6785 Check if `self == other` can ever have non-False entries.
6786 """
6788 # NB: we use inferred_type rather than is_bool_dtype to catch
6789 # object_dtype_of_bool and categorical[object_dtype_of_bool] cases
6790 if (
6791 other.inferred_type == "boolean" and is_any_real_numeric_dtype(self.dtype)
6792 ) or (
6793 self.inferred_type == "boolean" and is_any_real_numeric_dtype(other.dtype)
6794 ):
6795 # GH#16877 Treat boolean labels passed to a numeric index as not
6796 # found. Without this fix False and True would be treated as 0 and 1
6797 # respectively.
6798 return False
6800 dtype = _unpack_nested_dtype(other)
6801 return (
6802 self._is_comparable_dtype(dtype)
6803 or is_object_dtype(dtype)
6804 or is_string_dtype(dtype)
6805 )
6807 def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
6808 """
6809 Can we compare values of the given dtype to our own?
6810 """
6811 if self.dtype.kind == "b":
6812 return dtype.kind == "b"
6813 elif is_numeric_dtype(self.dtype):
6814 return is_numeric_dtype(dtype)
6815 # TODO: this was written assuming we only get here with object-dtype,
6816 # which is no longer correct. Can we specialize for EA?
6817 return True
6819 @final
6820 def groupby(self, values: Axes) -> PrettyDict[Hashable, Index]:
6821 """
6822 Group the index labels by a given array of values.
6824 Parameters
6825 ----------
6826 values : array
6827 Values used to determine the groups.
6829 Returns
6830 -------
6831 dict
6832 {group name -> group labels}
6833 """
6834 # TODO: if we are a MultiIndex, we can do better
6835 # that converting to tuples
6836 if isinstance(values, ABCMultiIndex):
6837 values = values._values
6838 values = Categorical(values)
6839 result = values._reverse_indexer()
6841 # map to the label
6842 result = {k: self.take(v) for k, v in result.items()} # type: ignore[misc]
6844 return PrettyDict(result) # type: ignore[arg-type]
6846 def map(
6847 self,
6848 mapper: Callable | dict | Series,
6849 na_action: Literal["ignore"] | None = None,
6850 ) -> Index:
6851 """
6852 Map values using an input mapping or function.
6854 Applies a function, dict, or Series to each element of the Index,
6855 returning a new Index (or MultiIndex if tuples are returned).
6857 Parameters
6858 ----------
6859 mapper : function, dict, or Series
6860 Mapping correspondence.
6861 na_action : {None, 'ignore'}
6862 If 'ignore', propagate NA values, without passing them to the
6863 mapping correspondence.
6865 Returns
6866 -------
6867 Union[Index, MultiIndex]
6868 The output of the mapping function applied to the index.
6869 If the function returns a tuple with more than one element
6870 a MultiIndex will be returned.
6872 See Also
6873 --------
6874 Index.where : Replace values where the condition is False.
6876 Examples
6877 --------
6878 >>> idx = pd.Index([1, 2, 3])
6879 >>> idx.map({1: "a", 2: "b", 3: "c"})
6880 Index(['a', 'b', 'c'], dtype='str')
6882 Using `map` with a function:
6884 >>> idx = pd.Index([1, 2, 3])
6885 >>> idx.map("I am a {}".format)
6886 Index(['I am a 1', 'I am a 2', 'I am a 3'], dtype='str')
6888 >>> idx = pd.Index(["a", "b", "c"])
6889 >>> idx.map(lambda x: x.upper())
6890 Index(['A', 'B', 'C'], dtype='str')
6891 """
6892 from pandas.core.indexes.multi import MultiIndex
6894 new_values = self._map_values(mapper, na_action=na_action) # type: ignore[no-untyped-call]
6896 # we can return a MultiIndex
6897 if new_values.size and isinstance(new_values[0], tuple):
6898 if isinstance(self, MultiIndex):
6899 names = self.names
6900 elif self.name:
6901 names = [self.name] * len(new_values[0])
6902 else:
6903 names = None
6904 return MultiIndex.from_tuples(new_values, names=names)
6906 dtype = None
6907 if not new_values.size:
6908 # empty
6909 dtype = self.dtype
6910 elif isinstance(new_values, Categorical):
6911 # cast_pointwise_result is unnecessary
6912 dtype = new_values.dtype
6913 else:
6914 if isinstance(self, MultiIndex):
6915 arr = self[:0].to_flat_index().array
6916 else:
6917 arr = self[:0].array
6918 # e.g. if we are floating and new_values is all ints, then we
6919 # don't want to cast back to floating. But if we are UInt64
6920 # and new_values is all ints, we want to try.
6921 new_values = arr._cast_pointwise_result(new_values)
6922 dtype = new_values.dtype
6923 return Index(new_values, dtype=dtype, copy=False, name=self.name)
6925 def replace(
6926 self, to_replace: Any = None, value: Any = lib.no_default, regex: Any = False
6927 ) -> Index:
6928 """
6929 Replace values in the Index.
6931 Replace values given in `to_replace` with `value`. Values of the Index
6932 that are not in `to_replace` are left unchanged.
6934 Parameters
6935 ----------
6936 to_replace : str, regex, list, dict, Series, scalar, or None
6937 The value(s) to be replaced. If a dict is provided, `value` must be omitted.
6938 value : scalar, dict, list, str, regex, default None
6939 The value to replace occurrences of `to_replace` with.
6940 regex : bool or same types as `to_replace`, default False
6941 Whether to interpret `to_replace` and/or `value` as regular expressions.
6943 Returns
6944 -------
6945 Index
6946 Index with replaced values.
6948 See Also
6949 --------
6950 Series.replace : Replace values in a Series.
6951 DataFrame.replace : Replace values in a DataFrame.
6953 Examples
6954 --------
6955 >>> idx = pd.Index([1, 2, 3, 4, 5])
6956 >>> idx.replace(3, 30)
6957 Index([1, 2, 30, 4, 5], dtype='int64')
6958 """
6959 if self._is_multi:
6960 raise NotImplementedError("replace is not implemented for MultiIndex")
6962 from pandas import Series
6964 # Pass pandas objects (not their underlying arrays) so that CoW
6965 # references are tracked in the no-copy cases (GH#65265).
6966 ser = Series(self, copy=False)
6967 replaced = ser.replace(to_replace, value, regex=regex)
6969 return Index(replaced, dtype=replaced.dtype, name=self.name, copy=False)
6971 # TODO: De-duplicate with map, xref GH#32349
6972 @final
6973 def _transform_index(self, func: Callable, *, level: int | None = None) -> Index:
6974 """
6975 Apply function to all values found in index.
6977 This includes transforming multiindex entries separately.
6978 Only apply function to one level of the MultiIndex if level is specified.
6979 """
6980 if isinstance(self, ABCMultiIndex):
6981 values = [
6982 (
6983 self.get_level_values(i).map(func)
6984 if i == level or level is None
6985 else self.get_level_values(i)
6986 )
6987 for i in range(self.nlevels)
6988 ]
6989 return type(self).from_arrays(values)
6990 else:
6991 items = [func(x) for x in self]
6992 if not items:
6993 return Index(
6994 items, dtype=self.dtype, name=self.name, tupleize_cols=False
6995 )
6996 new_values = self.array._cast_pointwise_result(items)
6997 return Index(
6998 new_values,
6999 dtype=new_values.dtype,
7000 copy=False,
7001 name=self.name,
7002 tupleize_cols=False,
7003 )
7005 def isin(
7006 self, values: Axes | set, level: str_t | int | None = None
7007 ) -> npt.NDArray[np.bool_]:
7008 """
7009 Return a boolean array where the index values are in `values`.
7011 Compute boolean array of whether each index value is found in the
7012 passed set of values. The length of the returned boolean array matches
7013 the length of the index.
7015 Parameters
7016 ----------
7017 values : set or list-like
7018 Sought values.
7019 level : str or int, optional
7020 Name or position of the index level to use (if the index is a
7021 `MultiIndex`).
7023 Returns
7024 -------
7025 np.ndarray[bool]
7026 NumPy array of boolean values.
7028 See Also
7029 --------
7030 Series.isin : Same for Series.
7031 DataFrame.isin : Same method for DataFrames.
7033 Notes
7034 -----
7035 In the case of `MultiIndex` you must either specify `values` as a
7036 list-like object containing tuples that are the same length as the
7037 number of levels, or specify `level`. Otherwise it will raise a
7038 ``ValueError``.
7040 If `level` is specified:
7042 - if it is the name of one *and only one* index level, use that level;
7043 - otherwise it should be a number indicating level position.
7045 Examples
7046 --------
7047 >>> idx = pd.Index([1, 2, 3])
7048 >>> idx
7049 Index([1, 2, 3], dtype='int64')
7051 Check whether each index value in a list of values.
7053 >>> idx.isin([1, 4])
7054 array([ True, False, False])
7056 >>> midx = pd.MultiIndex.from_arrays(
7057 ... [[1, 2, 3], ["red", "blue", "green"]], names=["number", "color"]
7058 ... )
7059 >>> midx
7060 MultiIndex([(1, 'red'),
7061 (2, 'blue'),
7062 (3, 'green')],
7063 names=['number', 'color'])
7065 Check whether the strings in the 'color' level of the MultiIndex
7066 are in a list of colors.
7068 >>> midx.isin(["red", "orange", "yellow"], level="color")
7069 array([ True, False, False])
7071 To check across the levels of a MultiIndex, pass a list of tuples:
7073 >>> midx.isin([(1, "red"), (3, "red")])
7074 array([ True, False, False])
7075 """
7076 if level is not None:
7077 self._validate_index_level(level)
7078 return algos.isin(self._values, values) # type: ignore[arg-type]
7080 def _get_string_slice(self, key: str_t) -> slice:
7081 # this is for partial string indexing,
7082 # overridden in DatetimeIndex, TimedeltaIndex and PeriodIndex
7083 raise NotImplementedError
7085 def slice_indexer(
7086 self,
7087 start: Hashable | None = None,
7088 end: Hashable | None = None,
7089 step: int | None = None,
7090 ) -> slice:
7091 """
7092 Compute the slice indexer for input labels and step.
7094 Index needs to be ordered and unique.
7096 Parameters
7097 ----------
7098 start : label, default None
7099 If None, defaults to the beginning.
7100 end : label, default None
7101 If None, defaults to the end.
7102 step : int, default None
7103 If None, defaults to 1.
7105 Returns
7106 -------
7107 slice
7108 A slice object.
7110 Raises
7111 ------
7112 KeyError : If key does not exist, or key is not unique and index is
7113 not ordered.
7115 See Also
7116 --------
7117 Index.slice_locs : Computes slice locations for input labels.
7118 Index.get_slice_bound : Retrieves slice bound that corresponds to given label.
7120 Notes
7121 -----
7122 This function assumes that the data is sorted, so use at your own peril.
7124 Examples
7125 --------
7126 This is a method on all index types. For example you can do:
7128 >>> idx = pd.Index(list("abcd"))
7129 >>> idx.slice_indexer(start="b", end="c")
7130 slice(1, 3, None)
7132 >>> idx = pd.MultiIndex.from_arrays([list("abcd"), list("efgh")])
7133 >>> idx.slice_indexer(start="b", end=("c", "g"))
7134 slice(1, 3, None)
7135 """
7136 start_slice, end_slice = self.slice_locs(start, end, step=step)
7138 # return a slice
7139 if not is_scalar(start_slice):
7140 raise AssertionError("Start slice bound is non-scalar")
7141 if not is_scalar(end_slice):
7142 raise AssertionError("End slice bound is non-scalar")
7144 return slice(start_slice, end_slice, step)
7146 def _maybe_cast_indexer(self, key: Hashable) -> Hashable:
7147 """
7148 If we have a float key and are not a floating index, then try to cast
7149 to an int if equivalent.
7150 """
7151 if (
7152 is_float(key)
7153 and np.isnan(key)
7154 and isinstance(self.dtype, FloatingDtype)
7155 and is_nan_na()
7156 ):
7157 # TODO: better place to do this?
7158 key = self.dtype.na_value
7159 return key
7161 def _maybe_cast_listlike_indexer(self, target: Axes) -> Index:
7162 """
7163 Analogue to maybe_cast_indexer for get_indexer instead of get_loc.
7164 """
7165 target_index = ensure_index(target)
7166 if (
7167 not hasattr(target, "dtype")
7168 and self.dtype == object
7169 and target_index.dtype == "string"
7170 ):
7171 # If we started with a list-like, avoid inference to string dtype if self
7172 # is object dtype (coercing to string dtype will alter the missing values)
7173 target_index = Index(target, dtype=self.dtype)
7174 elif (
7175 not hasattr(target, "dtype")
7176 and isinstance(self.dtype, StringDtype)
7177 and self.dtype.na_value is np.nan
7178 and using_string_dtype()
7179 ):
7180 # Fill missing values to ensure consistent missing value representation
7181 target_index = target_index.fillna(np.nan)
7182 return target_index
7184 @final
7185 def _validate_indexer(
7186 self,
7187 form: Literal["positional", "slice"],
7188 key: object,
7189 kind: Literal["getitem", "iloc"],
7190 ) -> None:
7191 """
7192 If we are positional indexer, validate that we have appropriate
7193 typed bounds must be an integer.
7194 """
7195 if not lib.is_int_or_none(key):
7196 self._raise_invalid_indexer(form, key)
7198 def _maybe_cast_slice_bound(self, label: Hashable, side: str_t) -> Hashable:
7199 """
7200 This function should be overloaded in subclasses that allow non-trivial
7201 casting on label-slice bounds, e.g. datetime-like indices allowing
7202 strings containing formatted datetimes.
7204 Parameters
7205 ----------
7206 label : object
7207 side : {'left', 'right'}
7209 Returns
7210 -------
7211 label : object
7213 Notes
7214 -----
7215 Value of `side` parameter should be validated in caller.
7216 """
7218 # We are a plain index here (sub-class override this method if they
7219 # wish to have special treatment for floats/ints, e.g. datetimelike Indexes
7221 if is_numeric_dtype(self.dtype):
7222 return self._maybe_cast_indexer(label)
7224 # reject them, if index does not contain label
7225 if (is_float(label) or is_integer(label)) and label not in self:
7226 self._raise_invalid_indexer("slice", label)
7228 return label
7230 def _searchsorted_monotonic(
7231 self, label: Hashable, side: Literal["left", "right"] = "left"
7232 ) -> int:
7233 if self.is_monotonic_increasing:
7234 return self.searchsorted(label, side=side) # type: ignore[call-overload]
7235 elif self.is_monotonic_decreasing:
7236 # np.searchsorted expects ascending sort order, have to reverse
7237 # everything for it to work (element ordering, search side and
7238 # resulting value).
7239 pos = self[::-1].searchsorted( # type: ignore[call-overload]
7240 label, # pyright: ignore[reportArgumentType]
7241 side="right" if side == "left" else "left",
7242 )
7243 return maybe_unbox_numpy_scalar(len(self) - pos)
7245 raise ValueError("index must be monotonic increasing or decreasing")
7247 def get_slice_bound(self, label: Hashable, side: Literal["left", "right"]) -> int:
7248 """
7249 Calculate slice bound that corresponds to given label.
7251 Returns leftmost (one-past-the-rightmost if ``side=='right'``) position
7252 of given label.
7254 Parameters
7255 ----------
7256 label : object
7257 The label for which to calculate the slice bound.
7258 side : {'left', 'right'}
7259 if 'left' return leftmost position of given label.
7260 if 'right' return one-past-the-rightmost position of given label.
7262 Returns
7263 -------
7264 int
7265 Index of label.
7267 See Also
7268 --------
7269 Index.get_loc : Get integer location, slice or boolean mask for requested
7270 label.
7272 Examples
7273 --------
7274 >>> idx = pd.RangeIndex(5)
7275 >>> idx.get_slice_bound(3, "left")
7276 3
7278 >>> idx.get_slice_bound(3, "right")
7279 4
7281 If ``label`` is non-unique in the index, an error will be raised.
7283 >>> idx_duplicate = pd.Index(["a", "b", "a", "c", "d"])
7284 >>> idx_duplicate.get_slice_bound("a", "left")
7285 Traceback (most recent call last):
7286 KeyError: Cannot get left slice bound for non-unique label: 'a'
7287 """
7289 if side not in ("left", "right"):
7290 raise ValueError(
7291 "Invalid value for side kwarg, must be either "
7292 f"'left' or 'right': {side}"
7293 )
7295 original_label = label
7297 # For datetime indices label may be a string that has to be converted
7298 # to datetime boundary according to its resolution.
7299 label = self._maybe_cast_slice_bound(label, side)
7301 # we need to look up the label
7302 try:
7303 slc = self.get_loc(label)
7304 except KeyError:
7305 try:
7306 return self._searchsorted_monotonic(label, side)
7307 except ValueError:
7308 raise KeyError(
7309 f"Cannot get {side} slice bound for non-monotonic index "
7310 f"with a missing label {original_label!r}. "
7311 "Either sort the index or specify an existing label."
7312 ) from None
7314 if isinstance(slc, np.ndarray):
7315 # get_loc may return a boolean array, which
7316 # is OK as long as they are representable by a slice.
7317 assert is_bool_dtype(slc.dtype)
7318 slc = lib.maybe_booleans_to_slice(slc.view("u1")) # type: ignore[assignment]
7319 if isinstance(slc, np.ndarray):
7320 raise KeyError(
7321 f"Cannot get {side} slice bound for non-unique "
7322 f"label: {original_label!r}"
7323 )
7325 if isinstance(slc, slice):
7326 if side == "left":
7327 return slc.start
7328 else:
7329 return slc.stop
7330 elif side == "right":
7331 return slc + 1
7332 else:
7333 return slc
7335 def slice_locs(
7336 self,
7337 start: SliceType = None,
7338 end: SliceType = None,
7339 step: int | None = None,
7340 ) -> tuple[int, int]:
7341 """
7342 Compute slice locations for input labels.
7344 This method determines the integer start and end positions for a
7345 label-based slice, which is useful for translating label-based
7346 slicing into positional slicing on the underlying data.
7348 Parameters
7349 ----------
7350 start : label, default None
7351 If None, defaults to the beginning.
7352 end : label, default None
7353 If None, defaults to the end.
7354 step : int, defaults None
7355 If None, defaults to 1.
7357 Returns
7358 -------
7359 tuple[int, int]
7360 Returns a tuple of two integers representing the slice locations for the
7361 input labels within the index.
7363 See Also
7364 --------
7365 Index.get_loc : Get location for a single label.
7366 Index.get_slice_bound : Calculate the slice bound for a single label;
7367 used internally by ``slice_locs`` for each of ``start`` and ``end``.
7369 Notes
7370 -----
7371 This method only works if the index is monotonic or unique.
7373 If ``start`` or ``end`` is not present in a monotonic index, the
7374 returned position is the insertion point that preserves ordering
7375 (analogous to :func:`numpy.searchsorted`); no error is raised. If the
7376 index is not monotonic and the label is missing, a ``KeyError`` is
7377 raised.
7379 Examples
7380 --------
7381 >>> idx = pd.Index(list("abcd"))
7382 >>> idx.slice_locs(start="b", end="c")
7383 (1, 3)
7385 ``start`` and ``end`` need not be present in the index; for a
7386 monotonic index they are mapped to the appropriate insertion
7387 positions:
7389 >>> idx = pd.Index(list("bcde"))
7390 >>> idx.slice_locs(start="a", end="c")
7391 (0, 2)
7392 """
7393 inc = step is None or step >= 0
7395 if not inc:
7396 # If it's a reverse slice, temporarily swap bounds.
7397 start, end = end, start
7399 # GH 16785: If start and end happen to be date strings with UTC offsets
7400 # attempt to parse and check that the offsets are the same
7401 if isinstance(start, (str, datetime)) and isinstance(end, (str, datetime)):
7402 try:
7403 tz_start = _slice_bound_tzinfo(start)
7404 tz_end = _slice_bound_tzinfo(end)
7405 except (ValueError, TypeError):
7406 pass
7407 else:
7408 if not tz_compare(tz_start, tz_end):
7409 raise ValueError("Both dates must have the same UTC offset")
7411 start_slice = None
7412 if start is not None:
7413 start_slice = self.get_slice_bound(start, "left")
7414 if start_slice is None:
7415 start_slice = 0
7417 end_slice = None
7418 if end is not None:
7419 end_slice = self.get_slice_bound(end, "right")
7420 if end_slice is None:
7421 end_slice = len(self)
7423 if not inc:
7424 # Bounds at this moment are swapped, swap them back and shift by 1.
7425 #
7426 # slice_locs('B', 'A', step=-1): s='B', e='A'
7427 #
7428 # s='A' e='B'
7429 # AFTER SWAP: | |
7430 # v ------------------> V
7431 # -----------------------------------
7432 # | | |A|A|A|A| | | | | |B|B| | | | |
7433 # -----------------------------------
7434 # ^ <------------------ ^
7435 # SHOULD BE: | |
7436 # end=s-1 start=e-1
7437 #
7438 end_slice, start_slice = start_slice - 1, end_slice - 1
7440 # i == -1 triggers ``len(self) + i`` selection that points to the
7441 # last element, not before-the-first one, subtracting len(self)
7442 # compensates that.
7443 if end_slice == -1:
7444 end_slice -= len(self)
7445 if start_slice == -1:
7446 start_slice -= len(self)
7448 start_slice = maybe_unbox_numpy_scalar(start_slice)
7449 end_slice = maybe_unbox_numpy_scalar(end_slice)
7450 return start_slice, end_slice
7452 def delete(
7453 self, loc: int | np.integer | list[int] | npt.NDArray[np.integer]
7454 ) -> Self:
7455 """
7456 Make new Index with passed location(-s) deleted.
7458 Returns a new Index with the entries at the given positional
7459 location(s) removed. The original Index is not modified.
7461 Parameters
7462 ----------
7463 loc : int or list of int
7464 Location of item(-s) which will be deleted.
7465 Use a list of locations to delete more than one value at the same time.
7467 Returns
7468 -------
7469 Index
7470 Will be same type as self, except for RangeIndex.
7472 See Also
7473 --------
7474 numpy.delete : Delete any rows and column from NumPy array (ndarray).
7476 Examples
7477 --------
7478 >>> idx = pd.Index(["a", "b", "c"])
7479 >>> idx.delete(1)
7480 Index(['a', 'c'], dtype='str')
7482 >>> idx = pd.Index(["a", "b", "c"])
7483 >>> idx.delete([0, 2])
7484 Index(['b'], dtype='str')
7485 """
7486 values = self._values
7487 res_values: ArrayLike
7488 if isinstance(values, np.ndarray):
7489 # TODO(__array_function__): special casing will be unnecessary
7490 res_values = np.delete(values, loc)
7491 else:
7492 res_values = values.delete(loc)
7494 # _constructor so RangeIndex-> Index with an int64 dtype
7495 return self._constructor._simple_new(res_values, name=self.name)
7497 def insert(self, loc: int, item: Hashable) -> Index:
7498 """
7499 Make new Index inserting new item at location.
7501 Follows Python numpy.insert semantics for negative values.
7503 Parameters
7504 ----------
7505 loc : int
7506 The integer location where the new item will be inserted.
7507 item : object
7508 The new item to be inserted into the Index.
7510 Returns
7511 -------
7512 Index
7513 Returns a new Index object resulting from inserting the specified item at
7514 the specified location within the original Index.
7516 See Also
7517 --------
7518 Index.append : Append a collection of Indexes together.
7520 Examples
7521 --------
7522 >>> idx = pd.Index(["a", "b", "c"])
7523 >>> idx.insert(1, "x")
7524 Index(['a', 'x', 'b', 'c'], dtype='str')
7525 """
7526 item = lib.item_from_zerodim(item)
7527 if is_valid_na_for_dtype(item, self.dtype) and self.dtype != object:
7528 item = self._na_value
7530 arr = self._values
7532 if using_string_dtype() and len(self) == 0 and self.dtype == np.object_:
7533 # special case: if we are an empty object-dtype Index, also
7534 # take into account the inserted item for the resulting dtype
7535 # (https://github.com/pandas-dev/pandas/pull/60797)
7536 dtype = self._find_common_type_compat(item)
7537 if dtype != self.dtype:
7538 return self.astype(dtype).insert(loc, item)
7540 try:
7541 if isinstance(arr, ExtensionArray):
7542 res_values = arr.insert(loc, item)
7543 return type(self)._simple_new(res_values, name=self.name)
7544 else:
7545 item = self._validate_fill_value(item)
7546 except (TypeError, ValueError, LossySetitemError):
7547 # e.g. trying to insert an integer into a DatetimeIndex
7548 # We cannot keep the same dtype, so cast to the (often object)
7549 # minimal shared dtype before doing the insert.
7550 dtype = self._find_common_type_compat(item)
7551 if dtype == self.dtype:
7552 # EA's might run into recursion errors if loc is invalid
7553 raise
7554 return self.astype(dtype).insert(loc, item)
7556 if arr.dtype != object or not isinstance(
7557 item, (tuple, np.datetime64, np.timedelta64)
7558 ):
7559 # with object-dtype we need to worry about numpy incorrectly casting
7560 # dt64/td64 to integer, also about treating tuples as sequences
7561 # special-casing dt64/td64 https://github.com/numpy/numpy/issues/12550
7562 casted = arr.dtype.type(item)
7563 new_values = np.insert(arr, loc, casted)
7565 else:
7566 # error: No overload variant of "insert" matches argument types
7567 # "ndarray[Any, Any]", "int", "None"
7568 new_values = np.insert(arr, loc, None) # type: ignore[call-overload]
7569 loc = loc if loc >= 0 else loc - 1
7570 new_values[loc] = item
7572 # GH#51363 stopped doing dtype inference here
7573 out = Index(new_values, dtype=new_values.dtype, name=self.name, copy=False)
7574 return out
7576 def drop(
7577 self,
7578 labels: Index | np.ndarray | Iterable[Hashable],
7579 errors: IgnoreRaise = "raise",
7580 ) -> Index:
7581 """
7582 Make new Index with passed list of labels deleted.
7584 Unlike :meth:`Index.delete`, which accepts positional indices, this
7585 method removes entries by their label values. The original Index is
7586 not modified.
7588 Parameters
7589 ----------
7590 labels : array-like or scalar
7591 Array-like object or a scalar value, representing the labels to be removed
7592 from the Index.
7593 errors : {'ignore', 'raise'}, default 'raise'
7594 If 'ignore', suppress error and existing labels are dropped.
7596 Returns
7597 -------
7598 Index
7599 Will be same type as self, except for RangeIndex.
7601 Raises
7602 ------
7603 KeyError
7604 If not all of the labels are found in the selected axis
7606 See Also
7607 --------
7608 Index.dropna : Return Index without NA/NaN values.
7609 Index.drop_duplicates : Return Index with duplicate values removed.
7611 Examples
7612 --------
7613 >>> idx = pd.Index(["a", "b", "c"])
7614 >>> idx.drop(["a"])
7615 Index(['b', 'c'], dtype='str')
7616 """
7617 if not isinstance(labels, Index):
7618 # avoid materializing e.g. RangeIndex
7619 arr_dtype = "object" if self.dtype == "object" else None
7620 labels = com.index_labels_to_array(labels, dtype=arr_dtype)
7622 indexer = self.get_indexer_for(labels)
7623 mask = indexer == -1
7624 if mask.any():
7625 if errors != "ignore":
7626 raise KeyError(f"{labels[mask].tolist()} not found in axis")
7627 indexer = indexer[~mask]
7628 return self.delete(indexer)
7630 @final
7631 def infer_objects(self, copy: bool = True) -> Index:
7632 """
7633 If we have an object dtype, try to infer a non-object dtype.
7635 For an Index with ``object`` dtype, this attempts to determine a
7636 more specific dtype (e.g., ``int64`` or ``float64``) from the values.
7638 Parameters
7639 ----------
7640 copy : bool, default True
7641 Whether to make a copy in cases where no inference occurs.
7643 Returns
7644 -------
7645 Index
7646 An Index with a new dtype if the dtype was inferred
7647 or a shallow copy if the dtype could not be inferred.
7649 See Also
7650 --------
7651 Index.inferred_type: Return a string of the type inferred from the values.
7653 Examples
7654 --------
7655 >>> pd.Index(["a", 1]).infer_objects()
7656 Index(['a', 1], dtype='object')
7657 >>> pd.Index([1, 2], dtype="object").infer_objects()
7658 Index([1, 2], dtype='int64')
7659 """
7660 if self._is_multi:
7661 raise NotImplementedError(
7662 "infer_objects is not implemented for MultiIndex. "
7663 "Use index.to_frame().infer_objects() instead."
7664 )
7665 if self.dtype != object:
7666 return self.copy() if copy else self
7668 values = self._values
7669 values = cast("npt.NDArray[np.object_]", values)
7670 res_values = lib.maybe_convert_objects(
7671 values,
7672 convert_non_numeric=True,
7673 )
7674 if copy and res_values is values:
7675 return self.copy()
7676 result = Index(res_values, name=self.name, copy=False)
7677 if not copy and res_values is values and self._references is not None:
7678 result._references = self._references
7679 result._references.add_index_reference(result)
7680 return result
7682 @final
7683 def diff(self, periods: int = 1) -> Index:
7684 """
7685 Computes the difference between consecutive values in the Index object.
7687 If periods is greater than 1, computes the difference between values that
7688 are `periods` number of positions apart.
7690 Parameters
7691 ----------
7692 periods : int, optional
7693 The number of positions between the current and previous
7694 value to compute the difference with. Default is 1.
7696 Returns
7697 -------
7698 Index
7699 A new Index object with the computed differences.
7701 See Also
7702 --------
7703 Series.diff : First discrete difference of Series elements.
7704 DataFrame.diff : First discrete difference of element.
7706 Examples
7707 --------
7708 >>> import pandas as pd
7709 >>> idx = pd.Index([10, 20, 30, 40, 50])
7710 >>> idx.diff()
7711 Index([nan, 10.0, 10.0, 10.0, 10.0], dtype='float64')
7713 """
7714 return Index(self.to_series().diff(periods))
7716 def round(self, decimals: int = 0) -> Self:
7717 """
7718 Round each value in the Index to the given number of decimals.
7720 Parameters
7721 ----------
7722 decimals : int, optional
7723 Number of decimal places to round to. If decimals is negative,
7724 it specifies the number of positions to the left of the decimal point.
7726 Returns
7727 -------
7728 Index
7729 A new Index with the rounded values.
7731 Examples
7732 --------
7733 >>> import pandas as pd
7734 >>> idx = pd.Index([10.1234, 20.5678, 30.9123, 40.4567, 50.7890])
7735 >>> idx.round(decimals=2)
7736 Index([10.12, 20.57, 30.91, 40.46, 50.79], dtype='float64')
7738 """
7739 return self._constructor(self.to_series().round(decimals))
7741 # --------------------------------------------------------------------
7742 # Generated Arithmetic, Comparison, and Unary Methods
7744 def _cmp_method(self, other: object, op: Callable) -> Any:
7745 """
7746 Wrapper used to dispatch comparison operations.
7747 """
7748 if isinstance(other, Index) and self.is_(other):
7749 # fastpath
7750 if op in {operator.eq, operator.le, operator.ge}:
7751 arr = np.ones(len(self), dtype=bool)
7752 if self._can_hold_na and not isinstance(self, ABCMultiIndex):
7753 # TODO: should set MultiIndex._can_hold_na = False?
7754 arr[self.isna()] = False
7755 return arr
7756 elif op is operator.ne:
7757 arr = np.zeros(len(self), dtype=bool)
7758 if self._can_hold_na and not isinstance(self, ABCMultiIndex):
7759 arr[self.isna()] = True
7760 return arr
7762 if isinstance(other, (np.ndarray, Index, ABCSeries, ExtensionArray)) and len(
7763 self
7764 ) != len(other):
7765 raise ValueError("Lengths must match to compare")
7767 if (
7768 not isinstance(self, ABCMultiIndex)
7769 and not isinstance(self._values, ExtensionArray)
7770 and self._values.dtype != object
7771 ):
7772 # MultiIndex and object dtype already treat a non-standard listlike
7773 # as scalar-like; for EA-backed values the warning is emitted
7774 # downstream by the EA (GH#62423).
7775 ops.maybe_warn_listlike(other)
7777 if not isinstance(other, ABCMultiIndex):
7778 other = extract_array(other, extract_numpy=True)
7779 else:
7780 other = np.asarray(other)
7782 result = ops.comparison_op(self._values, other, op)
7784 return result
7786 @final
7787 def _logical_method(self, other: object, op: Callable) -> Index:
7788 res_name = ops.get_op_result_name(self, other) # type: ignore[no-untyped-call]
7790 lvalues = self._values
7791 rvalues = extract_array(other, extract_numpy=True, extract_range=True)
7793 res_values = ops.logical_op(lvalues, rvalues, op)
7794 return self._construct_result(res_values, name=res_name, other=other) # type: ignore[return-value]
7796 @final
7797 def _construct_result(
7798 self, result: ArrayLike, name: Hashable, other: object
7799 ) -> Index | tuple[Index, Index]:
7800 if isinstance(result, tuple):
7801 return (
7802 Index(result[0], name=name, dtype=result[0].dtype, copy=False),
7803 Index(result[1], name=name, dtype=result[1].dtype, copy=False),
7804 )
7805 return Index(result, name=name, dtype=result.dtype, copy=False)
7807 def _arith_method(self, other: object, op: Callable) -> Index:
7808 if (
7809 isinstance(other, Index)
7810 and is_object_dtype(other.dtype)
7811 and type(other) is not Index
7812 ):
7813 # We return NotImplemented for object-dtype index *subclasses* so they have
7814 # a chance to implement ops before we unwrap them.
7815 # See https://github.com/pandas-dev/pandas/issues/31109
7816 return NotImplemented
7818 return super()._arith_method(other, op) # type: ignore[no-untyped-call]
7820 @final
7821 def _unary_method(self, op: Callable) -> Index:
7822 result = op(self._values)
7823 return Index(result, name=self.name, copy=False)
7825 def __abs__(self) -> Index:
7826 return self._unary_method(operator.abs)
7828 def __neg__(self) -> Index:
7829 return self._unary_method(operator.neg)
7831 def __pos__(self) -> Index:
7832 return self._unary_method(operator.pos)
7834 def __invert__(self) -> Index:
7835 # GH#8875
7836 return self._unary_method(operator.inv)
7838 # --------------------------------------------------------------------
7839 # Reductions
7841 def any(self, *args: Any, **kwargs: Any) -> Any:
7842 """
7843 Return whether any element is Truthy.
7845 This is equivalent to calling ``bool(self.any())``. An empty Index
7846 will return False.
7848 Parameters
7849 ----------
7850 *args
7851 Required for compatibility with numpy.
7852 **kwargs
7853 Required for compatibility with numpy.
7855 Returns
7856 -------
7857 bool or array-like (if axis is specified)
7858 A single element array-like may be converted to bool.
7860 See Also
7861 --------
7862 Index.all : Return whether all elements are True.
7863 Series.all : Return whether all elements are True.
7865 Notes
7866 -----
7867 Not a Number (NaN), positive infinity and negative infinity
7868 evaluate to True because these are not equal to zero.
7870 Examples
7871 --------
7872 >>> index = pd.Index([0, 1, 2])
7873 >>> index.any()
7874 True
7876 >>> index = pd.Index([0, 0, 0])
7877 >>> index.any()
7878 False
7879 """
7880 nv.validate_any(args, kwargs)
7881 self._maybe_disable_logical_methods("any")
7882 vals = self._values
7883 if not isinstance(vals, np.ndarray):
7884 # i.e. EA, call _reduce instead of "any" to get TypeError instead
7885 # of AttributeError
7886 return vals._reduce("any")
7887 return maybe_unbox_numpy_scalar(np.any(vals))
7889 def all(self, *args: Any, **kwargs: Any) -> Any:
7890 """
7891 Return whether all elements are Truthy.
7893 This is equivalent to calling ``bool(self.all())``. An empty Index
7894 will return True.
7896 Parameters
7897 ----------
7898 *args
7899 Required for compatibility with numpy.
7900 **kwargs
7901 Required for compatibility with numpy.
7903 Returns
7904 -------
7905 bool or array-like (if axis is specified)
7906 A single element array-like may be converted to bool.
7908 See Also
7909 --------
7910 Index.any : Return whether any element in an Index is True.
7911 Series.any : Return whether any element in a Series is True.
7912 Series.all : Return whether all elements in a Series are True.
7914 Notes
7915 -----
7916 Not a Number (NaN), positive infinity and negative infinity
7917 evaluate to True because these are not equal to zero.
7919 Examples
7920 --------
7921 True, because nonzero integers are considered True.
7923 >>> pd.Index([1, 2, 3]).all()
7924 True
7926 False, because ``0`` is considered False.
7928 >>> pd.Index([0, 1, 2]).all()
7929 False
7930 """
7931 nv.validate_all(args, kwargs)
7932 self._maybe_disable_logical_methods("all")
7933 vals = self._values
7934 if not isinstance(vals, np.ndarray):
7935 # i.e. EA, call _reduce instead of "all" to get TypeError instead
7936 # of AttributeError
7937 return vals._reduce("all")
7938 return maybe_unbox_numpy_scalar(np.all(vals))
7940 @final
7941 def _maybe_disable_logical_methods(self, opname: str_t) -> None:
7942 """
7943 raise if this Index subclass does not support any or all.
7944 """
7945 if isinstance(self, ABCMultiIndex):
7946 raise TypeError(f"cannot perform {opname} with {type(self).__name__}")
7948 def argmin(
7949 self,
7950 axis: AxisInt | None = None,
7951 skipna: bool = True,
7952 *args: Any,
7953 **kwargs: Any,
7954 ) -> int:
7955 """
7956 Return int position of the smallest value in the Index.
7958 If the minimum is achieved in multiple locations,
7959 the first row position is returned.
7961 Parameters
7962 ----------
7963 axis : None
7964 Unused. Parameter needed for compatibility with DataFrame.
7965 skipna : bool, default True
7966 Exclude NA/null values. If the entire Series is NA, or if ``skipna=False``
7967 and there is an NA value, this method will raise a ``ValueError``.
7968 *args, **kwargs
7969 Additional arguments and keywords for compatibility with NumPy.
7971 Returns
7972 -------
7973 int
7974 Row position of the minimum value.
7976 See Also
7977 --------
7978 Series.argmin : Return position of the minimum value.
7979 Series.argmax : Return position of the maximum value.
7980 numpy.ndarray.argmin : Equivalent method for numpy arrays.
7981 Series.idxmin : Return index label of the minimum values.
7982 Series.idxmax : Return index label of the maximum values.
7984 Examples
7985 --------
7986 Consider dataset containing cereal calories
7988 >>> idx = pd.Index([100.0, 110.0, 120.0, 110.0])
7989 >>> idx
7990 Index([100.0, 110.0, 120.0, 110.0], dtype='float64')
7992 >>> idx.argmax()
7993 2
7994 >>> idx.argmin()
7995 0
7997 The maximum cereal calories is the third element and
7998 the minimum cereal calories is the first element,
7999 since index is zero-indexed.
8000 """
8001 nv.validate_argmin(args, kwargs)
8002 nv.validate_minmax_axis(axis)
8004 if not self._is_multi and self.hasnans:
8005 if not skipna:
8006 raise ValueError("Encountered an NA value with skipna=False")
8007 elif self._isnan.all():
8008 raise ValueError("Encountered all NA values")
8010 return super().argmin(skipna=skipna)
8012 def argmax(
8013 self,
8014 axis: AxisInt | None = None,
8015 skipna: bool = True,
8016 *args: Any,
8017 **kwargs: Any,
8018 ) -> int:
8019 """
8020 Return int position of the largest value in the Index.
8022 If the maximum is achieved in multiple locations,
8023 the first row position is returned.
8025 Parameters
8026 ----------
8027 axis : None
8028 Unused. Parameter needed for compatibility with DataFrame.
8029 skipna : bool, default True
8030 Exclude NA/null values. If the entire Series is NA, or if ``skipna=False``
8031 and there is an NA value, this method will raise a ``ValueError``.
8032 *args, **kwargs
8033 Additional arguments and keywords for compatibility with NumPy.
8035 Returns
8036 -------
8037 int
8038 Row position of the maximum value.
8040 See Also
8041 --------
8042 Series.argmax : Return position of the maximum value.
8043 Series.argmin : Return position of the minimum value.
8044 numpy.ndarray.argmax : Equivalent method for numpy arrays.
8045 Series.idxmax : Return index label of the maximum values.
8046 Series.idxmin : Return index label of the minimum values.
8048 Examples
8049 --------
8050 Consider dataset containing cereal calories
8052 >>> idx = pd.Index([100.0, 110.0, 120.0, 110.0])
8053 >>> idx
8054 Index([100.0, 110.0, 120.0, 110.0], dtype='float64')
8056 >>> idx.argmax()
8057 2
8058 >>> idx.argmin()
8059 0
8061 The maximum cereal calories is the third element and
8062 the minimum cereal calories is the first element,
8063 since index is zero-indexed.
8064 """
8065 nv.validate_argmax(args, kwargs)
8066 nv.validate_minmax_axis(axis)
8068 if not self._is_multi and self.hasnans:
8069 if not skipna:
8070 raise ValueError("Encountered an NA value with skipna=False")
8071 elif self._isnan.all():
8072 raise ValueError("Encountered all NA values")
8073 return super().argmax(skipna=skipna)
8075 def min(
8076 self,
8077 axis: AxisInt | None = None,
8078 skipna: bool = True,
8079 *args: Any,
8080 **kwargs: Any,
8081 ) -> object:
8082 """
8083 Return the minimum value of the Index.
8085 The minimum is computed by comparing all values in the Index.
8086 NA/null values are excluded by default.
8088 Parameters
8089 ----------
8090 axis : {None}
8091 Dummy argument for consistency with Series.
8092 skipna : bool, default True
8093 Exclude NA/null values when showing the result.
8094 *args, **kwargs
8095 Additional arguments and keywords for compatibility with NumPy.
8097 Returns
8098 -------
8099 scalar
8100 Minimum value.
8102 See Also
8103 --------
8104 Index.max : Return the maximum value of the object.
8105 Series.min : Return the minimum value in a Series.
8106 DataFrame.min : Return the minimum values in a DataFrame.
8108 Examples
8109 --------
8110 >>> idx = pd.Index([3, 2, 1])
8111 >>> idx.min()
8112 1
8114 >>> idx = pd.Index(["c", "b", "a"])
8115 >>> idx.min()
8116 'a'
8118 For a MultiIndex, the minimum is determined lexicographically.
8120 >>> idx = pd.MultiIndex.from_product([("a", "b"), (2, 1)])
8121 >>> idx.min()
8122 ('a', 1)
8123 """
8124 nv.validate_min(args, kwargs)
8125 nv.validate_minmax_axis(axis)
8127 if not len(self):
8128 return self._na_value
8130 if len(self) and self.is_monotonic_increasing:
8131 # quick check
8132 first = self[0]
8133 if not isna(first):
8134 return maybe_unbox_numpy_scalar(first, object_with_dtype=self)
8136 if not self._is_multi and self.hasnans:
8137 # Take advantage of cache
8138 mask = self._isnan
8139 if not skipna or mask.all():
8140 return self._na_value
8142 if not self._is_multi and not isinstance(self._values, np.ndarray):
8143 return self._values._reduce(name="min", skipna=skipna)
8145 return maybe_unbox_numpy_scalar(
8146 nanops.nanmin(self._values, skipna=skipna), object_with_dtype=self
8147 )
8149 def max(
8150 self,
8151 axis: AxisInt | None = None,
8152 skipna: bool = True,
8153 *args: Any,
8154 **kwargs: Any,
8155 ) -> object:
8156 """
8157 Return the maximum value of the Index.
8159 The maximum is computed by comparing all values in the Index.
8160 NA/null values are excluded by default.
8162 Parameters
8163 ----------
8164 axis : int, optional
8165 For compatibility with NumPy. Only 0 or None are allowed.
8166 skipna : bool, default True
8167 Exclude NA/null values when showing the result.
8168 *args, **kwargs
8169 Additional arguments and keywords for compatibility with NumPy.
8171 Returns
8172 -------
8173 scalar
8174 Maximum value.
8176 See Also
8177 --------
8178 Index.min : Return the minimum value in an Index.
8179 Series.max : Return the maximum value in a Series.
8180 DataFrame.max : Return the maximum values in a DataFrame.
8182 Examples
8183 --------
8184 >>> idx = pd.Index([3, 2, 1])
8185 >>> idx.max()
8186 3
8188 >>> idx = pd.Index(["c", "b", "a"])
8189 >>> idx.max()
8190 'c'
8192 For a MultiIndex, the maximum is determined lexicographically.
8194 >>> idx = pd.MultiIndex.from_product([("a", "b"), (2, 1)])
8195 >>> idx.max()
8196 ('b', 2)
8197 """
8199 nv.validate_max(args, kwargs)
8200 nv.validate_minmax_axis(axis)
8202 if not len(self):
8203 return self._na_value
8205 if len(self) and self.is_monotonic_increasing:
8206 # quick check
8207 last = self[-1]
8208 if not isna(last):
8209 return maybe_unbox_numpy_scalar(last, object_with_dtype=self)
8211 if not self._is_multi and self.hasnans:
8212 # Take advantage of cache
8213 mask = self._isnan
8214 if not skipna or mask.all():
8215 return maybe_unbox_numpy_scalar(self._na_value)
8217 if not self._is_multi and not isinstance(self._values, np.ndarray):
8218 return self._values._reduce(name="max", skipna=skipna)
8220 return maybe_unbox_numpy_scalar(
8221 nanops.nanmax(self._values, skipna=skipna), object_with_dtype=self
8222 )
8224 # --------------------------------------------------------------------
8226 @final
8227 @property
8228 def shape(self) -> Shape:
8229 """
8230 Return a tuple of the shape of the underlying data.
8232 Since an Index is always one-dimensional, this returns a tuple
8233 with a single element equal to the number of elements in the Index.
8235 See Also
8236 --------
8237 Index.size: Return the number of elements in the underlying data.
8238 Index.ndim: Number of dimensions of the underlying data, by definition 1.
8239 Index.dtype: Return the dtype object of the underlying data.
8240 Index.values: Return an array representing the data in the Index.
8242 Examples
8243 --------
8244 >>> idx = pd.Index([1, 2, 3])
8245 >>> idx
8246 Index([1, 2, 3], dtype='int64')
8247 >>> idx.shape
8248 (3,)
8249 """
8250 # See GH#27775, GH#27384 for history/reasoning in how this is defined.
8251 return (len(self),)
8254def maybe_sequence_to_range(sequence: Axes) -> Axes:
8255 """
8256 Convert a 1D, non-pandas sequence to a range if possible.
8258 Returns the input if not possible.
8260 Parameters
8261 ----------
8262 sequence : 1D sequence
8264 Returns
8265 -------
8266 Any : input or range
8267 """
8268 if isinstance(sequence, (range, ExtensionArray)):
8269 return sequence
8270 elif len(sequence) == 1 or lib.infer_dtype(sequence, skipna=False) != "integer":
8271 return sequence
8272 elif isinstance(sequence, (ABCSeries, Index)) and not (
8273 isinstance(sequence.dtype, np.dtype) and sequence.dtype.kind == "i"
8274 ):
8275 return sequence
8276 if len(sequence) == 0:
8277 return range(0)
8278 try:
8279 np_sequence = np.asarray(sequence, dtype=np.int64)
8280 except OverflowError:
8281 return sequence
8282 if (
8283 isinstance(sequence, np.ndarray)
8284 and sequence.dtype.kind == "u"
8285 and (np_sequence < 0).any()
8286 ):
8287 # GH#64148: uint64 values above INT64_MAX wrapped negative in the cast
8288 return sequence
8289 # GH#64148: use python ints so widely-separated values (e.g. INT64_MIN and
8290 # INT64_MAX) don't overflow the int64 domain RangeIndex relies on.
8291 first = int(np_sequence[0])
8292 diff = int(np_sequence[1]) - first
8293 if diff == 0 or not -(2**63) <= diff < 2**63:
8294 return sequence
8295 if len(sequence) != 2 and not lib.is_sequence_range(np_sequence, diff):
8296 return sequence
8297 stop = int(np_sequence[-1]) + diff
8298 if not -(2**63) <= stop < 2**63:
8299 return sequence
8300 # is_sequence_range matches modularly, so a sequence that wraps past
8301 # INT64_MAX can slip through; such a match has the wrong length.
8302 candidate = range(first, stop, diff)
8303 if len(candidate) == len(sequence):
8304 return candidate
8305 return sequence
8308def ensure_index_from_sequences(
8309 sequences: Sequence[Axes], names: Sequence[Hashable] | None = None
8310) -> Index:
8311 """
8312 Construct an index from sequences of data.
8314 A single sequence returns an Index. Many sequences returns a
8315 MultiIndex.
8317 Parameters
8318 ----------
8319 sequences : sequence of sequences
8320 names : sequence of str
8322 Returns
8323 -------
8324 index : Index or MultiIndex
8326 Examples
8327 --------
8328 >>> ensure_index_from_sequences([[1, 2, 4]], names=["name"])
8329 Index([1, 2, 4], dtype='int64', name='name')
8331 >>> ensure_index_from_sequences([["a", "a"], ["a", "b"]], names=["L1", "L2"])
8332 MultiIndex([('a', 'a'),
8333 ('a', 'b')],
8334 names=['L1', 'L2'])
8336 See Also
8337 --------
8338 ensure_index
8339 """
8340 from pandas.core.indexes.api import default_index
8341 from pandas.core.indexes.multi import MultiIndex
8343 if len(sequences) == 0:
8344 return default_index(0)
8345 elif len(sequences) == 1:
8346 if names is not None:
8347 names = names[0] # type: ignore[assignment]
8348 return Index(maybe_sequence_to_range(sequences[0]), name=names)
8349 else:
8350 # TODO: Apply maybe_sequence_to_range to sequences?
8351 return MultiIndex.from_arrays(sequences, names=names)
8354def ensure_index(index_like: Axes, copy: bool = False) -> Index:
8355 """
8356 Ensure that we have an index from some index-like object.
8358 Parameters
8359 ----------
8360 index_like : sequence
8361 An Index or other sequence
8362 copy : bool, default False
8364 Returns
8365 -------
8366 index : Index or MultiIndex
8368 See Also
8369 --------
8370 ensure_index_from_sequences
8372 Examples
8373 --------
8374 >>> ensure_index(["a", "b"])
8375 Index(['a', 'b'], dtype='str')
8377 >>> ensure_index([("a", "a"), ("b", "c")])
8378 Index([('a', 'a'), ('b', 'c')], dtype='object')
8380 >>> ensure_index([["a", "a"], ["b", "c"]])
8381 MultiIndex([('a', 'b'),
8382 ('a', 'c')],
8383 )
8384 """
8385 if isinstance(index_like, Index):
8386 if copy:
8387 index_like = index_like.copy()
8388 return index_like
8390 if isinstance(index_like, ABCSeries):
8391 name = index_like.name
8392 return Index(index_like, name=name, copy=copy)
8394 if is_iterator(index_like):
8395 index_like = list(index_like)
8397 if isinstance(index_like, list):
8398 if type(index_like) is not list:
8399 # must check for exactly list here because of strict type
8400 # check in clean_index_list
8401 index_like = list(index_like)
8403 if index_like and lib.is_all_arraylike(index_like):
8404 from pandas.core.indexes.multi import MultiIndex
8406 return MultiIndex.from_arrays(index_like)
8407 else:
8408 return Index(index_like, copy=copy, tupleize_cols=False)
8409 else:
8410 return Index(index_like, copy=copy)
8413def trim_front(strings: list[str]) -> list[str]:
8414 """
8415 Trims leading spaces evenly among all strings.
8417 Examples
8418 --------
8419 >>> trim_front([" a", " b"])
8420 ['a', 'b']
8422 >>> trim_front([" a", " "])
8423 ['a', '']
8424 """
8425 if not strings:
8426 return strings
8427 smallest_leading_space = min(len(x) - len(x.lstrip()) for x in strings)
8428 if smallest_leading_space > 0:
8429 strings = [x[smallest_leading_space:] for x in strings]
8430 return strings
8433def _slice_bound_tzinfo(bound: str | datetime) -> tzinfo | None:
8434 """
8435 tzinfo of a ``slice_locs`` bound, for the GH#16785 UTC-offset comparison.
8437 GH#50907: this parse is internal, so a quarterly string bound must not emit
8438 the deprecation -- on a :class:`DatetimeIndex` that would duplicate the one
8439 ``get_slice_bound`` gives, and on a :class:`PeriodIndex` or an object Index
8440 it would be spurious. A quarterly string is always naive, so the opted-out
8441 parse is needed only to recognize one; anything else falls through to
8442 ``Timestamp``, which does not warn.
8443 """
8444 if isinstance(bound, str) and ("Q" in bound or "q" in bound):
8445 # GH#45580 parse_datetime_string_with_reso rejects a non-exact str
8446 parsed, reso = parse_datetime_string_with_reso(str(bound), warn_quarter=False)
8447 if reso == "quarter":
8448 return parsed.tzinfo
8449 return Timestamp(bound).tzinfo
8452def _validate_join_method(method: str) -> None:
8453 if method not in ["left", "right", "inner", "outer"]:
8454 raise ValueError(f"do not recognize join method {method}")
8457def maybe_extract_name(name: Hashable, obj: object, cls: type) -> Hashable:
8458 """
8459 If no name is passed, then extract it from data, validating hashability.
8460 """
8461 if name is None and isinstance(obj, (Index, ABCSeries)):
8462 # Note we don't just check for "name" attribute since that would
8463 # pick up e.g. dtype.name
8464 name = obj.name
8466 # GH#29069
8467 if not is_hashable(name):
8468 raise TypeError(f"{cls.__name__}.name must be a hashable type")
8470 return name
8473def get_unanimous_names(*indexes: Index) -> tuple[Hashable, ...]:
8474 """
8475 Return common name if all indices agree, otherwise None (level-by-level).
8477 Parameters
8478 ----------
8479 indexes : list of Index objects
8481 Returns
8482 -------
8483 list
8484 A list representing the unanimous 'names' found.
8485 """
8486 name_tups = (tuple(i.names) for i in indexes)
8487 name_sets = ({*ns} for ns in zip_longest(*name_tups))
8488 names = tuple(ns.pop() if len(ns) == 1 else None for ns in name_sets)
8489 return names
8492def droplevel_result(
8493 new_levels: list[Index],
8494 new_codes: list[npt.NDArray[np.intp]],
8495 new_names: list[Hashable],
8496) -> Index:
8497 """
8498 Construct the result of dropping MultiIndex levels from the
8499 retained levels, codes, and names.
8500 """
8501 if len(new_levels) == 1:
8502 lev = new_levels[0]
8504 if len(lev) == 0 and len(new_codes[0]) == 0:
8505 # GH#42055, GH#45230 preserve RangeIndex here
8506 result = lev[:0]
8507 else:
8508 result = lev.take(new_codes[0], allow_fill=True)
8509 result._name = new_names[0]
8511 return result
8512 else:
8513 from pandas.core.indexes.multi import MultiIndex
8515 return MultiIndex(
8516 levels=new_levels,
8517 codes=new_codes,
8518 names=new_names,
8519 verify_integrity=False,
8520 )
8523def _unpack_nested_dtype(other: Index) -> DtypeObj:
8524 """
8525 When checking if our dtype is comparable with another, we need
8526 to unpack CategoricalDtype to look at its categories.dtype.
8528 Parameters
8529 ----------
8530 other : Index
8532 Returns
8533 -------
8534 np.dtype or ExtensionDtype
8535 """
8536 dtype = other.dtype
8537 if isinstance(dtype, CategoricalDtype):
8538 # If there is ever a SparseIndex, this could get dispatched
8539 # here too.
8540 return dtype.categories.dtype
8541 elif isinstance(dtype, ArrowDtype):
8542 # GH 53617
8543 import pyarrow as pa
8545 if pa.types.is_dictionary(dtype.pyarrow_dtype):
8546 other = other[:0].astype(ArrowDtype(dtype.pyarrow_dtype.value_type))
8547 return other.dtype
8550def _maybe_try_sort(result: Index | ArrayLike, sort: bool | None) -> Index | ArrayLike:
8551 if sort is not False:
8552 try:
8553 # error: Incompatible types in assignment (expression has type
8554 # "Union[ExtensionArray, ndarray[Any, Any], Index, Series,
8555 # Tuple[Union[Union[ExtensionArray, ndarray[Any, Any]], Index, Series],
8556 # ndarray[Any, Any]]]", variable has type "Union[Index,
8557 # Union[ExtensionArray, ndarray[Any, Any]]]")
8558 result = algos.safe_sort(result) # type: ignore[assignment]
8559 except TypeError as err:
8560 if sort is True:
8561 raise
8562 warnings.warn(
8563 f"{err}, sort order is undefined for incomparable objects.",
8564 RuntimeWarning,
8565 stacklevel=find_stack_level(),
8566 )
8567 return result
8570def get_values_for_csv(
8571 values: ArrayLike,
8572 *,
8573 date_format: str_t | None,
8574 na_rep: str = "nan",
8575 quoting: int | None = None,
8576 float_format: str_t | None = None,
8577 decimal: str = ".",
8578) -> npt.NDArray[np.object_]:
8579 """
8580 Convert to types which can be consumed by the standard library's
8581 csv.writer.writerows.
8582 """
8583 if isinstance(values, Categorical) and values.categories.dtype.kind in "Mm":
8584 # GH#40754 Convert categorical datetimes to datetime array.
8585 # Fill missing with NaT (not na_rep) to avoid dtype promotion (GH#53910);
8586 # downstream _format_native_types handles NaT -> na_rep.
8587 values = algos.take_nd(
8588 values.categories._values,
8589 ensure_platform_int(values._codes),
8590 )
8592 values = ensure_wrapped_if_datetimelike(values) # type: ignore[no-untyped-call]
8594 if isinstance(values, (DatetimeArray, TimedeltaArray, PeriodArray)):
8595 if values.ndim == 1:
8596 result = values._format_native_types(na_rep=na_rep, date_format=date_format)
8597 result = result.astype(object, copy=False)
8598 return result
8600 # GH#21734 Process every column separately, they might have different formats
8601 results_converted = []
8602 for i in range(len(values)):
8603 result = values[i, :]._format_native_types(
8604 na_rep=na_rep, date_format=date_format
8605 )
8606 results_converted.append(result.astype(object, copy=False))
8607 return np.vstack(results_converted)
8609 elif isinstance(values.dtype, IntervalDtype):
8610 values = cast("IntervalArray", values)
8611 mask = values.isna()
8612 if not quoting:
8613 result = np.asarray(values).astype(str)
8614 else:
8615 result = np.array(values, dtype=object, copy=True)
8617 result[mask] = na_rep
8618 return result
8620 elif values.dtype.kind == "f" and not isinstance(values.dtype, SparseDtype):
8621 # see GH#13418: no special formatting is desired at the
8622 # output (important for appropriate 'quoting' behaviour),
8623 # so do not pass it through the FloatArrayFormatter
8624 if float_format is None and decimal == ".":
8625 mask = isna(values)
8627 if not quoting:
8628 if isinstance(values, ExtensionArray) and values.ndim == 2:
8629 # e.g. test_to_csv_2d_float_ea
8630 values = np.asarray(values.to_numpy(), dtype=str)
8631 else:
8632 values = values.astype(str)
8633 else:
8634 if isinstance(values, ExtensionArray):
8635 values = values.to_numpy(na_value=np.nan)
8636 if values.dtype.itemsize < 8:
8637 # GH#60699 keep numpy scalars; Python floats would be
8638 # written with the float64 repr, e.g. 8.569999694824219
8639 result = np.empty(values.size, dtype=object)
8640 result[:] = list(values.ravel())
8641 values = result.reshape(values.shape)
8642 else:
8643 values = np.array(values, dtype="object")
8645 values[mask] = na_rep
8646 values = values.astype(object, copy=False)
8647 return values
8649 from pandas.io.formats.format import FloatArrayFormatter
8651 formatter = FloatArrayFormatter(
8652 values,
8653 na_rep=na_rep,
8654 float_format=float_format,
8655 decimal=decimal,
8656 quoting=quoting,
8657 fixed_width=False,
8658 )
8659 res = formatter.get_result_as_array()
8660 res = res.astype(object, copy=False)
8661 return res
8663 elif isinstance(values, ExtensionArray):
8664 mask = isna(values)
8666 new_values = np.asarray(values.astype(object))
8667 new_values[mask] = na_rep
8668 return new_values
8670 else:
8671 mask = isna(values)
8672 itemsize = writers.word_len(na_rep)
8674 if values.dtype != _dtype_obj and not quoting and itemsize:
8675 values = values.astype(str)
8676 if values.dtype.itemsize / np.dtype("U1").itemsize < itemsize:
8677 # enlarge for the na_rep
8678 values = values.astype(f"<U{itemsize}")
8679 elif date_format is not None and values.dtype == _dtype_obj:
8680 # GH#27306 match the formatting of datetime64 and Period arrays
8681 def _format(val: object) -> object:
8682 if val is not NaT and isinstance(val, (datetime, Period)):
8683 return val.strftime(date_format)
8684 return val
8686 values = lib.map_infer(values.ravel(), _format, convert=False)
8687 values = values.reshape(mask.shape)
8688 else:
8689 values = np.array(values, dtype="object")
8691 values[mask] = na_rep
8692 values = values.astype(object, copy=False)
8693 return values