Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from typing import (
4 TYPE_CHECKING,
5 Any,
6 Literal,
7 Self,
8 cast,
9)
11import numpy as np
13from pandas._libs import index as libindex
14from pandas.util._decorators import (
15 cache_readonly,
16 set_module,
17)
19from pandas.core.dtypes.common import is_scalar
20from pandas.core.dtypes.dtypes import CategoricalDtype
21from pandas.core.dtypes.missing import (
22 is_valid_na_for_dtype,
23)
25from pandas.core.arrays.categorical import (
26 Categorical,
27 contains,
28)
29from pandas.core.construction import extract_array
30from pandas.core.indexes.base import (
31 Index,
32 maybe_extract_name,
33)
34from pandas.core.indexes.extension import (
35 NDArrayBackedExtensionIndex,
36 inherit_names,
37)
39if TYPE_CHECKING:
40 from collections.abc import Hashable
42 from pandas._typing import (
43 Dtype,
44 DtypeObj,
45 npt,
46 )
49@inherit_names(
50 [
51 "argsort",
52 "tolist",
53 "codes",
54 "categories",
55 "ordered",
56 "_reverse_indexer",
57 "searchsorted",
58 "min",
59 "max",
60 ],
61 Categorical,
62)
63@inherit_names(
64 [
65 "rename_categories",
66 "reorder_categories",
67 "add_categories",
68 "remove_categories",
69 "remove_unused_categories",
70 "set_categories",
71 "as_ordered",
72 "as_unordered",
73 ],
74 Categorical,
75 wrap=True,
76)
77@set_module("pandas")
78class CategoricalIndex(NDArrayBackedExtensionIndex):
79 """
80 Index based on an underlying :class:`Categorical`.
82 CategoricalIndex, like Categorical, can only take on a limited,
83 and usually fixed, number of possible values (`categories`). Also,
84 like Categorical, it might have an order, but numerical operations
85 (additions, divisions, ...) are not possible.
87 Parameters
88 ----------
89 data : array-like (1-dimensional)
90 The values of the categorical. If `categories` are given, values not in
91 `categories` will be replaced with NaN.
92 categories : index-like, optional
93 The categories for the categorical. Items need to be unique.
94 If the categories are not given here (and also not in `dtype`), they
95 will be inferred from the `data`.
96 ordered : bool, optional
97 Whether or not this categorical is treated as an ordered
98 categorical. If not given here or in `dtype`, the resulting
99 categorical will be unordered.
100 dtype : CategoricalDtype or "category", optional
101 If :class:`CategoricalDtype`, cannot be used together with
102 `categories` or `ordered`.
103 copy : bool, default False
104 Make a copy of input ndarray.
105 name : object, optional
106 Name to be stored in the index.
108 Attributes
109 ----------
110 codes
111 categories
112 ordered
114 Methods
115 -------
116 rename_categories
117 reorder_categories
118 add_categories
119 remove_categories
120 remove_unused_categories
121 set_categories
122 as_ordered
123 as_unordered
124 map
126 Raises
127 ------
128 ValueError
129 If the categories do not validate.
130 TypeError
131 If an explicit ``ordered=True`` is given but no `categories` and the
132 `values` are not sortable.
134 See Also
135 --------
136 Index : The base pandas Index type.
137 Categorical : A categorical array.
138 CategoricalDtype : Type for categorical data.
140 Notes
141 -----
142 See the `user guide
143 <https://pandas.pydata.org/pandas-docs/stable/user_guide/advanced.html#categoricalindex>`__
144 for more.
146 Examples
147 --------
148 >>> pd.CategoricalIndex(["a", "b", "c", "a", "b", "c"])
149 CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
150 categories=['a', 'b', 'c'], ordered=False, dtype='category')
152 ``CategoricalIndex`` can also be instantiated from a ``Categorical``:
154 >>> c = pd.Categorical(["a", "b", "c", "a", "b", "c"])
155 >>> pd.CategoricalIndex(c)
156 CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
157 categories=['a', 'b', 'c'], ordered=False, dtype='category')
159 Ordered ``CategoricalIndex`` can have a min and max value.
161 >>> ci = pd.CategoricalIndex(
162 ... ["a", "b", "c", "a", "b", "c"], ordered=True, categories=["c", "b", "a"]
163 ... )
164 >>> ci
165 CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
166 categories=['c', 'b', 'a'], ordered=True, dtype='category')
167 >>> ci.min()
168 'c'
169 """
171 _typ = "categoricalindex"
172 _data_cls = Categorical
174 @property
175 def _can_hold_strings(self):
176 return self.categories._can_hold_strings
178 @cache_readonly
179 def _should_fallback_to_positional(self) -> bool:
180 return self.categories._should_fallback_to_positional
182 codes: np.ndarray
183 categories: Index
184 ordered: bool | None
185 _data: Categorical
186 _values: Categorical
188 @property
189 def _engine_type(self) -> type[libindex.IndexEngine]:
190 # self.codes can have dtype int8, int16, int32 or int64, so we need
191 # to return the corresponding engine type (libindex.Int8Engine, etc.).
192 return {
193 np.int8: libindex.Int8Engine,
194 np.int16: libindex.Int16Engine,
195 np.int32: libindex.Int32Engine,
196 np.int64: libindex.Int64Engine,
197 }[self.codes.dtype.type]
199 # --------------------------------------------------------------------
200 # Constructors
202 def __new__(
203 cls,
204 data=None,
205 categories=None,
206 ordered=None,
207 dtype: Dtype | None = None,
208 copy: bool = False,
209 name: Hashable | None = None,
210 ) -> Self:
211 name = maybe_extract_name(name, data, cls)
213 if is_scalar(data):
214 # GH#38944 include None here, which pre-2.0 subbed in []
215 cls._raise_scalar_data_error(data)
217 data = Categorical(
218 data, categories=categories, ordered=ordered, dtype=dtype, copy=copy
219 )
221 return cls._simple_new(data, name=name)
223 # --------------------------------------------------------------------
225 def _is_dtype_compat(self, other: Index) -> Categorical:
226 """
227 *this is an internal non-public method*
229 provide a comparison between the dtype of self and other (coercing if
230 needed)
232 Parameters
233 ----------
234 other : Index
236 Returns
237 -------
238 Categorical
240 Raises
241 ------
242 TypeError if the dtypes are not compatible
243 """
244 if isinstance(other.dtype, CategoricalDtype):
245 cat = extract_array(other)
246 cat = cast(Categorical, cat)
247 if not cat._categories_match_up_to_permutation(self._values):
248 raise TypeError(
249 "categories must match existing categories when appending"
250 )
252 elif other._is_multi:
253 # preempt raising NotImplementedError in isna call
254 raise TypeError("MultiIndex is not dtype-compatible with CategoricalIndex")
255 else:
256 values = other
258 codes = self.categories.get_indexer(values)
259 if ((codes == -1) & ~values.isna()).any():
260 # GH#37667 see test_equals_non_category
261 raise TypeError(
262 "categories must match existing categories when appending"
263 )
264 cat = Categorical(other, dtype=self.dtype)
265 other = CategoricalIndex(cat)
266 if not other.isin(values).all():
267 raise TypeError(
268 "cannot append a non-category item to a CategoricalIndex"
269 )
270 cat = other._values
272 return cat
274 def equals(self, other: object) -> bool:
275 """
276 Determine if two CategoricalIndex objects contain the same elements.
278 The order and orderedness of elements matters. The categories matter,
279 but the order of the categories matters only when ``ordered=True``.
281 Parameters
282 ----------
283 other : object
284 The CategoricalIndex object to compare with.
286 Returns
287 -------
288 bool
289 ``True`` if two :class:`pandas.CategoricalIndex` objects have equal
290 elements, ``False`` otherwise.
292 See Also
293 --------
294 Categorical.equals : Returns True if categorical arrays are equal.
296 Examples
297 --------
298 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a", "b", "c"])
299 >>> ci2 = pd.CategoricalIndex(pd.Categorical(["a", "b", "c", "a", "b", "c"]))
300 >>> ci.equals(ci2)
301 True
303 The order of elements matters.
305 >>> ci3 = pd.CategoricalIndex(["c", "b", "a", "a", "b", "c"])
306 >>> ci.equals(ci3)
307 False
309 The orderedness also matters.
311 >>> ci4 = ci.as_ordered()
312 >>> ci.equals(ci4)
313 False
315 The categories matter, but the order of the categories matters only when
316 ``ordered=True``.
318 >>> ci5 = ci.set_categories(["a", "b", "c", "d"])
319 >>> ci.equals(ci5)
320 False
322 >>> ci6 = ci.set_categories(["b", "c", "a"])
323 >>> ci.equals(ci6)
324 True
325 >>> ci_ordered = pd.CategoricalIndex(
326 ... ["a", "b", "c", "a", "b", "c"], ordered=True
327 ... )
328 >>> ci2_ordered = ci_ordered.set_categories(["b", "c", "a"])
329 >>> ci_ordered.equals(ci2_ordered)
330 False
331 """
332 if self.is_(other):
333 return True
335 if not isinstance(other, Index):
336 return False
338 try:
339 other = self._is_dtype_compat(other)
340 except (TypeError, ValueError):
341 return False
343 return self._data.equals(other)
345 # --------------------------------------------------------------------
346 # Rendering Methods
348 @property
349 def _formatter_func(self):
350 return self.categories._formatter_func
352 def _format_attrs(self):
353 """
354 Return a list of tuples of the (attr,formatted_value)
355 """
356 attrs: list[tuple[str, str | int | bool | None]]
358 attrs = [
359 (
360 "categories",
361 f"[{', '.join(self._data._repr_categories())}]",
362 ),
363 ("ordered", self.ordered),
364 ]
365 extra = super()._format_attrs()
366 return attrs + extra
368 # --------------------------------------------------------------------
370 @property
371 def inferred_type(self) -> str:
372 return "categorical"
374 def __contains__(self, key: Any) -> bool:
375 """
376 Return a boolean indicating whether the provided key is in the index.
378 Parameters
379 ----------
380 key : label
381 The key to check if it is present in the index.
383 Returns
384 -------
385 bool
386 Whether the key search is in the index.
388 Raises
389 ------
390 TypeError
391 If the key is not hashable.
393 See Also
394 --------
395 Index.isin : Returns an ndarray of boolean dtype indicating whether the
396 list-like key is in the index.
398 Examples
399 --------
400 >>> idx = pd.Index([1, 2, 3, 4])
401 >>> idx
402 Index([1, 2, 3, 4], dtype='int64')
404 >>> 2 in idx
405 True
406 >>> 6 in idx
407 False
408 """
409 # if key is a NaN, check if any NaN is in self.
410 if is_valid_na_for_dtype(key, self.categories.dtype):
411 return self.hasnans
412 if self.categories._typ == "rangeindex":
413 container: Index | libindex.IndexEngine | libindex.ExtensionEngine = (
414 self.categories
415 )
416 else:
417 container = self._engine
418 return contains(self, key, container=container)
420 def reindex(
421 self, target, method=None, level=None, limit: int | None = None, tolerance=None
422 ) -> tuple[Index, npt.NDArray[np.intp] | None]:
423 """
424 Create index with target's values (move/add/delete values as necessary)
426 Returns
427 -------
428 new_index : pd.Index
429 Resulting index
430 indexer : np.ndarray[np.intp] or None
431 Indices of output values in original index
433 """
434 if method is not None:
435 raise NotImplementedError(
436 "argument method is not implemented for CategoricalIndex.reindex"
437 )
438 if level is not None:
439 raise NotImplementedError(
440 "argument level is not implemented for CategoricalIndex.reindex"
441 )
442 if limit is not None:
443 raise NotImplementedError(
444 "argument limit is not implemented for CategoricalIndex.reindex"
445 )
446 return super().reindex(target)
448 # --------------------------------------------------------------------
449 # Indexing Methods
451 def _maybe_cast_indexer(self, key) -> int:
452 # GH#41933: we have to do this instead of self._data._validate_scalar
453 # because this will correctly get partial-indexing on Interval categories
454 try:
455 return self._data._unbox_scalar(key)
456 except KeyError:
457 if is_valid_na_for_dtype(key, self.categories.dtype):
458 return -1
459 raise
461 def _maybe_cast_listlike_indexer(self, values) -> CategoricalIndex:
462 if isinstance(values, CategoricalIndex):
463 values = values._data
464 if isinstance(values, Categorical):
465 # Indexing on codes is more efficient if categories are the same,
466 # so we can apply some optimizations based on the degree of
467 # dtype-matching.
468 cat = self._data._encode_with_my_categories(values)
469 codes = cat._codes
470 else:
471 codes = self.categories.get_indexer(values)
472 codes = codes.astype(self.codes.dtype, copy=False)
473 cat = self._data._from_backing_data(codes)
474 return type(self)._simple_new(cat)
476 # --------------------------------------------------------------------
478 def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
479 return self.categories._is_comparable_dtype(dtype)
481 def map(self, mapper, na_action: Literal["ignore"] | None = None):
482 """
483 Map values using input an input mapping or function.
485 Maps the values (their categories, not the codes) of the index to new
486 categories. If the mapping correspondence is one-to-one the result is a
487 :class:`~pandas.CategoricalIndex` which has the same order property as
488 the original, otherwise an :class:`~pandas.Index` is returned.
490 If a `dict` or :class:`~pandas.Series` is used any unmapped category is
491 mapped to `NaN`. Note that if this happens an :class:`~pandas.Index`
492 will be returned.
494 Parameters
495 ----------
496 mapper : function, dict, or Series
497 Mapping correspondence.
498 na_action : {None, 'ignore'}, default 'ignore'
499 If 'ignore', propagate NaN values, without passing them to
500 the mapping correspondence.
502 Returns
503 -------
504 pandas.CategoricalIndex or pandas.Index
505 Mapped index.
507 See Also
508 --------
509 Index.map : Apply a mapping correspondence on an
510 :class:`~pandas.Index`.
511 Series.map : Apply a mapping correspondence on a
512 :class:`~pandas.Series`.
513 Series.apply : Apply more complex functions on a
514 :class:`~pandas.Series`.
516 Examples
517 --------
518 >>> idx = pd.CategoricalIndex(["a", "b", "c"])
519 >>> idx
520 CategoricalIndex(['a', 'b', 'c'], categories=['a', 'b', 'c'],
521 ordered=False, dtype='category')
522 >>> idx.map(lambda x: x.upper())
523 CategoricalIndex(['A', 'B', 'C'], categories=['A', 'B', 'C'],
524 ordered=False, dtype='category')
525 >>> idx.map({"a": "first", "b": "second", "c": "third"})
526 CategoricalIndex(['first', 'second', 'third'], categories=['first',
527 'second', 'third'], ordered=False, dtype='category')
529 If the mapping is one-to-one the ordering of the categories is
530 preserved:
532 >>> idx = pd.CategoricalIndex(["a", "b", "c"], ordered=True)
533 >>> idx
534 CategoricalIndex(['a', 'b', 'c'], categories=['a', 'b', 'c'],
535 ordered=True, dtype='category')
536 >>> idx.map({"a": 3, "b": 2, "c": 1})
537 CategoricalIndex([3, 2, 1], categories=[3, 2, 1], ordered=True,
538 dtype='category')
540 If the mapping is not one-to-one an :class:`~pandas.Index` is returned:
542 >>> idx.map({"a": "first", "b": "second", "c": "first"})
543 Index(['first', 'second', 'first'], dtype='str')
545 If a `dict` is used, all unmapped categories are mapped to `NaN` and
546 the result is an :class:`~pandas.Index`:
548 >>> idx.map({"a": "first", "b": "second"})
549 Index(['first', 'second', nan], dtype='str')
550 """
551 mapped = self._values.map(mapper, na_action=na_action)
552 return Index(mapped, name=self.name, copy=False)