Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from csv import QUOTE_NONNUMERIC
4from functools import partial
5import itertools
6import operator
7from shutil import get_terminal_size
8from typing import (
9 TYPE_CHECKING,
10 Literal,
11 Self,
12 cast,
13 overload,
14)
15import warnings
17import numpy as np
19from pandas._config.config import _global_config as config
21from pandas._libs import (
22 NaT,
23 algos as libalgos,
24 lib,
25 ops as libops,
26)
27from pandas._libs.arrays import NDArrayBacked
28from pandas.compat.numpy import function as nv
29from pandas.errors import Pandas4Warning
30from pandas.util._decorators import set_module
31from pandas.util._exceptions import find_stack_level
32from pandas.util._validators import validate_bool_kwarg
34from pandas.core.dtypes.cast import (
35 coerce_indexer_dtype,
36 find_common_type,
37)
38from pandas.core.dtypes.common import (
39 ensure_int64,
40 ensure_platform_int,
41 is_any_real_numeric_dtype,
42 is_bool_dtype,
43 is_dict_like,
44 is_hashable,
45 is_integer_dtype,
46 is_list_like,
47 is_scalar,
48 needs_i8_conversion,
49 pandas_dtype,
50)
51from pandas.core.dtypes.dtypes import (
52 ArrowDtype,
53 CategoricalDtype,
54 CategoricalDtypeType,
55 ExtensionDtype,
56)
57from pandas.core.dtypes.generic import (
58 ABCIndex,
59 ABCMultiIndex,
60 ABCSeries,
61)
62from pandas.core.dtypes.missing import (
63 is_valid_na_for_dtype,
64 isna,
65)
67from pandas.core import (
68 algorithms,
69 arraylike,
70 ops,
71)
72from pandas.core.accessor import (
73 PandasDelegate,
74 delegate_names,
75)
76from pandas.core.algorithms import (
77 factorize,
78 take_nd,
79)
80from pandas.core.arrays._mixins import (
81 NDArrayBackedExtensionArray,
82 ravel_compat,
83)
84from pandas.core.base import (
85 ExtensionArray,
86 NoNewAttributesMixin,
87 PandasObject,
88)
89import pandas.core.common as com
90from pandas.core.construction import (
91 extract_array,
92 sanitize_array,
93)
94from pandas.core.ops.common import unpack_zerodim_and_defer
95from pandas.core.sorting import nargsort
96from pandas.core.strings.object_array import ObjectStringArrayMixin
98from pandas.io.formats import console
100if TYPE_CHECKING:
101 from collections.abc import (
102 Callable,
103 Hashable,
104 Iterator,
105 Sequence,
106 )
107 from typing import Any
109 from pandas._typing import (
110 ArrayLike,
111 AstypeArg,
112 AxisInt,
113 Dtype,
114 NpDtype,
115 Ordered,
116 RankMethod,
117 RankNaOption,
118 Shape,
119 SortKind,
120 npt,
121 )
123 from pandas import (
124 DataFrame,
125 Index,
126 Series,
127 )
130def _cat_compare_op(op):
131 opname = f"__{op.__name__}__"
132 fill_value = op is operator.ne
134 @unpack_zerodim_and_defer(opname)
135 def func(self, other):
136 hashable = is_hashable(other)
137 if is_list_like(other) and len(other) != len(self) and not hashable:
138 # in hashable case we may have a tuple that is itself a category
139 raise ValueError("Lengths must match.")
141 if not self.ordered:
142 if opname in ["__lt__", "__gt__", "__le__", "__ge__"]:
143 raise TypeError(
144 "Unordered Categoricals can only compare equality or not"
145 )
146 if isinstance(other, Categorical):
147 # Two Categoricals can only be compared if the categories are
148 # the same (maybe up to ordering, depending on ordered)
150 msg = "Categoricals can only be compared if 'categories' are the same."
151 if not self._categories_match_up_to_permutation(other):
152 raise TypeError(msg)
154 if not self.ordered and not self.categories.equals(other.categories):
155 # both unordered and different order
156 other_codes = recode_for_categories(
157 other.codes, other.categories, self.categories, copy=False
158 )
159 else:
160 other_codes = other._codes
162 ret = op(self._codes, other_codes)
163 mask = (self._codes == -1) | (other_codes == -1)
164 if mask.any():
165 ret[mask] = fill_value
166 return ret
168 if hashable:
169 if other in self.categories:
170 i = self._unbox_scalar(other)
171 ret = op(self._codes, i)
173 if opname not in {"__eq__", "__ge__", "__gt__"}:
174 # GH#29820 performance trick; get_loc will always give i>=0,
175 # so in the cases (__ne__, __le__, __lt__) the setting
176 # here is a no-op, so can be skipped.
177 mask = self._codes == -1
178 ret[mask] = fill_value
179 return ret
180 else:
181 return ops.invalid_comparison(self, other, op)
182 else:
183 # allow categorical vs object dtype array comparisons for equality
184 # these are only positional comparisons
185 # (hashable list-likes such as tuple/range take the branch above and
186 # are already treated as scalar-like, so only non-standard
187 # positional list-likes like ``deque`` warn here, GH#62423)
188 ops.maybe_warn_listlike(other)
189 if opname not in ["__eq__", "__ne__"]:
190 raise TypeError(
191 f"Cannot compare a Categorical for op {opname} with "
192 f"type {type(other)}.\nIf you want to compare values, "
193 "use 'np.asarray(cat) <op> other'."
194 )
196 if isinstance(other, ExtensionArray) and needs_i8_conversion(other.dtype):
197 # We would return NotImplemented here, but that messes up
198 # ExtensionIndex's wrapped methods
199 return op(other, self)
200 return getattr(np.array(self), opname)(np.array(other))
202 func.__name__ = opname
204 return func
207def contains(cat, key, container) -> bool:
208 """
209 Helper for membership check for ``key`` in ``cat``.
211 This is a helper method for :meth:`__contains__`
212 and :meth:`CategoricalIndex.__contains__`.
214 Returns True if ``key`` is in ``cat.categories`` and the
215 location of ``key`` in ``categories`` is in ``container``.
217 Parameters
218 ----------
219 cat : :class:`Categorical` or :class:`CategoricalIndex`
220 key : a hashable object
221 The key to check membership for.
222 container : Container (e.g. list-like or mapping)
223 The container to check for membership in.
225 Returns
226 -------
227 is_in : bool
228 True if ``key`` is in ``self.categories`` and location of
229 ``key`` in ``categories`` is in ``container``, else False.
231 Notes
232 -----
233 This method does not check for NaN values. Do that separately
234 before calling this method.
235 """
236 hash(key)
238 # get location of key in categories.
239 # If a KeyError, the key isn't in categories, so logically
240 # can't be in container either.
241 try:
242 loc = cat.categories.get_loc(key)
243 except (KeyError, TypeError):
244 return False
246 # loc is the location of key in categories, but also the *value*
247 # for key in container. So, `key` may be in categories,
248 # but still not in `container`. Example ('b' in categories,
249 # but not in values):
250 # 'b' in Categorical(['a'], categories=['a', 'b']) # False
251 if is_scalar(loc):
252 return loc in container
253 else:
254 # if categories is an IntervalIndex, loc is an array.
255 return any(loc_ in container for loc_ in loc)
258@set_module("pandas")
259class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMixin):
260 """
261 Represent a categorical variable in classic R / S-plus fashion.
263 `Categoricals` can only take on a limited, and usually fixed, number
264 of possible values (`categories`). In contrast to statistical categorical
265 variables, a `Categorical` might have an order, but numerical operations
266 (additions, divisions, ...) are not possible.
268 All values of the `Categorical` are either in `categories` or `np.nan`.
269 Assigning values outside of `categories` will raise a `ValueError`. Order
270 is defined by the order of the `categories`, not lexical order of the
271 values.
273 Parameters
274 ----------
275 values : list-like
276 The values of the categorical. If categories are given, values not in
277 categories will be replaced with NaN.
278 categories : Index-like (unique), optional
279 The unique categories for this categorical. If not given, the
280 categories are assumed to be the unique values of `values` (sorted, if
281 possible, otherwise in the order in which they appear).
282 ordered : bool, default False
283 Whether or not this categorical is treated as an ordered categorical.
284 If True, the resulting categorical will be ordered.
285 An ordered categorical respects, when sorted, the order of its
286 `categories` attribute (which in turn is the `categories` argument, if
287 provided).
288 dtype : CategoricalDtype
289 An instance of ``CategoricalDtype`` to use for this categorical.
290 copy : bool, default True
291 Whether to copy if the codes are unchanged.
293 Attributes
294 ----------
295 categories : Index
296 The categories of this categorical.
297 codes : ndarray
298 The codes (integer positions, which point to the categories) of this
299 categorical, read only.
300 ordered : bool
301 Whether or not this Categorical is ordered.
302 dtype : CategoricalDtype
303 The instance of ``CategoricalDtype`` storing the ``categories``
304 and ``ordered``.
306 Methods
307 -------
308 from_codes
309 as_ordered
310 as_unordered
311 set_categories
312 rename_categories
313 reorder_categories
314 add_categories
315 remove_categories
316 remove_unused_categories
317 map
318 __array__
320 Raises
321 ------
322 ValueError
323 If the categories do not validate.
324 TypeError
325 If an explicit ``ordered=True`` is given but no `categories` and the
326 `values` are not sortable.
328 See Also
329 --------
330 CategoricalDtype : Type for categorical data.
331 CategoricalIndex : An Index with an underlying ``Categorical``.
333 Notes
334 -----
335 See the `user guide
336 <https://pandas.pydata.org/pandas-docs/stable/user_guide/categorical.html>`__
337 for more.
339 Examples
340 --------
341 >>> pd.Categorical([1, 2, 3, 1, 2, 3])
342 [1, 2, 3, 1, 2, 3]
343 Categories (3, int64): [1, 2, 3]
345 >>> pd.Categorical(["a", "b", "c", "a", "b", "c"])
346 ['a', 'b', 'c', 'a', 'b', 'c']
347 Categories (3, str): ['a', 'b', 'c']
349 Missing values are not included as a category.
351 >>> c = pd.Categorical([1, 2, 3, 1, 2, 3, np.nan])
352 >>> c
353 [1, 2, 3, 1, 2, 3, NaN]
354 Categories (3, int64): [1, 2, 3]
356 However, their presence is indicated in the `codes` attribute
357 by code `-1`.
359 >>> c.codes
360 array([ 0, 1, 2, 0, 1, 2, -1], dtype=int8)
362 Ordered `Categoricals` can be sorted according to the custom order
363 of the categories and can have a min and max value.
365 >>> c = pd.Categorical(
366 ... ["a", "b", "c", "a", "b", "c"], ordered=True, categories=["c", "b", "a"]
367 ... )
368 >>> c
369 ['a', 'b', 'c', 'a', 'b', 'c']
370 Categories (3, str): ['c' < 'b' < 'a']
371 >>> c.min()
372 'c'
373 """
375 # For comparisons, so that numpy uses our implementation if the compare
376 # ops, which raise
377 __array_priority__ = 1000
378 # tolist is not actually deprecated, just suppressed in the __dir__
379 _hidden_attrs = PandasObject._hidden_attrs | frozenset(["tolist"])
380 _typ = "categorical"
382 _dtype: CategoricalDtype
384 @classmethod
385 # error: Argument 2 of "_simple_new" is incompatible with supertype
386 # "NDArrayBacked"; supertype defines the argument type as
387 # "Union[dtype[Any], ExtensionDtype]"
388 def _simple_new( # type: ignore[override]
389 cls, codes: np.ndarray, dtype: CategoricalDtype
390 ) -> Self:
391 # NB: This is not _quite_ as simple as the "usual" _simple_new
392 codes = coerce_indexer_dtype(codes, dtype.categories)
393 if dtype.ordered is None:
394 dtype = CategoricalDtype._from_fastpath(dtype.categories, ordered=False)
395 return super()._simple_new(codes, dtype)
397 def __init__(
398 self,
399 values,
400 categories=None,
401 ordered=None,
402 dtype: Dtype | None = None,
403 copy: bool = True,
404 ) -> None:
405 dtype = CategoricalDtype._from_values_or_dtype(
406 values, categories, ordered, dtype
407 )
408 # At this point, dtype is always a CategoricalDtype, but
409 # we may have dtype.categories be None, and we need to
410 # infer categories in a factorization step further below
412 if not is_list_like(values):
413 # GH#38433
414 raise TypeError("Categorical input must be list-like")
416 # null_mask indicates missing values we want to exclude from inference.
417 # This means: only missing values in list-likes (not arrays/ndframes).
418 null_mask = np.array(False)
420 # sanitize input
421 vdtype = getattr(values, "dtype", None)
422 if isinstance(vdtype, CategoricalDtype):
423 if dtype.categories is None:
424 dtype = CategoricalDtype(values.categories, dtype.ordered)
425 elif isinstance(values, range):
426 from pandas.core.indexes.range import RangeIndex
428 values = RangeIndex(values)
429 elif not isinstance(values, (ABCIndex, ABCSeries, ExtensionArray)):
430 values = com.convert_to_list_like(values)
431 if isinstance(values, list) and len(values) == 0:
432 # By convention, empty lists result in object dtype:
433 values = np.array([], dtype=object)
434 elif isinstance(values, np.ndarray):
435 if values.ndim > 1:
436 # preempt sanitize_array from raising ValueError
437 raise NotImplementedError(
438 "> 1 ndim Categorical are not supported at this time"
439 )
440 values = sanitize_array(values, None)
441 else:
442 # i.e. must be a list
443 arr = sanitize_array(values, None)
444 null_mask = isna(arr)
445 if null_mask.any():
446 # We remove null values here, then below will re-insert
447 # them, grep "full_codes"
448 arr_list = [values[idx] for idx in np.where(~null_mask)[0]]
450 # GH#44900 Do not cast to float if we have only missing values
451 if arr_list or arr.dtype == "object":
452 sanitize_dtype = None
453 else:
454 sanitize_dtype = arr.dtype
456 arr = sanitize_array(arr_list, None, dtype=sanitize_dtype)
457 values = arr
459 if dtype.categories is None:
460 if isinstance(values.dtype, ArrowDtype) and issubclass(
461 values.dtype.type, CategoricalDtypeType
462 ):
463 from pandas import Index
465 if isinstance(values, Index):
466 arr = values._data._pa_array.combine_chunks()
467 else:
468 arr = extract_array(values)._pa_array.combine_chunks()
469 categories = arr.dictionary.to_pandas(types_mapper=ArrowDtype)
470 codes = arr.indices.to_numpy()
471 dtype = CategoricalDtype(categories, values.dtype.pyarrow_dtype.ordered)
472 else:
473 preserve_object = False
474 if isinstance(values, (ABCIndex, ABCSeries)) and values.dtype == object:
475 # GH#61778
476 preserve_object = True
477 if not isinstance(values, ABCIndex):
478 # in particular RangeIndex xref test_index_equal_range_categories
479 values = sanitize_array(values, None)
480 try:
481 codes, categories = factorize(values, sort=True)
482 except TypeError as err:
483 codes, categories = factorize(values, sort=False)
484 if dtype.ordered:
485 # raise, as we don't have a sortable data structure and so
486 # the user should give us one by specifying categories
487 raise TypeError(
488 "'values' is not ordered, please "
489 "explicitly specify the categories order "
490 "by passing in a categories argument."
491 ) from err
493 if preserve_object:
494 # GH#61778 wrap categories in an Index to prevent dtype
495 # inference in the CategoricalDtype constructor
496 from pandas import Index
498 categories = Index(categories, dtype=object, copy=False)
500 # if not preserve_object, we're inferring from values
501 dtype = CategoricalDtype(categories, dtype.ordered)
503 elif isinstance(values.dtype, CategoricalDtype):
504 old_codes = extract_array(values)._codes
505 codes = recode_for_categories(
506 old_codes,
507 values.dtype.categories,
508 dtype.categories,
509 copy=copy,
510 warn=True,
511 )
513 else:
514 codes = _get_codes_for_values(values, dtype.categories)
516 if null_mask.any():
517 # Reinsert -1 placeholders for previously removed missing values
518 full_codes = -np.ones(null_mask.shape, dtype=codes.dtype)
519 full_codes[~null_mask] = codes
520 codes = full_codes
522 if dtype.ordered is None:
523 dtype = CategoricalDtype._from_fastpath(dtype.categories, ordered=False)
524 arr = coerce_indexer_dtype(codes, dtype.categories)
525 super().__init__(arr, dtype)
527 @property
528 def dtype(self) -> CategoricalDtype:
529 """
530 The :class:`~pandas.api.types.CategoricalDtype` for this instance.
532 This property returns the CategoricalDtype which contains information
533 about the categories and whether the categorical is ordered.
535 See Also
536 --------
537 astype : Cast argument to a specified dtype.
538 CategoricalDtype : Type for categorical data.
540 Examples
541 --------
542 >>> cat = pd.Categorical(["a", "b"], ordered=True)
543 >>> cat
544 ['a', 'b']
545 Categories (2, str): ['a' < 'b']
546 >>> cat.dtype
547 CategoricalDtype(categories=['a', 'b'], ordered=True, categories_dtype=str)
548 """
549 return self._dtype
551 @property
552 def _internal_fill_value(self) -> int:
553 # using the specific numpy integer instead of python int to get
554 # the correct dtype back from _quantile in the all-NA case
555 dtype = self._ndarray.dtype
556 return dtype.type(-1)
558 @classmethod
559 def _from_sequence(
560 cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
561 ) -> Self:
562 return cls(scalars, dtype=dtype, copy=copy)
564 def _cast_pointwise_result(self, values) -> ArrayLike:
565 res = super()._cast_pointwise_result(values)
566 with warnings.catch_warnings():
567 warnings.filterwarnings(
568 "ignore",
569 "Constructing a Categorical with a dtype and values containing",
570 )
571 try:
572 cat = type(self)._from_sequence(res, dtype=self.dtype)
573 except (TypeError, ValueError):
574 return res
575 if (cat.isna() == isna(res)).all():
576 # i.e. the conversion was non-lossy
577 return cat
578 return res
580 @overload
581 def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
583 @overload
584 def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
586 @overload
587 def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
589 def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
590 """
591 Coerce this type to another dtype
593 Parameters
594 ----------
595 dtype : numpy dtype or pandas type
596 copy : bool, default True
597 By default, astype always returns a newly allocated object.
598 If copy is set to False and dtype is categorical, the original
599 object is returned.
600 """
601 dtype = pandas_dtype(dtype)
602 result: Categorical | np.ndarray
603 if self.dtype is dtype:
604 result = self.copy() if copy else self
606 elif isinstance(dtype, CategoricalDtype):
607 # GH 10696/18593/18630
608 dtype = self.dtype.update_dtype(dtype)
609 self = self.copy() if copy else self
610 result = self._set_dtype(dtype, copy=False)
611 wrong = result.isna() & ~self.isna()
612 if wrong.any():
613 warnings.warn(
614 "Constructing a Categorical with a dtype and values containing "
615 "non-null entries not in that dtype's categories is deprecated "
616 "and will raise in a future version.",
617 Pandas4Warning,
618 stacklevel=find_stack_level(),
619 )
621 elif isinstance(dtype, ExtensionDtype):
622 return super().astype(dtype, copy=copy)
624 elif dtype.kind in "iu" and self.isna().any():
625 raise ValueError("Cannot convert float NaN to integer")
627 elif len(self.codes) == 0 or len(self.categories) == 0:
628 # For NumPy 1.x compatibility we cannot use copy=None. And
629 # `copy=False` has the meaning of `copy=None` here:
630 if not copy:
631 result = np.asarray(self, dtype=dtype)
632 else:
633 result = np.array(self, dtype=dtype)
635 else:
636 # GH8628 (PERF): astype category codes instead of astyping array
637 new_cats = self.categories._values
639 try:
640 new_cats = new_cats.astype(dtype=dtype, copy=copy)
641 fill_value = self.categories._na_value
642 if not is_valid_na_for_dtype(fill_value, dtype):
643 fill_value = lib.item_from_zerodim(
644 np.array(self.categories._na_value).astype(dtype)
645 )
646 except (
647 TypeError, # downstream error msg for CategoricalIndex is misleading
648 ValueError,
649 ) as err:
650 msg = f"Cannot cast {self.categories.dtype} dtype to {dtype}"
651 raise ValueError(msg) from err
653 result = take_nd(
654 new_cats, ensure_platform_int(self._codes), fill_value=fill_value
655 )
657 return result
659 @staticmethod
660 def _maybe_convert_categories(
661 cats: Index,
662 true_values=None,
663 false_values=None,
664 convert_numeric: bool = False,
665 convert_bool: bool = False,
666 float_only: bool = False,
667 bool_case_insensitive: bool = False,
668 ) -> Index | None:
669 """
670 Try converting string categories to numeric or boolean, mirroring
671 the type inference read_csv performs on non-categorical columns.
673 Parameters
674 ----------
675 cats : Index
676 true_values : list, optional
677 Strings recognized as True, in addition to the defaults
678 "True", "TRUE", and "true."
679 false_values : list, optional
680 Strings recognized as False, in addition to the defaults
681 "False", "FALSE", and "false."
682 convert_numeric : bool, default False
683 Whether to attempt numeric conversion.
684 convert_bool : bool, default False
685 Whether to attempt boolean conversion.
686 float_only : bool, default False
687 Whether numeric conversion must produce floats, as with
688 ``quoting=csv.QUOTE_NONNUMERIC``.
689 bool_case_insensitive : bool, default False
690 Whether the default True/False spellings match in any case, as
691 they do for the c parser's tokenizer. `true_values` and
692 `false_values` always match exactly, as they do there too.
694 Returns
695 -------
696 Index or None
697 The converted categories, which may contain duplicates (e.g. "1"
698 and "1.0"), or None if no conversion applies.
699 """
700 from pandas import (
701 Index,
702 to_numeric,
703 )
705 # "U" covers arrow-backed strings, which arrive with dtype_backend="pyarrow"
706 if len(cats) == 0 or cats.dtype.kind not in "OU":
707 # empty categories stay object dtype, matching the zero-row case
708 return None
710 if convert_numeric:
711 try:
712 converted = Index(to_numeric(cats, errors="raise"), copy=False)
713 except (ValueError, TypeError):
714 pass
715 else:
716 # test the values rather than Index.hasnans: arrow-backed
717 # strings convert to a float NaN value rather than a null,
718 # which the validity bitmap does not report
719 if not isna(np.asarray(converted)).any():
720 if float_only and converted.dtype.kind != "f":
721 # cast before the caller de-duplicates, so that values
722 # colliding at float64 precision merge into one category
723 if converted.dtype.kind == "O":
724 # ints too large for int64/uint64 arrive as Python
725 # ints and overflow a direct cast; parsing the
726 # strings gives inf, as the tokenizer does
727 converted = Index(
728 np.asarray(cats, dtype="U").astype(np.float64),
729 copy=False,
730 )
731 else:
732 converted = converted.astype(np.float64)
733 return converted
734 # to_numeric converts "" to NaN, which cannot be a category
736 if convert_bool:
737 values = np.asarray(cats)
738 if bool_case_insensitive:
739 # fold only the default spellings, and only where the user has
740 # not spelled the value out: the tokenizer consults
741 # true_values and false_values before its case-insensitive
742 # comparison, so those win over the default spelling
743 spelled_out = (*(true_values or ()), *(false_values or ()))
744 values = np.array(
745 [
746 val.lower()
747 if isinstance(val, str)
748 and val.lower() in ("true", "false")
749 and val not in spelled_out
750 else val
751 for val in values
752 ],
753 dtype=object,
754 )
755 inferred_bool, _ = libops.maybe_convert_bool(
756 values,
757 true_values=true_values,
758 false_values=false_values,
759 )
760 if inferred_bool.dtype.kind == "b":
761 if isinstance(cats.dtype, ArrowDtype):
762 # keep the backing that to_numeric preserves for the
763 # numeric branch; maybe_convert_bool only speaks numpy
764 import pyarrow as pa
766 return Index(inferred_bool, dtype=ArrowDtype(pa.bool_()))
767 return Index(inferred_bool, copy=False)
769 return None
771 @classmethod
772 def _from_converted_categories(
773 cls, converted: Index, codes: np.ndarray, ordered: bool | None = False
774 ) -> Self:
775 """
776 Construct a Categorical from ``_maybe_convert_categories`` output,
777 merging categories that converted to the same value, sorting, and
778 recoding ``codes`` accordingly.
779 """
780 # unique() because distinct strings can convert to the same value,
781 # e.g. "1"/"1.0" -> 1.0 or "True"/"TRUE" -> True
782 target = converted.unique().sort_values()
783 codes = recode_for_categories(codes, converted, target, copy=False)
784 return cls._simple_new(codes, dtype=CategoricalDtype(target, ordered=ordered))
786 @classmethod
787 def _from_inferred_categories(
788 cls,
789 inferred_categories,
790 inferred_codes,
791 dtype,
792 true_values=None,
793 sort_categories: bool = True,
794 ) -> Self:
795 """
796 Construct a Categorical from inferred values.
798 For inferred categories (`dtype` is None) the categories are sorted
799 unless `sort_categories` is False. For explicit `dtype`, the
800 `inferred_categories` are cast to the appropriate type.
802 Callers that want read_csv's numeric/boolean inference for categories
803 not supplied by `dtype` should call ``_maybe_convert_categories`` and
804 ``_from_converted_categories`` themselves.
806 Parameters
807 ----------
808 inferred_categories : Index
809 inferred_codes : Index
810 dtype : CategoricalDtype or 'category'
811 true_values : list, optional
812 Strings recognized as True when `dtype` provides boolean
813 categories. If none are provided, the default ones are
814 "True", "TRUE", and "true."
815 sort_categories : bool, default True
816 Whether to sort inferred categories. The deferred low-memory path
817 passes False so that every chunk keeps observation order, matching
818 what the non-chunked path sees; it sorts once after concatenation.
820 Returns
821 -------
822 Categorical
823 """
824 from pandas import (
825 Index,
826 to_datetime,
827 to_numeric,
828 to_timedelta,
829 )
831 cats = Index(inferred_categories, copy=False)
832 known_categories = (
833 isinstance(dtype, CategoricalDtype) and dtype.categories is not None
834 )
835 ordered = dtype.ordered if isinstance(dtype, CategoricalDtype) else False
837 if known_categories:
838 # Convert to a specialized type with `dtype` if specified.
839 if is_any_real_numeric_dtype(dtype.categories.dtype):
840 cats = to_numeric(inferred_categories, errors="coerce")
841 elif lib.is_np_dtype(dtype.categories.dtype, "M"):
842 cats = to_datetime(inferred_categories, errors="coerce")
843 elif lib.is_np_dtype(dtype.categories.dtype, "m"):
844 cats = to_timedelta(inferred_categories, errors="coerce")
845 elif is_bool_dtype(dtype.categories.dtype):
846 if true_values is None:
847 true_values = ["True", "TRUE", "true"]
849 # error: Incompatible types in assignment (expression has type
850 # "ndarray", variable has type "Index")
851 cats = cats.isin(true_values) # type: ignore[assignment]
853 if known_categories:
854 # Recode from observation order to dtype.categories order.
855 categories = dtype.categories
856 codes = recode_for_categories(
857 inferred_codes, cats, categories, copy=False, warn=True
858 )
859 elif sort_categories and not cats.is_monotonic_increasing:
860 # Sort categories and recode for unknown categories.
861 unsorted = cats.copy()
862 categories = cats.sort_values()
864 codes = recode_for_categories(
865 inferred_codes, unsorted, categories, copy=False, warn=True
866 )
867 dtype = CategoricalDtype(categories, ordered=ordered)
868 else:
869 dtype = CategoricalDtype(cats, ordered=ordered)
870 codes = inferred_codes
872 return cls._simple_new(codes, dtype=dtype)
874 @classmethod
875 def from_codes(
876 cls,
877 codes,
878 categories=None,
879 ordered=None,
880 dtype: Dtype | None = None,
881 validate: bool = True,
882 ) -> Self:
883 """
884 Make a Categorical type from codes and categories or dtype.
886 This constructor is useful if you already have codes and
887 categories/dtype and so do not need the (computation intensive)
888 factorization step, which is usually done on the constructor.
890 If your data does not follow this convention, please use the normal
891 constructor.
893 Parameters
894 ----------
895 codes : array-like of int
896 An integer array, where each integer points to a category in
897 categories or dtype.categories, or else is -1 for NaN.
898 categories : index-like, optional
899 The categories for the categorical. Items need to be unique.
900 If the categories are not given here, then they must be provided
901 in `dtype`.
902 ordered : bool, optional
903 Whether or not this categorical is treated as an ordered
904 categorical. If not given here or in `dtype`, the resulting
905 categorical will be unordered.
906 dtype : CategoricalDtype or "category", optional
907 If :class:`CategoricalDtype`, cannot be used together with
908 `categories` or `ordered`.
909 validate : bool, default True
910 If True, validate that the codes are valid for the dtype.
911 If False, don't validate that the codes are valid. Be careful about skipping
912 validation, as invalid codes can lead to severe problems, such as segfaults.
914 .. versionadded:: 2.1.0
916 Returns
917 -------
918 Categorical
920 See Also
921 --------
922 codes : The category codes of the categorical.
923 CategoricalIndex : An Index with an underlying ``Categorical``.
925 Examples
926 --------
927 >>> dtype = pd.CategoricalDtype(["a", "b"], ordered=True)
928 >>> pd.Categorical.from_codes(codes=[0, 1, 0, 1], dtype=dtype)
929 ['a', 'b', 'a', 'b']
930 Categories (2, str): ['a' < 'b']
931 """
932 dtype = CategoricalDtype._from_values_or_dtype(
933 categories=categories, ordered=ordered, dtype=dtype
934 )
935 if dtype.categories is None:
936 msg = (
937 "The categories must be provided in 'categories' or "
938 "'dtype'. Both were None."
939 )
940 raise ValueError(msg)
942 if validate:
943 # beware: non-valid codes may segfault
944 codes = cls._validate_codes_for_dtype(codes, dtype=dtype)
946 return cls._simple_new(codes, dtype=dtype)
948 # ------------------------------------------------------------------
949 # Categories/Codes/Ordered
951 @property
952 def categories(self) -> Index:
953 """
954 The categories of this categorical.
956 Setting assigns new values to each category (effectively a rename of
957 each individual category).
959 The assigned value has to be a list-like object. All items must be
960 unique and the number of items in the new categories must be the same
961 as the number of items in the old categories.
963 Raises
964 ------
965 ValueError
966 If the new categories do not validate as categories or if the
967 number of new categories is unequal the number of old categories
969 See Also
970 --------
971 rename_categories : Rename categories.
972 reorder_categories : Reorder categories.
973 add_categories : Add new categories.
974 remove_categories : Remove the specified categories.
975 remove_unused_categories : Remove categories which are not used.
976 set_categories : Set the categories to the specified ones.
978 Examples
979 --------
980 For :class:`pandas.Series`:
982 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
983 >>> ser.cat.categories
984 Index(['a', 'b', 'c'], dtype='str')
986 >>> raw_cat = pd.Categorical([None, "b", "c", None], categories=["b", "c", "d"])
987 >>> ser = pd.Series(raw_cat)
988 >>> ser.cat.categories
989 Index(['b', 'c', 'd'], dtype='str')
991 For :class:`pandas.Categorical`:
993 >>> cat = pd.Categorical(["a", "b"], ordered=True)
994 >>> cat.categories
995 Index(['a', 'b'], dtype='str')
997 For :class:`pandas.CategoricalIndex`:
999 >>> ci = pd.CategoricalIndex(["a", "c", "b", "a", "c", "b"])
1000 >>> ci.categories
1001 Index(['a', 'b', 'c'], dtype='str')
1003 >>> ci = pd.CategoricalIndex(["a", "c"], categories=["c", "b", "a"])
1004 >>> ci.categories
1005 Index(['c', 'b', 'a'], dtype='str')
1006 """
1007 return self.dtype.categories
1009 @property
1010 def ordered(self) -> Ordered:
1011 """
1012 Whether the categories have an ordered relationship.
1014 This property returns True if the categories are ordered, meaning
1015 they have a meaningful order that allows comparison operations.
1017 See Also
1018 --------
1019 set_ordered : Set the ordered attribute.
1020 as_ordered : Set the Categorical to be ordered.
1021 as_unordered : Set the Categorical to be unordered.
1023 Examples
1024 --------
1025 For :class:`pandas.Series`:
1027 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
1028 >>> ser.cat.ordered
1029 False
1031 >>> raw_cat = pd.Categorical(["a", "b", "c", "a"], ordered=True)
1032 >>> ser = pd.Series(raw_cat)
1033 >>> ser.cat.ordered
1034 True
1036 For :class:`pandas.Categorical`:
1038 >>> cat = pd.Categorical(["a", "b"], ordered=True)
1039 >>> cat.ordered
1040 True
1042 >>> cat = pd.Categorical(["a", "b"], ordered=False)
1043 >>> cat.ordered
1044 False
1046 For :class:`pandas.CategoricalIndex`:
1048 >>> ci = pd.CategoricalIndex(["a", "b"], ordered=True)
1049 >>> ci.ordered
1050 True
1052 >>> ci = pd.CategoricalIndex(["a", "b"], ordered=False)
1053 >>> ci.ordered
1054 False
1055 """
1056 return self.dtype.ordered
1058 @property
1059 def codes(self) -> np.ndarray:
1060 """
1061 The category codes of this categorical index.
1063 Codes are an array of integers which are the positions of the actual
1064 values in the categories array.
1066 There is no setter, use the other categorical methods and the normal item
1067 setter to change values in the categorical.
1069 Returns
1070 -------
1071 ndarray[int]
1072 A non-writable view of the ``codes`` array.
1074 See Also
1075 --------
1076 Categorical.from_codes : Make a Categorical from codes.
1077 CategoricalIndex : An Index with an underlying ``Categorical``.
1079 Examples
1080 --------
1081 For :class:`pandas.Categorical`:
1083 >>> cat = pd.Categorical(["a", "b"], ordered=True)
1084 >>> cat.codes
1085 array([0, 1], dtype=int8)
1087 For :class:`pandas.CategoricalIndex`:
1089 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a", "b", "c"])
1090 >>> ci.codes
1091 array([0, 1, 2, 0, 1, 2], dtype=int8)
1093 >>> ci = pd.CategoricalIndex(["a", "c"], categories=["c", "b", "a"])
1094 >>> ci.codes
1095 array([2, 0], dtype=int8)
1096 """
1097 v = self._codes.view()
1098 v.flags.writeable = False
1099 return v
1101 def _set_categories(self, categories, fastpath: bool = False) -> None:
1102 """
1103 Sets new categories inplace
1105 Parameters
1106 ----------
1107 fastpath : bool, default False
1108 Don't perform validation of the categories for uniqueness or nulls
1110 Examples
1111 --------
1112 >>> c = pd.Categorical(["a", "b"])
1113 >>> c
1114 ['a', 'b']
1115 Categories (2, str): ['a', 'b']
1117 >>> c._set_categories(pd.Index(["a", "c"]))
1118 >>> c
1119 ['a', 'c']
1120 Categories (2, str): ['a', 'c']
1121 """
1122 if fastpath:
1123 new_dtype = CategoricalDtype._from_fastpath(categories, self.ordered)
1124 else:
1125 new_dtype = CategoricalDtype(categories, ordered=self.ordered)
1126 if (
1127 not fastpath
1128 and self.dtype.categories is not None
1129 and len(new_dtype.categories) != len(self.dtype.categories)
1130 ):
1131 raise ValueError(
1132 "new categories need to have the same number of "
1133 "items as the old categories!"
1134 )
1136 super().__init__(self._ndarray, new_dtype)
1138 def _set_dtype(self, dtype: CategoricalDtype, *, copy: bool) -> Self:
1139 """
1140 Internal method for directly updating the CategoricalDtype
1142 Parameters
1143 ----------
1144 dtype : CategoricalDtype
1146 Notes
1147 -----
1148 We don't do any validation here. It's assumed that the dtype is
1149 a (valid) instance of `CategoricalDtype`.
1150 """
1151 codes = recode_for_categories(
1152 self.codes, self.categories, dtype.categories, copy=copy
1153 )
1154 return type(self)._simple_new(codes, dtype=dtype)
1156 def set_ordered(self, value: bool) -> Self:
1157 """
1158 Set the ordered attribute to the boolean value.
1160 Parameters
1161 ----------
1162 value : bool
1163 Set whether this categorical is ordered (True) or not (False).
1164 """
1165 new_dtype = CategoricalDtype(self.categories, ordered=value)
1166 cat = self.copy()
1167 NDArrayBacked.__init__(cat, cat._ndarray, new_dtype)
1168 return cat
1170 def as_ordered(self) -> Self:
1171 """
1172 Set the Categorical to be ordered.
1174 This method returns a new Categorical with the ordered attribute set
1175 to True, enabling comparison operations between categories.
1177 Returns
1178 -------
1179 Categorical
1180 Ordered Categorical.
1182 See Also
1183 --------
1184 as_unordered : Set the Categorical to be unordered.
1186 Examples
1187 --------
1188 For :class:`pandas.Series`:
1190 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
1191 >>> ser.cat.ordered
1192 False
1193 >>> ser = ser.cat.as_ordered()
1194 >>> ser.cat.ordered
1195 True
1197 For :class:`pandas.CategoricalIndex`:
1199 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"])
1200 >>> ci.ordered
1201 False
1202 >>> ci = ci.as_ordered()
1203 >>> ci.ordered
1204 True
1205 """
1206 return self.set_ordered(True)
1208 def as_unordered(self) -> Self:
1209 """
1210 Set the Categorical to be unordered.
1212 This method returns a new Categorical with the ordered attribute set
1213 to False, disabling comparison operations between categories.
1215 Returns
1216 -------
1217 Categorical
1218 Unordered Categorical.
1220 See Also
1221 --------
1222 as_ordered : Set the Categorical to be ordered.
1224 Examples
1225 --------
1226 For :class:`pandas.Series`:
1228 >>> raw_cat = pd.Categorical(["a", "b", "c", "a"], ordered=True)
1229 >>> ser = pd.Series(raw_cat)
1230 >>> ser.cat.ordered
1231 True
1232 >>> ser = ser.cat.as_unordered()
1233 >>> ser.cat.ordered
1234 False
1236 For :class:`pandas.CategoricalIndex`:
1238 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"], ordered=True)
1239 >>> ci.ordered
1240 True
1241 >>> ci = ci.as_unordered()
1242 >>> ci.ordered
1243 False
1244 """
1245 return self.set_ordered(False)
1247 def set_categories(
1248 self, new_categories, ordered=None, rename: bool = False
1249 ) -> Self:
1250 """
1251 Set the categories to the specified new categories.
1253 ``new_categories`` can include new categories (which will result in
1254 unused categories) or remove old categories (which results in values
1255 set to ``NaN``). If ``rename=True``, the categories will simply be renamed
1256 (less or more items than in old categories will result in values set to
1257 ``NaN`` or in unused categories respectively).
1259 This method can be used to perform more than one action of adding,
1260 removing, and reordering simultaneously and is therefore faster than
1261 performing the individual steps via the more specialised methods.
1263 On the other hand this methods does not do checks (e.g., whether the
1264 old categories are included in the new categories on a reorder), which
1265 can result in surprising changes, for example when using special string
1266 dtypes, which do not consider a S1 string equal to a single char
1267 python string.
1269 Parameters
1270 ----------
1271 new_categories : Index-like
1272 The categories in new order.
1273 ordered : bool, default None
1274 Whether or not the categorical is treated as an ordered categorical.
1275 If not given, do not change the ordered information.
1276 rename : bool, default False
1277 Whether or not the new_categories should be considered as a rename
1278 of the old categories or as reordered categories.
1280 Returns
1281 -------
1282 Categorical
1283 New categories to be used, with optional ordering changes.
1285 Raises
1286 ------
1287 ValueError
1288 If new_categories does not validate as categories
1290 See Also
1291 --------
1292 rename_categories : Rename categories.
1293 reorder_categories : Reorder categories.
1294 add_categories : Add new categories.
1295 remove_categories : Remove the specified categories.
1296 remove_unused_categories : Remove categories which are not used.
1298 Examples
1299 --------
1300 For :class:`pandas.Series`:
1302 >>> raw_cat = pd.Categorical(
1303 ... ["a", "b", "c", None], categories=["a", "b", "c"], ordered=True
1304 ... )
1305 >>> ser = pd.Series(raw_cat)
1306 >>> ser
1307 0 a
1308 1 b
1309 2 c
1310 3 NaN
1311 dtype: category
1312 Categories (3, str): ['a' < 'b' < 'c']
1314 >>> ser.cat.set_categories(["A", "B", "C"], rename=True)
1315 0 A
1316 1 B
1317 2 C
1318 3 NaN
1319 dtype: category
1320 Categories (3, str): ['A' < 'B' < 'C']
1322 For :class:`pandas.CategoricalIndex`:
1324 >>> ci = pd.CategoricalIndex(
1325 ... ["a", "b", "c", None], categories=["a", "b", "c"], ordered=True
1326 ... )
1327 >>> ci
1328 CategoricalIndex(['a', 'b', 'c', nan], categories=['a', 'b', 'c'],
1329 ordered=True, dtype='category')
1331 >>> ci.set_categories(["A", "b", "c"])
1332 CategoricalIndex([nan, 'b', 'c', nan], categories=['A', 'b', 'c'],
1333 ordered=True, dtype='category')
1334 >>> ci.set_categories(["A", "b", "c"], rename=True)
1335 CategoricalIndex(['A', 'b', 'c', nan], categories=['A', 'b', 'c'],
1336 ordered=True, dtype='category')
1337 """
1339 if ordered is None:
1340 ordered = self.dtype.ordered
1341 new_dtype = CategoricalDtype(new_categories, ordered=ordered)
1343 cat = self.copy()
1344 if rename:
1345 if cat.dtype.categories is not None and len(new_dtype.categories) < len(
1346 cat.dtype.categories
1347 ):
1348 # remove all _codes which are larger and set to -1/NaN
1349 cat._codes[cat._codes >= len(new_dtype.categories)] = -1
1350 codes = cat._codes
1351 else:
1352 codes = recode_for_categories(
1353 cat.codes, cat.categories, new_dtype.categories, copy=False, warn=False
1354 )
1355 NDArrayBacked.__init__(cat, codes, new_dtype)
1356 return cat
1358 def rename_categories(self, new_categories) -> Self:
1359 """
1360 Rename categories.
1362 This method is commonly used to re-label or adjust the
1363 category names in categorical data without changing the
1364 underlying data. It is useful in situations where you want
1365 to modify the labels used for clarity, consistency,
1366 or readability.
1368 Parameters
1369 ----------
1370 new_categories : list-like, dict-like or callable
1372 New categories which will replace old categories.
1374 * list-like: all items must be unique and the number of items in
1375 the new categories must match the existing number of categories.
1377 * dict-like: specifies a mapping from
1378 old categories to new. Categories not contained in the mapping
1379 are passed through and extra categories in the mapping are
1380 ignored.
1382 * callable : a callable that is called on all items in the old
1383 categories and whose return values comprise the new categories.
1385 Returns
1386 -------
1387 Categorical
1388 Categorical with renamed categories.
1390 Raises
1391 ------
1392 ValueError
1393 If new categories are list-like and do not have the same number of
1394 items than the current categories or do not validate as categories
1396 See Also
1397 --------
1398 reorder_categories : Reorder categories.
1399 add_categories : Add new categories.
1400 remove_categories : Remove the specified categories.
1401 remove_unused_categories : Remove categories which are not used.
1402 set_categories : Set the categories to the specified ones.
1403 Series.replace : Replace values, e.g. to merge several categories into one.
1405 Examples
1406 --------
1407 >>> c = pd.Categorical(["a", "a", "b"])
1408 >>> c.rename_categories([0, 1])
1409 [0, 0, 1]
1410 Categories (2, int64): [0, 1]
1412 For dict-like ``new_categories``, extra keys are ignored and
1413 categories not in the dictionary are passed through
1415 >>> c.rename_categories({"a": "A", "c": "C"})
1416 ['A', 'A', 'b']
1417 Categories (2, str): ['A', 'b']
1419 You may also provide a callable to create the new categories
1421 >>> c.rename_categories(lambda x: x.upper())
1422 ['A', 'A', 'B']
1423 Categories (2, str): ['A', 'B']
1424 """
1426 if is_dict_like(new_categories):
1427 new_categories = [
1428 new_categories.get(item, item) for item in self.categories
1429 ]
1430 elif callable(new_categories):
1431 new_categories = [new_categories(item) for item in self.categories]
1433 cat = self.copy()
1434 cat._set_categories(new_categories)
1435 return cat
1437 def reorder_categories(self, new_categories, ordered=None) -> Self:
1438 """
1439 Reorder categories as specified in new_categories.
1441 ``new_categories`` need to include all old categories and no new category
1442 items.
1444 Parameters
1445 ----------
1446 new_categories : Index-like
1447 The categories in new order.
1448 ordered : bool, optional
1449 Whether or not the categorical is treated as an ordered categorical.
1450 If not given, do not change the ordered information.
1452 Returns
1453 -------
1454 Categorical
1455 Categorical with reordered categories.
1457 Raises
1458 ------
1459 ValueError
1460 If the new categories do not contain all old category items or any
1461 new ones
1463 See Also
1464 --------
1465 rename_categories : Rename categories.
1466 add_categories : Add new categories.
1467 remove_categories : Remove the specified categories.
1468 remove_unused_categories : Remove categories which are not used.
1469 set_categories : Set the categories to the specified ones.
1471 Examples
1472 --------
1473 For :class:`pandas.Series`:
1475 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
1476 >>> ser = ser.cat.reorder_categories(["c", "b", "a"], ordered=True)
1477 >>> ser
1478 0 a
1479 1 b
1480 2 c
1481 3 a
1482 dtype: category
1483 Categories (3, str): ['c' < 'b' < 'a']
1485 >>> ser.sort_values()
1486 2 c
1487 1 b
1488 0 a
1489 3 a
1490 dtype: category
1491 Categories (3, str): ['c' < 'b' < 'a']
1493 For :class:`pandas.CategoricalIndex`:
1495 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"])
1496 >>> ci
1497 CategoricalIndex(['a', 'b', 'c', 'a'], categories=['a', 'b', 'c'],
1498 ordered=False, dtype='category')
1499 >>> ci.reorder_categories(["c", "b", "a"], ordered=True)
1500 CategoricalIndex(['a', 'b', 'c', 'a'], categories=['c', 'b', 'a'],
1501 ordered=True, dtype='category')
1502 """
1503 if (
1504 len(self.categories) != len(new_categories)
1505 or not self.categories.difference(new_categories).empty
1506 ):
1507 raise ValueError(
1508 "items in new_categories are not the same as in old categories"
1509 )
1510 return self.set_categories(new_categories, ordered=ordered)
1512 def add_categories(self, new_categories) -> Self:
1513 """
1514 Add new categories.
1516 `new_categories` will be included at the last/highest place in the
1517 categories and will be unused directly after this call.
1519 Parameters
1520 ----------
1521 new_categories : category or list-like of category
1522 The new categories to be included.
1524 Returns
1525 -------
1526 Categorical
1527 Categorical with new categories added.
1529 Raises
1530 ------
1531 ValueError
1532 If the new categories include old categories or do not validate as
1533 categories
1535 See Also
1536 --------
1537 rename_categories : Rename categories.
1538 reorder_categories : Reorder categories.
1539 remove_categories : Remove the specified categories.
1540 remove_unused_categories : Remove categories which are not used.
1541 set_categories : Set the categories to the specified ones.
1543 Examples
1544 --------
1545 >>> c = pd.Categorical(["c", "b", "c"])
1546 >>> c
1547 ['c', 'b', 'c']
1548 Categories (2, str): ['b', 'c']
1550 >>> c.add_categories(["d", "a"])
1551 ['c', 'b', 'c']
1552 Categories (4, str): ['b', 'c', 'd', 'a']
1553 """
1555 if not is_list_like(new_categories):
1556 new_categories = [new_categories]
1557 already_included = set(new_categories) & set(self.dtype.categories)
1558 if len(already_included) != 0:
1559 raise ValueError(
1560 f"new categories must not include old categories: {already_included}"
1561 )
1563 if hasattr(new_categories, "dtype"):
1564 from pandas import Series
1566 dtype = find_common_type(
1567 [self.dtype.categories.dtype, new_categories.dtype]
1568 )
1569 new_categories = Series(
1570 list(self.dtype.categories) + list(new_categories), dtype=dtype
1571 )
1572 else:
1573 new_categories = list(self.dtype.categories) + list(new_categories)
1575 new_dtype = CategoricalDtype(new_categories, self.ordered)
1576 cat = self.copy()
1577 codes = coerce_indexer_dtype(cat._ndarray, new_dtype.categories)
1578 NDArrayBacked.__init__(cat, codes, new_dtype)
1579 return cat
1581 def remove_categories(self, removals) -> Self:
1582 """
1583 Remove the specified categories.
1585 The ``removals`` argument must be a subset of the current categories.
1586 Any values that were part of the removed categories will be set to NaN.
1588 Parameters
1589 ----------
1590 removals : category or list of categories
1591 The categories which should be removed.
1593 Returns
1594 -------
1595 Categorical
1596 Categorical with removed categories.
1598 Raises
1599 ------
1600 ValueError
1601 If the removals are not contained in the categories
1603 See Also
1604 --------
1605 rename_categories : Rename categories.
1606 reorder_categories : Reorder categories.
1607 add_categories : Add new categories.
1608 remove_unused_categories : Remove categories which are not used.
1609 set_categories : Set the categories to the specified ones.
1611 Examples
1612 --------
1613 >>> c = pd.Categorical(["a", "c", "b", "c", "d"])
1614 >>> c
1615 ['a', 'c', 'b', 'c', 'd']
1616 Categories (4, str): ['a', 'b', 'c', 'd']
1618 >>> c.remove_categories(["d", "a"])
1619 [NaN, 'c', 'b', 'c', NaN]
1620 Categories (2, str): ['b', 'c']
1621 """
1622 from pandas import Index
1624 if not is_list_like(removals):
1625 removals = [removals]
1627 removals = Index(removals).unique().dropna()
1628 new_categories = (
1629 self.dtype.categories.difference(removals, sort=False)
1630 if self.dtype.ordered is True
1631 else self.dtype.categories.difference(removals)
1632 )
1633 not_included = removals.difference(self.dtype.categories)
1635 if len(not_included) != 0:
1636 not_included = set(not_included)
1637 raise ValueError(f"removals must all be in old categories: {not_included}")
1639 return self.set_categories(new_categories, ordered=self.ordered, rename=False)
1641 def remove_unused_categories(self) -> Self:
1642 """
1643 Remove categories which are not used.
1645 This method is useful when working with datasets
1646 that undergo dynamic changes where categories may no longer be
1647 relevant, allowing to maintain a clean, efficient data structure.
1649 Returns
1650 -------
1651 Categorical
1652 Categorical with unused categories dropped.
1654 See Also
1655 --------
1656 rename_categories : Rename categories.
1657 reorder_categories : Reorder categories.
1658 add_categories : Add new categories.
1659 remove_categories : Remove the specified categories.
1660 set_categories : Set the categories to the specified ones.
1662 Examples
1663 --------
1664 >>> c = pd.Categorical(["a", "c", "b", "c", "d"])
1665 >>> c
1666 ['a', 'c', 'b', 'c', 'd']
1667 Categories (4, str): ['a', 'b', 'c', 'd']
1669 >>> c[2] = "a"
1670 >>> c[4] = "c"
1671 >>> c
1672 ['a', 'c', 'a', 'c', 'c']
1673 Categories (4, str): ['a', 'b', 'c', 'd']
1675 >>> c.remove_unused_categories()
1676 ['a', 'c', 'a', 'c', 'c']
1677 Categories (2, str): ['a', 'c']
1678 """
1679 idx, inv = np.unique(self._codes, return_inverse=True)
1681 if idx.size != 0 and idx[0] == -1: # na sentinel
1682 idx, inv = idx[1:], inv - 1
1684 new_categories = self.dtype.categories.take(idx)
1685 new_dtype = CategoricalDtype._from_fastpath(
1686 new_categories, ordered=self.ordered
1687 )
1688 new_codes = coerce_indexer_dtype(inv, new_dtype.categories)
1690 cat = self.copy()
1691 NDArrayBacked.__init__(cat, new_codes, new_dtype)
1692 return cat
1694 # ------------------------------------------------------------------
1696 def map(
1697 self,
1698 mapper,
1699 na_action: Literal["ignore"] | None = None,
1700 ):
1701 """
1702 Map categories using an input mapping or function.
1704 Maps the categories to new categories. If the mapping correspondence is
1705 one-to-one the result is a :class:`~pandas.Categorical` which has the
1706 same order property as the original, otherwise a :class:`~pandas.Index`
1707 is returned. NaN values are unaffected.
1709 If a `dict` or :class:`~pandas.Series` is used any unmapped category is
1710 mapped to `NaN`. Note that if this happens an :class:`~pandas.Index`
1711 will be returned.
1713 Parameters
1714 ----------
1715 mapper : function, dict, or Series
1716 Mapping correspondence.
1717 na_action : {None, 'ignore'}, default None
1718 If 'ignore', propagate NaN values, without passing them to the
1719 mapping correspondence.
1721 Returns
1722 -------
1723 pandas.Categorical or pandas.Index
1724 Mapped categorical.
1726 See Also
1727 --------
1728 CategoricalIndex.map : Apply a mapping correspondence on a
1729 :class:`~pandas.CategoricalIndex`.
1730 Index.map : Apply a mapping correspondence on an
1731 :class:`~pandas.Index`.
1732 Series.map : Apply a mapping correspondence on a
1733 :class:`~pandas.Series`.
1734 Series.apply : Apply more complex functions on a
1735 :class:`~pandas.Series`.
1737 Examples
1738 --------
1739 >>> cat = pd.Categorical(["a", "b", "c"])
1740 >>> cat
1741 ['a', 'b', 'c']
1742 Categories (3, str): ['a', 'b', 'c']
1743 >>> cat.map(lambda x: x.upper(), na_action=None)
1744 ['A', 'B', 'C']
1745 Categories (3, str): ['A', 'B', 'C']
1746 >>> cat.map({"a": "first", "b": "second", "c": "third"}, na_action=None)
1747 ['first', 'second', 'third']
1748 Categories (3, str): ['first', 'second', 'third']
1750 If the mapping is one-to-one the ordering of the categories is
1751 preserved:
1753 >>> cat = pd.Categorical(["a", "b", "c"], ordered=True)
1754 >>> cat
1755 ['a', 'b', 'c']
1756 Categories (3, str): ['a' < 'b' < 'c']
1757 >>> cat.map({"a": 3, "b": 2, "c": 1}, na_action=None)
1758 [3, 2, 1]
1759 Categories (3, int64): [3 < 2 < 1]
1761 If the mapping is not one-to-one an :class:`~pandas.Index` is returned:
1763 >>> cat.map({"a": "first", "b": "second", "c": "first"}, na_action=None)
1764 Index(['first', 'second', 'first'], dtype='str')
1766 If a `dict` is used, all unmapped categories are mapped to `NaN` and
1767 the result is an :class:`~pandas.Index`:
1769 >>> cat.map({"a": "first", "b": "second"}, na_action=None)
1770 Index(['first', 'second', nan], dtype='str')
1772 The mapping function is applied to categories, not to each value. It is
1773 therefore only called once per unique category, and the result reused for
1774 all occurrences:
1776 >>> cat = pd.Categorical(["a", "a", "b"])
1777 >>> calls = []
1778 >>> def f(x):
1779 ... calls.append(x)
1780 ... return x.upper()
1781 >>> result = cat.map(f)
1782 >>> result
1783 ['A', 'A', 'B']
1784 Categories (2, str): ['A', 'B']
1785 >>> calls
1786 ['a', 'b']
1787 """
1788 assert callable(mapper) or is_dict_like(mapper)
1790 new_categories = self.categories.map(mapper)
1792 has_nans = np.any(self._codes == -1)
1794 na_val = np.nan
1795 if na_action is None and has_nans:
1796 if callable(mapper):
1797 na_val = mapper(np.nan)
1798 else:
1799 try:
1800 na_val = mapper[np.nan]
1801 except KeyError:
1802 na_val = np.nan
1804 # A MultiIndex (i.e. mapper returned tuples) can't back a
1805 # CategoricalDtype, so it must take the slow path below regardless
1806 # of uniqueness/na checks.
1807 if (
1808 not isinstance(new_categories, ABCMultiIndex)
1809 and new_categories.is_unique
1810 and not new_categories.hasnans
1811 and na_val is np.nan
1812 ):
1813 new_dtype = CategoricalDtype(new_categories, ordered=self.ordered)
1814 return self.from_codes(self._codes.copy(), dtype=new_dtype, validate=False)
1816 if isinstance(new_categories, ABCMultiIndex):
1817 # mapper returned tuples; a CategoricalDtype/Categorical cannot be
1818 # constructed from a MultiIndex, so fall back to a flat
1819 # object-dtype Index of tuples.
1820 new_categories = new_categories.to_flat_index()
1822 if has_nans:
1823 new_categories = new_categories.insert(len(new_categories), na_val)
1825 return np.take(new_categories, self._codes)
1827 __eq__ = _cat_compare_op(operator.eq)
1828 __ne__ = _cat_compare_op(operator.ne)
1829 __lt__ = _cat_compare_op(operator.lt)
1830 __gt__ = _cat_compare_op(operator.gt)
1831 __le__ = _cat_compare_op(operator.le)
1832 __ge__ = _cat_compare_op(operator.ge)
1834 # -------------------------------------------------------------
1835 # Validators; ideally these can be de-duplicated
1837 def _validate_setitem_value(self, value):
1838 if not is_hashable(value):
1839 # wrap scalars and hashable-listlikes in list
1840 return self._validate_listlike(value)
1841 else:
1842 return self._validate_scalar(value)
1844 def _validate_scalar(self, fill_value):
1845 """
1846 Convert a user-facing fill_value to a representation to use with our
1847 underlying ndarray, raising TypeError if this is not possible.
1849 Parameters
1850 ----------
1851 fill_value : object
1853 Returns
1854 -------
1855 fill_value : int
1857 Raises
1858 ------
1859 TypeError
1860 """
1862 if is_valid_na_for_dtype(fill_value, self.categories.dtype):
1863 fill_value = -1
1864 elif fill_value in self.categories:
1865 fill_value = self._unbox_scalar(fill_value)
1866 else:
1867 raise TypeError(
1868 "Cannot setitem on a Categorical with a new "
1869 f"category ({fill_value}), set the categories first"
1870 ) from None
1871 return fill_value
1873 @classmethod
1874 def _validate_codes_for_dtype(cls, codes, *, dtype: CategoricalDtype) -> np.ndarray:
1875 if isinstance(codes, ExtensionArray) and is_integer_dtype(codes.dtype):
1876 # Avoid the implicit conversion of Int to object
1877 if isna(codes).any():
1878 raise ValueError("codes cannot contain NA values")
1879 codes = codes.to_numpy(dtype=np.int64)
1880 else:
1881 codes = np.asarray(codes)
1882 if len(codes) and codes.dtype.kind not in "iu":
1883 raise ValueError("codes need to be array-like integers")
1885 if len(codes) and (codes.max() >= len(dtype.categories) or codes.min() < -1):
1886 raise ValueError("codes need to be between -1 and len(categories)-1")
1887 return codes
1889 # -------------------------------------------------------------
1891 @ravel_compat
1892 def __array__(
1893 self, dtype: NpDtype | None = None, copy: bool | None = None
1894 ) -> np.ndarray:
1895 """
1896 The numpy array interface.
1898 Users should not call this directly. Rather, it is invoked by
1899 :func:`numpy.array` and :func:`numpy.asarray`.
1901 Parameters
1902 ----------
1903 dtype : np.dtype or None
1904 Specifies the dtype for the array.
1906 copy : bool or None, optional
1907 See :func:`numpy.asarray`.
1909 Returns
1910 -------
1911 numpy.array
1912 A numpy array of either the specified dtype or,
1913 if dtype==None (default), the same dtype as
1914 categorical.categories.dtype.
1916 See Also
1917 --------
1918 numpy.asarray : Convert input to numpy.ndarray.
1920 Examples
1921 --------
1923 >>> cat = pd.Categorical(["a", "b"], ordered=True)
1925 The following calls ``cat.__array__``
1927 >>> np.asarray(cat)
1928 array(['a', 'b'], dtype=object)
1929 """
1930 if copy is False:
1931 raise ValueError(
1932 "Unable to avoid copy while creating an array as requested."
1933 )
1935 ret = take_nd(self.categories._values, self._codes)
1936 # When we're a Categorical[ExtensionArray], like Interval,
1937 # we need to ensure __array__ gets all the way to an
1938 # ndarray.
1940 # `take_nd` should already make a copy, so don't force again.
1941 return np.asarray(ret, dtype=dtype)
1943 def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
1944 # for binary ops, use our custom dunder methods
1945 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(
1946 self, ufunc, method, *inputs, **kwargs
1947 )
1948 if result is not NotImplemented:
1949 return result
1951 if "out" in kwargs:
1952 # e.g. test_numpy_ufuncs_out
1953 return arraylike.dispatch_ufunc_with_out(
1954 self, ufunc, method, *inputs, **kwargs
1955 )
1957 if method == "reduce":
1958 # e.g. TestCategoricalAnalytics::test_min_max_ordered
1959 result = arraylike.dispatch_reduction_ufunc(
1960 self, ufunc, method, *inputs, **kwargs
1961 )
1962 if result is not NotImplemented:
1963 return result
1965 # for all other cases, raise for now (similarly as what happens in
1966 # Series.__array_prepare__)
1967 raise TypeError(
1968 f"Object with dtype {self.dtype} cannot perform "
1969 f"the numpy op {ufunc.__name__}"
1970 )
1972 def __setstate__(self, state) -> None:
1973 """Necessary for making this object picklable"""
1974 if not isinstance(state, dict):
1975 return super().__setstate__(state)
1977 if "_dtype" not in state:
1978 state["_dtype"] = CategoricalDtype(state["_categories"], state["_ordered"])
1980 if "_codes" in state and "_ndarray" not in state:
1981 # backward compat, changed what is property vs attribute
1982 state["_ndarray"] = state.pop("_codes")
1984 super().__setstate__(state)
1986 @property
1987 def nbytes(self) -> int:
1988 return self._codes.nbytes + self.dtype.categories.values.nbytes
1990 def memory_usage(self, deep: bool = False) -> int:
1991 """
1992 Memory usage of my values
1994 Parameters
1995 ----------
1996 deep : bool
1997 Introspect the data deeply, interrogate
1998 `object` dtypes for system-level memory consumption
2000 Returns
2001 -------
2002 bytes used
2004 Notes
2005 -----
2006 Memory usage does not include memory consumed by elements that
2007 are not components of the array if deep=False
2009 See Also
2010 --------
2011 numpy.ndarray.nbytes
2012 """
2013 return self._codes.nbytes + self.dtype.categories.memory_usage(deep=deep)
2015 def isna(self) -> npt.NDArray[np.bool_]:
2016 """
2017 Detect missing values
2019 Missing values (-1 in .codes) are detected.
2021 Returns
2022 -------
2023 np.ndarray[bool] of whether my values are null
2025 See Also
2026 --------
2027 isna : Top-level isna.
2028 isnull : Alias of isna.
2029 Categorical.notna : Boolean inverse of Categorical.isna.
2031 """
2032 return self._codes == -1
2034 isnull = isna
2036 def notna(self) -> npt.NDArray[np.bool_]:
2037 """
2038 Inverse of isna
2040 Both missing values (-1 in .codes) and NA as a category are detected as
2041 null.
2043 Returns
2044 -------
2045 np.ndarray[bool] of whether my values are not null
2047 See Also
2048 --------
2049 notna : Top-level notna.
2050 notnull : Alias of notna.
2051 Categorical.isna : Boolean inverse of Categorical.notna.
2053 """
2054 return ~self.isna()
2056 notnull = notna
2058 def value_counts(self, dropna: bool = True) -> Series:
2059 """
2060 Return a Series containing counts of each category.
2062 Every category will have an entry, even those with a count of 0.
2064 Parameters
2065 ----------
2066 dropna : bool, default True
2067 Don't include counts of NaN.
2069 Returns
2070 -------
2071 counts : Series
2073 See Also
2074 --------
2075 Series.value_counts
2076 """
2077 from pandas import (
2078 CategoricalIndex,
2079 Series,
2080 )
2082 code, cat = self._codes, self.categories
2083 ncat, mask = (len(cat), code >= 0)
2084 ix, clean = np.arange(ncat), mask.all()
2086 if dropna or clean:
2087 obs = code if clean else code[mask]
2088 count = np.bincount(obs, minlength=ncat or 0)
2089 else:
2090 count = np.bincount(np.where(mask, code, ncat))
2091 ix = np.append(ix, -1)
2093 ix = coerce_indexer_dtype(ix, self.dtype.categories)
2094 ix_categorical = self._from_backing_data(ix)
2096 return Series(
2097 count,
2098 index=CategoricalIndex(ix_categorical),
2099 dtype="int64",
2100 name="count",
2101 copy=False,
2102 )
2104 # error: Argument 2 of "_empty" is incompatible with supertype
2105 # "NDArrayBackedExtensionArray"; supertype defines the argument type as
2106 # "ExtensionDtype"
2107 @classmethod
2108 def _empty( # type: ignore[override]
2109 cls, shape: Shape, dtype: CategoricalDtype
2110 ) -> Self:
2111 """
2112 Analogous to np.empty(shape, dtype=dtype)
2114 Parameters
2115 ----------
2116 shape : tuple[int]
2117 dtype : CategoricalDtype
2118 """
2119 arr = cls._from_sequence([], dtype=dtype)
2121 # We have to use np.zeros instead of np.empty otherwise the resulting
2122 # ndarray may contain codes not supported by this dtype, in which
2123 # case repr(result) could segfault.
2124 backing = np.zeros(shape, dtype=arr._ndarray.dtype)
2126 return arr._from_backing_data(backing)
2128 def _internal_get_values(self) -> ArrayLike:
2129 """
2130 Return the values.
2132 For internal compatibility with pandas formatting.
2134 Returns
2135 -------
2136 np.ndarray or ExtensionArray
2137 A numpy array or ExtensionArray of the same dtype as
2138 categorical.categories.dtype.
2139 """
2140 # if we are a datetime and period index, return Index to keep metadata
2141 if needs_i8_conversion(self.categories.dtype):
2142 return self.categories.take(
2143 self._codes, allow_fill=True, fill_value=NaT
2144 )._values
2145 elif is_integer_dtype(self.categories.dtype) and -1 in self._codes:
2146 return (
2147 self.categories.astype("object")
2148 .take(self._codes, allow_fill=True, fill_value=np.nan)
2149 ._values
2150 )
2151 return np.array(self)
2153 def check_for_ordered(self, op) -> None:
2154 """assert that we are ordered"""
2155 if not self.ordered:
2156 raise TypeError(
2157 f"Categorical is not ordered for operation {op}\n"
2158 "you can use .as_ordered() to change the "
2159 "Categorical to an ordered one\n"
2160 )
2162 def argsort(
2163 self, *, ascending: bool = True, kind: SortKind = "quicksort", **kwargs
2164 ) -> npt.NDArray[np.intp]:
2165 """
2166 Return the indices that would sort the Categorical.
2168 Missing values are sorted at the end.
2170 Parameters
2171 ----------
2172 ascending : bool, default True
2173 Whether the indices should result in an ascending
2174 or descending sort.
2175 kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional
2176 Sorting algorithm.
2177 **kwargs:
2178 passed through to :func:`numpy.argsort`.
2180 Returns
2181 -------
2182 np.ndarray[np.intp]
2184 See Also
2185 --------
2186 numpy.ndarray.argsort
2188 Notes
2189 -----
2190 While an ordering is applied to the category values, arg-sorting
2191 in this context refers more to organizing and grouping together
2192 based on matching category values. Thus, this function can be
2193 called on an unordered Categorical instance unlike the functions
2194 'Categorical.min' and 'Categorical.max'.
2196 Examples
2197 --------
2198 >>> pd.Categorical(["b", "b", "a", "c"]).argsort()
2199 array([2, 0, 1, 3])
2201 >>> cat = pd.Categorical(
2202 ... ["b", "b", "a", "c"], categories=["c", "b", "a"], ordered=True
2203 ... )
2204 >>> cat.argsort()
2205 array([3, 0, 1, 2])
2207 Missing values are placed at the end
2209 >>> cat = pd.Categorical([2, None, 1])
2210 >>> cat.argsort()
2211 array([2, 0, 1])
2212 """
2213 return super().argsort(ascending=ascending, kind=kind, **kwargs)
2215 @overload
2216 def sort_values(
2217 self,
2218 *,
2219 inplace: Literal[False] = ...,
2220 ascending: bool = ...,
2221 na_position: str = ...,
2222 ) -> Self: ...
2224 @overload
2225 def sort_values(
2226 self, *, inplace: Literal[True], ascending: bool = ..., na_position: str = ...
2227 ) -> None: ...
2229 def sort_values(
2230 self,
2231 *,
2232 inplace: bool = False,
2233 ascending: bool = True,
2234 na_position: str = "last",
2235 ) -> Self | None:
2236 """
2237 Sort the Categorical by category value returning a new
2238 Categorical by default.
2240 While an ordering is applied to the category values, sorting in this
2241 context refers more to organizing and grouping together based on
2242 matching category values. Thus, this function can be called on an
2243 unordered Categorical instance unlike the functions 'Categorical.min'
2244 and 'Categorical.max'.
2246 Parameters
2247 ----------
2248 inplace : bool, default False
2249 Do operation in place.
2250 ascending : bool, default True
2251 Order ascending. Passing False orders descending. The
2252 ordering parameter provides the method by which the
2253 category values are organized.
2254 na_position : {'first', 'last'} (optional, default='last')
2255 'first' puts NaNs at the beginning
2256 'last' puts NaNs at the end
2258 Returns
2259 -------
2260 Categorical or None
2262 See Also
2263 --------
2264 Categorical.sort
2265 Series.sort_values
2267 Examples
2268 --------
2269 >>> c = pd.Categorical([1, 2, 2, 1, 5])
2270 >>> c
2271 [1, 2, 2, 1, 5]
2272 Categories (3, int64): [1, 2, 5]
2273 >>> c.sort_values()
2274 [1, 1, 2, 2, 5]
2275 Categories (3, int64): [1, 2, 5]
2276 >>> c.sort_values(ascending=False)
2277 [5, 2, 2, 1, 1]
2278 Categories (3, int64): [1, 2, 5]
2280 >>> c = pd.Categorical([1, 2, 2, 1, 5])
2282 'sort_values' behaviour with NaNs. Note that 'na_position'
2283 is independent of the 'ascending' parameter:
2285 >>> c = pd.Categorical([np.nan, 2, 2, np.nan, 5])
2286 >>> c
2287 [NaN, 2, 2, NaN, 5]
2288 Categories (2, int64): [2, 5]
2289 >>> c.sort_values()
2290 [2, 2, 5, NaN, NaN]
2291 Categories (2, int64): [2, 5]
2292 >>> c.sort_values(ascending=False)
2293 [5, 2, 2, NaN, NaN]
2294 Categories (2, int64): [2, 5]
2295 >>> c.sort_values(na_position="first")
2296 [NaN, NaN, 2, 2, 5]
2297 Categories (2, int64): [2, 5]
2298 >>> c.sort_values(ascending=False, na_position="first")
2299 [NaN, NaN, 5, 2, 2]
2300 Categories (2, int64): [2, 5]
2301 """
2302 inplace = validate_bool_kwarg(inplace, "inplace")
2303 if na_position not in ["last", "first"]:
2304 raise ValueError(f"invalid na_position: {na_position!r}")
2306 sorted_idx = nargsort(self, ascending=ascending, na_position=na_position)
2308 if not inplace:
2309 codes = self._codes[sorted_idx]
2310 return self._from_backing_data(codes)
2311 self._codes[:] = self._codes[sorted_idx]
2312 return None
2314 def _rank(
2315 self,
2316 *,
2317 axis: AxisInt = 0,
2318 method: RankMethod = "average",
2319 na_option: RankNaOption = "keep",
2320 ascending: bool = True,
2321 pct: bool = False,
2322 ):
2323 """
2324 See Series.rank.__doc__.
2325 """
2326 if axis != 0:
2327 raise NotImplementedError
2328 vff = self._values_for_rank()
2329 return algorithms.rank(
2330 vff,
2331 axis=axis,
2332 method=method,
2333 na_option=na_option,
2334 ascending=ascending,
2335 pct=pct,
2336 )
2338 def _values_for_rank(self) -> np.ndarray:
2339 """
2340 For correctly ranking ordered categorical data. See GH#15420
2342 Ordered categorical data should be ranked on the basis of
2343 codes with -1 translated to NaN.
2345 Returns
2346 -------
2347 numpy.array
2349 """
2350 from pandas import Series
2352 if self.ordered:
2353 values = self.codes
2354 mask = values == -1
2355 if mask.any():
2356 values = values.astype("float64")
2357 values[mask] = np.nan
2358 elif is_any_real_numeric_dtype(self.categories.dtype):
2359 values = np.array(self)
2360 else:
2361 # reorder the categories (so rank can use the float codes)
2362 # instead of passing an object array to rank
2363 values = np.array(
2364 self.rename_categories(
2365 Series(self.categories, copy=False).rank().values
2366 )
2367 )
2368 return values
2370 def _hash_pandas_object(
2371 self, *, encoding: str, hash_key: str, categorize: bool
2372 ) -> npt.NDArray[np.uint64]:
2373 """
2374 Hash a Categorical by hashing its categories, and then mapping the codes
2375 to the hashes.
2377 Parameters
2378 ----------
2379 encoding : str
2380 hash_key : str
2381 categorize : bool
2382 Ignored for Categorical.
2384 Returns
2385 -------
2386 np.ndarray[uint64]
2387 """
2388 # Note we ignore categorize, as we are already Categorical.
2389 from pandas.core.util.hashing import hash_array
2391 # Convert ExtensionArrays to ndarrays
2392 values = np.asarray(self.categories._values)
2393 hashed = hash_array(values, encoding, hash_key, categorize=False)
2395 # we have uint64, as we don't directly support missing values
2396 # we don't want to use take_nd which will coerce to float
2397 # instead, directly construct the result with a
2398 # max(np.uint64) as the missing value indicator
2399 #
2400 # TODO: GH#15362
2402 mask = self.isna()
2403 if len(hashed):
2404 result = hashed.take(self._codes)
2405 else:
2406 result = np.zeros(len(mask), dtype="uint64")
2408 if mask.any():
2409 result[mask] = lib.u8max
2411 return result
2413 # ------------------------------------------------------------------
2414 # NDArrayBackedExtensionArray compat
2416 @property
2417 def _codes(self) -> np.ndarray:
2418 return self._ndarray
2420 def _box_func(self, i: int):
2421 if i == -1:
2422 return np.nan
2423 return self.categories[i]
2425 def _unbox_scalar(self, key) -> int:
2426 # searchsorted is very performance sensitive. By converting codes
2427 # to same dtype as self.codes, we get much faster performance.
2428 code = self.categories.get_loc(key)
2429 code = self._ndarray.dtype.type(code)
2430 return code
2432 # ------------------------------------------------------------------
2434 def __iter__(self) -> Iterator[Any]:
2435 """
2436 Returns an Iterator over the values of this Categorical.
2437 """
2438 if self.ndim == 1:
2439 return iter(self._internal_get_values().tolist())
2440 else:
2441 return (self[n] for n in range(len(self)))
2443 def __contains__(self, key) -> bool:
2444 """
2445 Returns True if `key` is in this Categorical.
2446 """
2447 # if key is a NaN, check if any NaN is in self.
2448 if is_valid_na_for_dtype(key, self.categories.dtype):
2449 return bool(self.isna().any())
2451 return contains(self, key, container=self._codes)
2453 # ------------------------------------------------------------------
2454 # Rendering Methods
2456 # error: Return type "None" of "_formatter" incompatible with return
2457 # type "Callable[[Any], str | None]" in supertype "ExtensionArray"
2458 def _formatter(self, boxed: bool = False) -> None: # type: ignore[override]
2459 # Returning None here will cause format_array to do inference.
2460 return None
2462 def _repr_categories(self) -> list[str]:
2463 """
2464 return the base repr for the categories
2465 """
2466 max_categories = (
2467 10
2468 if config["display"]["max_categories"] == 0
2469 else config["display"]["max_categories"]
2470 )
2471 from pandas.io.formats import format as fmt
2473 formatter = None
2474 if self.categories.dtype == "str" or self.categories.dtype == "string": # noqa: PLR1714 (repeated-equality-comparison)
2475 # the extension array formatter defaults to boxed=True in format_array
2476 # override here to boxed=False to be consistent with QUOTE_NONNUMERIC
2477 formatter = cast("ExtensionArray", self.categories._values)._formatter(
2478 boxed=False
2479 )
2481 format_array = partial(
2482 fmt.format_array, formatter=formatter, quoting=QUOTE_NONNUMERIC
2483 )
2484 if len(self.categories) > max_categories:
2485 num = max_categories // 2
2486 head = format_array(self.categories[:num]._values)
2487 tail = format_array(self.categories[-num:]._values)
2488 category_strs = [*head, "...", *tail]
2489 else:
2490 category_strs = format_array(self.categories._values)
2492 # Strip all leading spaces, which format_array adds for columns...
2493 category_strs = [x.strip() for x in category_strs]
2494 return category_strs
2496 def _get_repr_footer(self) -> str:
2497 """
2498 Returns a string representation of the footer.
2499 """
2500 category_strs = self._repr_categories()
2501 dtype = str(self.categories.dtype)
2502 levheader = f"Categories ({len(self.categories)}, {dtype}): "
2503 width, _ = get_terminal_size()
2504 max_width = config["display"]["width"] or width
2505 if console.in_ipython_frontend():
2506 # 0 = no breaks
2507 max_width = 0
2508 parts: list[str] = []
2509 start = True
2510 cur_col_len = len(levheader) # header
2511 sep_len, sep = (3, " < ") if self.ordered else (2, ", ")
2512 linesep = f"{sep.rstrip()}\n" # remove whitespace
2513 for val in category_strs:
2514 if max_width != 0 and cur_col_len + sep_len + len(val) > max_width:
2515 parts.append(linesep + (" " * (len(levheader) + 1)))
2516 cur_col_len = len(levheader) + 1 # header + a whitespace
2517 elif not start:
2518 parts.append(sep)
2519 cur_col_len += sep_len
2520 parts.append(val)
2521 cur_col_len += len(val)
2522 start = False
2523 levstring = "".join(parts)
2524 # replace to simple save space by
2525 return f"{levheader}[{levstring.replace(' < ... < ', ' ... ')}]"
2527 def _get_formatted_values(self) -> list[str]:
2528 """Return formatted string representations of values."""
2529 from pandas.io.formats import format as fmt
2531 assert len(self) > 0
2533 vals = self._internal_get_values()
2534 fmt_values = fmt.format_array(
2535 vals,
2536 None,
2537 float_format=None,
2538 na_rep="NaN",
2539 quoting=QUOTE_NONNUMERIC,
2540 )
2541 return [val.strip() for val in fmt_values]
2543 @staticmethod
2544 def _format_values_line(fmt_values: list[str], max_width: int) -> str:
2545 """Format a list of values into a bracketed, width-respecting string."""
2546 if max_width == 0:
2547 return "[" + ", ".join(fmt_values) + "]"
2549 parts = ["["]
2550 cur_col_len = 1 # account for the opening bracket
2551 start = True
2552 for val in fmt_values:
2553 if not start:
2554 if cur_col_len + 2 + len(val) > max_width:
2555 parts.append(",\n ")
2556 cur_col_len = 1 # 1 space indent
2557 else:
2558 parts.append(", ")
2559 cur_col_len += 2
2560 parts.append(val)
2561 cur_col_len += len(val)
2562 start = False
2563 parts.append("]")
2564 return "".join(parts)
2566 def __repr__(self) -> str:
2567 """
2568 String representation.
2569 """
2570 footer = self._get_repr_footer()
2571 length = len(self)
2572 max_len = 10
2574 width, _ = get_terminal_size()
2575 max_width = config["display"]["width"] or width
2576 if console.in_ipython_frontend():
2577 max_width = 0
2579 if length > max_len:
2580 # In long cases we do not display all entries, so we add Length
2581 # information to the __repr__.
2582 num = max_len // 2
2583 head_vals = self[:num]._get_formatted_values()
2584 tail_vals = self[-(max_len - num) :]._get_formatted_values()
2585 all_vals = [*head_vals, "...", *tail_vals]
2586 body = self._format_values_line(all_vals, max_width)
2587 length_info = f"Length: {len(self)}"
2588 result = f"{body}\n{length_info}\n{footer}"
2589 elif length > 0:
2590 fmt_values = self._get_formatted_values()
2591 body = self._format_values_line(fmt_values, max_width)
2592 result = f"{body}\n{footer}"
2593 else:
2594 # In the empty case we use a comma instead of newline to get
2595 # a more compact __repr__
2596 body = "[]"
2597 result = f"{body}, {footer}"
2599 return result
2601 # ------------------------------------------------------------------
2603 def _validate_listlike(self, value):
2604 # NB: here we assume scalar-like tuples have already been excluded
2605 value = extract_array(value, extract_numpy=True)
2607 # require identical categories set
2608 if isinstance(value, Categorical):
2609 if self.dtype != value.dtype:
2610 raise TypeError(
2611 "Cannot set a Categorical with another, "
2612 "without identical categories"
2613 )
2614 # dtype equality implies categories_match_up_to_permutation
2615 value = self._encode_with_my_categories(value)
2616 return value._codes
2618 from pandas import Index
2620 # tupleize_cols=False for e.g. test_fillna_iterable_category GH#41914
2621 to_add = Index._with_infer(value, tupleize_cols=False, copy=False).difference(
2622 self.categories
2623 )
2625 # no assignments of values not in categories, but it's always ok to set
2626 # something to np.nan
2627 if len(to_add) and not isna(to_add).all():
2628 raise TypeError(
2629 "Cannot setitem on a Categorical with a new "
2630 "category, set the categories first"
2631 )
2633 codes = self.categories.get_indexer(value)
2634 return codes.astype(self._ndarray.dtype, copy=False)
2636 def _reverse_indexer(self) -> dict[Hashable, npt.NDArray[np.intp]]:
2637 """
2638 Compute the inverse of a categorical, returning
2639 a dict of categories -> indexers.
2641 *This is an internal function*
2643 Returns
2644 -------
2645 Dict[Hashable, np.ndarray[np.intp]]
2646 dict of categories -> indexers
2648 Examples
2649 --------
2650 >>> c = pd.Categorical(list("aabca"))
2651 >>> c
2652 ['a', 'a', 'b', 'c', 'a']
2653 Categories (3, str): ['a', 'b', 'c']
2654 >>> c.categories
2655 Index(['a', 'b', 'c'], dtype='str')
2656 >>> c.codes
2657 array([0, 0, 1, 2, 0], dtype=int8)
2658 >>> c._reverse_indexer()
2659 {'a': array([0, 1, 4]), 'b': array([2]), 'c': array([3])}
2661 """
2662 categories = self.categories
2663 r, counts = libalgos.groupsort_indexer(
2664 ensure_platform_int(self.codes), categories.size
2665 )
2666 counts = ensure_int64(counts).cumsum()
2667 _result = (r[start:end] for start, end in itertools.pairwise(counts))
2668 return dict(zip(categories, _result, strict=True))
2670 # ------------------------------------------------------------------
2671 # Reductions
2673 def _reduce(
2674 self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
2675 ):
2676 result = super()._reduce(name, skipna=skipna, keepdims=keepdims, **kwargs)
2677 if name in ["argmax", "argmin"]:
2678 # don't wrap in Categorical!
2679 return result
2680 if keepdims:
2681 return type(self)(result, dtype=self.dtype)
2682 else:
2683 return result
2685 def min(self, *, skipna: bool = True, **kwargs):
2686 """
2687 The minimum value of the object.
2689 Only ordered `Categoricals` have a minimum!
2691 Raises
2692 ------
2693 TypeError
2694 If the `Categorical` is not `ordered`.
2696 Returns
2697 -------
2698 min : the minimum of this `Categorical`, NA value if empty
2699 """
2700 nv.validate_minmax_axis(kwargs.get("axis", 0))
2701 nv.validate_min((), kwargs)
2702 self.check_for_ordered("min")
2704 if not len(self._codes):
2705 return self.dtype.na_value
2707 good = self._codes != -1
2708 if not good.all():
2709 if skipna and good.any():
2710 pointer = self._codes[good].min()
2711 else:
2712 return np.nan
2713 else:
2714 pointer = self._codes.min()
2715 return self._wrap_reduction_result(None, pointer)
2717 def max(self, *, skipna: bool = True, **kwargs):
2718 """
2719 The maximum value of the object.
2721 Only ordered `Categoricals` have a maximum!
2723 Raises
2724 ------
2725 TypeError
2726 If the `Categorical` is not `ordered`.
2728 Returns
2729 -------
2730 max : the maximum of this `Categorical`, NA if array is empty
2731 """
2732 nv.validate_minmax_axis(kwargs.get("axis", 0))
2733 nv.validate_max((), kwargs)
2734 self.check_for_ordered("max")
2736 if not len(self._codes):
2737 return self.dtype.na_value
2739 good = self._codes != -1
2740 if not good.all():
2741 if skipna and good.any():
2742 pointer = self._codes[good].max()
2743 else:
2744 return np.nan
2745 else:
2746 pointer = self._codes.max()
2747 return self._wrap_reduction_result(None, pointer)
2749 def _mode(self, dropna: bool = True) -> Categorical:
2750 codes = self._codes
2751 mask = None
2752 if dropna:
2753 mask = self.isna()
2755 res_codes, _ = algorithms.mode(codes, mask=mask)
2756 res_codes = cast("np.ndarray", res_codes)
2757 assert res_codes.dtype == codes.dtype
2758 res = self._from_backing_data(res_codes)
2759 return res
2761 # ------------------------------------------------------------------
2762 # ExtensionArray Interface
2764 def unique(self) -> Self:
2765 """
2766 Return the ``Categorical`` which ``categories`` and ``codes`` are
2767 unique.
2769 Returns
2770 -------
2771 Categorical
2773 See Also
2774 --------
2775 pandas.unique
2776 CategoricalIndex.unique
2777 Series.unique : Return unique values of Series object.
2779 Examples
2780 --------
2781 >>> pd.Categorical(list("baabc")).unique()
2782 ['b', 'a', 'c']
2783 Categories (3, str): ['a', 'b', 'c']
2784 >>> pd.Categorical(list("baab"), categories=list("abc"), ordered=True).unique()
2785 ['b', 'a']
2786 Categories (3, str): ['a' < 'b' < 'c']
2787 """
2788 return super().unique()
2790 def equals(self, other: object) -> bool:
2791 """
2792 Returns True if categorical arrays are equal.
2794 Parameters
2795 ----------
2796 other : `Categorical`
2798 Returns
2799 -------
2800 bool
2801 """
2802 if not isinstance(other, Categorical):
2803 return False
2804 elif self._categories_match_up_to_permutation(other):
2805 other = self._encode_with_my_categories(other)
2806 return lib.array_equivalent_bytes(self._codes, other._codes)
2807 return False
2809 def _accumulate(self, name: str, skipna: bool = True, **kwargs) -> Self:
2810 func: Callable[..., Any]
2811 if name == "cummin":
2812 func = np.minimum.accumulate
2813 elif name == "cummax":
2814 func = np.maximum.accumulate
2815 else:
2816 raise TypeError(f"Accumulation {name} not supported for {type(self)}")
2817 self.check_for_ordered(name)
2819 codes = self.codes.copy()
2820 mask = self.isna()
2821 if func == np.minimum.accumulate:
2822 codes[mask] = np.iinfo(codes.dtype.type).max
2823 # no need to change codes for maximum because codes[mask] is already -1
2824 if not skipna:
2825 mask = np.maximum.accumulate(mask)
2827 codes = func(codes)
2828 codes[mask] = -1
2829 return self._simple_new(codes, dtype=self._dtype)
2831 @classmethod
2832 def _concat_same_type(cls, to_concat: Sequence[Self], axis: AxisInt = 0) -> Self:
2833 from pandas.core.dtypes.concat import union_categoricals
2835 first = to_concat[0]
2836 if axis >= first.ndim:
2837 raise ValueError(
2838 f"axis {axis} is out of bounds for array of dimension {first.ndim}"
2839 )
2841 if axis == 1:
2842 # Flatten, concatenate then reshape
2843 if not all(x.ndim == 2 for x in to_concat):
2844 raise ValueError
2846 # pass correctly-shaped to union_categoricals
2847 tc_flat = []
2848 for obj in to_concat:
2849 tc_flat.extend([obj[:, i] for i in range(obj.shape[1])])
2851 res_flat = cls._concat_same_type(tc_flat, axis=0)
2853 result = res_flat.reshape(len(first), -1, order="F")
2854 return result
2856 # error: Incompatible types in assignment (expression has type "Categorical",
2857 # variable has type "Self")
2858 result = union_categoricals(to_concat) # type: ignore[assignment]
2859 return result
2861 # ------------------------------------------------------------------
2863 def _encode_with_my_categories(self, other: Categorical) -> Categorical:
2864 """
2865 Re-encode another categorical using this Categorical's categories.
2867 Notes
2868 -----
2869 This assumes we have already checked
2870 self._categories_match_up_to_permutation(other).
2871 """
2872 # Indexing on codes is more efficient if categories are the same,
2873 # so we can apply some optimizations based on the degree of
2874 # dtype-matching.
2875 codes = recode_for_categories(
2876 other.codes, other.categories, self.categories, copy=False
2877 )
2878 return self._from_backing_data(codes)
2880 def _categories_match_up_to_permutation(self, other: Categorical) -> bool:
2881 """
2882 Returns True if categoricals are the same dtype
2883 same categories, and same ordered
2885 Parameters
2886 ----------
2887 other : Categorical
2889 Returns
2890 -------
2891 bool
2892 """
2893 return hash(self.dtype) == hash(other.dtype)
2895 def describe(self) -> DataFrame:
2896 """
2897 Describes this Categorical
2899 Returns
2900 -------
2901 description: `DataFrame`
2902 A dataframe with frequency and counts by category.
2903 """
2904 counts = self.value_counts(dropna=False)
2905 freqs = counts / counts.sum()
2907 from pandas import Index
2908 from pandas.core.reshape.concat import concat
2910 result = concat([counts, freqs], ignore_index=True, axis=1)
2911 result.columns = Index(["counts", "freqs"])
2912 result.index.name = "categories"
2914 return result
2916 def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
2917 """
2918 Check whether `values` are contained in Categorical.
2920 Return a boolean NumPy Array showing whether each element in
2921 the Categorical matches an element in the passed sequence of
2922 `values` exactly.
2924 Parameters
2925 ----------
2926 values : np.ndarray or ExtensionArray
2927 The sequence of values to test. Passing in a single string will
2928 raise a ``TypeError``. Instead, turn a single string into a
2929 list of one element.
2931 Returns
2932 -------
2933 np.ndarray[bool]
2935 Raises
2936 ------
2937 TypeError
2938 * If `values` is not a set or list-like
2940 See Also
2941 --------
2942 pandas.Series.isin : Equivalent method on Series.
2944 Examples
2945 --------
2946 >>> s = pd.Categorical(["llama", "cow", "llama", "beetle", "llama", "hippo"])
2947 >>> s.isin(["cow", "llama"])
2948 array([ True, True, True, False, True, False])
2950 Passing a single string as ``s.isin('llama')`` will raise an error. Use
2951 a list of one element instead:
2953 >>> s.isin(["llama"])
2954 array([ True, False, True, False, True, False])
2955 """
2956 null_mask = np.asarray(isna(values))
2957 code_values = self.categories.get_indexer_for(values)
2958 code_values = code_values[null_mask | (code_values >= 0)]
2959 return algorithms.isin(self.codes, code_values)
2961 # ------------------------------------------------------------------------
2962 # String methods interface
2963 def _str_map(
2964 self, f, na_value=lib.no_default, dtype=np.dtype("object"), convert: bool = True
2965 ):
2966 # Optimization to apply the callable `f` to the categories once
2967 # and rebuild the result by `take`ing from the result with the codes.
2968 # Returns the same type as the object-dtype implementation though.
2969 categories = self.categories
2970 codes = self.codes
2971 if categories.dtype == "string":
2972 result = categories.array._str_map(f, na_value, dtype) # type: ignore[attr-defined]
2973 if (
2974 categories.dtype.na_value is np.nan # type: ignore[union-attr]
2975 and is_bool_dtype(dtype)
2976 and (na_value is lib.no_default or isna(na_value))
2977 ):
2978 # NaN propagates as False for functions with boolean return type
2979 na_value = False
2980 else:
2981 from pandas.core.arrays import NumpyExtensionArray
2983 result = NumpyExtensionArray(categories.to_numpy())._str_map(
2984 f, na_value, dtype
2985 )
2986 return take_nd(result, codes, fill_value=na_value)
2988 def _str_get_dummies(self, sep: str = "|", dtype: NpDtype | None = None):
2989 # sep may not be in categories. Just bail on this.
2990 from pandas.core.arrays import NumpyExtensionArray
2992 return NumpyExtensionArray(self.to_numpy(str, na_value="NaN"))._str_get_dummies(
2993 sep, dtype
2994 )
2996 # ------------------------------------------------------------------------
2997 # GroupBy Methods
2999 def _groupby_op(
3000 self,
3001 *,
3002 how: str,
3003 has_dropped_na: bool,
3004 min_count: int,
3005 ngroups: int,
3006 ids: npt.NDArray[np.intp],
3007 **kwargs,
3008 ):
3009 from pandas.core.groupby.ops import WrappedCythonOp
3011 op = WrappedCythonOp(how=how, has_dropped_na=has_dropped_na)
3013 dtype = self.dtype
3014 if how in ["sum", "prod", "cumsum", "cumprod", "skew", "kurt"]:
3015 raise TypeError(f"{dtype} type does not support {how} operations")
3016 if how in ["min", "max", "rank", "idxmin", "idxmax"] and not dtype.ordered:
3017 # raise TypeError instead of NotImplementedError to ensure we
3018 # don't go down a group-by-group path, since in the empty-groups
3019 # case that would fail to raise
3020 raise TypeError(f"Cannot perform {how} with non-ordered Categorical")
3021 if how not in [
3022 "rank",
3023 "any",
3024 "all",
3025 "first",
3026 "last",
3027 "min",
3028 "max",
3029 "idxmin",
3030 "idxmax",
3031 ]:
3032 if op.kind == "transform":
3033 raise TypeError(f"{dtype} type does not support {how} operations")
3034 raise TypeError(f"{dtype} dtype does not support aggregation '{how}'")
3036 result_mask = None
3037 mask = self.isna()
3038 if how == "rank":
3039 assert self.ordered # checked earlier
3040 npvalues = self._ndarray
3041 elif how in ["first", "last", "min", "max", "idxmin", "idxmax"]:
3042 npvalues = self._ndarray
3043 result_mask = np.zeros(ngroups, dtype=bool)
3044 else:
3045 # any/all
3046 npvalues = self.astype(bool)
3048 res_values = op._cython_op_ndim_compat(
3049 npvalues,
3050 min_count=min_count,
3051 ngroups=ngroups,
3052 comp_ids=ids,
3053 mask=mask,
3054 result_mask=result_mask,
3055 **kwargs,
3056 )
3058 if how in op.cast_blocklist:
3059 return res_values
3060 elif how in ["first", "last", "min", "max"]:
3061 res_values[result_mask == 1] = -1
3062 return self._from_backing_data(res_values)
3065# The Series.cat accessor
3068@delegate_names(
3069 delegate=Categorical, accessors=["categories", "ordered"], typ="property"
3070)
3071@delegate_names(
3072 delegate=Categorical,
3073 accessors=[
3074 "rename_categories",
3075 "reorder_categories",
3076 "add_categories",
3077 "remove_categories",
3078 "remove_unused_categories",
3079 "set_categories",
3080 "as_ordered",
3081 "as_unordered",
3082 ],
3083 typ="method",
3084)
3085class CategoricalAccessor(PandasDelegate, PandasObject, NoNewAttributesMixin):
3086 """
3087 Accessor object for categorical properties of the Series values.
3089 This accessor provides methods and properties to interact with the
3090 underlying categorical data, such as accessing categories, renaming
3091 them, reordering, or adding and removing categories.
3093 Parameters
3094 ----------
3095 data : Series or CategoricalIndex
3096 The object to which the categorical accessor is attached.
3098 See Also
3099 --------
3100 Series.dt : Accessor object for datetimelike properties of the Series values.
3101 Series.sparse : Accessor for sparse matrix data types.
3103 Examples
3104 --------
3105 >>> s = pd.Series(list("abbccc")).astype("category")
3106 >>> s
3107 0 a
3108 1 b
3109 2 b
3110 3 c
3111 4 c
3112 5 c
3113 dtype: category
3114 Categories (3, str): ['a', 'b', 'c']
3116 >>> s.cat.categories
3117 Index(['a', 'b', 'c'], dtype='str')
3119 >>> s.cat.rename_categories(list("cba"))
3120 0 c
3121 1 b
3122 2 b
3123 3 a
3124 4 a
3125 5 a
3126 dtype: category
3127 Categories (3, str): ['c', 'b', 'a']
3129 >>> s.cat.reorder_categories(list("cba"))
3130 0 a
3131 1 b
3132 2 b
3133 3 c
3134 4 c
3135 5 c
3136 dtype: category
3137 Categories (3, str): ['c', 'b', 'a']
3139 >>> s.cat.add_categories(["d", "e"])
3140 0 a
3141 1 b
3142 2 b
3143 3 c
3144 4 c
3145 5 c
3146 dtype: category
3147 Categories (5, str): ['a', 'b', 'c', 'd', 'e']
3149 >>> s.cat.remove_categories(["a", "c"])
3150 0 NaN
3151 1 b
3152 2 b
3153 3 NaN
3154 4 NaN
3155 5 NaN
3156 dtype: category
3157 Categories (1, str): ['b']
3159 >>> s1 = s.cat.add_categories(["d", "e"])
3160 >>> s1.cat.remove_unused_categories()
3161 0 a
3162 1 b
3163 2 b
3164 3 c
3165 4 c
3166 5 c
3167 dtype: category
3168 Categories (3, str): ['a', 'b', 'c']
3170 >>> s.cat.set_categories(list("abcde"))
3171 0 a
3172 1 b
3173 2 b
3174 3 c
3175 4 c
3176 5 c
3177 dtype: category
3178 Categories (5, str): ['a', 'b', 'c', 'd', 'e']
3180 >>> s.cat.as_ordered()
3181 0 a
3182 1 b
3183 2 b
3184 3 c
3185 4 c
3186 5 c
3187 dtype: category
3188 Categories (3, str): ['a' < 'b' < 'c']
3190 >>> s.cat.as_unordered()
3191 0 a
3192 1 b
3193 2 b
3194 3 c
3195 4 c
3196 5 c
3197 dtype: category
3198 Categories (3, str): ['a', 'b', 'c']
3199 """
3201 def __init__(self, data) -> None:
3202 self._validate(data)
3203 self._parent = data.values
3204 self._index = data.index
3205 self._name = data.name
3206 self._freeze()
3208 @staticmethod
3209 def _validate(data) -> None:
3210 if not isinstance(data.dtype, CategoricalDtype):
3211 raise AttributeError("Can only use .cat accessor with a 'category' dtype")
3213 def _delegate_property_get(self, name: str):
3214 return getattr(self._parent, name)
3216 def _delegate_property_set(self, name: str, new_values) -> None:
3217 setattr(self._parent, name, new_values)
3219 @property
3220 def codes(self) -> Series:
3221 """
3222 Return Series of codes as well as the index.
3224 The codes are integer indicators for the position of each value in
3225 the categories. Uncategorized values (i.e., NaN) are assigned a code
3226 of ``-1``.
3228 See Also
3229 --------
3230 Series.cat.categories : Return the categories of this categorical.
3231 Series.cat.as_ordered : Set the Categorical to be ordered.
3232 Series.cat.as_unordered : Set the Categorical to be unordered.
3234 Examples
3235 --------
3236 >>> raw_cate = pd.Categorical(["a", "b", None, "a"], categories=["a", "b"])
3237 >>> ser = pd.Series(raw_cate)
3238 >>> ser.cat.codes
3239 0 0
3240 1 1
3241 2 -1
3242 3 0
3243 dtype: int8
3244 """
3245 from pandas import Series
3247 return Series(self._parent.codes, index=self._index)
3249 def _delegate_method(self, name: str, *args, **kwargs):
3250 from pandas import Series
3252 method = getattr(self._parent, name)
3253 res = method(*args, **kwargs)
3254 if res is not None:
3255 return Series(res, index=self._index, name=self._name)
3258# utility routines
3261def _get_codes_for_values(
3262 values: Index | Series | ExtensionArray | np.ndarray,
3263 categories: Index,
3264) -> np.ndarray:
3265 """
3266 utility routine to turn values into codes given the specified categories
3268 If `values` is known to be a Categorical, use recode_for_categories instead.
3269 """
3270 codes = categories.get_indexer_for(values)
3271 wrong = (codes == -1) & ~isna(values)
3272 if wrong.any():
3273 warnings.warn(
3274 "Constructing a Categorical with a dtype and values containing "
3275 "non-null entries not in that dtype's categories is deprecated "
3276 "and will raise in a future version.",
3277 Pandas4Warning,
3278 stacklevel=find_stack_level(),
3279 )
3280 return coerce_indexer_dtype(codes, categories)
3283def recode_for_categories(
3284 codes: np.ndarray,
3285 old_categories,
3286 new_categories,
3287 *,
3288 copy: bool = True,
3289 warn: bool = False,
3290) -> np.ndarray:
3291 """
3292 Convert a set of codes for to a new set of categories
3294 Parameters
3295 ----------
3296 codes : np.ndarray
3297 old_categories, new_categories : Index
3298 copy: bool, default True
3299 Whether to copy if the codes are unchanged.
3300 warn : bool, default False
3301 Whether to warn on silent-NA mapping.
3303 Returns
3304 -------
3305 new_codes : np.ndarray[np.int64]
3307 Examples
3308 --------
3309 >>> old_cat = pd.Index(["b", "a", "c"])
3310 >>> new_cat = pd.Index(["a", "b"])
3311 >>> codes = np.array([0, 1, 1, 2])
3312 >>> recode_for_categories(codes, old_cat, new_cat, copy=True)
3313 array([ 1, 0, 0, -1], dtype=int8)
3314 """
3315 if len(old_categories) == 0:
3316 # All null anyway, so just retain the nulls
3317 if copy:
3318 return codes.copy()
3319 return codes
3320 elif new_categories.equals(old_categories):
3321 # Same categories, so no need to actually recode
3322 if copy:
3323 return codes.copy()
3324 return codes
3326 codes_in_old_cats = new_categories.get_indexer_for(old_categories)
3327 if warn:
3328 wrong = codes_in_old_cats == -1
3329 if wrong.any():
3330 warnings.warn(
3331 "Constructing a Categorical with a dtype and values containing "
3332 "non-null entries not in that dtype's categories is deprecated "
3333 "and will raise in a future version.",
3334 Pandas4Warning,
3335 stacklevel=find_stack_level(),
3336 )
3337 indexer = coerce_indexer_dtype(codes_in_old_cats, new_categories)
3338 new_codes = take_nd(indexer, codes, fill_value=-1)
3339 return new_codes
3342def factorize_from_iterable(values) -> tuple[np.ndarray, Index]:
3343 """
3344 Factorize an input `values` into `categories` and `codes`. Preserves
3345 categorical dtype in `categories`.
3347 Parameters
3348 ----------
3349 values : list-like
3351 Returns
3352 -------
3353 codes : ndarray
3354 categories : Index
3355 If `values` has a categorical dtype, then `categories` is
3356 a CategoricalIndex keeping the categories and order of `values`.
3357 """
3358 from pandas import CategoricalIndex
3360 if not is_list_like(values):
3361 raise TypeError("Input must be list-like")
3363 categories: Index
3365 vdtype = getattr(values, "dtype", None)
3366 if isinstance(vdtype, CategoricalDtype):
3367 values = extract_array(values)
3368 # The Categorical we want to build has the same categories
3369 # as values but its codes are by def [0, ..., len(n_categories) - 1]
3370 cat_codes = np.arange(len(values.categories), dtype=values.codes.dtype)
3371 cat = Categorical.from_codes(cat_codes, dtype=values.dtype, validate=False)
3373 categories = CategoricalIndex(cat)
3374 codes = values.codes
3375 else:
3376 # The value of ordered is irrelevant since we don't use cat as such,
3377 # but only the resulting categories, the order of which is independent
3378 # from ordered. Set ordered to False as default. See GH #15457
3379 cat = Categorical(values, ordered=False)
3380 categories = cat.categories
3381 codes = cat.codes
3382 return codes, categories
3385def factorize_from_iterables(iterables) -> tuple[list[np.ndarray], list[Index]]:
3386 """
3387 A higher-level wrapper over `factorize_from_iterable`.
3389 Parameters
3390 ----------
3391 iterables : list-like of list-likes
3393 Returns
3394 -------
3395 codes : list of ndarrays
3396 categories : list of Indexes
3398 Notes
3399 -----
3400 See `factorize_from_iterable` for more info.
3401 """
3402 if len(iterables) == 0:
3403 # For consistency, it should return two empty lists.
3404 return [], []
3406 codes, categories = zip(
3407 *(factorize_from_iterable(it) for it in iterables),
3408 strict=True,
3409 )
3410 return list(codes), list(categories)