Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from csv import QUOTE_NONNUMERIC
4from functools import partial
5import itertools
6import operator
7from shutil import get_terminal_size
8from typing import (
9 TYPE_CHECKING,
10 Literal,
11 Self,
12 cast,
13 overload,
14)
15import warnings
17import numpy as np
19from pandas._config import get_option
21from pandas._libs import (
22 NaT,
23 algos as libalgos,
24 lib,
25)
26from pandas._libs.arrays import NDArrayBacked
27from pandas.compat.numpy import function as nv
28from pandas.errors import Pandas4Warning
29from pandas.util._decorators import set_module
30from pandas.util._exceptions import find_stack_level
31from pandas.util._validators import validate_bool_kwarg
33from pandas.core.dtypes.cast import (
34 coerce_indexer_dtype,
35 find_common_type,
36)
37from pandas.core.dtypes.common import (
38 ensure_int64,
39 ensure_platform_int,
40 is_any_real_numeric_dtype,
41 is_bool_dtype,
42 is_dict_like,
43 is_hashable,
44 is_integer_dtype,
45 is_list_like,
46 is_scalar,
47 needs_i8_conversion,
48 pandas_dtype,
49)
50from pandas.core.dtypes.dtypes import (
51 ArrowDtype,
52 CategoricalDtype,
53 CategoricalDtypeType,
54 ExtensionDtype,
55)
56from pandas.core.dtypes.generic import (
57 ABCIndex,
58 ABCSeries,
59)
60from pandas.core.dtypes.missing import (
61 is_valid_na_for_dtype,
62 isna,
63)
65from pandas.core import (
66 algorithms,
67 arraylike,
68 ops,
69)
70from pandas.core.accessor import (
71 PandasDelegate,
72 delegate_names,
73)
74from pandas.core.algorithms import (
75 factorize,
76 take_nd,
77)
78from pandas.core.arrays._mixins import (
79 NDArrayBackedExtensionArray,
80 ravel_compat,
81)
82from pandas.core.base import (
83 ExtensionArray,
84 NoNewAttributesMixin,
85 PandasObject,
86)
87import pandas.core.common as com
88from pandas.core.construction import (
89 extract_array,
90 sanitize_array,
91)
92from pandas.core.ops.common import unpack_zerodim_and_defer
93from pandas.core.sorting import nargsort
94from pandas.core.strings.object_array import ObjectStringArrayMixin
96from pandas.io.formats import console
98if TYPE_CHECKING:
99 from collections.abc import (
100 Callable,
101 Hashable,
102 Iterator,
103 Sequence,
104 )
106 from pandas._typing import (
107 ArrayLike,
108 AstypeArg,
109 AxisInt,
110 Dtype,
111 NpDtype,
112 Ordered,
113 Shape,
114 SortKind,
115 npt,
116 )
118 from pandas import (
119 DataFrame,
120 Index,
121 Series,
122 )
125def _cat_compare_op(op):
126 opname = f"__{op.__name__}__"
127 fill_value = op is operator.ne
129 @unpack_zerodim_and_defer(opname)
130 def func(self, other):
131 hashable = is_hashable(other)
132 if is_list_like(other) and len(other) != len(self) and not hashable:
133 # in hashable case we may have a tuple that is itself a category
134 raise ValueError("Lengths must match.")
136 if not self.ordered:
137 if opname in ["__lt__", "__gt__", "__le__", "__ge__"]:
138 raise TypeError(
139 "Unordered Categoricals can only compare equality or not"
140 )
141 if isinstance(other, Categorical):
142 # Two Categoricals can only be compared if the categories are
143 # the same (maybe up to ordering, depending on ordered)
145 msg = "Categoricals can only be compared if 'categories' are the same."
146 if not self._categories_match_up_to_permutation(other):
147 raise TypeError(msg)
149 if not self.ordered and not self.categories.equals(other.categories):
150 # both unordered and different order
151 other_codes = recode_for_categories(
152 other.codes, other.categories, self.categories, copy=False
153 )
154 else:
155 other_codes = other._codes
157 ret = op(self._codes, other_codes)
158 mask = (self._codes == -1) | (other_codes == -1)
159 if mask.any():
160 ret[mask] = fill_value
161 return ret
163 if hashable:
164 if other in self.categories:
165 i = self._unbox_scalar(other)
166 ret = op(self._codes, i)
168 if opname not in {"__eq__", "__ge__", "__gt__"}:
169 # GH#29820 performance trick; get_loc will always give i>=0,
170 # so in the cases (__ne__, __le__, __lt__) the setting
171 # here is a no-op, so can be skipped.
172 mask = self._codes == -1
173 ret[mask] = fill_value
174 return ret
175 else:
176 return ops.invalid_comparison(self, other, op)
177 else:
178 # allow categorical vs object dtype array comparisons for equality
179 # these are only positional comparisons
180 if opname not in ["__eq__", "__ne__"]:
181 raise TypeError(
182 f"Cannot compare a Categorical for op {opname} with "
183 f"type {type(other)}.\nIf you want to compare values, "
184 "use 'np.asarray(cat) <op> other'."
185 )
187 if isinstance(other, ExtensionArray) and needs_i8_conversion(other.dtype):
188 # We would return NotImplemented here, but that messes up
189 # ExtensionIndex's wrapped methods
190 return op(other, self)
191 return getattr(np.array(self), opname)(np.array(other))
193 func.__name__ = opname
195 return func
198def contains(cat, key, container) -> bool:
199 """
200 Helper for membership check for ``key`` in ``cat``.
202 This is a helper method for :method:`__contains__`
203 and :class:`CategoricalIndex.__contains__`.
205 Returns True if ``key`` is in ``cat.categories`` and the
206 location of ``key`` in ``categories`` is in ``container``.
208 Parameters
209 ----------
210 cat : :class:`Categorical`or :class:`categoricalIndex`
211 key : a hashable object
212 The key to check membership for.
213 container : Container (e.g. list-like or mapping)
214 The container to check for membership in.
216 Returns
217 -------
218 is_in : bool
219 True if ``key`` is in ``self.categories`` and location of
220 ``key`` in ``categories`` is in ``container``, else False.
222 Notes
223 -----
224 This method does not check for NaN values. Do that separately
225 before calling this method.
226 """
227 hash(key)
229 # get location of key in categories.
230 # If a KeyError, the key isn't in categories, so logically
231 # can't be in container either.
232 try:
233 loc = cat.categories.get_loc(key)
234 except (KeyError, TypeError):
235 return False
237 # loc is the location of key in categories, but also the *value*
238 # for key in container. So, `key` may be in categories,
239 # but still not in `container`. Example ('b' in categories,
240 # but not in values):
241 # 'b' in Categorical(['a'], categories=['a', 'b']) # False
242 if is_scalar(loc):
243 return loc in container
244 else:
245 # if categories is an IntervalIndex, loc is an array.
246 return any(loc_ in container for loc_ in loc)
249@set_module("pandas")
250class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMixin):
251 """
252 Represent a categorical variable in classic R / S-plus fashion.
254 `Categoricals` can only take on a limited, and usually fixed, number
255 of possible values (`categories`). In contrast to statistical categorical
256 variables, a `Categorical` might have an order, but numerical operations
257 (additions, divisions, ...) are not possible.
259 All values of the `Categorical` are either in `categories` or `np.nan`.
260 Assigning values outside of `categories` will raise a `ValueError`. Order
261 is defined by the order of the `categories`, not lexical order of the
262 values.
264 Parameters
265 ----------
266 values : list-like
267 The values of the categorical. If categories are given, values not in
268 categories will be replaced with NaN.
269 categories : Index-like (unique), optional
270 The unique categories for this categorical. If not given, the
271 categories are assumed to be the unique values of `values` (sorted, if
272 possible, otherwise in the order in which they appear).
273 ordered : bool, default False
274 Whether or not this categorical is treated as an ordered categorical.
275 If True, the resulting categorical will be ordered.
276 An ordered categorical respects, when sorted, the order of its
277 `categories` attribute (which in turn is the `categories` argument, if
278 provided).
279 dtype : CategoricalDtype
280 An instance of ``CategoricalDtype`` to use for this categorical.
281 copy : bool, default True
282 Whether to copy if the codes are unchanged.
284 Attributes
285 ----------
286 categories : Index
287 The categories of this categorical.
288 codes : ndarray
289 The codes (integer positions, which point to the categories) of this
290 categorical, read only.
291 ordered : bool
292 Whether or not this Categorical is ordered.
293 dtype : CategoricalDtype
294 The instance of ``CategoricalDtype`` storing the ``categories``
295 and ``ordered``.
297 Methods
298 -------
299 from_codes
300 as_ordered
301 as_unordered
302 set_categories
303 rename_categories
304 reorder_categories
305 add_categories
306 remove_categories
307 remove_unused_categories
308 map
309 __array__
311 Raises
312 ------
313 ValueError
314 If the categories do not validate.
315 TypeError
316 If an explicit ``ordered=True`` is given but no `categories` and the
317 `values` are not sortable.
319 See Also
320 --------
321 CategoricalDtype : Type for categorical data.
322 CategoricalIndex : An Index with an underlying ``Categorical``.
324 Notes
325 -----
326 See the `user guide
327 <https://pandas.pydata.org/pandas-docs/stable/user_guide/categorical.html>`__
328 for more.
330 Examples
331 --------
332 >>> pd.Categorical([1, 2, 3, 1, 2, 3])
333 [1, 2, 3, 1, 2, 3]
334 Categories (3, int64): [1, 2, 3]
336 >>> pd.Categorical(["a", "b", "c", "a", "b", "c"])
337 ['a', 'b', 'c', 'a', 'b', 'c']
338 Categories (3, str): ['a', 'b', 'c']
340 Missing values are not included as a category.
342 >>> c = pd.Categorical([1, 2, 3, 1, 2, 3, np.nan])
343 >>> c
344 [1, 2, 3, 1, 2, 3, NaN]
345 Categories (3, int64): [1, 2, 3]
347 However, their presence is indicated in the `codes` attribute
348 by code `-1`.
350 >>> c.codes
351 array([ 0, 1, 2, 0, 1, 2, -1], dtype=int8)
353 Ordered `Categoricals` can be sorted according to the custom order
354 of the categories and can have a min and max value.
356 >>> c = pd.Categorical(
357 ... ["a", "b", "c", "a", "b", "c"], ordered=True, categories=["c", "b", "a"]
358 ... )
359 >>> c
360 ['a', 'b', 'c', 'a', 'b', 'c']
361 Categories (3, str): ['c' < 'b' < 'a']
362 >>> c.min()
363 'c'
364 """
366 # For comparisons, so that numpy uses our implementation if the compare
367 # ops, which raise
368 __array_priority__ = 1000
369 # tolist is not actually deprecated, just suppressed in the __dir__
370 _hidden_attrs = PandasObject._hidden_attrs | frozenset(["tolist"])
371 _typ = "categorical"
373 _dtype: CategoricalDtype
375 @classmethod
376 # error: Argument 2 of "_simple_new" is incompatible with supertype
377 # "NDArrayBacked"; supertype defines the argument type as
378 # "Union[dtype[Any], ExtensionDtype]"
379 def _simple_new( # type: ignore[override]
380 cls, codes: np.ndarray, dtype: CategoricalDtype
381 ) -> Self:
382 # NB: This is not _quite_ as simple as the "usual" _simple_new
383 codes = coerce_indexer_dtype(codes, dtype.categories)
384 dtype = CategoricalDtype(ordered=False).update_dtype(dtype)
385 return super()._simple_new(codes, dtype)
387 def __init__(
388 self,
389 values,
390 categories=None,
391 ordered=None,
392 dtype: Dtype | None = None,
393 copy: bool = True,
394 ) -> None:
395 dtype = CategoricalDtype._from_values_or_dtype(
396 values, categories, ordered, dtype
397 )
398 # At this point, dtype is always a CategoricalDtype, but
399 # we may have dtype.categories be None, and we need to
400 # infer categories in a factorization step further below
402 if not is_list_like(values):
403 # GH#38433
404 raise TypeError("Categorical input must be list-like")
406 # null_mask indicates missing values we want to exclude from inference.
407 # This means: only missing values in list-likes (not arrays/ndframes).
408 null_mask = np.array(False)
410 # sanitize input
411 vdtype = getattr(values, "dtype", None)
412 if isinstance(vdtype, CategoricalDtype):
413 if dtype.categories is None:
414 dtype = CategoricalDtype(values.categories, dtype.ordered)
415 elif isinstance(values, range):
416 from pandas.core.indexes.range import RangeIndex
418 values = RangeIndex(values)
419 elif not isinstance(values, (ABCIndex, ABCSeries, ExtensionArray)):
420 values = com.convert_to_list_like(values)
421 if isinstance(values, list) and len(values) == 0:
422 # By convention, empty lists result in object dtype:
423 values = np.array([], dtype=object)
424 elif isinstance(values, np.ndarray):
425 if values.ndim > 1:
426 # preempt sanitize_array from raising ValueError
427 raise NotImplementedError(
428 "> 1 ndim Categorical are not supported at this time"
429 )
430 values = sanitize_array(values, None)
431 else:
432 # i.e. must be a list
433 arr = sanitize_array(values, None)
434 null_mask = isna(arr)
435 if null_mask.any():
436 # We remove null values here, then below will re-insert
437 # them, grep "full_codes"
438 arr_list = [values[idx] for idx in np.where(~null_mask)[0]]
440 # GH#44900 Do not cast to float if we have only missing values
441 if arr_list or arr.dtype == "object":
442 sanitize_dtype = None
443 else:
444 sanitize_dtype = arr.dtype
446 arr = sanitize_array(arr_list, None, dtype=sanitize_dtype)
447 values = arr
449 if dtype.categories is None:
450 if isinstance(values.dtype, ArrowDtype) and issubclass(
451 values.dtype.type, CategoricalDtypeType
452 ):
453 from pandas import Index
455 if isinstance(values, Index):
456 arr = values._data._pa_array.combine_chunks()
457 else:
458 arr = extract_array(values)._pa_array.combine_chunks()
459 categories = arr.dictionary.to_pandas(types_mapper=ArrowDtype)
460 codes = arr.indices.to_numpy()
461 dtype = CategoricalDtype(categories, values.dtype.pyarrow_dtype.ordered)
462 else:
463 preserve_object = False
464 if isinstance(values, (ABCIndex, ABCSeries)) and values.dtype == object:
465 # GH#61778
466 preserve_object = True
467 if not isinstance(values, ABCIndex):
468 # in particular RangeIndex xref test_index_equal_range_categories
469 values = sanitize_array(values, None)
470 try:
471 codes, categories = factorize(values, sort=True)
472 except TypeError as err:
473 codes, categories = factorize(values, sort=False)
474 if dtype.ordered:
475 # raise, as we don't have a sortable data structure and so
476 # the user should give us one by specifying categories
477 raise TypeError(
478 "'values' is not ordered, please "
479 "explicitly specify the categories order "
480 "by passing in a categories argument."
481 ) from err
483 if preserve_object:
484 # GH#61778 wrap categories in an Index to prevent dtype
485 # inference in the CategoricalDtype constructor
486 from pandas import Index
488 categories = Index(categories, dtype=object, copy=False)
490 # if not preserve_obejct, we're inferring from values
491 dtype = CategoricalDtype(categories, dtype.ordered)
493 elif isinstance(values.dtype, CategoricalDtype):
494 old_codes = extract_array(values)._codes
495 codes = recode_for_categories(
496 old_codes,
497 values.dtype.categories,
498 dtype.categories,
499 copy=copy,
500 warn=True,
501 )
503 else:
504 codes = _get_codes_for_values(values, dtype.categories)
506 if null_mask.any():
507 # Reinsert -1 placeholders for previously removed missing values
508 full_codes = -np.ones(null_mask.shape, dtype=codes.dtype)
509 full_codes[~null_mask] = codes
510 codes = full_codes
512 dtype = CategoricalDtype(ordered=False).update_dtype(dtype)
513 arr = coerce_indexer_dtype(codes, dtype.categories)
514 super().__init__(arr, dtype)
516 @property
517 def dtype(self) -> CategoricalDtype:
518 """
519 The :class:`~pandas.api.types.CategoricalDtype` for this instance.
521 See Also
522 --------
523 astype : Cast argument to a specified dtype.
524 CategoricalDtype : Type for categorical data.
526 Examples
527 --------
528 >>> cat = pd.Categorical(["a", "b"], ordered=True)
529 >>> cat
530 ['a', 'b']
531 Categories (2, str): ['a' < 'b']
532 >>> cat.dtype
533 CategoricalDtype(categories=['a', 'b'], ordered=True, categories_dtype=str)
534 """
535 return self._dtype
537 @property
538 def _internal_fill_value(self) -> int:
539 # using the specific numpy integer instead of python int to get
540 # the correct dtype back from _quantile in the all-NA case
541 dtype = self._ndarray.dtype
542 return dtype.type(-1)
544 @classmethod
545 def _from_sequence(
546 cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
547 ) -> Self:
548 return cls(scalars, dtype=dtype, copy=copy)
550 def _cast_pointwise_result(self, values) -> ArrayLike:
551 res = super()._cast_pointwise_result(values)
552 with warnings.catch_warnings():
553 warnings.filterwarnings(
554 "ignore",
555 "Constructing a Categorical with a dtype and values containing",
556 )
557 cat = type(self)._from_sequence(res, dtype=self.dtype)
558 if (cat.isna() == isna(res)).all():
559 # i.e. the conversion was non-lossy
560 return cat
561 return res
563 @overload
564 def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
566 @overload
567 def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
569 @overload
570 def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
572 def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
573 """
574 Coerce this type to another dtype
576 Parameters
577 ----------
578 dtype : numpy dtype or pandas type
579 copy : bool, default True
580 By default, astype always returns a newly allocated object.
581 If copy is set to False and dtype is categorical, the original
582 object is returned.
583 """
584 dtype = pandas_dtype(dtype)
585 result: Categorical | np.ndarray
586 if self.dtype is dtype:
587 result = self.copy() if copy else self
589 elif isinstance(dtype, CategoricalDtype):
590 # GH 10696/18593/18630
591 dtype = self.dtype.update_dtype(dtype)
592 self = self.copy() if copy else self
593 result = self._set_dtype(dtype, copy=False)
594 wrong = result.isna() & ~self.isna()
595 if wrong.any():
596 warnings.warn(
597 "Constructing a Categorical with a dtype and values containing "
598 "non-null entries not in that dtype's categories is deprecated "
599 "and will raise in a future version.",
600 Pandas4Warning,
601 stacklevel=find_stack_level(),
602 )
604 elif isinstance(dtype, ExtensionDtype):
605 return super().astype(dtype, copy=copy)
607 elif dtype.kind in "iu" and self.isna().any():
608 raise ValueError("Cannot convert float NaN to integer")
610 elif len(self.codes) == 0 or len(self.categories) == 0:
611 # For NumPy 1.x compatibility we cannot use copy=None. And
612 # `copy=False` has the meaning of `copy=None` here:
613 if not copy:
614 result = np.asarray(self, dtype=dtype)
615 else:
616 result = np.array(self, dtype=dtype)
618 else:
619 # GH8628 (PERF): astype category codes instead of astyping array
620 new_cats = self.categories._values
622 try:
623 new_cats = new_cats.astype(dtype=dtype, copy=copy)
624 fill_value = self.categories._na_value
625 if not is_valid_na_for_dtype(fill_value, dtype):
626 fill_value = lib.item_from_zerodim(
627 np.array(self.categories._na_value).astype(dtype)
628 )
629 except (
630 TypeError, # downstream error msg for CategoricalIndex is misleading
631 ValueError,
632 ) as err:
633 msg = f"Cannot cast {self.categories.dtype} dtype to {dtype}"
634 raise ValueError(msg) from err
636 result = take_nd(
637 new_cats, ensure_platform_int(self._codes), fill_value=fill_value
638 )
640 return result
642 @classmethod
643 def _from_inferred_categories(
644 cls, inferred_categories, inferred_codes, dtype, true_values=None
645 ) -> Self:
646 """
647 Construct a Categorical from inferred values.
649 For inferred categories (`dtype` is None) the categories are sorted.
650 For explicit `dtype`, the `inferred_categories` are cast to the
651 appropriate type.
653 Parameters
654 ----------
655 inferred_categories : Index
656 inferred_codes : Index
657 dtype : CategoricalDtype or 'category'
658 true_values : list, optional
659 If none are provided, the default ones are
660 "True", "TRUE", and "true."
662 Returns
663 -------
664 Categorical
665 """
666 from pandas import (
667 Index,
668 to_datetime,
669 to_numeric,
670 to_timedelta,
671 )
673 cats = Index(inferred_categories, copy=False)
674 known_categories = (
675 isinstance(dtype, CategoricalDtype) and dtype.categories is not None
676 )
678 if known_categories:
679 # Convert to a specialized type with `dtype` if specified.
680 if is_any_real_numeric_dtype(dtype.categories.dtype):
681 cats = to_numeric(inferred_categories, errors="coerce")
682 elif lib.is_np_dtype(dtype.categories.dtype, "M"):
683 cats = to_datetime(inferred_categories, errors="coerce")
684 elif lib.is_np_dtype(dtype.categories.dtype, "m"):
685 cats = to_timedelta(inferred_categories, errors="coerce")
686 elif is_bool_dtype(dtype.categories.dtype):
687 if true_values is None:
688 true_values = ["True", "TRUE", "true"]
690 # error: Incompatible types in assignment (expression has type
691 # "ndarray", variable has type "Index")
692 cats = cats.isin(true_values) # type: ignore[assignment]
694 if known_categories:
695 # Recode from observation order to dtype.categories order.
696 categories = dtype.categories
697 codes = recode_for_categories(
698 inferred_codes, cats, categories, copy=False, warn=True
699 )
700 elif not cats.is_monotonic_increasing:
701 # Sort categories and recode for unknown categories.
702 unsorted = cats.copy()
703 categories = cats.sort_values()
705 codes = recode_for_categories(
706 inferred_codes, unsorted, categories, copy=False, warn=True
707 )
708 dtype = CategoricalDtype(categories, ordered=False)
709 else:
710 dtype = CategoricalDtype(cats, ordered=False)
711 codes = inferred_codes
713 return cls._simple_new(codes, dtype=dtype)
715 @classmethod
716 def from_codes(
717 cls,
718 codes,
719 categories=None,
720 ordered=None,
721 dtype: Dtype | None = None,
722 validate: bool = True,
723 ) -> Self:
724 """
725 Make a Categorical type from codes and categories or dtype.
727 This constructor is useful if you already have codes and
728 categories/dtype and so do not need the (computation intensive)
729 factorization step, which is usually done on the constructor.
731 If your data does not follow this convention, please use the normal
732 constructor.
734 Parameters
735 ----------
736 codes : array-like of int
737 An integer array, where each integer points to a category in
738 categories or dtype.categories, or else is -1 for NaN.
739 categories : index-like, optional
740 The categories for the categorical. Items need to be unique.
741 If the categories are not given here, then they must be provided
742 in `dtype`.
743 ordered : bool, optional
744 Whether or not this categorical is treated as an ordered
745 categorical. If not given here or in `dtype`, the resulting
746 categorical will be unordered.
747 dtype : CategoricalDtype or "category", optional
748 If :class:`CategoricalDtype`, cannot be used together with
749 `categories` or `ordered`.
750 validate : bool, default True
751 If True, validate that the codes are valid for the dtype.
752 If False, don't validate that the codes are valid. Be careful about skipping
753 validation, as invalid codes can lead to severe problems, such as segfaults.
755 .. versionadded:: 2.1.0
757 Returns
758 -------
759 Categorical
761 See Also
762 --------
763 codes : The category codes of the categorical.
764 CategoricalIndex : An Index with an underlying ``Categorical``.
766 Examples
767 --------
768 >>> dtype = pd.CategoricalDtype(["a", "b"], ordered=True)
769 >>> pd.Categorical.from_codes(codes=[0, 1, 0, 1], dtype=dtype)
770 ['a', 'b', 'a', 'b']
771 Categories (2, str): ['a' < 'b']
772 """
773 dtype = CategoricalDtype._from_values_or_dtype(
774 categories=categories, ordered=ordered, dtype=dtype
775 )
776 if dtype.categories is None:
777 msg = (
778 "The categories must be provided in 'categories' or "
779 "'dtype'. Both were None."
780 )
781 raise ValueError(msg)
783 if validate:
784 # beware: non-valid codes may segfault
785 codes = cls._validate_codes_for_dtype(codes, dtype=dtype)
787 return cls._simple_new(codes, dtype=dtype)
789 # ------------------------------------------------------------------
790 # Categories/Codes/Ordered
792 @property
793 def categories(self) -> Index:
794 """
795 The categories of this categorical.
797 Setting assigns new values to each category (effectively a rename of
798 each individual category).
800 The assigned value has to be a list-like object. All items must be
801 unique and the number of items in the new categories must be the same
802 as the number of items in the old categories.
804 Raises
805 ------
806 ValueError
807 If the new categories do not validate as categories or if the
808 number of new categories is unequal the number of old categories
810 See Also
811 --------
812 rename_categories : Rename categories.
813 reorder_categories : Reorder categories.
814 add_categories : Add new categories.
815 remove_categories : Remove the specified categories.
816 remove_unused_categories : Remove categories which are not used.
817 set_categories : Set the categories to the specified ones.
819 Examples
820 --------
821 For :class:`pandas.Series`:
823 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
824 >>> ser.cat.categories
825 Index(['a', 'b', 'c'], dtype='str')
827 >>> raw_cat = pd.Categorical([None, "b", "c", None], categories=["b", "c", "d"])
828 >>> ser = pd.Series(raw_cat)
829 >>> ser.cat.categories
830 Index(['b', 'c', 'd'], dtype='str')
832 For :class:`pandas.Categorical`:
834 >>> cat = pd.Categorical(["a", "b"], ordered=True)
835 >>> cat.categories
836 Index(['a', 'b'], dtype='str')
838 For :class:`pandas.CategoricalIndex`:
840 >>> ci = pd.CategoricalIndex(["a", "c", "b", "a", "c", "b"])
841 >>> ci.categories
842 Index(['a', 'b', 'c'], dtype='str')
844 >>> ci = pd.CategoricalIndex(["a", "c"], categories=["c", "b", "a"])
845 >>> ci.categories
846 Index(['c', 'b', 'a'], dtype='str')
847 """
848 return self.dtype.categories
850 @property
851 def ordered(self) -> Ordered:
852 """
853 Whether the categories have an ordered relationship.
855 See Also
856 --------
857 set_ordered : Set the ordered attribute.
858 as_ordered : Set the Categorical to be ordered.
859 as_unordered : Set the Categorical to be unordered.
861 Examples
862 --------
863 For :class:`pandas.Series`:
865 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
866 >>> ser.cat.ordered
867 False
869 >>> raw_cat = pd.Categorical(["a", "b", "c", "a"], ordered=True)
870 >>> ser = pd.Series(raw_cat)
871 >>> ser.cat.ordered
872 True
874 For :class:`pandas.Categorical`:
876 >>> cat = pd.Categorical(["a", "b"], ordered=True)
877 >>> cat.ordered
878 True
880 >>> cat = pd.Categorical(["a", "b"], ordered=False)
881 >>> cat.ordered
882 False
884 For :class:`pandas.CategoricalIndex`:
886 >>> ci = pd.CategoricalIndex(["a", "b"], ordered=True)
887 >>> ci.ordered
888 True
890 >>> ci = pd.CategoricalIndex(["a", "b"], ordered=False)
891 >>> ci.ordered
892 False
893 """
894 return self.dtype.ordered
896 @property
897 def codes(self) -> np.ndarray:
898 """
899 The category codes of this categorical index.
901 Codes are an array of integers which are the positions of the actual
902 values in the categories array.
904 There is no setter, use the other categorical methods and the normal item
905 setter to change values in the categorical.
907 Returns
908 -------
909 ndarray[int]
910 A non-writable view of the ``codes`` array.
912 See Also
913 --------
914 Categorical.from_codes : Make a Categorical from codes.
915 CategoricalIndex : An Index with an underlying ``Categorical``.
917 Examples
918 --------
919 For :class:`pandas.Categorical`:
921 >>> cat = pd.Categorical(["a", "b"], ordered=True)
922 >>> cat.codes
923 array([0, 1], dtype=int8)
925 For :class:`pandas.CategoricalIndex`:
927 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a", "b", "c"])
928 >>> ci.codes
929 array([0, 1, 2, 0, 1, 2], dtype=int8)
931 >>> ci = pd.CategoricalIndex(["a", "c"], categories=["c", "b", "a"])
932 >>> ci.codes
933 array([2, 0], dtype=int8)
934 """
935 v = self._codes.view()
936 v.flags.writeable = False
937 return v
939 def _set_categories(self, categories, fastpath: bool = False) -> None:
940 """
941 Sets new categories inplace
943 Parameters
944 ----------
945 fastpath : bool, default False
946 Don't perform validation of the categories for uniqueness or nulls
948 Examples
949 --------
950 >>> c = pd.Categorical(["a", "b"])
951 >>> c
952 ['a', 'b']
953 Categories (2, str): ['a', 'b']
955 >>> c._set_categories(pd.Index(["a", "c"]))
956 >>> c
957 ['a', 'c']
958 Categories (2, str): ['a', 'c']
959 """
960 if fastpath:
961 new_dtype = CategoricalDtype._from_fastpath(categories, self.ordered)
962 else:
963 new_dtype = CategoricalDtype(categories, ordered=self.ordered)
964 if (
965 not fastpath
966 and self.dtype.categories is not None
967 and len(new_dtype.categories) != len(self.dtype.categories)
968 ):
969 raise ValueError(
970 "new categories need to have the same number of "
971 "items as the old categories!"
972 )
974 super().__init__(self._ndarray, new_dtype)
976 def _set_dtype(self, dtype: CategoricalDtype, *, copy: bool) -> Self:
977 """
978 Internal method for directly updating the CategoricalDtype
980 Parameters
981 ----------
982 dtype : CategoricalDtype
984 Notes
985 -----
986 We don't do any validation here. It's assumed that the dtype is
987 a (valid) instance of `CategoricalDtype`.
988 """
989 codes = recode_for_categories(
990 self.codes, self.categories, dtype.categories, copy=copy
991 )
992 return type(self)._simple_new(codes, dtype=dtype)
994 def set_ordered(self, value: bool) -> Self:
995 """
996 Set the ordered attribute to the boolean value.
998 Parameters
999 ----------
1000 value : bool
1001 Set whether this categorical is ordered (True) or not (False).
1002 """
1003 new_dtype = CategoricalDtype(self.categories, ordered=value)
1004 cat = self.copy()
1005 NDArrayBacked.__init__(cat, cat._ndarray, new_dtype)
1006 return cat
1008 def as_ordered(self) -> Self:
1009 """
1010 Set the Categorical to be ordered.
1012 Returns
1013 -------
1014 Categorical
1015 Ordered Categorical.
1017 See Also
1018 --------
1019 as_unordered : Set the Categorical to be unordered.
1021 Examples
1022 --------
1023 For :class:`pandas.Series`:
1025 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
1026 >>> ser.cat.ordered
1027 False
1028 >>> ser = ser.cat.as_ordered()
1029 >>> ser.cat.ordered
1030 True
1032 For :class:`pandas.CategoricalIndex`:
1034 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"])
1035 >>> ci.ordered
1036 False
1037 >>> ci = ci.as_ordered()
1038 >>> ci.ordered
1039 True
1040 """
1041 return self.set_ordered(True)
1043 def as_unordered(self) -> Self:
1044 """
1045 Set the Categorical to be unordered.
1047 Returns
1048 -------
1049 Categorical
1050 Unordered Categorical.
1052 See Also
1053 --------
1054 as_ordered : Set the Categorical to be ordered.
1056 Examples
1057 --------
1058 For :class:`pandas.Series`:
1060 >>> raw_cat = pd.Categorical(["a", "b", "c", "a"], ordered=True)
1061 >>> ser = pd.Series(raw_cat)
1062 >>> ser.cat.ordered
1063 True
1064 >>> ser = ser.cat.as_unordered()
1065 >>> ser.cat.ordered
1066 False
1068 For :class:`pandas.CategoricalIndex`:
1070 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"], ordered=True)
1071 >>> ci.ordered
1072 True
1073 >>> ci = ci.as_unordered()
1074 >>> ci.ordered
1075 False
1076 """
1077 return self.set_ordered(False)
1079 def set_categories(
1080 self, new_categories, ordered=None, rename: bool = False
1081 ) -> Self:
1082 """
1083 Set the categories to the specified new categories.
1085 ``new_categories`` can include new categories (which will result in
1086 unused categories) or remove old categories (which results in values
1087 set to ``NaN``). If ``rename=True``, the categories will simply be renamed
1088 (less or more items than in old categories will result in values set to
1089 ``NaN`` or in unused categories respectively).
1091 This method can be used to perform more than one action of adding,
1092 removing, and reordering simultaneously and is therefore faster than
1093 performing the individual steps via the more specialised methods.
1095 On the other hand this methods does not do checks (e.g., whether the
1096 old categories are included in the new categories on a reorder), which
1097 can result in surprising changes, for example when using special string
1098 dtypes, which do not consider a S1 string equal to a single char
1099 python string.
1101 Parameters
1102 ----------
1103 new_categories : Index-like
1104 The categories in new order.
1105 ordered : bool, default None
1106 Whether or not the categorical is treated as an ordered categorical.
1107 If not given, do not change the ordered information.
1108 rename : bool, default False
1109 Whether or not the new_categories should be considered as a rename
1110 of the old categories or as reordered categories.
1112 Returns
1113 -------
1114 Categorical
1115 New categories to be used, with optional ordering changes.
1117 Raises
1118 ------
1119 ValueError
1120 If new_categories does not validate as categories
1122 See Also
1123 --------
1124 rename_categories : Rename categories.
1125 reorder_categories : Reorder categories.
1126 add_categories : Add new categories.
1127 remove_categories : Remove the specified categories.
1128 remove_unused_categories : Remove categories which are not used.
1130 Examples
1131 --------
1132 For :class:`pandas.Series`:
1134 >>> raw_cat = pd.Categorical(
1135 ... ["a", "b", "c", None], categories=["a", "b", "c"], ordered=True
1136 ... )
1137 >>> ser = pd.Series(raw_cat)
1138 >>> ser
1139 0 a
1140 1 b
1141 2 c
1142 3 NaN
1143 dtype: category
1144 Categories (3, str): ['a' < 'b' < 'c']
1146 >>> ser.cat.set_categories(["A", "B", "C"], rename=True)
1147 0 A
1148 1 B
1149 2 C
1150 3 NaN
1151 dtype: category
1152 Categories (3, str): ['A' < 'B' < 'C']
1154 For :class:`pandas.CategoricalIndex`:
1156 >>> ci = pd.CategoricalIndex(
1157 ... ["a", "b", "c", None], categories=["a", "b", "c"], ordered=True
1158 ... )
1159 >>> ci
1160 CategoricalIndex(['a', 'b', 'c', nan], categories=['a', 'b', 'c'],
1161 ordered=True, dtype='category')
1163 >>> ci.set_categories(["A", "b", "c"])
1164 CategoricalIndex([nan, 'b', 'c', nan], categories=['A', 'b', 'c'],
1165 ordered=True, dtype='category')
1166 >>> ci.set_categories(["A", "b", "c"], rename=True)
1167 CategoricalIndex(['A', 'b', 'c', nan], categories=['A', 'b', 'c'],
1168 ordered=True, dtype='category')
1169 """
1171 if ordered is None:
1172 ordered = self.dtype.ordered
1173 new_dtype = CategoricalDtype(new_categories, ordered=ordered)
1175 cat = self.copy()
1176 if rename:
1177 if cat.dtype.categories is not None and len(new_dtype.categories) < len(
1178 cat.dtype.categories
1179 ):
1180 # remove all _codes which are larger and set to -1/NaN
1181 cat._codes[cat._codes >= len(new_dtype.categories)] = -1
1182 codes = cat._codes
1183 else:
1184 codes = recode_for_categories(
1185 cat.codes, cat.categories, new_dtype.categories, copy=False, warn=False
1186 )
1187 NDArrayBacked.__init__(cat, codes, new_dtype)
1188 return cat
1190 def rename_categories(self, new_categories) -> Self:
1191 """
1192 Rename categories.
1194 This method is commonly used to re-label or adjust the
1195 category names in categorical data without changing the
1196 underlying data. It is useful in situations where you want
1197 to modify the labels used for clarity, consistency,
1198 or readability.
1200 Parameters
1201 ----------
1202 new_categories : list-like, dict-like or callable
1204 New categories which will replace old categories.
1206 * list-like: all items must be unique and the number of items in
1207 the new categories must match the existing number of categories.
1209 * dict-like: specifies a mapping from
1210 old categories to new. Categories not contained in the mapping
1211 are passed through and extra categories in the mapping are
1212 ignored.
1214 * callable : a callable that is called on all items in the old
1215 categories and whose return values comprise the new categories.
1217 Returns
1218 -------
1219 Categorical
1220 Categorical with renamed categories.
1222 Raises
1223 ------
1224 ValueError
1225 If new categories are list-like and do not have the same number of
1226 items than the current categories or do not validate as categories
1228 See Also
1229 --------
1230 reorder_categories : Reorder categories.
1231 add_categories : Add new categories.
1232 remove_categories : Remove the specified categories.
1233 remove_unused_categories : Remove categories which are not used.
1234 set_categories : Set the categories to the specified ones.
1236 Examples
1237 --------
1238 >>> c = pd.Categorical(["a", "a", "b"])
1239 >>> c.rename_categories([0, 1])
1240 [0, 0, 1]
1241 Categories (2, int64): [0, 1]
1243 For dict-like ``new_categories``, extra keys are ignored and
1244 categories not in the dictionary are passed through
1246 >>> c.rename_categories({"a": "A", "c": "C"})
1247 ['A', 'A', 'b']
1248 Categories (2, str): ['A', 'b']
1250 You may also provide a callable to create the new categories
1252 >>> c.rename_categories(lambda x: x.upper())
1253 ['A', 'A', 'B']
1254 Categories (2, str): ['A', 'B']
1255 """
1257 if is_dict_like(new_categories):
1258 new_categories = [
1259 new_categories.get(item, item) for item in self.categories
1260 ]
1261 elif callable(new_categories):
1262 new_categories = [new_categories(item) for item in self.categories]
1264 cat = self.copy()
1265 cat._set_categories(new_categories)
1266 return cat
1268 def reorder_categories(self, new_categories, ordered=None) -> Self:
1269 """
1270 Reorder categories as specified in new_categories.
1272 ``new_categories`` need to include all old categories and no new category
1273 items.
1275 Parameters
1276 ----------
1277 new_categories : Index-like
1278 The categories in new order.
1279 ordered : bool, optional
1280 Whether or not the categorical is treated as an ordered categorical.
1281 If not given, do not change the ordered information.
1283 Returns
1284 -------
1285 Categorical
1286 Categorical with reordered categories.
1288 Raises
1289 ------
1290 ValueError
1291 If the new categories do not contain all old category items or any
1292 new ones
1294 See Also
1295 --------
1296 rename_categories : Rename categories.
1297 add_categories : Add new categories.
1298 remove_categories : Remove the specified categories.
1299 remove_unused_categories : Remove categories which are not used.
1300 set_categories : Set the categories to the specified ones.
1302 Examples
1303 --------
1304 For :class:`pandas.Series`:
1306 >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
1307 >>> ser = ser.cat.reorder_categories(["c", "b", "a"], ordered=True)
1308 >>> ser
1309 0 a
1310 1 b
1311 2 c
1312 3 a
1313 dtype: category
1314 Categories (3, str): ['c' < 'b' < 'a']
1316 >>> ser.sort_values()
1317 2 c
1318 1 b
1319 0 a
1320 3 a
1321 dtype: category
1322 Categories (3, str): ['c' < 'b' < 'a']
1324 For :class:`pandas.CategoricalIndex`:
1326 >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"])
1327 >>> ci
1328 CategoricalIndex(['a', 'b', 'c', 'a'], categories=['a', 'b', 'c'],
1329 ordered=False, dtype='category')
1330 >>> ci.reorder_categories(["c", "b", "a"], ordered=True)
1331 CategoricalIndex(['a', 'b', 'c', 'a'], categories=['c', 'b', 'a'],
1332 ordered=True, dtype='category')
1333 """
1334 if (
1335 len(self.categories) != len(new_categories)
1336 or not self.categories.difference(new_categories).empty
1337 ):
1338 raise ValueError(
1339 "items in new_categories are not the same as in old categories"
1340 )
1341 return self.set_categories(new_categories, ordered=ordered)
1343 def add_categories(self, new_categories) -> Self:
1344 """
1345 Add new categories.
1347 `new_categories` will be included at the last/highest place in the
1348 categories and will be unused directly after this call.
1350 Parameters
1351 ----------
1352 new_categories : category or list-like of category
1353 The new categories to be included.
1355 Returns
1356 -------
1357 Categorical
1358 Categorical with new categories added.
1360 Raises
1361 ------
1362 ValueError
1363 If the new categories include old categories or do not validate as
1364 categories
1366 See Also
1367 --------
1368 rename_categories : Rename categories.
1369 reorder_categories : Reorder categories.
1370 remove_categories : Remove the specified categories.
1371 remove_unused_categories : Remove categories which are not used.
1372 set_categories : Set the categories to the specified ones.
1374 Examples
1375 --------
1376 >>> c = pd.Categorical(["c", "b", "c"])
1377 >>> c
1378 ['c', 'b', 'c']
1379 Categories (2, str): ['b', 'c']
1381 >>> c.add_categories(["d", "a"])
1382 ['c', 'b', 'c']
1383 Categories (4, str): ['b', 'c', 'd', 'a']
1384 """
1386 if not is_list_like(new_categories):
1387 new_categories = [new_categories]
1388 already_included = set(new_categories) & set(self.dtype.categories)
1389 if len(already_included) != 0:
1390 raise ValueError(
1391 f"new categories must not include old categories: {already_included}"
1392 )
1394 if hasattr(new_categories, "dtype"):
1395 from pandas import Series
1397 dtype = find_common_type(
1398 [self.dtype.categories.dtype, new_categories.dtype]
1399 )
1400 new_categories = Series(
1401 list(self.dtype.categories) + list(new_categories), dtype=dtype
1402 )
1403 else:
1404 new_categories = list(self.dtype.categories) + list(new_categories)
1406 new_dtype = CategoricalDtype(new_categories, self.ordered)
1407 cat = self.copy()
1408 codes = coerce_indexer_dtype(cat._ndarray, new_dtype.categories)
1409 NDArrayBacked.__init__(cat, codes, new_dtype)
1410 return cat
1412 def remove_categories(self, removals) -> Self:
1413 """
1414 Remove the specified categories.
1416 The ``removals`` argument must be a subset of the current categories.
1417 Any values that were part of the removed categories will be set to NaN.
1419 Parameters
1420 ----------
1421 removals : category or list of categories
1422 The categories which should be removed.
1424 Returns
1425 -------
1426 Categorical
1427 Categorical with removed categories.
1429 Raises
1430 ------
1431 ValueError
1432 If the removals are not contained in the categories
1434 See Also
1435 --------
1436 rename_categories : Rename categories.
1437 reorder_categories : Reorder categories.
1438 add_categories : Add new categories.
1439 remove_unused_categories : Remove categories which are not used.
1440 set_categories : Set the categories to the specified ones.
1442 Examples
1443 --------
1444 >>> c = pd.Categorical(["a", "c", "b", "c", "d"])
1445 >>> c
1446 ['a', 'c', 'b', 'c', 'd']
1447 Categories (4, str): ['a', 'b', 'c', 'd']
1449 >>> c.remove_categories(["d", "a"])
1450 [NaN, 'c', 'b', 'c', NaN]
1451 Categories (2, str): ['b', 'c']
1452 """
1453 from pandas import Index
1455 if not is_list_like(removals):
1456 removals = [removals]
1458 removals = Index(removals).unique().dropna()
1459 new_categories = (
1460 self.dtype.categories.difference(removals, sort=False)
1461 if self.dtype.ordered is True
1462 else self.dtype.categories.difference(removals)
1463 )
1464 not_included = removals.difference(self.dtype.categories)
1466 if len(not_included) != 0:
1467 not_included = set(not_included)
1468 raise ValueError(f"removals must all be in old categories: {not_included}")
1470 return self.set_categories(new_categories, ordered=self.ordered, rename=False)
1472 def remove_unused_categories(self) -> Self:
1473 """
1474 Remove categories which are not used.
1476 This method is useful when working with datasets
1477 that undergo dynamic changes where categories may no longer be
1478 relevant, allowing to maintain a clean, efficient data structure.
1480 Returns
1481 -------
1482 Categorical
1483 Categorical with unused categories dropped.
1485 See Also
1486 --------
1487 rename_categories : Rename categories.
1488 reorder_categories : Reorder categories.
1489 add_categories : Add new categories.
1490 remove_categories : Remove the specified categories.
1491 set_categories : Set the categories to the specified ones.
1493 Examples
1494 --------
1495 >>> c = pd.Categorical(["a", "c", "b", "c", "d"])
1496 >>> c
1497 ['a', 'c', 'b', 'c', 'd']
1498 Categories (4, str): ['a', 'b', 'c', 'd']
1500 >>> c[2] = "a"
1501 >>> c[4] = "c"
1502 >>> c
1503 ['a', 'c', 'a', 'c', 'c']
1504 Categories (4, str): ['a', 'b', 'c', 'd']
1506 >>> c.remove_unused_categories()
1507 ['a', 'c', 'a', 'c', 'c']
1508 Categories (2, str): ['a', 'c']
1509 """
1510 idx, inv = np.unique(self._codes, return_inverse=True)
1512 if idx.size != 0 and idx[0] == -1: # na sentinel
1513 idx, inv = idx[1:], inv - 1
1515 new_categories = self.dtype.categories.take(idx)
1516 new_dtype = CategoricalDtype._from_fastpath(
1517 new_categories, ordered=self.ordered
1518 )
1519 new_codes = coerce_indexer_dtype(inv, new_dtype.categories)
1521 cat = self.copy()
1522 NDArrayBacked.__init__(cat, new_codes, new_dtype)
1523 return cat
1525 # ------------------------------------------------------------------
1527 def map(
1528 self,
1529 mapper,
1530 na_action: Literal["ignore"] | None = None,
1531 ):
1532 """
1533 Map categories using an input mapping or function.
1535 Maps the categories to new categories. If the mapping correspondence is
1536 one-to-one the result is a :class:`~pandas.Categorical` which has the
1537 same order property as the original, otherwise a :class:`~pandas.Index`
1538 is returned. NaN values are unaffected.
1540 If a `dict` or :class:`~pandas.Series` is used any unmapped category is
1541 mapped to `NaN`. Note that if this happens an :class:`~pandas.Index`
1542 will be returned.
1544 Parameters
1545 ----------
1546 mapper : function, dict, or Series
1547 Mapping correspondence.
1548 na_action : {None, 'ignore'}, default None
1549 If 'ignore', propagate NaN values, without passing them to the
1550 mapping correspondence.
1552 Returns
1553 -------
1554 pandas.Categorical or pandas.Index
1555 Mapped categorical.
1557 See Also
1558 --------
1559 CategoricalIndex.map : Apply a mapping correspondence on a
1560 :class:`~pandas.CategoricalIndex`.
1561 Index.map : Apply a mapping correspondence on an
1562 :class:`~pandas.Index`.
1563 Series.map : Apply a mapping correspondence on a
1564 :class:`~pandas.Series`.
1565 Series.apply : Apply more complex functions on a
1566 :class:`~pandas.Series`.
1568 Examples
1569 --------
1570 >>> cat = pd.Categorical(["a", "b", "c"])
1571 >>> cat
1572 ['a', 'b', 'c']
1573 Categories (3, str): ['a', 'b', 'c']
1574 >>> cat.map(lambda x: x.upper(), na_action=None)
1575 ['A', 'B', 'C']
1576 Categories (3, str): ['A', 'B', 'C']
1577 >>> cat.map({"a": "first", "b": "second", "c": "third"}, na_action=None)
1578 ['first', 'second', 'third']
1579 Categories (3, str): ['first', 'second', 'third']
1581 If the mapping is one-to-one the ordering of the categories is
1582 preserved:
1584 >>> cat = pd.Categorical(["a", "b", "c"], ordered=True)
1585 >>> cat
1586 ['a', 'b', 'c']
1587 Categories (3, str): ['a' < 'b' < 'c']
1588 >>> cat.map({"a": 3, "b": 2, "c": 1}, na_action=None)
1589 [3, 2, 1]
1590 Categories (3, int64): [3 < 2 < 1]
1592 If the mapping is not one-to-one an :class:`~pandas.Index` is returned:
1594 >>> cat.map({"a": "first", "b": "second", "c": "first"}, na_action=None)
1595 Index(['first', 'second', 'first'], dtype='str')
1597 If a `dict` is used, all unmapped categories are mapped to `NaN` and
1598 the result is an :class:`~pandas.Index`:
1600 >>> cat.map({"a": "first", "b": "second"}, na_action=None)
1601 Index(['first', 'second', nan], dtype='str')
1603 The mapping function is applied to categories, not to each value. It is
1604 therefore only called once per unique category, and the result reused for
1605 all occurrences:
1607 >>> cat = pd.Categorical(["a", "a", "b"])
1608 >>> calls = []
1609 >>> def f(x):
1610 ... calls.append(x)
1611 ... return x.upper()
1612 >>> result = cat.map(f)
1613 >>> result
1614 ['A', 'A', 'B']
1615 Categories (2, str): ['A', 'B']
1616 >>> calls
1617 ['a', 'b']
1618 """
1619 assert callable(mapper) or is_dict_like(mapper)
1621 new_categories = self.categories.map(mapper)
1623 has_nans = np.any(self._codes == -1)
1625 na_val = np.nan
1626 if na_action is None and has_nans:
1627 na_val = mapper(np.nan) if callable(mapper) else mapper.get(np.nan, np.nan)
1629 if new_categories.is_unique and not new_categories.hasnans and na_val is np.nan:
1630 new_dtype = CategoricalDtype(new_categories, ordered=self.ordered)
1631 return self.from_codes(self._codes.copy(), dtype=new_dtype, validate=False)
1633 if has_nans:
1634 new_categories = new_categories.insert(len(new_categories), na_val)
1636 return np.take(new_categories, self._codes)
1638 __eq__ = _cat_compare_op(operator.eq)
1639 __ne__ = _cat_compare_op(operator.ne)
1640 __lt__ = _cat_compare_op(operator.lt)
1641 __gt__ = _cat_compare_op(operator.gt)
1642 __le__ = _cat_compare_op(operator.le)
1643 __ge__ = _cat_compare_op(operator.ge)
1645 # -------------------------------------------------------------
1646 # Validators; ideally these can be de-duplicated
1648 def _validate_setitem_value(self, value):
1649 if not is_hashable(value):
1650 # wrap scalars and hashable-listlikes in list
1651 return self._validate_listlike(value)
1652 else:
1653 return self._validate_scalar(value)
1655 def _validate_scalar(self, fill_value):
1656 """
1657 Convert a user-facing fill_value to a representation to use with our
1658 underlying ndarray, raising TypeError if this is not possible.
1660 Parameters
1661 ----------
1662 fill_value : object
1664 Returns
1665 -------
1666 fill_value : int
1668 Raises
1669 ------
1670 TypeError
1671 """
1673 if is_valid_na_for_dtype(fill_value, self.categories.dtype):
1674 fill_value = -1
1675 elif fill_value in self.categories:
1676 fill_value = self._unbox_scalar(fill_value)
1677 else:
1678 raise TypeError(
1679 "Cannot setitem on a Categorical with a new "
1680 f"category ({fill_value}), set the categories first"
1681 ) from None
1682 return fill_value
1684 @classmethod
1685 def _validate_codes_for_dtype(cls, codes, *, dtype: CategoricalDtype) -> np.ndarray:
1686 if isinstance(codes, ExtensionArray) and is_integer_dtype(codes.dtype):
1687 # Avoid the implicit conversion of Int to object
1688 if isna(codes).any():
1689 raise ValueError("codes cannot contain NA values")
1690 codes = codes.to_numpy(dtype=np.int64)
1691 else:
1692 codes = np.asarray(codes)
1693 if len(codes) and codes.dtype.kind not in "iu":
1694 raise ValueError("codes need to be array-like integers")
1696 if len(codes) and (codes.max() >= len(dtype.categories) or codes.min() < -1):
1697 raise ValueError("codes need to be between -1 and len(categories)-1")
1698 return codes
1700 # -------------------------------------------------------------
1702 @ravel_compat
1703 def __array__(
1704 self, dtype: NpDtype | None = None, copy: bool | None = None
1705 ) -> np.ndarray:
1706 """
1707 The numpy array interface.
1709 Users should not call this directly. Rather, it is invoked by
1710 :func:`numpy.array` and :func:`numpy.asarray`.
1712 Parameters
1713 ----------
1714 dtype : np.dtype or None
1715 Specifies the dtype for the array.
1717 copy : bool or None, optional
1718 See :func:`numpy.asarray`.
1720 Returns
1721 -------
1722 numpy.array
1723 A numpy array of either the specified dtype or,
1724 if dtype==None (default), the same dtype as
1725 categorical.categories.dtype.
1727 See Also
1728 --------
1729 numpy.asarray : Convert input to numpy.ndarray.
1731 Examples
1732 --------
1734 >>> cat = pd.Categorical(["a", "b"], ordered=True)
1736 The following calls ``cat.__array__``
1738 >>> np.asarray(cat)
1739 array(['a', 'b'], dtype=object)
1740 """
1741 if copy is False:
1742 raise ValueError(
1743 "Unable to avoid copy while creating an array as requested."
1744 )
1746 ret = take_nd(self.categories._values, self._codes)
1747 # When we're a Categorical[ExtensionArray], like Interval,
1748 # we need to ensure __array__ gets all the way to an
1749 # ndarray.
1751 # `take_nd` should already make a copy, so don't force again.
1752 return np.asarray(ret, dtype=dtype)
1754 def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
1755 # for binary ops, use our custom dunder methods
1756 result = arraylike.maybe_dispatch_ufunc_to_dunder_op(
1757 self, ufunc, method, *inputs, **kwargs
1758 )
1759 if result is not NotImplemented:
1760 return result
1762 if "out" in kwargs:
1763 # e.g. test_numpy_ufuncs_out
1764 return arraylike.dispatch_ufunc_with_out(
1765 self, ufunc, method, *inputs, **kwargs
1766 )
1768 if method == "reduce":
1769 # e.g. TestCategoricalAnalytics::test_min_max_ordered
1770 result = arraylike.dispatch_reduction_ufunc(
1771 self, ufunc, method, *inputs, **kwargs
1772 )
1773 if result is not NotImplemented:
1774 return result
1776 # for all other cases, raise for now (similarly as what happens in
1777 # Series.__array_prepare__)
1778 raise TypeError(
1779 f"Object with dtype {self.dtype} cannot perform "
1780 f"the numpy op {ufunc.__name__}"
1781 )
1783 def __setstate__(self, state) -> None:
1784 """Necessary for making this object picklable"""
1785 if not isinstance(state, dict):
1786 return super().__setstate__(state)
1788 if "_dtype" not in state:
1789 state["_dtype"] = CategoricalDtype(state["_categories"], state["_ordered"])
1791 if "_codes" in state and "_ndarray" not in state:
1792 # backward compat, changed what is property vs attribute
1793 state["_ndarray"] = state.pop("_codes")
1795 super().__setstate__(state)
1797 @property
1798 def nbytes(self) -> int:
1799 return self._codes.nbytes + self.dtype.categories.values.nbytes
1801 def memory_usage(self, deep: bool = False) -> int:
1802 """
1803 Memory usage of my values
1805 Parameters
1806 ----------
1807 deep : bool
1808 Introspect the data deeply, interrogate
1809 `object` dtypes for system-level memory consumption
1811 Returns
1812 -------
1813 bytes used
1815 Notes
1816 -----
1817 Memory usage does not include memory consumed by elements that
1818 are not components of the array if deep=False
1820 See Also
1821 --------
1822 numpy.ndarray.nbytes
1823 """
1824 return self._codes.nbytes + self.dtype.categories.memory_usage(deep=deep)
1826 def isna(self) -> npt.NDArray[np.bool_]:
1827 """
1828 Detect missing values
1830 Missing values (-1 in .codes) are detected.
1832 Returns
1833 -------
1834 np.ndarray[bool] of whether my values are null
1836 See Also
1837 --------
1838 isna : Top-level isna.
1839 isnull : Alias of isna.
1840 Categorical.notna : Boolean inverse of Categorical.isna.
1842 """
1843 return self._codes == -1
1845 isnull = isna
1847 def notna(self) -> npt.NDArray[np.bool_]:
1848 """
1849 Inverse of isna
1851 Both missing values (-1 in .codes) and NA as a category are detected as
1852 null.
1854 Returns
1855 -------
1856 np.ndarray[bool] of whether my values are not null
1858 See Also
1859 --------
1860 notna : Top-level notna.
1861 notnull : Alias of notna.
1862 Categorical.isna : Boolean inverse of Categorical.notna.
1864 """
1865 return ~self.isna()
1867 notnull = notna
1869 def value_counts(self, dropna: bool = True) -> Series:
1870 """
1871 Return a Series containing counts of each category.
1873 Every category will have an entry, even those with a count of 0.
1875 Parameters
1876 ----------
1877 dropna : bool, default True
1878 Don't include counts of NaN.
1880 Returns
1881 -------
1882 counts : Series
1884 See Also
1885 --------
1886 Series.value_counts
1887 """
1888 from pandas import (
1889 CategoricalIndex,
1890 Series,
1891 )
1893 code, cat = self._codes, self.categories
1894 ncat, mask = (len(cat), code >= 0)
1895 ix, clean = np.arange(ncat), mask.all()
1897 if dropna or clean:
1898 obs = code if clean else code[mask]
1899 count = np.bincount(obs, minlength=ncat or 0)
1900 else:
1901 count = np.bincount(np.where(mask, code, ncat))
1902 ix = np.append(ix, -1)
1904 ix = coerce_indexer_dtype(ix, self.dtype.categories)
1905 ix_categorical = self._from_backing_data(ix)
1907 return Series(
1908 count,
1909 index=CategoricalIndex(ix_categorical),
1910 dtype="int64",
1911 name="count",
1912 copy=False,
1913 )
1915 # error: Argument 2 of "_empty" is incompatible with supertype
1916 # "NDArrayBackedExtensionArray"; supertype defines the argument type as
1917 # "ExtensionDtype"
1918 @classmethod
1919 def _empty( # type: ignore[override]
1920 cls, shape: Shape, dtype: CategoricalDtype
1921 ) -> Self:
1922 """
1923 Analogous to np.empty(shape, dtype=dtype)
1925 Parameters
1926 ----------
1927 shape : tuple[int]
1928 dtype : CategoricalDtype
1929 """
1930 arr = cls._from_sequence([], dtype=dtype)
1932 # We have to use np.zeros instead of np.empty otherwise the resulting
1933 # ndarray may contain codes not supported by this dtype, in which
1934 # case repr(result) could segfault.
1935 backing = np.zeros(shape, dtype=arr._ndarray.dtype)
1937 return arr._from_backing_data(backing)
1939 def _internal_get_values(self) -> ArrayLike:
1940 """
1941 Return the values.
1943 For internal compatibility with pandas formatting.
1945 Returns
1946 -------
1947 np.ndarray or ExtensionArray
1948 A numpy array or ExtensionArray of the same dtype as
1949 categorical.categories.dtype.
1950 """
1951 # if we are a datetime and period index, return Index to keep metadata
1952 if needs_i8_conversion(self.categories.dtype):
1953 return self.categories.take(self._codes, fill_value=NaT)._values
1954 elif is_integer_dtype(self.categories.dtype) and -1 in self._codes:
1955 return (
1956 self.categories.astype("object")
1957 .take(self._codes, fill_value=np.nan)
1958 ._values
1959 )
1960 return np.array(self)
1962 def check_for_ordered(self, op) -> None:
1963 """assert that we are ordered"""
1964 if not self.ordered:
1965 raise TypeError(
1966 f"Categorical is not ordered for operation {op}\n"
1967 "you can use .as_ordered() to change the "
1968 "Categorical to an ordered one\n"
1969 )
1971 def argsort(
1972 self, *, ascending: bool = True, kind: SortKind = "quicksort", **kwargs
1973 ) -> npt.NDArray[np.intp]:
1974 """
1975 Return the indices that would sort the Categorical.
1977 Missing values are sorted at the end.
1979 Parameters
1980 ----------
1981 ascending : bool, default True
1982 Whether the indices should result in an ascending
1983 or descending sort.
1984 kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional
1985 Sorting algorithm.
1986 **kwargs:
1987 passed through to :func:`numpy.argsort`.
1989 Returns
1990 -------
1991 np.ndarray[np.intp]
1993 See Also
1994 --------
1995 numpy.ndarray.argsort
1997 Notes
1998 -----
1999 While an ordering is applied to the category values, arg-sorting
2000 in this context refers more to organizing and grouping together
2001 based on matching category values. Thus, this function can be
2002 called on an unordered Categorical instance unlike the functions
2003 'Categorical.min' and 'Categorical.max'.
2005 Examples
2006 --------
2007 >>> pd.Categorical(["b", "b", "a", "c"]).argsort()
2008 array([2, 0, 1, 3])
2010 >>> cat = pd.Categorical(
2011 ... ["b", "b", "a", "c"], categories=["c", "b", "a"], ordered=True
2012 ... )
2013 >>> cat.argsort()
2014 array([3, 0, 1, 2])
2016 Missing values are placed at the end
2018 >>> cat = pd.Categorical([2, None, 1])
2019 >>> cat.argsort()
2020 array([2, 0, 1])
2021 """
2022 return super().argsort(ascending=ascending, kind=kind, **kwargs)
2024 @overload
2025 def sort_values(
2026 self,
2027 *,
2028 inplace: Literal[False] = ...,
2029 ascending: bool = ...,
2030 na_position: str = ...,
2031 ) -> Self: ...
2033 @overload
2034 def sort_values(
2035 self, *, inplace: Literal[True], ascending: bool = ..., na_position: str = ...
2036 ) -> None: ...
2038 def sort_values(
2039 self,
2040 *,
2041 inplace: bool = False,
2042 ascending: bool = True,
2043 na_position: str = "last",
2044 ) -> Self | None:
2045 """
2046 Sort the Categorical by category value returning a new
2047 Categorical by default.
2049 While an ordering is applied to the category values, sorting in this
2050 context refers more to organizing and grouping together based on
2051 matching category values. Thus, this function can be called on an
2052 unordered Categorical instance unlike the functions 'Categorical.min'
2053 and 'Categorical.max'.
2055 Parameters
2056 ----------
2057 inplace : bool, default False
2058 Do operation in place.
2059 ascending : bool, default True
2060 Order ascending. Passing False orders descending. The
2061 ordering parameter provides the method by which the
2062 category values are organized.
2063 na_position : {'first', 'last'} (optional, default='last')
2064 'first' puts NaNs at the beginning
2065 'last' puts NaNs at the end
2067 Returns
2068 -------
2069 Categorical or None
2071 See Also
2072 --------
2073 Categorical.sort
2074 Series.sort_values
2076 Examples
2077 --------
2078 >>> c = pd.Categorical([1, 2, 2, 1, 5])
2079 >>> c
2080 [1, 2, 2, 1, 5]
2081 Categories (3, int64): [1, 2, 5]
2082 >>> c.sort_values()
2083 [1, 1, 2, 2, 5]
2084 Categories (3, int64): [1, 2, 5]
2085 >>> c.sort_values(ascending=False)
2086 [5, 2, 2, 1, 1]
2087 Categories (3, int64): [1, 2, 5]
2089 >>> c = pd.Categorical([1, 2, 2, 1, 5])
2091 'sort_values' behaviour with NaNs. Note that 'na_position'
2092 is independent of the 'ascending' parameter:
2094 >>> c = pd.Categorical([np.nan, 2, 2, np.nan, 5])
2095 >>> c
2096 [NaN, 2, 2, NaN, 5]
2097 Categories (2, int64): [2, 5]
2098 >>> c.sort_values()
2099 [2, 2, 5, NaN, NaN]
2100 Categories (2, int64): [2, 5]
2101 >>> c.sort_values(ascending=False)
2102 [5, 2, 2, NaN, NaN]
2103 Categories (2, int64): [2, 5]
2104 >>> c.sort_values(na_position="first")
2105 [NaN, NaN, 2, 2, 5]
2106 Categories (2, int64): [2, 5]
2107 >>> c.sort_values(ascending=False, na_position="first")
2108 [NaN, NaN, 5, 2, 2]
2109 Categories (2, int64): [2, 5]
2110 """
2111 inplace = validate_bool_kwarg(inplace, "inplace")
2112 if na_position not in ["last", "first"]:
2113 raise ValueError(f"invalid na_position: {na_position!r}")
2115 sorted_idx = nargsort(self, ascending=ascending, na_position=na_position)
2117 if not inplace:
2118 codes = self._codes[sorted_idx]
2119 return self._from_backing_data(codes)
2120 self._codes[:] = self._codes[sorted_idx]
2121 return None
2123 def _rank(
2124 self,
2125 *,
2126 axis: AxisInt = 0,
2127 method: str = "average",
2128 na_option: str = "keep",
2129 ascending: bool = True,
2130 pct: bool = False,
2131 ):
2132 """
2133 See Series.rank.__doc__.
2134 """
2135 if axis != 0:
2136 raise NotImplementedError
2137 vff = self._values_for_rank()
2138 return algorithms.rank(
2139 vff,
2140 axis=axis,
2141 method=method,
2142 na_option=na_option,
2143 ascending=ascending,
2144 pct=pct,
2145 )
2147 def _values_for_rank(self) -> np.ndarray:
2148 """
2149 For correctly ranking ordered categorical data. See GH#15420
2151 Ordered categorical data should be ranked on the basis of
2152 codes with -1 translated to NaN.
2154 Returns
2155 -------
2156 numpy.array
2158 """
2159 from pandas import Series
2161 if self.ordered:
2162 values = self.codes
2163 mask = values == -1
2164 if mask.any():
2165 values = values.astype("float64")
2166 values[mask] = np.nan
2167 elif is_any_real_numeric_dtype(self.categories.dtype):
2168 values = np.array(self)
2169 else:
2170 # reorder the categories (so rank can use the float codes)
2171 # instead of passing an object array to rank
2172 values = np.array(
2173 self.rename_categories(
2174 Series(self.categories, copy=False).rank().values
2175 )
2176 )
2177 return values
2179 def _hash_pandas_object(
2180 self, *, encoding: str, hash_key: str, categorize: bool
2181 ) -> npt.NDArray[np.uint64]:
2182 """
2183 Hash a Categorical by hashing its categories, and then mapping the codes
2184 to the hashes.
2186 Parameters
2187 ----------
2188 encoding : str
2189 hash_key : str
2190 categorize : bool
2191 Ignored for Categorical.
2193 Returns
2194 -------
2195 np.ndarray[uint64]
2196 """
2197 # Note we ignore categorize, as we are already Categorical.
2198 from pandas.core.util.hashing import hash_array
2200 # Convert ExtensionArrays to ndarrays
2201 values = np.asarray(self.categories._values)
2202 hashed = hash_array(values, encoding, hash_key, categorize=False)
2204 # we have uint64, as we don't directly support missing values
2205 # we don't want to use take_nd which will coerce to float
2206 # instead, directly construct the result with a
2207 # max(np.uint64) as the missing value indicator
2208 #
2209 # TODO: GH#15362
2211 mask = self.isna()
2212 if len(hashed):
2213 result = hashed.take(self._codes)
2214 else:
2215 result = np.zeros(len(mask), dtype="uint64")
2217 if mask.any():
2218 result[mask] = lib.u8max
2220 return result
2222 # ------------------------------------------------------------------
2223 # NDArrayBackedExtensionArray compat
2225 @property
2226 def _codes(self) -> np.ndarray:
2227 return self._ndarray
2229 def _box_func(self, i: int):
2230 if i == -1:
2231 return np.nan
2232 return self.categories[i]
2234 def _unbox_scalar(self, key) -> int:
2235 # searchsorted is very performance sensitive. By converting codes
2236 # to same dtype as self.codes, we get much faster performance.
2237 code = self.categories.get_loc(key)
2238 code = self._ndarray.dtype.type(code)
2239 return code
2241 # ------------------------------------------------------------------
2243 def __iter__(self) -> Iterator:
2244 """
2245 Returns an Iterator over the values of this Categorical.
2246 """
2247 if self.ndim == 1:
2248 return iter(self._internal_get_values().tolist())
2249 else:
2250 return (self[n] for n in range(len(self)))
2252 def __contains__(self, key) -> bool:
2253 """
2254 Returns True if `key` is in this Categorical.
2255 """
2256 # if key is a NaN, check if any NaN is in self.
2257 if is_valid_na_for_dtype(key, self.categories.dtype):
2258 return bool(self.isna().any())
2260 return contains(self, key, container=self._codes)
2262 # ------------------------------------------------------------------
2263 # Rendering Methods
2265 # error: Return type "None" of "_formatter" incompatible with return
2266 # type "Callable[[Any], str | None]" in supertype "ExtensionArray"
2267 def _formatter(self, boxed: bool = False) -> None: # type: ignore[override]
2268 # Returning None here will cause format_array to do inference.
2269 return None
2271 def _repr_categories(self) -> list[str]:
2272 """
2273 return the base repr for the categories
2274 """
2275 max_categories = (
2276 10
2277 if get_option("display.max_categories") == 0
2278 else get_option("display.max_categories")
2279 )
2280 from pandas.io.formats import format as fmt
2282 formatter = None
2283 if self.categories.dtype == "str" or self.categories.dtype == "string": # noqa: PLR1714 (repeated-equality-comparison)
2284 # the extension array formatter defaults to boxed=True in format_array
2285 # override here to boxed=False to be consistent with QUOTE_NONNUMERIC
2286 formatter = cast(ExtensionArray, self.categories._values)._formatter(
2287 boxed=False
2288 )
2290 format_array = partial(
2291 fmt.format_array, formatter=formatter, quoting=QUOTE_NONNUMERIC
2292 )
2293 if len(self.categories) > max_categories:
2294 num = max_categories // 2
2295 head = format_array(self.categories[:num]._values)
2296 tail = format_array(self.categories[-num:]._values)
2297 category_strs = [*head, "...", *tail]
2298 else:
2299 category_strs = format_array(self.categories._values)
2301 # Strip all leading spaces, which format_array adds for columns...
2302 category_strs = [x.strip() for x in category_strs]
2303 return category_strs
2305 def _get_repr_footer(self) -> str:
2306 """
2307 Returns a string representation of the footer.
2308 """
2309 category_strs = self._repr_categories()
2310 dtype = str(self.categories.dtype)
2311 levheader = f"Categories ({len(self.categories)}, {dtype}): "
2312 width, _ = get_terminal_size()
2313 max_width = get_option("display.width") or width
2314 if console.in_ipython_frontend():
2315 # 0 = no breaks
2316 max_width = 0
2317 levstring = ""
2318 start = True
2319 cur_col_len = len(levheader) # header
2320 sep_len, sep = (3, " < ") if self.ordered else (2, ", ")
2321 linesep = f"{sep.rstrip()}\n" # remove whitespace
2322 for val in category_strs:
2323 if max_width != 0 and cur_col_len + sep_len + len(val) > max_width:
2324 levstring += linesep + (" " * (len(levheader) + 1))
2325 cur_col_len = len(levheader) + 1 # header + a whitespace
2326 elif not start:
2327 levstring += sep
2328 cur_col_len += len(val)
2329 levstring += val
2330 start = False
2331 # replace to simple save space by
2332 return f"{levheader}[{levstring.replace(' < ... < ', ' ... ')}]"
2334 def _get_values_repr(self) -> str:
2335 from pandas.io.formats import format as fmt
2337 assert len(self) > 0
2339 vals = self._internal_get_values()
2340 fmt_values = fmt.format_array(
2341 vals,
2342 None,
2343 float_format=None,
2344 na_rep="NaN",
2345 quoting=QUOTE_NONNUMERIC,
2346 )
2348 fmt_values = [i.strip() for i in fmt_values]
2349 joined = ", ".join(fmt_values)
2350 result = "[" + joined + "]"
2351 return result
2353 def __repr__(self) -> str:
2354 """
2355 String representation.
2356 """
2357 footer = self._get_repr_footer()
2358 length = len(self)
2359 max_len = 10
2360 if length > max_len:
2361 # In long cases we do not display all entries, so we add Length
2362 # information to the __repr__.
2363 num = max_len // 2
2364 head = self[:num]._get_values_repr()
2365 tail = self[-(max_len - num) :]._get_values_repr()
2366 body = f"{head[:-1]}, ..., {tail[1:]}"
2367 length_info = f"Length: {len(self)}"
2368 result = f"{body}\n{length_info}\n{footer}"
2369 elif length > 0:
2370 body = self._get_values_repr()
2371 result = f"{body}\n{footer}"
2372 else:
2373 # In the empty case we use a comma instead of newline to get
2374 # a more compact __repr__
2375 body = "[]"
2376 result = f"{body}, {footer}"
2378 return result
2380 # ------------------------------------------------------------------
2382 def _validate_listlike(self, value):
2383 # NB: here we assume scalar-like tuples have already been excluded
2384 value = extract_array(value, extract_numpy=True)
2386 # require identical categories set
2387 if isinstance(value, Categorical):
2388 if self.dtype != value.dtype:
2389 raise TypeError(
2390 "Cannot set a Categorical with another, "
2391 "without identical categories"
2392 )
2393 # dtype equality implies categories_match_up_to_permutation
2394 value = self._encode_with_my_categories(value)
2395 return value._codes
2397 from pandas import Index
2399 # tupleize_cols=False for e.g. test_fillna_iterable_category GH#41914
2400 to_add = Index._with_infer(value, tupleize_cols=False, copy=False).difference(
2401 self.categories
2402 )
2404 # no assignments of values not in categories, but it's always ok to set
2405 # something to np.nan
2406 if len(to_add) and not isna(to_add).all():
2407 raise TypeError(
2408 "Cannot setitem on a Categorical with a new "
2409 "category, set the categories first"
2410 )
2412 codes = self.categories.get_indexer(value)
2413 return codes.astype(self._ndarray.dtype, copy=False)
2415 def _reverse_indexer(self) -> dict[Hashable, npt.NDArray[np.intp]]:
2416 """
2417 Compute the inverse of a categorical, returning
2418 a dict of categories -> indexers.
2420 *This is an internal function*
2422 Returns
2423 -------
2424 Dict[Hashable, np.ndarray[np.intp]]
2425 dict of categories -> indexers
2427 Examples
2428 --------
2429 >>> c = pd.Categorical(list("aabca"))
2430 >>> c
2431 ['a', 'a', 'b', 'c', 'a']
2432 Categories (3, str): ['a', 'b', 'c']
2433 >>> c.categories
2434 Index(['a', 'b', 'c'], dtype='str')
2435 >>> c.codes
2436 array([0, 0, 1, 2, 0], dtype=int8)
2437 >>> c._reverse_indexer()
2438 {'a': array([0, 1, 4]), 'b': array([2]), 'c': array([3])}
2440 """
2441 categories = self.categories
2442 r, counts = libalgos.groupsort_indexer(
2443 ensure_platform_int(self.codes), categories.size
2444 )
2445 counts = ensure_int64(counts).cumsum()
2446 _result = (r[start:end] for start, end in itertools.pairwise(counts))
2447 return dict(zip(categories, _result, strict=True))
2449 # ------------------------------------------------------------------
2450 # Reductions
2452 def _reduce(
2453 self, name: str, *, skipna: bool = True, keepdims: bool = False, **kwargs
2454 ):
2455 result = super()._reduce(name, skipna=skipna, keepdims=keepdims, **kwargs)
2456 if name in ["argmax", "argmin"]:
2457 # don't wrap in Categorical!
2458 return result
2459 if keepdims:
2460 return type(self)(result, dtype=self.dtype)
2461 else:
2462 return result
2464 def min(self, *, skipna: bool = True, **kwargs):
2465 """
2466 The minimum value of the object.
2468 Only ordered `Categoricals` have a minimum!
2470 Raises
2471 ------
2472 TypeError
2473 If the `Categorical` is not `ordered`.
2475 Returns
2476 -------
2477 min : the minimum of this `Categorical`, NA value if empty
2478 """
2479 nv.validate_minmax_axis(kwargs.get("axis", 0))
2480 nv.validate_min((), kwargs)
2481 self.check_for_ordered("min")
2483 if not len(self._codes):
2484 return self.dtype.na_value
2486 good = self._codes != -1
2487 if not good.all():
2488 if skipna and good.any():
2489 pointer = self._codes[good].min()
2490 else:
2491 return np.nan
2492 else:
2493 pointer = self._codes.min()
2494 return self._wrap_reduction_result(None, pointer)
2496 def max(self, *, skipna: bool = True, **kwargs):
2497 """
2498 The maximum value of the object.
2500 Only ordered `Categoricals` have a maximum!
2502 Raises
2503 ------
2504 TypeError
2505 If the `Categorical` is not `ordered`.
2507 Returns
2508 -------
2509 max : the maximum of this `Categorical`, NA if array is empty
2510 """
2511 nv.validate_minmax_axis(kwargs.get("axis", 0))
2512 nv.validate_max((), kwargs)
2513 self.check_for_ordered("max")
2515 if not len(self._codes):
2516 return self.dtype.na_value
2518 good = self._codes != -1
2519 if not good.all():
2520 if skipna and good.any():
2521 pointer = self._codes[good].max()
2522 else:
2523 return np.nan
2524 else:
2525 pointer = self._codes.max()
2526 return self._wrap_reduction_result(None, pointer)
2528 def _mode(self, dropna: bool = True) -> Categorical:
2529 codes = self._codes
2530 mask = None
2531 if dropna:
2532 mask = self.isna()
2534 res_codes, _ = algorithms.mode(codes, mask=mask)
2535 res_codes = cast(np.ndarray, res_codes)
2536 assert res_codes.dtype == codes.dtype
2537 res = self._from_backing_data(res_codes)
2538 return res
2540 # ------------------------------------------------------------------
2541 # ExtensionArray Interface
2543 def unique(self) -> Self:
2544 """
2545 Return the ``Categorical`` which ``categories`` and ``codes`` are
2546 unique.
2548 Returns
2549 -------
2550 Categorical
2552 See Also
2553 --------
2554 pandas.unique
2555 CategoricalIndex.unique
2556 Series.unique : Return unique values of Series object.
2558 Examples
2559 --------
2560 >>> pd.Categorical(list("baabc")).unique()
2561 ['b', 'a', 'c']
2562 Categories (3, str): ['a', 'b', 'c']
2563 >>> pd.Categorical(list("baab"), categories=list("abc"), ordered=True).unique()
2564 ['b', 'a']
2565 Categories (3, str): ['a' < 'b' < 'c']
2566 """
2567 return super().unique()
2569 def equals(self, other: object) -> bool:
2570 """
2571 Returns True if categorical arrays are equal.
2573 Parameters
2574 ----------
2575 other : `Categorical`
2577 Returns
2578 -------
2579 bool
2580 """
2581 if not isinstance(other, Categorical):
2582 return False
2583 elif self._categories_match_up_to_permutation(other):
2584 other = self._encode_with_my_categories(other)
2585 return np.array_equal(self._codes, other._codes)
2586 return False
2588 def _accumulate(self, name: str, skipna: bool = True, **kwargs) -> Self:
2589 func: Callable
2590 if name == "cummin":
2591 func = np.minimum.accumulate
2592 elif name == "cummax":
2593 func = np.maximum.accumulate
2594 else:
2595 raise TypeError(f"Accumulation {name} not supported for {type(self)}")
2596 self.check_for_ordered(name)
2598 codes = self.codes.copy()
2599 mask = self.isna()
2600 if func == np.minimum.accumulate:
2601 codes[mask] = np.iinfo(codes.dtype.type).max
2602 # no need to change codes for maximum because codes[mask] is already -1
2603 if not skipna:
2604 mask = np.maximum.accumulate(mask)
2606 codes = func(codes)
2607 codes[mask] = -1
2608 return self._simple_new(codes, dtype=self._dtype)
2610 @classmethod
2611 def _concat_same_type(cls, to_concat: Sequence[Self], axis: AxisInt = 0) -> Self:
2612 from pandas.core.dtypes.concat import union_categoricals
2614 first = to_concat[0]
2615 if axis >= first.ndim:
2616 raise ValueError(
2617 f"axis {axis} is out of bounds for array of dimension {first.ndim}"
2618 )
2620 if axis == 1:
2621 # Flatten, concatenate then reshape
2622 if not all(x.ndim == 2 for x in to_concat):
2623 raise ValueError
2625 # pass correctly-shaped to union_categoricals
2626 tc_flat = []
2627 for obj in to_concat:
2628 tc_flat.extend([obj[:, i] for i in range(obj.shape[1])])
2630 res_flat = cls._concat_same_type(tc_flat, axis=0)
2632 result = res_flat.reshape(len(first), -1, order="F")
2633 return result
2635 # error: Incompatible types in assignment (expression has type "Categorical",
2636 # variable has type "Self")
2637 result = union_categoricals(to_concat) # type: ignore[assignment]
2638 return result
2640 # ------------------------------------------------------------------
2642 def _encode_with_my_categories(self, other: Categorical) -> Categorical:
2643 """
2644 Re-encode another categorical using this Categorical's categories.
2646 Notes
2647 -----
2648 This assumes we have already checked
2649 self._categories_match_up_to_permutation(other).
2650 """
2651 # Indexing on codes is more efficient if categories are the same,
2652 # so we can apply some optimizations based on the degree of
2653 # dtype-matching.
2654 codes = recode_for_categories(
2655 other.codes, other.categories, self.categories, copy=False
2656 )
2657 return self._from_backing_data(codes)
2659 def _categories_match_up_to_permutation(self, other: Categorical) -> bool:
2660 """
2661 Returns True if categoricals are the same dtype
2662 same categories, and same ordered
2664 Parameters
2665 ----------
2666 other : Categorical
2668 Returns
2669 -------
2670 bool
2671 """
2672 return hash(self.dtype) == hash(other.dtype)
2674 def describe(self) -> DataFrame:
2675 """
2676 Describes this Categorical
2678 Returns
2679 -------
2680 description: `DataFrame`
2681 A dataframe with frequency and counts by category.
2682 """
2683 counts = self.value_counts(dropna=False)
2684 freqs = counts / counts.sum()
2686 from pandas import Index
2687 from pandas.core.reshape.concat import concat
2689 result = concat([counts, freqs], ignore_index=True, axis=1)
2690 result.columns = Index(["counts", "freqs"])
2691 result.index.name = "categories"
2693 return result
2695 def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
2696 """
2697 Check whether `values` are contained in Categorical.
2699 Return a boolean NumPy Array showing whether each element in
2700 the Categorical matches an element in the passed sequence of
2701 `values` exactly.
2703 Parameters
2704 ----------
2705 values : np.ndarray or ExtensionArray
2706 The sequence of values to test. Passing in a single string will
2707 raise a ``TypeError``. Instead, turn a single string into a
2708 list of one element.
2710 Returns
2711 -------
2712 np.ndarray[bool]
2714 Raises
2715 ------
2716 TypeError
2717 * If `values` is not a set or list-like
2719 See Also
2720 --------
2721 pandas.Series.isin : Equivalent method on Series.
2723 Examples
2724 --------
2725 >>> s = pd.Categorical(["llama", "cow", "llama", "beetle", "llama", "hippo"])
2726 >>> s.isin(["cow", "llama"])
2727 array([ True, True, True, False, True, False])
2729 Passing a single string as ``s.isin('llama')`` will raise an error. Use
2730 a list of one element instead:
2732 >>> s.isin(["llama"])
2733 array([ True, False, True, False, True, False])
2734 """
2735 null_mask = np.asarray(isna(values))
2736 code_values = self.categories.get_indexer_for(values)
2737 code_values = code_values[null_mask | (code_values >= 0)]
2738 return algorithms.isin(self.codes, code_values)
2740 # ------------------------------------------------------------------------
2741 # String methods interface
2742 def _str_map(
2743 self, f, na_value=lib.no_default, dtype=np.dtype("object"), convert: bool = True
2744 ):
2745 # Optimization to apply the callable `f` to the categories once
2746 # and rebuild the result by `take`ing from the result with the codes.
2747 # Returns the same type as the object-dtype implementation though.
2748 categories = self.categories
2749 codes = self.codes
2750 if categories.dtype == "string":
2751 result = categories.array._str_map(f, na_value, dtype) # type: ignore[attr-defined]
2752 if (
2753 categories.dtype.na_value is np.nan # type: ignore[union-attr]
2754 and is_bool_dtype(dtype)
2755 and (na_value is lib.no_default or isna(na_value))
2756 ):
2757 # NaN propagates as False for functions with boolean return type
2758 na_value = False
2759 else:
2760 from pandas.core.arrays import NumpyExtensionArray
2762 result = NumpyExtensionArray(categories.to_numpy())._str_map(
2763 f, na_value, dtype
2764 )
2765 return take_nd(result, codes, fill_value=na_value)
2767 def _str_get_dummies(self, sep: str = "|", dtype: NpDtype | None = None):
2768 # sep may not be in categories. Just bail on this.
2769 from pandas.core.arrays import NumpyExtensionArray
2771 return NumpyExtensionArray(self.to_numpy(str, na_value="NaN"))._str_get_dummies(
2772 sep, dtype
2773 )
2775 # ------------------------------------------------------------------------
2776 # GroupBy Methods
2778 def _groupby_op(
2779 self,
2780 *,
2781 how: str,
2782 has_dropped_na: bool,
2783 min_count: int,
2784 ngroups: int,
2785 ids: npt.NDArray[np.intp],
2786 **kwargs,
2787 ):
2788 from pandas.core.groupby.ops import WrappedCythonOp
2790 kind = WrappedCythonOp.get_kind_from_how(how)
2791 op = WrappedCythonOp(how=how, kind=kind, has_dropped_na=has_dropped_na)
2793 dtype = self.dtype
2794 if how in ["sum", "prod", "cumsum", "cumprod", "skew", "kurt"]:
2795 raise TypeError(f"{dtype} type does not support {how} operations")
2796 if how in ["min", "max", "rank", "idxmin", "idxmax"] and not dtype.ordered:
2797 # raise TypeError instead of NotImplementedError to ensure we
2798 # don't go down a group-by-group path, since in the empty-groups
2799 # case that would fail to raise
2800 raise TypeError(f"Cannot perform {how} with non-ordered Categorical")
2801 if how not in [
2802 "rank",
2803 "any",
2804 "all",
2805 "first",
2806 "last",
2807 "min",
2808 "max",
2809 "idxmin",
2810 "idxmax",
2811 ]:
2812 if kind == "transform":
2813 raise TypeError(f"{dtype} type does not support {how} operations")
2814 raise TypeError(f"{dtype} dtype does not support aggregation '{how}'")
2816 result_mask = None
2817 mask = self.isna()
2818 if how == "rank":
2819 assert self.ordered # checked earlier
2820 npvalues = self._ndarray
2821 elif how in ["first", "last", "min", "max", "idxmin", "idxmax"]:
2822 npvalues = self._ndarray
2823 result_mask = np.zeros(ngroups, dtype=bool)
2824 else:
2825 # any/all
2826 npvalues = self.astype(bool)
2828 res_values = op._cython_op_ndim_compat(
2829 npvalues,
2830 min_count=min_count,
2831 ngroups=ngroups,
2832 comp_ids=ids,
2833 mask=mask,
2834 result_mask=result_mask,
2835 **kwargs,
2836 )
2838 if how in op.cast_blocklist:
2839 return res_values
2840 elif how in ["first", "last", "min", "max"]:
2841 res_values[result_mask == 1] = -1
2842 return self._from_backing_data(res_values)
2845# The Series.cat accessor
2848@delegate_names(
2849 delegate=Categorical, accessors=["categories", "ordered"], typ="property"
2850)
2851@delegate_names(
2852 delegate=Categorical,
2853 accessors=[
2854 "rename_categories",
2855 "reorder_categories",
2856 "add_categories",
2857 "remove_categories",
2858 "remove_unused_categories",
2859 "set_categories",
2860 "as_ordered",
2861 "as_unordered",
2862 ],
2863 typ="method",
2864)
2865class CategoricalAccessor(PandasDelegate, PandasObject, NoNewAttributesMixin):
2866 """
2867 Accessor object for categorical properties of the Series values.
2869 Parameters
2870 ----------
2871 data : Series or CategoricalIndex
2872 The object to which the categorical accessor is attached.
2874 See Also
2875 --------
2876 Series.dt : Accessor object for datetimelike properties of the Series values.
2877 Series.sparse : Accessor for sparse matrix data types.
2879 Examples
2880 --------
2881 >>> s = pd.Series(list("abbccc")).astype("category")
2882 >>> s
2883 0 a
2884 1 b
2885 2 b
2886 3 c
2887 4 c
2888 5 c
2889 dtype: category
2890 Categories (3, str): ['a', 'b', 'c']
2892 >>> s.cat.categories
2893 Index(['a', 'b', 'c'], dtype='str')
2895 >>> s.cat.rename_categories(list("cba"))
2896 0 c
2897 1 b
2898 2 b
2899 3 a
2900 4 a
2901 5 a
2902 dtype: category
2903 Categories (3, str): ['c', 'b', 'a']
2905 >>> s.cat.reorder_categories(list("cba"))
2906 0 a
2907 1 b
2908 2 b
2909 3 c
2910 4 c
2911 5 c
2912 dtype: category
2913 Categories (3, str): ['c', 'b', 'a']
2915 >>> s.cat.add_categories(["d", "e"])
2916 0 a
2917 1 b
2918 2 b
2919 3 c
2920 4 c
2921 5 c
2922 dtype: category
2923 Categories (5, str): ['a', 'b', 'c', 'd', 'e']
2925 >>> s.cat.remove_categories(["a", "c"])
2926 0 NaN
2927 1 b
2928 2 b
2929 3 NaN
2930 4 NaN
2931 5 NaN
2932 dtype: category
2933 Categories (1, str): ['b']
2935 >>> s1 = s.cat.add_categories(["d", "e"])
2936 >>> s1.cat.remove_unused_categories()
2937 0 a
2938 1 b
2939 2 b
2940 3 c
2941 4 c
2942 5 c
2943 dtype: category
2944 Categories (3, str): ['a', 'b', 'c']
2946 >>> s.cat.set_categories(list("abcde"))
2947 0 a
2948 1 b
2949 2 b
2950 3 c
2951 4 c
2952 5 c
2953 dtype: category
2954 Categories (5, str): ['a', 'b', 'c', 'd', 'e']
2956 >>> s.cat.as_ordered()
2957 0 a
2958 1 b
2959 2 b
2960 3 c
2961 4 c
2962 5 c
2963 dtype: category
2964 Categories (3, str): ['a' < 'b' < 'c']
2966 >>> s.cat.as_unordered()
2967 0 a
2968 1 b
2969 2 b
2970 3 c
2971 4 c
2972 5 c
2973 dtype: category
2974 Categories (3, str): ['a', 'b', 'c']
2975 """
2977 def __init__(self, data) -> None:
2978 self._validate(data)
2979 self._parent = data.values
2980 self._index = data.index
2981 self._name = data.name
2982 self._freeze()
2984 @staticmethod
2985 def _validate(data) -> None:
2986 if not isinstance(data.dtype, CategoricalDtype):
2987 raise AttributeError("Can only use .cat accessor with a 'category' dtype")
2989 def _delegate_property_get(self, name: str):
2990 return getattr(self._parent, name)
2992 def _delegate_property_set(self, name: str, new_values) -> None:
2993 setattr(self._parent, name, new_values)
2995 @property
2996 def codes(self) -> Series:
2997 """
2998 Return Series of codes as well as the index.
3000 See Also
3001 --------
3002 Series.cat.categories : Return the categories of this categorical.
3003 Series.cat.as_ordered : Set the Categorical to be ordered.
3004 Series.cat.as_unordered : Set the Categorical to be unordered.
3006 Examples
3007 --------
3008 >>> raw_cate = pd.Categorical(["a", "b", None, "a"], categories=["a", "b"])
3009 >>> ser = pd.Series(raw_cate)
3010 >>> ser.cat.codes
3011 0 0
3012 1 1
3013 2 -1
3014 3 0
3015 dtype: int8
3016 """
3017 from pandas import Series
3019 return Series(self._parent.codes, index=self._index)
3021 def _delegate_method(self, name: str, *args, **kwargs):
3022 from pandas import Series
3024 method = getattr(self._parent, name)
3025 res = method(*args, **kwargs)
3026 if res is not None:
3027 return Series(res, index=self._index, name=self._name)
3030# utility routines
3033def _get_codes_for_values(
3034 values: Index | Series | ExtensionArray | np.ndarray,
3035 categories: Index,
3036) -> np.ndarray:
3037 """
3038 utility routine to turn values into codes given the specified categories
3040 If `values` is known to be a Categorical, use recode_for_categories instead.
3041 """
3042 codes = categories.get_indexer_for(values)
3043 wrong = (codes == -1) & ~isna(values)
3044 if wrong.any():
3045 warnings.warn(
3046 "Constructing a Categorical with a dtype and values containing "
3047 "non-null entries not in that dtype's categories is deprecated "
3048 "and will raise in a future version.",
3049 Pandas4Warning,
3050 stacklevel=find_stack_level(),
3051 )
3052 return coerce_indexer_dtype(codes, categories)
3055def recode_for_categories(
3056 codes: np.ndarray,
3057 old_categories,
3058 new_categories,
3059 *,
3060 copy: bool = True,
3061 warn: bool = False,
3062) -> np.ndarray:
3063 """
3064 Convert a set of codes for to a new set of categories
3066 Parameters
3067 ----------
3068 codes : np.ndarray
3069 old_categories, new_categories : Index
3070 copy: bool, default True
3071 Whether to copy if the codes are unchanged.
3072 warn : bool, default False
3073 Whether to warn on silent-NA mapping.
3075 Returns
3076 -------
3077 new_codes : np.ndarray[np.int64]
3079 Examples
3080 --------
3081 >>> old_cat = pd.Index(["b", "a", "c"])
3082 >>> new_cat = pd.Index(["a", "b"])
3083 >>> codes = np.array([0, 1, 1, 2])
3084 >>> recode_for_categories(codes, old_cat, new_cat, copy=True)
3085 array([ 1, 0, 0, -1], dtype=int8)
3086 """
3087 if len(old_categories) == 0:
3088 # All null anyway, so just retain the nulls
3089 if copy:
3090 return codes.copy()
3091 return codes
3092 elif new_categories.equals(old_categories):
3093 # Same categories, so no need to actually recode
3094 if copy:
3095 return codes.copy()
3096 return codes
3098 codes_in_old_cats = new_categories.get_indexer_for(old_categories)
3099 if warn:
3100 wrong = codes_in_old_cats == -1
3101 if wrong.any():
3102 warnings.warn(
3103 "Constructing a Categorical with a dtype and values containing "
3104 "non-null entries not in that dtype's categories is deprecated "
3105 "and will raise in a future version.",
3106 Pandas4Warning,
3107 stacklevel=find_stack_level(),
3108 )
3109 indexer = coerce_indexer_dtype(codes_in_old_cats, new_categories)
3110 new_codes = take_nd(indexer, codes, fill_value=-1)
3111 return new_codes
3114def factorize_from_iterable(values) -> tuple[np.ndarray, Index]:
3115 """
3116 Factorize an input `values` into `categories` and `codes`. Preserves
3117 categorical dtype in `categories`.
3119 Parameters
3120 ----------
3121 values : list-like
3123 Returns
3124 -------
3125 codes : ndarray
3126 categories : Index
3127 If `values` has a categorical dtype, then `categories` is
3128 a CategoricalIndex keeping the categories and order of `values`.
3129 """
3130 from pandas import CategoricalIndex
3132 if not is_list_like(values):
3133 raise TypeError("Input must be list-like")
3135 categories: Index
3137 vdtype = getattr(values, "dtype", None)
3138 if isinstance(vdtype, CategoricalDtype):
3139 values = extract_array(values)
3140 # The Categorical we want to build has the same categories
3141 # as values but its codes are by def [0, ..., len(n_categories) - 1]
3142 cat_codes = np.arange(len(values.categories), dtype=values.codes.dtype)
3143 cat = Categorical.from_codes(cat_codes, dtype=values.dtype, validate=False)
3145 categories = CategoricalIndex(cat)
3146 codes = values.codes
3147 else:
3148 # The value of ordered is irrelevant since we don't use cat as such,
3149 # but only the resulting categories, the order of which is independent
3150 # from ordered. Set ordered to False as default. See GH #15457
3151 cat = Categorical(values, ordered=False)
3152 categories = cat.categories
3153 codes = cat.codes
3154 return codes, categories
3157def factorize_from_iterables(iterables) -> tuple[list[np.ndarray], list[Index]]:
3158 """
3159 A higher-level wrapper over `factorize_from_iterable`.
3161 Parameters
3162 ----------
3163 iterables : list-like of list-likes
3165 Returns
3166 -------
3167 codes : list of ndarrays
3168 categories : list of Indexes
3170 Notes
3171 -----
3172 See `factorize_from_iterable` for more info.
3173 """
3174 if len(iterables) == 0:
3175 # For consistency, it should return two empty lists.
3176 return [], []
3178 codes, categories = zip(
3179 *(factorize_from_iterable(it) for it in iterables),
3180 strict=True,
3181 )
3182 return list(codes), list(categories)