Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Quantilization functions and related stuff
3"""
5from __future__ import annotations
7from typing import (
8 TYPE_CHECKING,
9 Any,
10 Literal,
11 cast,
12)
14import numpy as np
16from pandas._libs import (
17 Timedelta,
18 Timestamp,
19 lib,
20)
21from pandas.util._decorators import set_module
23from pandas.core.dtypes.common import (
24 ensure_platform_int,
25 is_bool_dtype,
26 is_integer,
27 is_list_like,
28 is_numeric_dtype,
29 is_scalar,
30)
31from pandas.core.dtypes.dtypes import (
32 CategoricalDtype,
33 DatetimeTZDtype,
34 ExtensionDtype,
35)
36from pandas.core.dtypes.generic import ABCSeries
37from pandas.core.dtypes.missing import isna
39from pandas import (
40 Categorical,
41 Index,
42 IntervalIndex,
43)
44import pandas.core.algorithms as algos
45from pandas.core.arrays.datetimelike import dtype_to_unit
46from pandas.core.col import Expression
48if TYPE_CHECKING:
49 from collections.abc import Callable
51 from pandas._typing import (
52 DtypeObj,
53 IntervalLeftRight,
54 TimeUnit,
55 )
58@set_module("pandas")
59def cut(
60 x,
61 bins,
62 right: bool = True,
63 labels=None,
64 retbins: bool = False,
65 precision: int = 3,
66 include_lowest: bool = False,
67 duplicates: str = "raise",
68 ordered: bool = True,
69):
70 """
71 Bin values into discrete intervals.
73 Use `cut` when you need to segment and sort data values into bins. This
74 function is also useful for going from a continuous variable to a
75 categorical variable. For example, `cut` could convert ages to groups of
76 age ranges. Supports binning into an equal number of bins, or a
77 pre-specified array of bins.
79 Parameters
80 ----------
81 x : 1d ndarray or Series
82 The input array to be binned. Must be 1-dimensional.
83 bins : int, sequence of scalars, or IntervalIndex
84 The criteria to bin by.
86 * int : Defines the number of equal-width bins in the range of `x`. The
87 range of `x` is extended by .1% on each side to include the minimum
88 and maximum values of `x`.
89 * sequence of scalars : Defines the bin edges allowing for non-uniform
90 width. No extension of the range of `x` is done.
91 * IntervalIndex : Defines the exact bins to be used. Note that
92 IntervalIndex for `bins` must be non-overlapping.
94 right : bool, default True
95 Indicates whether `bins` includes the rightmost edge or not. If
96 ``right == True`` (the default), then the `bins` ``[1, 2, 3, 4]``
97 indicate (1,2], (2,3], (3,4]. This argument is ignored when
98 `bins` is an IntervalIndex.
99 labels : array or False, default None
100 Specifies the labels for the returned bins. Must be the same length as
101 the resulting bins. If False, returns only integer indicators of the
102 bins. This affects the type of the output container (see below).
103 This argument is ignored when `bins` is an IntervalIndex. If True,
104 raises an error. When `ordered=False`, labels must be provided.
105 retbins : bool, default False
106 Whether to return the bins or not. Useful when bins is provided
107 as a scalar.
108 precision : int, default 3
109 The precision at which to store and display the bins labels.
110 include_lowest : bool, default False
111 Whether the first interval should be left-inclusive or not.
112 duplicates : {'raise', 'drop'}, default 'raise'
113 If bin edges are not unique, raise ValueError or drop non-uniques.
114 ordered : bool, default True
115 Whether the labels are ordered or not. Applies to returned types
116 Categorical and Series (with Categorical dtype). If True,
117 the resulting categorical will be ordered. If False, the resulting
118 categorical will be unordered (labels must be provided).
120 Returns
121 -------
122 out : Categorical, Series, or ndarray
123 An array-like object representing the respective bin for each value
124 of `x`. The type depends on the value of `labels`.
126 * None (default) : returns a Series for Series `x` or a
127 Categorical for all other inputs. The values stored within
128 are Interval dtype.
130 * sequence of scalars : returns a Series for Series `x` or a
131 Categorical for all other inputs. The values stored within
132 are whatever the type in the sequence is.
134 * False : returns a 1d ndarray or Series of integers.
136 bins : numpy.ndarray or IntervalIndex.
137 The computed or specified bins. Only returned when `retbins=True`.
138 For scalar or sequence `bins`, this is an ndarray with the computed
139 bins. If set `duplicates=drop`, `bins` will drop non-unique bin. For
140 an IntervalIndex `bins`, this is equal to `bins`.
142 See Also
143 --------
144 qcut : Discretize variable into equal-sized buckets based on rank
145 or based on sample quantiles.
146 Categorical : Array type for storing data that come from a
147 fixed set of values.
148 Series : One-dimensional array with axis labels (including time series).
149 IntervalIndex : Immutable Index implementing an ordered, sliceable set.
150 numpy.histogram_bin_edges: Function to calculate only the edges of the bins
151 used by the histogram function.
153 Notes
154 -----
155 Any NA values will be NA in the result. Out of bounds values will be NA in
156 the resulting Series or Categorical object.
158 ``numpy.histogram_bin_edges`` can be used along with cut to calculate bins according
159 to some predefined methods.
161 Reference :ref:`the user guide <reshaping.tile.cut>` for more examples.
163 Examples
164 --------
165 Discretize into three equal-sized bins.
167 >>> pd.cut(np.array([1, 7, 5, 4, 6, 3]), 3)
168 ... # doctest: +ELLIPSIS
169 [(0.994, 3.0], (5.0, 7.0], (3.0, 5.0], (3.0, 5.0], (5.0, 7.0], ...
170 Categories (3, interval[float64, right]): [(0.994, 3.0] < (3.0, 5.0] ...
172 >>> pd.cut(np.array([1, 7, 5, 4, 6, 3]), 3, retbins=True)
173 ... # doctest: +ELLIPSIS
174 ([(0.994, 3.0], (5.0, 7.0], (3.0, 5.0], (3.0, 5.0], (5.0, 7.0], ...
175 Categories (3, interval[float64, right]): [(0.994, 3.0] < (3.0, 5.0] ...
176 array([0.994, 3. , 5. , 7. ]))
178 Discovers the same bins, but assign them specific labels. Notice that
179 the returned Categorical's categories are `labels` and is ordered.
181 >>> pd.cut(np.array([1, 7, 5, 4, 6, 3]), 3, labels=["bad", "medium", "good"])
182 ['bad', 'good', 'medium', 'medium', 'good', 'bad']
183 Categories (3, str): ['bad' < 'medium' < 'good']
185 ``ordered=False`` will result in unordered categories when labels are passed.
186 This parameter can be used to allow non-unique labels:
188 >>> pd.cut(np.array([1, 7, 5, 4, 6, 3]), 3, labels=["B", "A", "B"], ordered=False)
189 ['B', 'B', 'A', 'A', 'B', 'B']
190 Categories (2, str): ['A', 'B']
192 ``labels=False`` implies you just want the bins back.
194 >>> pd.cut([0, 1, 1, 2], bins=4, labels=False)
195 array([0, 1, 1, 3])
197 Passing a Series as an input returns a Series with categorical dtype:
199 >>> s = pd.Series(np.array([2, 4, 6, 8, 10]), index=["a", "b", "c", "d", "e"])
200 >>> pd.cut(s, 3)
201 ... # doctest: +ELLIPSIS
202 a (1.992, 4.667]
203 b (1.992, 4.667]
204 c (4.667, 7.333]
205 d (7.333, 10.0]
206 e (7.333, 10.0]
207 dtype: category
208 Categories (3, interval[float64, right]): [(1.992, 4.667] < (4.667, ...
210 Passing a Series as an input returns a Series with mapping value.
211 It is used to map numerically to intervals based on bins.
213 >>> s = pd.Series(np.array([2, 4, 6, 8, 10]), index=["a", "b", "c", "d", "e"])
214 >>> pd.cut(s, [0, 2, 4, 6, 8, 10], labels=False, retbins=True, right=False)
215 ... # doctest: +ELLIPSIS
216 (a 1.0
217 b 2.0
218 c 3.0
219 d 4.0
220 e NaN
221 dtype: float64,
222 array([ 0, 2, 4, 6, 8, 10]))
224 Use `drop` optional when bins is not unique
226 >>> pd.cut(
227 ... s,
228 ... [0, 2, 4, 6, 10, 10],
229 ... labels=False,
230 ... retbins=True,
231 ... right=False,
232 ... duplicates="drop",
233 ... )
234 ... # doctest: +ELLIPSIS
235 (a 1.0
236 b 2.0
237 c 3.0
238 d 3.0
239 e NaN
240 dtype: float64,
241 array([ 0, 2, 4, 6, 10]))
243 Passing an IntervalIndex for `bins` results in those categories exactly.
244 Notice that values not covered by the IntervalIndex are set to NaN. 0
245 is to the left of the first bin (which is closed on the right), and 1.5
246 falls between two bins.
248 >>> bins = pd.IntervalIndex.from_tuples([(0, 1), (2, 3), (4, 5)])
249 >>> pd.cut([0, 0.5, 1.5, 2.5, 4.5], bins)
250 [NaN, (0.0, 1.0], NaN, (2.0, 3.0], (4.0, 5.0]]
251 Categories (3, interval[int64, right]): [(0, 1] < (2, 3] < (4, 5]]
253 Using np.histogram_bin_edges with cut
255 >>> pd.cut(
256 ... np.array([1, 7, 5, 4]),
257 ... bins=np.histogram_bin_edges(np.array([1, 7, 5, 4]), bins="auto"),
258 ... )
259 ... # doctest: +ELLIPSIS
260 [NaN, (5.0, 7.0], (3.0, 5.0], (3.0, 5.0]]
261 Categories (3, interval[float64, right]): [(1.0, 3.0] < (3.0, 5.0] < (5.0, 7.0]]
262 """
263 # NOTE: this binning code is changed a bit from histogram for var(x) == 0
265 original = x
266 x_idx = _preprocess_for_cut(x)
267 x_idx, _ = _coerce_to_type(x_idx)
269 if not np.iterable(bins):
270 bins = _nbins_to_bins(x_idx, bins, right)
272 elif isinstance(bins, IntervalIndex):
273 if bins.is_overlapping:
274 raise ValueError("Overlapping IntervalIndex is not accepted.")
276 else:
277 bins = Index(bins)
278 if not bins.is_monotonic_increasing:
279 raise ValueError("bins must increase monotonically.")
281 fac, bins = _bins_to_cuts(
282 x_idx,
283 bins,
284 right=right,
285 labels=labels,
286 precision=precision,
287 include_lowest=include_lowest,
288 duplicates=duplicates,
289 ordered=ordered,
290 )
292 return _postprocess_for_cut(fac, bins, retbins, original)
295@set_module("pandas")
296def qcut(
297 x,
298 q,
299 labels=None,
300 retbins: bool = False,
301 precision: int = 3,
302 duplicates: str = "raise",
303):
304 """
305 Quantile-based discretization function.
307 Discretize variable into equal-sized buckets based on rank or based
308 on sample quantiles. For example 1000 values for 10 quantiles would
309 produce a Categorical object indicating quantile membership for each data point.
311 Parameters
312 ----------
313 x : 1d ndarray or Series
314 Input Numpy array or pandas Series object to be discretized.
315 q : int or list-like of float
316 Number of quantiles. 10 for deciles, 4 for quartiles, etc. Alternately
317 array of quantiles, e.g. [0, .25, .5, .75, 1.] for quartiles.
318 labels : array or False, default None
319 Used as labels for the resulting bins. Must be of the same length as
320 the resulting bins. If False, return only integer indicators of the
321 bins. If True, raises an error.
322 retbins : bool, optional
323 Whether to return the (bins, labels) or not. Can be useful if bins
324 is given as a scalar.
325 precision : int, optional
326 The precision at which to store and display the bins labels.
327 duplicates : {default 'raise', 'drop'}, optional
328 If bin edges are not unique, raise ValueError or drop non-uniques.
330 Returns
331 -------
332 out : Categorical or Series or array of integers if labels is False
333 The return type (Categorical or Series) depends on the input: a Series
334 of type category if input is a Series else Categorical. Bins are
335 represented as categories when categorical data is returned.
336 bins : ndarray of floats
337 Returned only if `retbins` is True.
339 See Also
340 --------
341 cut : Bin values into discrete intervals.
342 Series.quantile : Return value at the given quantile.
344 Notes
345 -----
346 Out of bounds values will be NA in the resulting Categorical object
348 Examples
349 --------
350 >>> pd.qcut(range(5), 4)
351 ... # doctest: +ELLIPSIS
352 [(-0.001, 1.0], (-0.001, 1.0], (1.0, 2.0], (2.0, 3.0], (3.0, 4.0]]
353 Categories (4, interval[float64, right]): [(-0.001, 1.0] < (1.0, 2.0] ...
355 >>> pd.qcut(range(5), 3, labels=["good", "medium", "bad"])
356 ... # doctest: +SKIP
357 [good, good, medium, bad, bad]
358 Categories (3, str): [good < medium < bad]
360 >>> pd.qcut(range(5), 4, labels=False)
361 array([0, 0, 1, 2, 3])
362 """
363 if isinstance(x, Expression):
364 return x._call_with_func(
365 qcut, x=x, q=q, labels=labels, retbins=retbins, precision=precision
366 )
367 original = x
368 x_idx = _preprocess_for_cut(x)
369 x_idx, _ = _coerce_to_type(x_idx)
371 if is_integer(q):
372 quantiles = np.linspace(0, 1, q + 1)
373 # Round up rather than to nearest if not representable in base 2
374 np.putmask(
375 quantiles,
376 q * quantiles != np.arange(q + 1),
377 np.nextafter(quantiles, 1),
378 )
379 else:
380 quantiles = q
382 bins = x_idx.to_series().dropna().quantile(quantiles)
384 fac, bins = _bins_to_cuts(
385 x_idx,
386 Index(bins),
387 labels=labels,
388 precision=precision,
389 include_lowest=True,
390 duplicates=duplicates,
391 )
393 return _postprocess_for_cut(fac, bins, retbins, original)
396def _nbins_to_bins(x_idx: Index, nbins: int, right: bool) -> Index:
397 """
398 If a user passed an integer N for bins, convert this to a sequence of N
399 equal(ish)-sized bins.
400 """
401 if is_scalar(nbins) and nbins < 1:
402 raise ValueError("`bins` should be a positive integer.")
404 if x_idx.size == 0:
405 raise ValueError("Cannot cut empty array")
407 rng = (x_idx.min(), x_idx.max())
408 mn, mx = rng
410 if is_numeric_dtype(x_idx.dtype) and (np.isinf(mn) or np.isinf(mx)):
411 # GH#24314
412 raise ValueError(
413 "cannot specify integer `bins` when input data contains infinity"
414 )
416 if mn == mx: # adjust end points before binning
417 if _is_dt_or_td(x_idx.dtype):
418 # using seconds=1 is pretty arbitrary here
419 # error: Argument 1 to "dtype_to_unit" has incompatible type
420 # "dtype[Any] | ExtensionDtype"; expected "DatetimeTZDtype | dtype[Any]"
421 unit = dtype_to_unit(x_idx.dtype) # type: ignore[arg-type]
422 td = Timedelta(seconds=1).as_unit(cast("TimeUnit", unit))
423 # Use DatetimeArray/TimedeltaArray method instead of linspace
424 # error: Item "ExtensionArray" of "ExtensionArray | ndarray[Any, Any]"
425 # has no attribute "_generate_range"
426 bins = x_idx._values._generate_range( # type: ignore[union-attr]
427 start=mn - td, end=mx + td, periods=nbins + 1, freq=None, unit=unit
428 )
429 else:
430 mn -= 0.001 * abs(mn) if mn != 0 else 0.001
431 mx += 0.001 * abs(mx) if mx != 0 else 0.001
433 bins = np.linspace(mn, mx, nbins + 1, endpoint=True)
434 else: # adjust end points after binning
435 if _is_dt_or_td(x_idx.dtype):
436 # Use DatetimeArray/TimedeltaArray method instead of linspace
438 # error: Argument 1 to "dtype_to_unit" has incompatible type
439 # "dtype[Any] | ExtensionDtype"; expected "DatetimeTZDtype | dtype[Any]"
440 unit = dtype_to_unit(x_idx.dtype) # type: ignore[arg-type]
441 # error: Item "ExtensionArray" of "ExtensionArray | ndarray[Any, Any]"
442 # has no attribute "_generate_range"
443 bins = x_idx._values._generate_range( # type: ignore[union-attr]
444 start=mn, end=mx, periods=nbins + 1, freq=None, unit=unit
445 )
446 else:
447 bins = np.linspace(mn, mx, nbins + 1, endpoint=True)
448 adj = (mx - mn) * 0.001 # 0.1% of the range
449 if right:
450 bins[0] -= adj
451 else:
452 bins[-1] += adj
454 return Index(bins, copy=False)
457def _bins_to_cuts(
458 x_idx: Index,
459 bins: Index,
460 right: bool = True,
461 labels=None,
462 precision: int = 3,
463 include_lowest: bool = False,
464 duplicates: str = "raise",
465 ordered: bool = True,
466):
467 if not ordered and labels is None:
468 raise ValueError("'labels' must be provided if 'ordered = False'")
470 if duplicates not in ["raise", "drop"]:
471 raise ValueError(
472 "invalid value for 'duplicates' parameter, valid options are: raise, drop"
473 )
475 result: Categorical | np.ndarray
477 if isinstance(bins, IntervalIndex):
478 # we have a fast-path here
479 ids = bins.get_indexer(x_idx)
480 cat_dtype = CategoricalDtype(bins, ordered=True)
481 result = Categorical.from_codes(ids, dtype=cat_dtype, validate=False)
482 return result, bins
484 unique_bins = algos.unique(bins)
485 if len(unique_bins) < len(bins) and len(bins) != 2:
486 if duplicates == "raise":
487 raise ValueError(
488 f"Bin edges must be unique: {bins!r}.\n"
489 f"You can drop duplicate edges by setting the 'duplicates' kwarg"
490 )
491 bins = unique_bins
493 side: Literal["left", "right"] = "left" if right else "right"
495 try:
496 ids = bins.searchsorted(x_idx, side=side)
497 except TypeError as err:
498 # e.g. test_datetime_nan_error if bins are DatetimeArray and x_idx
499 # is integers
500 if x_idx.dtype.kind == "m":
501 raise ValueError("bins must be of timedelta64 dtype") from err
502 elif x_idx.dtype.kind == bins.dtype.kind == "M":
503 raise ValueError(
504 "Cannot use timezone-naive bins with timezone-aware values, "
505 "or vice-versa"
506 ) from err
507 elif x_idx.dtype.kind == "M":
508 raise ValueError("bins must be of datetime64 dtype") from err
509 else:
510 raise
511 ids = ensure_platform_int(ids)
513 if include_lowest:
514 ids[x_idx == bins[0]] = 1
516 na_mask = isna(x_idx) | (ids == len(bins)) | (ids == 0)
517 has_nas = na_mask.any()
519 if labels is not False:
520 if not (labels is None or is_list_like(labels)):
521 raise ValueError(
522 "Bin labels must either be False, None or passed in as a "
523 "list-like argument"
524 )
526 if labels is None:
527 labels = _format_labels(
528 bins, precision, right=right, include_lowest=include_lowest
529 )
530 elif ordered and len(set(labels)) != len(labels):
531 raise ValueError(
532 "labels must be unique if ordered=True; pass ordered=False "
533 "for duplicate labels"
534 )
535 elif len(labels) != len(bins) - 1:
536 raise ValueError(
537 "Bin labels must be one fewer than the number of bin edges"
538 )
540 if not isinstance(getattr(labels, "dtype", None), CategoricalDtype):
541 labels = Categorical(
542 labels,
543 categories=labels if len(set(labels)) == len(labels) else None,
544 ordered=ordered,
545 )
546 # TODO: handle mismatch between categorical label order and pandas.cut order.
547 np.putmask(ids, na_mask, 0)
548 result = algos.take_nd(labels, ids - 1)
550 else:
551 result = ids - 1
552 if has_nas:
553 result = result.astype(np.float64)
554 np.putmask(result, na_mask, np.nan)
556 return result, bins
559def _coerce_to_type(x: Index) -> tuple[Index, DtypeObj | None]:
560 """
561 if the passed data is of datetime/timedelta, bool or nullable int type,
562 this method converts it to numeric so that cut or qcut method can
563 handle it
564 """
565 dtype: DtypeObj | None = None
567 if _is_dt_or_td(x.dtype):
568 dtype = x.dtype
569 elif is_bool_dtype(x.dtype):
570 # GH 20303
571 x = x.astype(np.int64)
572 # To support cut and qcut for IntegerArray we convert to float dtype.
573 # Will properly support in the future.
574 # https://github.com/pandas-dev/pandas/pull/31290
575 # https://github.com/pandas-dev/pandas/issues/31389
576 elif isinstance(x.dtype, ExtensionDtype) and is_numeric_dtype(x.dtype):
577 x_arr = x.to_numpy(dtype=np.float64, na_value=np.nan)
578 x = Index(x_arr, copy=False)
580 return Index(x), dtype
583def _is_dt_or_td(dtype: DtypeObj) -> bool:
584 # Note: the dtype here comes from an Index.dtype, so we know that that any
585 # dt64/td64 dtype is of a supported unit.
586 return isinstance(dtype, DatetimeTZDtype) or lib.is_np_dtype(dtype, "mM")
589def _format_labels(
590 bins: Index,
591 precision: int,
592 right: bool = True,
593 include_lowest: bool = False,
594) -> IntervalIndex:
595 """based on the dtype, return our labels"""
596 closed: IntervalLeftRight = "right" if right else "left"
598 formatter: Callable[[Any], Timestamp] | Callable[[Any], Timedelta]
600 if _is_dt_or_td(bins.dtype):
601 # error: Argument 1 to "dtype_to_unit" has incompatible type
602 # "dtype[Any] | ExtensionDtype"; expected "DatetimeTZDtype | dtype[Any]"
603 unit = dtype_to_unit(bins.dtype) # type: ignore[arg-type]
604 unit = cast("TimeUnit", unit)
605 formatter = lambda x: x
606 adjust = lambda x: x - Timedelta(1, unit=unit).as_unit(unit)
607 else:
608 precision = _infer_precision(precision, bins)
609 formatter = lambda x: _round_frac(x, precision)
610 adjust = lambda x: x - 10 ** (-precision)
612 breaks = [formatter(b) for b in bins]
613 if right and include_lowest:
614 # adjust lhs of first interval by precision to account for being right closed
615 breaks[0] = adjust(breaks[0])
617 if _is_dt_or_td(bins.dtype):
618 # error: "Index" has no attribute "as_unit"
619 breaks = type(bins)(breaks).as_unit(unit) # type: ignore[attr-defined]
621 return IntervalIndex.from_breaks(breaks, closed=closed)
624def _preprocess_for_cut(x) -> Index:
625 """
626 handles preprocessing for cut where we convert passed
627 input to array, strip the index information and store it
628 separately
629 """
630 # Check that the passed array is a Pandas or Numpy object
631 # We don't want to strip away a Pandas data-type here (e.g. datetimetz)
632 ndim = getattr(x, "ndim", None)
633 if ndim is None:
634 x = np.asarray(x)
635 if x.ndim != 1:
636 raise ValueError("Input array must be 1 dimensional")
638 return Index(x, copy=False)
641def _postprocess_for_cut(fac, bins, retbins: bool, original):
642 """
643 handles post processing for the cut method where
644 we combine the index information if the originally passed
645 datatype was a series
646 """
647 if isinstance(original, ABCSeries):
648 fac = original._constructor(fac, index=original.index, name=original.name)
650 if not retbins:
651 return fac
653 if isinstance(bins, Index) and is_numeric_dtype(bins.dtype):
654 bins = bins._values
656 return fac, bins
659def _round_frac(x, precision: int):
660 """
661 Round the fractional part of the given number
662 """
663 if not np.isfinite(x) or x == 0:
664 return x
665 else:
666 frac, whole = np.modf(x)
667 if whole == 0:
668 digits = -int(np.floor(np.log10(abs(frac)))) - 1 + precision
669 else:
670 digits = precision
671 return np.around(x, digits)
674def _infer_precision(base_precision: int, bins: Index) -> int:
675 """
676 Infer an appropriate precision for _round_frac
677 """
678 for precision in range(base_precision, 20):
679 levels = np.asarray([_round_frac(b, precision) for b in bins])
680 if algos.unique(levels).size == bins.size:
681 return precision
682 return base_precision # default