Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Provide user facing operators for doing the split part of the
3split-apply-combine paradigm.
4"""
6from __future__ import annotations
8from itertools import pairwise
9from typing import (
10 TYPE_CHECKING,
11 final,
12)
13import warnings
15import numpy as np
17from pandas._libs import (
18 algos as libalgos,
19)
20from pandas._libs.tslibs import OutOfBoundsDatetime
21from pandas.errors import (
22 InvalidIndexError,
23 Pandas4Warning,
24)
25from pandas.util._decorators import (
26 cache_readonly,
27 set_module,
28)
29from pandas.util._exceptions import find_stack_level
31from pandas.core.dtypes.common import (
32 ensure_int64,
33 ensure_platform_int,
34 is_list_like,
35 is_scalar,
36)
37from pandas.core.dtypes.dtypes import CategoricalDtype
39from pandas.core import algorithms
40from pandas.core.arrays import (
41 Categorical,
42 ExtensionArray,
43)
44import pandas.core.common as com
45from pandas.core.frame import DataFrame
46from pandas.core.groupby import ops
47from pandas.core.groupby.categorical import recode_for_groupby
48from pandas.core.indexes.api import (
49 Index,
50 MultiIndex,
51 default_index,
52)
53from pandas.core.series import Series
55from pandas.io.formats.printing import (
56 PrettyDict,
57 pprint_thing,
58)
60if TYPE_CHECKING:
61 from collections.abc import (
62 Hashable,
63 Iterator,
64 )
66 from pandas._typing import (
67 ArrayLike,
68 NDFrameT,
69 npt,
70 )
72 from pandas.core.generic import NDFrame
75@set_module("pandas")
76class Grouper:
77 """
78 A Grouper allows the user to specify a groupby instruction for an object.
80 This specification will select a column via the key parameter, or if the
81 level parameter is given, a level of the index of the target
82 object.
84 If ``level`` is passed as a keyword to both `Grouper` and
85 `groupby`, the values passed to `Grouper` take precedence.
87 Parameters
88 ----------
89 *args
90 Currently unused, reserved for future use.
91 **kwargs
92 Dictionary of the keyword arguments to pass to Grouper.
94 Attributes
95 ----------
96 key : str, defaults to None
97 Groupby key, which selects the grouping column of the target.
98 level : name/number, defaults to None
99 The level for the target index.
100 freq : str / frequency object, defaults to None
101 This will groupby the specified frequency if the target selection
102 (via key or level) is a datetime-like object. For full specification
103 of available frequencies, please see :ref:`here<timeseries.offset_aliases>`.
104 sort : bool, default False
105 Whether to sort the resulting labels.
106 closed : {'left' or 'right'}
107 Closed end of interval. Only when `freq` parameter is passed.
108 label : {'left' or 'right'}
109 Interval boundary to use for labeling.
110 Only when `freq` parameter is passed.
111 convention : {'start', 'end', 'e', 's'}
112 If grouper is PeriodIndex and `freq` parameter is passed.
114 origin : Timestamp or str, default 'start_day'
115 The timestamp on which to adjust the grouping. The timezone of origin must
116 match the timezone of the index.
117 If string, must be one of the following:
119 - 'epoch': `origin` is 1970-01-01
120 - 'start': `origin` is the first value of the timeseries
121 - 'start_day': `origin` is the first day at midnight of the timeseries
123 - 'end': `origin` is the last value of the timeseries
124 - 'end_day': `origin` is the ceiling midnight of the last day
126 offset : Timedelta or str, default None
127 An offset timedelta added to the origin.
129 dropna : bool, default True
130 If True, and if group keys contain NA values, NA values together with
131 row/column will be dropped. If False, NA values will also be treated as
132 the key in groups.
134 Returns
135 -------
136 Grouper or pandas.api.typing.TimeGrouper
137 A TimeGrouper is returned if ``freq`` is not ``None``. Otherwise, a Grouper
138 is returned.
140 See Also
141 --------
142 Series.groupby : Apply a function groupby to a Series.
143 DataFrame.groupby : Apply a function groupby.
145 Examples
146 --------
147 ``df.groupby(pd.Grouper(key="Animal"))`` is equivalent to ``df.groupby('Animal')``
149 >>> df = pd.DataFrame(
150 ... {
151 ... "Animal": ["Falcon", "Parrot", "Falcon", "Falcon", "Parrot"],
152 ... "Speed": [100, 5, 200, 300, 15],
153 ... }
154 ... )
155 >>> df
156 Animal Speed
157 0 Falcon 100
158 1 Parrot 5
159 2 Falcon 200
160 3 Falcon 300
161 4 Parrot 15
162 >>> df.groupby(pd.Grouper(key="Animal")).mean()
163 Speed
164 Animal
165 Falcon 200.0
166 Parrot 10.0
168 Specify a resample operation on the column 'Publish date'
170 >>> df = pd.DataFrame(
171 ... {
172 ... "Publish date": [
173 ... pd.Timestamp("2000-01-02"),
174 ... pd.Timestamp("2000-01-02"),
175 ... pd.Timestamp("2000-01-09"),
176 ... pd.Timestamp("2000-01-16"),
177 ... ],
178 ... "ID": [0, 1, 2, 3],
179 ... "Price": [10, 20, 30, 40],
180 ... }
181 ... )
182 >>> df
183 Publish date ID Price
184 0 2000-01-02 0 10
185 1 2000-01-02 1 20
186 2 2000-01-09 2 30
187 3 2000-01-16 3 40
188 >>> df.groupby(pd.Grouper(key="Publish date", freq="1W")).mean()
189 ID Price
190 Publish date
191 2000-01-02 0.5 15.0
192 2000-01-09 2.0 30.0
193 2000-01-16 3.0 40.0
195 If you want to adjust the start of the bins based on a fixed timestamp:
197 >>> start, end = "2000-10-01 23:30:00", "2000-10-02 00:30:00"
198 >>> rng = pd.date_range(start, end, freq="7min")
199 >>> ts = pd.Series(np.arange(len(rng)) * 3, index=rng)
200 >>> ts
201 2000-10-01 23:30:00 0
202 2000-10-01 23:37:00 3
203 2000-10-01 23:44:00 6
204 2000-10-01 23:51:00 9
205 2000-10-01 23:58:00 12
206 2000-10-02 00:05:00 15
207 2000-10-02 00:12:00 18
208 2000-10-02 00:19:00 21
209 2000-10-02 00:26:00 24
210 Freq: 7min, dtype: int64
212 >>> ts.groupby(pd.Grouper(freq="17min")).sum()
213 2000-10-01 23:14:00 0
214 2000-10-01 23:31:00 9
215 2000-10-01 23:48:00 21
216 2000-10-02 00:05:00 54
217 2000-10-02 00:22:00 24
218 Freq: 17min, dtype: int64
220 >>> ts.groupby(pd.Grouper(freq="17min", origin="epoch")).sum()
221 2000-10-01 23:18:00 0
222 2000-10-01 23:35:00 18
223 2000-10-01 23:52:00 27
224 2000-10-02 00:09:00 39
225 2000-10-02 00:26:00 24
226 Freq: 17min, dtype: int64
228 >>> ts.groupby(pd.Grouper(freq="17min", origin="2000-01-01")).sum()
229 2000-10-01 23:24:00 3
230 2000-10-01 23:41:00 15
231 2000-10-01 23:58:00 45
232 2000-10-02 00:15:00 45
233 Freq: 17min, dtype: int64
235 If you want to adjust the start of the bins with an `offset` Timedelta, the two
236 following lines are equivalent:
238 >>> ts.groupby(pd.Grouper(freq="17min", origin="start")).sum()
239 2000-10-01 23:30:00 9
240 2000-10-01 23:47:00 21
241 2000-10-02 00:04:00 54
242 2000-10-02 00:21:00 24
243 Freq: 17min, dtype: int64
245 >>> ts.groupby(pd.Grouper(freq="17min", offset="23h30min")).sum()
246 2000-10-01 23:30:00 9
247 2000-10-01 23:47:00 21
248 2000-10-02 00:04:00 54
249 2000-10-02 00:21:00 24
250 Freq: 17min, dtype: int64
252 To replace the use of the deprecated `base` argument, you can now use `offset`,
253 in this example it is equivalent to have `base=2`:
255 >>> ts.groupby(pd.Grouper(freq="17min", offset="2min")).sum()
256 2000-10-01 23:16:00 0
257 2000-10-01 23:33:00 9
258 2000-10-01 23:50:00 36
259 2000-10-02 00:07:00 39
260 2000-10-02 00:24:00 24
261 Freq: 17min, dtype: int64
262 """
264 sort: bool
265 dropna: bool
266 _grouper: Index | None
268 _attributes: tuple[str, ...] = ("key", "level", "freq", "sort", "dropna")
270 def __new__(cls, *args, **kwargs):
271 if kwargs.get("freq") is not None:
272 from pandas.core.resample import TimeGrouper
274 cls = TimeGrouper
275 return super().__new__(cls)
277 def __init__(
278 self,
279 key=None,
280 level=None,
281 freq=None,
282 sort: bool = False,
283 dropna: bool = True,
284 ) -> None:
285 self.key = key
286 self.level = level
287 self.freq = freq
288 self.sort = sort
289 self.dropna = dropna
291 self._indexer_deprecated: npt.NDArray[np.intp] | None = None
292 self.binner = None
293 self._grouper = None
294 self._indexer: npt.NDArray[np.intp] | None = None
296 def _get_grouper(
297 self, obj: NDFrameT, validate: bool = True, observed: bool = True
298 ) -> tuple[ops.BaseGrouper, NDFrameT]:
299 """
300 Parameters
301 ----------
302 obj : Series or DataFrame
303 Object being grouped.
304 validate : bool, default True
305 If True, validate the grouper.
306 observed : bool, default True
307 Whether only observed groups should be in the result. Only
308 has an impact when grouping on categorical data.
310 Returns
311 -------
312 A tuple of grouper, obj (possibly sorted)
313 """
314 obj, _, _ = self._set_grouper(obj)
315 grouper, _, obj = get_grouper(
316 obj,
317 [self.key],
318 level=self.level,
319 sort=self.sort,
320 validate=validate,
321 dropna=self.dropna,
322 observed=observed,
323 )
325 return grouper, obj
327 def _set_grouper(
328 self, obj: NDFrameT, sort: bool = False, *, gpr_index: Index | None = None
329 ) -> tuple[NDFrameT, Index, npt.NDArray[np.intp] | None]:
330 """
331 given an object and the specifications, setup the internal grouper
332 for this particular specification
334 Parameters
335 ----------
336 obj : Series or DataFrame
337 sort : bool, default False
338 whether the resulting grouper should be sorted
339 gpr_index : Index or None, default None
341 Returns
342 -------
343 NDFrame
344 Index
345 np.ndarray[np.intp] | None
346 """
347 assert obj is not None
349 if self.key is not None and self.level is not None:
350 raise ValueError("The Grouper cannot specify both a key and a level!")
352 # Keep self._grouper value before overriding
353 if self._grouper is None:
354 # TODO: What are we assuming about subsequent calls?
355 self._grouper = gpr_index
356 self._indexer = self._indexer_deprecated
358 # the key must be a valid info item
359 if self.key is not None:
360 key = self.key
361 # The 'on' is already defined
362 if getattr(gpr_index, "name", None) == key and isinstance(obj, Series):
363 # Sometimes self._grouper will have been resorted while
364 # obj has not. In this case there is a mismatch when we
365 # call self._grouper.take(obj.index) so we need to undo the sorting
366 # before we call _grouper.take.
367 assert self._grouper is not None
368 if self._indexer is not None:
369 reverse_indexer = self._indexer.argsort()
370 unsorted_ax = self._grouper.take(reverse_indexer)
371 ax = unsorted_ax.take(obj.index) # type: ignore[arg-type]
372 else:
373 ax = self._grouper.take(obj.index) # type: ignore[arg-type]
374 else:
375 if key not in obj._info_axis:
376 raise KeyError(f"The grouper name {key} is not found")
377 ax = Index(obj[key], name=key)
379 else:
380 ax = obj.index
381 if self.level is not None:
382 level = self.level
384 # if a level is given it must be a mi level or
385 # equivalent to the axis name
386 if isinstance(ax, MultiIndex):
387 level = ax._get_level_number(level)
388 ax = Index(ax._get_level_values(level), name=ax.names[level])
390 elif level not in (0, ax.name):
391 raise ValueError(f"The level {level} is not valid")
393 # possibly sort
394 indexer: npt.NDArray[np.intp] | None = None
395 if (self.sort or sort) and not ax.is_monotonic_increasing:
396 # use stable sort to support first, last, nth
397 # TODO: why does putting na_position="first" fix datetimelike cases?
398 indexer = self._indexer_deprecated = ax.array.argsort(
399 kind="mergesort", na_position="first"
400 )
401 ax = ax.take(indexer)
402 obj = obj.take(indexer, axis=0)
404 return obj, ax, indexer
406 @final
407 def __repr__(self) -> str:
408 attrs_list = (
409 f"{attr_name}={getattr(self, attr_name)!r}"
410 for attr_name in self._attributes
411 if getattr(self, attr_name) is not None
412 )
413 attrs = ", ".join(attrs_list)
414 cls_name = type(self).__name__
415 return f"{cls_name}({attrs})"
418@final
419class Grouping:
420 """
421 Holds the grouping information for a single key
423 Parameters
424 ----------
425 index : Index
426 grouper :
427 obj : DataFrame or Series
428 name : Label
429 level :
430 observed : bool, default True
431 If we are a Categorical, use the observed values
432 in_axis : if the Grouping is a column in self.obj and hence among
433 Groupby.exclusions list
434 dropna : bool, default True
435 Whether to drop NA groups.
436 uniques : Array-like, optional
437 When specified, will be used for unique values. Enables including empty groups
438 in the result for a BinGrouper. Must not contain duplicates.
440 Attributes
441 -------
442 indices : dict
443 Mapping of {group -> index_list}
444 codes : ndarray
445 Group codes
446 group_index : Index or None
447 unique groups
448 groups : dict
449 Mapping of {group -> label_list}
450 """
452 _codes: npt.NDArray[np.signedinteger] | None = None
453 _orig_cats: Index | None
454 _index: Index
456 def __init__(
457 self,
458 index: Index,
459 grouper=None,
460 obj: NDFrame | None = None,
461 level=None,
462 sort: bool = True,
463 observed: bool = True,
464 in_axis: bool = False,
465 dropna: bool = True,
466 uniques: ArrayLike | None = None,
467 ) -> None:
468 if isinstance(grouper, Series):
469 grouper = grouper.copy(deep=False)
470 self.level = level
471 self._orig_grouper = grouper
472 grouping_vector = _convert_grouper(index, grouper)
473 self._orig_cats = None
474 self._index = index
475 self._sort = sort
476 self.obj = obj
477 self._observed = observed
478 self.in_axis = in_axis
479 self._dropna = dropna
480 self._uniques = uniques
482 # we have a single grouper which may be a myriad of things,
483 # some of which are dependent on the passing in level
485 ilevel = self._ilevel
486 if ilevel is not None:
487 # In extant tests, the new self.grouping_vector matches
488 # `index.get_level_values(ilevel)` whenever
489 # mapper is None and isinstance(index, MultiIndex)
490 if isinstance(index, MultiIndex):
491 index_level = index.get_level_values(ilevel)
492 else:
493 index_level = index
495 if grouping_vector is None:
496 grouping_vector = index_level
497 else:
498 mapper = grouping_vector
499 grouping_vector = index_level.map(mapper)
501 # a passed Grouper like, directly get the grouper in the same way
502 # as single grouper groupby, use the group_info to get codes
503 elif isinstance(grouping_vector, Grouper):
504 # get the new grouper; we already have disambiguated
505 # what key/level refer to exactly, don't need to
506 # check again as we have by this point converted these
507 # to an actual value (rather than a pd.Grouper)
508 assert self.obj is not None # for mypy
509 newgrouper, newobj = grouping_vector._get_grouper(self.obj, validate=False)
510 self.obj = newobj
512 if isinstance(newgrouper, ops.BinGrouper):
513 # TODO: can we unwrap this and get a tighter typing
514 # for self.grouping_vector?
515 grouping_vector = newgrouper
516 else:
517 # ops.BaseGrouper
518 # TODO: 2023-02-03 no test cases with len(newgrouper.groupings) > 1.
519 # If that were to occur, would we be throwing out information?
520 # error: Cannot determine type of "grouping_vector" [has-type]
521 ng = newgrouper.groupings[0].grouping_vector # type: ignore[has-type]
522 # use Index instead of ndarray so we can recover the name
523 grouping_vector = Index(
524 ng, name=newgrouper.result_index.name, copy=False
525 )
527 elif not isinstance(
528 grouping_vector, (Series, Index, ExtensionArray, np.ndarray)
529 ):
530 # no level passed
531 if getattr(grouping_vector, "ndim", 1) != 1:
532 t = str(type(grouping_vector))
533 raise ValueError(f"Grouper for '{t}' not 1-dimensional")
535 grouping_vector = index.map(grouping_vector)
537 if not (
538 hasattr(grouping_vector, "__len__")
539 and len(grouping_vector) == len(index)
540 ):
541 grper = pprint_thing(grouping_vector)
542 errmsg = (
543 f"Grouper result violates len(labels) == len(data)\nresult: {grper}"
544 )
545 raise AssertionError(errmsg)
547 if isinstance(grouping_vector, np.ndarray):
548 if grouping_vector.dtype.kind in "mM":
549 # if we have a date/time-like grouper, make sure that we have
550 # Timestamps like
551 # TODO 2022-10-08 we only have one test that gets here and
552 # values are already in nanoseconds in that case.
553 grouping_vector = Series(grouping_vector).to_numpy()
554 elif isinstance(getattr(grouping_vector, "dtype", None), CategoricalDtype):
555 # a passed Categorical
556 self._orig_cats = grouping_vector.categories
557 grouping_vector = recode_for_groupby(grouping_vector, sort, observed)
559 self.grouping_vector = grouping_vector
561 def __repr__(self) -> str:
562 return f"Grouping({self.name})"
564 def __iter__(self) -> Iterator[Hashable]:
565 return iter(self.indices)
567 @cache_readonly
568 def _passed_categorical(self) -> bool:
569 dtype = getattr(self.grouping_vector, "dtype", None)
570 return isinstance(dtype, CategoricalDtype)
572 @cache_readonly
573 def name(self) -> Hashable:
574 ilevel = self._ilevel
575 if ilevel is not None:
576 return self._index.names[ilevel]
578 if isinstance(self._orig_grouper, (Index, Series)):
579 return self._orig_grouper.name
581 elif isinstance(self.grouping_vector, ops.BaseGrouper):
582 return self.grouping_vector.result_index.name
584 elif isinstance(self.grouping_vector, Index):
585 return self.grouping_vector.name
587 # otherwise we have ndarray or ExtensionArray -> no name
588 return None
590 @cache_readonly
591 def _ilevel(self) -> int | None:
592 """
593 If necessary, converted index level name to index level position.
594 """
595 level = self.level
596 if level is None:
597 return None
598 if not isinstance(level, int):
599 index = self._index
600 if level not in index.names:
601 raise AssertionError(f"Level {level} not in index")
602 if isinstance(index, MultiIndex) and index.names.count(level) > 1:
603 warnings.warn(
604 f"Grouping by index level '{level}' which matches multiple "
605 f"index levels is ambiguous. Currently the first matching "
606 f"level is used. In a future version of pandas, this will "
607 f"raise a ValueError. Use the level number instead.",
608 Pandas4Warning,
609 stacklevel=find_stack_level(),
610 )
611 return index.names.index(level)
612 return level
614 @property
615 def ngroups(self) -> int:
616 return len(self.uniques)
618 @cache_readonly
619 def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
620 # we have a list of groupers
621 if isinstance(self.grouping_vector, ops.BaseGrouper):
622 return self.grouping_vector.indices
624 values = Categorical(self.grouping_vector)
625 return values._reverse_indexer()
627 @property
628 def codes(self) -> npt.NDArray[np.signedinteger]:
629 return self._codes_and_uniques[0]
631 @property
632 def uniques(self) -> ArrayLike:
633 return self._codes_and_uniques[1]
635 @cache_readonly
636 def _codes_and_uniques(self) -> tuple[npt.NDArray[np.signedinteger], ArrayLike]:
637 uniques: ArrayLike
638 if self._passed_categorical:
639 # we make a CategoricalIndex out of the cat grouper
640 # preserving the categories / ordered attributes;
641 # doesn't (yet - GH#46909) handle dropna=False
642 cat = self.grouping_vector
643 categories = cat.categories
645 if self._observed:
646 ucodes = algorithms.unique1d(cat.codes)
647 ucodes = ucodes[ucodes != -1]
648 if self._sort:
649 ucodes = np.sort(ucodes)
650 else:
651 ucodes = np.arange(len(categories))
653 has_dropped_na = False
654 if not self._dropna:
655 na_mask = cat.isna()
656 if np.any(na_mask):
657 has_dropped_na = True
658 if self._sort:
659 # NA goes at the end, gets `largest non-NA code + 1`
660 na_code = len(categories)
661 else:
662 # Insert NA in result based on first appearance, need
663 # the number of unique codes prior
664 na_idx = na_mask.argmax()
665 na_code = algorithms.nunique_ints(cat.codes[:na_idx])
666 ucodes = np.insert(ucodes, na_code, -1)
668 uniques = Categorical.from_codes(
669 codes=ucodes, categories=categories, ordered=cat.ordered, validate=False
670 )
671 codes = cat.codes
673 if has_dropped_na:
674 if not self._sort:
675 # NA code is based on first appearance, increment higher codes
676 codes = np.where(codes >= na_code, codes + 1, codes)
677 codes = np.where(na_mask, na_code, codes)
679 return codes, uniques
681 elif isinstance(self.grouping_vector, ops.BaseGrouper):
682 # we have a list of groupers
683 codes = self.grouping_vector.codes_info
684 # Pass the full Index (not ._values) so DatetimeIndex/TimedeltaIndex
685 # freq survives the trip through _with_infer.
686 uniques = self.grouping_vector.result_index # type: ignore[assignment]
687 elif self._uniques is not None:
688 # GH#50486 Code grouping_vector using _uniques; allows
689 # including uniques that are not present in grouping_vector.
690 cat = Categorical(self.grouping_vector, categories=self._uniques)
691 codes = cat.codes
692 uniques = self._uniques
693 else:
694 result = _factorize_monotonic(self.grouping_vector, self._sort)
695 if result is not None:
696 codes, uniques = result
697 else:
698 # GH35667, replace dropna=False with use_na_sentinel=False
699 # error: Incompatible types in assignment (expression has type
700 # "Union[ndarray[Any, Any], Index]", variable has type
701 # "Categorical")
702 codes, uniques = algorithms.factorize( # type: ignore[assignment]
703 self.grouping_vector,
704 sort=self._sort,
705 use_na_sentinel=self._dropna,
706 )
707 return codes, uniques
709 @cache_readonly
710 def groups(self) -> dict[Hashable, Index]:
711 codes, uniques = self._codes_and_uniques
712 uniques = Index._with_infer(uniques, name=self.name, copy=False) # type: ignore[assignment]
714 r, counts = libalgos.groupsort_indexer(ensure_platform_int(codes), len(uniques))
715 counts = ensure_int64(counts).cumsum()
716 _result = (r[start:end] for start, end in pairwise(counts))
717 # map to the label
718 result = {k: self._index.take(v) for k, v in zip(uniques, _result, strict=True)}
720 return PrettyDict(result)
722 @property
723 def observed_grouping(self) -> Grouping:
724 if self._observed:
725 return self
727 return self._observed_grouping
729 @cache_readonly
730 def _observed_grouping(self) -> Grouping:
731 grouping = Grouping(
732 self._index,
733 self._orig_grouper,
734 obj=self.obj,
735 level=self.level,
736 sort=self._sort,
737 observed=True,
738 in_axis=self.in_axis,
739 dropna=self._dropna,
740 uniques=self._uniques,
741 )
742 return grouping
745def get_grouper(
746 obj: NDFrameT,
747 key=None,
748 level=None,
749 sort: bool = True,
750 observed: bool = True,
751 validate: bool = True,
752 dropna: bool = True,
753) -> tuple[ops.BaseGrouper, frozenset[Hashable], NDFrameT]:
754 """
755 Create and return a BaseGrouper, which is an internal
756 mapping of how to create the grouper indexers.
757 This may be composed of multiple Grouping objects, indicating
758 multiple groupers
760 Groupers are ultimately index mappings. They can originate as:
761 index mappings, keys to columns, functions, or Groupers
763 Groupers enable local references to level,sort, while
764 the passed in level, and sort are 'global'.
766 This routine tries to figure out what the passing in references
767 are and then creates a Grouping for each one, combined into
768 a BaseGrouper.
770 If observed & we have a categorical grouper, only show the observed
771 values.
773 If validate, then check for key/level overlaps.
775 """
776 group_axis = obj.index
778 # validate that the passed single level is compatible with the passed
779 # index of the object
780 if level is not None:
781 # TODO: These if-block and else-block are almost same.
782 # MultiIndex instance check is removable, but it seems that there are
783 # some processes only for non-MultiIndex in else-block,
784 # eg. `obj.index.name != level`. We have to consider carefully whether
785 # these are applicable for MultiIndex. Even if these are applicable,
786 # we need to check if it makes no side effect to subsequent processes
787 # on the outside of this condition.
788 # (GH 17621)
789 if isinstance(group_axis, MultiIndex):
790 if is_list_like(level) and len(level) == 1:
791 level = level[0]
793 if key is None and is_scalar(level):
794 # Get the level values from group_axis
795 key = group_axis.get_level_values(level)
796 level = None
798 else:
799 # allow level to be a length-one list-like object
800 # (e.g., level=[0])
801 # GH 13901
802 if is_list_like(level):
803 nlevels = len(level)
804 if nlevels == 1:
805 level = level[0]
806 elif nlevels == 0:
807 raise ValueError("No group keys passed!")
808 else:
809 raise ValueError("multiple levels only valid with MultiIndex")
811 if isinstance(level, str):
812 if obj.index.name != level:
813 raise ValueError(f"level name {level} is not the name of the index")
814 elif level > 0 or level < -1:
815 raise ValueError("level > 0 or level < -1 only valid with MultiIndex")
817 # NOTE: `group_axis` and `group_axis.get_level_values(level)`
818 # are same in this section.
819 level = None
820 key = group_axis
822 # a passed-in Grouper, directly convert
823 if isinstance(key, Grouper):
824 grouper, obj = key._get_grouper(obj, validate=False, observed=observed)
825 if key.key is None:
826 return grouper, frozenset(), obj
827 else:
828 return grouper, frozenset({key.key}), obj
830 # already have a BaseGrouper, just return it
831 elif isinstance(key, ops.BaseGrouper):
832 return key, frozenset(), obj
834 if not isinstance(key, list):
835 keys = [key]
836 match_axis_length = False
837 else:
838 keys = key
839 match_axis_length = len(keys) == len(group_axis)
841 # what are we after, exactly?
842 any_callable = any(callable(g) or isinstance(g, dict) for g in keys)
843 any_groupers = any(isinstance(g, (Grouper, Grouping)) for g in keys)
844 any_arraylike = any(
845 isinstance(g, (list, tuple, Series, Index, np.ndarray)) for g in keys
846 )
848 # is this an index replacement?
849 if (
850 not any_callable
851 and not any_arraylike
852 and not any_groupers
853 and match_axis_length
854 and level is None
855 ):
856 if isinstance(obj, DataFrame):
857 all_in_columns_index = all(
858 g in obj.columns or g in obj.index.names for g in keys
859 )
860 else:
861 assert isinstance(obj, Series)
862 all_in_columns_index = all(g in obj.index.names for g in keys)
864 if not all_in_columns_index:
865 keys = [com.asarray_tuplesafe(keys)]
867 if isinstance(level, (tuple, list)):
868 if key is None:
869 keys = [None] * len(level)
870 levels = level
871 else:
872 levels = [level] * len(keys)
874 groupings: list[Grouping] = []
875 exclusions: set[Hashable] = set()
877 # if the actual grouper should be obj[key]
878 def is_in_axis(key) -> bool:
879 if not _is_label_like(key):
880 if obj.ndim == 1:
881 return False
883 # items -> .columns for DataFrame, .index for Series
884 items = obj.axes[-1]
885 try:
886 items.get_loc(key)
887 except (KeyError, TypeError, InvalidIndexError):
888 # TypeError shows up here if we pass e.g. an Index
889 return False
891 return True
893 # if the grouper is obj[name]
894 def is_in_obj(gpr) -> bool:
895 if not hasattr(gpr, "name"):
896 return False
897 # We check the references to determine if the
898 # series is part of the object
899 try:
900 obj_gpr_column = obj[gpr.name]
901 except (KeyError, IndexError, InvalidIndexError, OutOfBoundsDatetime):
902 return False
903 if isinstance(gpr, Series) and isinstance(obj_gpr_column, Series):
904 return gpr._mgr.references_same_values(obj_gpr_column._mgr, 0)
905 return False
907 for gpr, level in zip(keys, levels, strict=True):
908 if is_in_obj(gpr): # df.groupby(df['name'])
909 in_axis = True
910 exclusions.add(gpr.name)
912 elif is_in_axis(gpr): # df.groupby('name')
913 if obj.ndim != 1 and gpr in obj:
914 if validate:
915 obj._check_label_or_level_ambiguity(gpr, axis=0)
916 in_axis, name, gpr = True, gpr, obj[gpr]
917 if gpr.ndim != 1:
918 # non-unique columns; raise here to get the name in the
919 # exception message
920 raise ValueError(f"Grouper for '{name}' not 1-dimensional")
921 exclusions.add(name)
922 elif obj._is_level_reference(gpr, axis=0):
923 in_axis, level, gpr = False, gpr, None
924 else:
925 raise KeyError(gpr)
926 elif isinstance(gpr, Grouper) and gpr.key is not None:
927 # Add key to exclusions
928 exclusions.add(gpr.key)
929 in_axis = True
930 else:
931 in_axis = False
933 # create the Grouping
934 # allow us to passing the actual Grouping as the gpr
935 ping = (
936 Grouping(
937 group_axis,
938 gpr,
939 obj=obj,
940 level=level,
941 sort=sort,
942 observed=observed,
943 in_axis=in_axis,
944 dropna=dropna,
945 )
946 if not isinstance(gpr, Grouping)
947 else gpr
948 )
950 groupings.append(ping)
952 if len(groupings) == 0 and len(obj):
953 raise ValueError("No group keys passed!")
954 if len(groupings) == 0:
955 groupings.append(Grouping(default_index(0), np.array([], dtype=np.intp)))
957 # create the internals grouper
958 grouper = ops.BaseGrouper(group_axis, groupings, sort=sort, dropna=dropna)
959 return grouper, frozenset(exclusions), obj
962def _is_label_like(val) -> bool:
963 return isinstance(val, (str, tuple)) or (val is not None and is_scalar(val))
966def _factorize_monotonic(
967 grouping_vector,
968 sort: bool,
969) -> tuple[npt.NDArray[np.intp], np.ndarray] | None:
970 """
971 Fast-path factorization for monotonic (sorted) grouping vectors.
973 Uses adjacent-element comparison instead of hash table construction.
974 Returns (codes, uniques) or None if the fast path is not applicable.
976 Since monotonic arrays contain no NA values (NAs break monotonicity
977 checks for n >= 2), NA handling is not needed here.
978 """
979 if isinstance(grouping_vector, (Series, Index)):
980 # Bail before np.asarray for extension dtypes (PeriodDtype,
981 # DatetimeTZDtype, etc.) — converting them would box every element.
982 dtype = grouping_vector.dtype
983 if not isinstance(dtype, np.dtype) or dtype.kind not in "iufmMb":
984 return None
985 if isinstance(grouping_vector, Index) and dtype.kind in "mM":
986 # DatetimeIndex/TimedeltaIndex.factorize has its own fastpaths
987 # (freq-based and monotonic) that retain freq and Index uniques
988 return None
989 ascending = grouping_vector.is_monotonic_increasing
990 if not ascending and not grouping_vector.is_monotonic_decreasing:
991 return None
992 arr = np.asarray(grouping_vector)
993 elif isinstance(grouping_vector, np.ndarray):
994 arr = grouping_vector
995 if arr.dtype.kind not in "iufmMb":
996 return None
997 if len(arr) <= 1:
998 return None
999 # Quick sample check: compare a few spaced elements to avoid
1000 # a full O(n) scan on clearly unsorted data.
1001 sample_idx = np.linspace(0, len(arr) - 1, num=min(8, len(arr)), dtype=np.intp)
1002 sample = arr[sample_idx]
1003 if not bool(np.all(sample[1:] >= sample[:-1])):
1004 if not bool(np.all(sample[1:] <= sample[:-1])):
1005 return None
1006 ascending = bool(np.all(arr[1:] >= arr[:-1]))
1007 if not ascending:
1008 if not bool(np.all(arr[1:] <= arr[:-1])):
1009 return None
1010 else:
1011 return None
1013 if len(arr) <= 1:
1014 return None
1016 codes, uniques_indexer = algorithms.factorize_monotonic_codes(arr, ascending, sort)
1017 return codes, arr[uniques_indexer]
1020def _convert_grouper(axis: Index, grouper):
1021 if isinstance(grouper, dict):
1022 return grouper.get
1023 elif isinstance(grouper, Series):
1024 if grouper.index.equals(axis):
1025 return grouper._values
1026 else:
1027 return grouper.reindex(axis)._values
1028 elif isinstance(grouper, MultiIndex):
1029 return grouper._values
1030 elif isinstance(grouper, (list, tuple, Index, Categorical, np.ndarray)):
1031 if len(grouper) != len(axis):
1032 raise ValueError("Grouper and axis must be same length")
1034 if isinstance(grouper, (list, tuple)):
1035 grouper = com.asarray_tuplesafe(grouper)
1036 return grouper
1037 else:
1038 return grouper