Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Provide user facing operators for doing the split part of the
3split-apply-combine paradigm.
4"""
6from __future__ import annotations
8from typing import (
9 TYPE_CHECKING,
10 final,
11)
13import numpy as np
15from pandas._libs import (
16 algos as libalgos,
17)
18from pandas._libs.tslibs import OutOfBoundsDatetime
19from pandas.errors import InvalidIndexError
20from pandas.util._decorators import (
21 cache_readonly,
22 set_module,
23)
25from pandas.core.dtypes.common import (
26 ensure_int64,
27 ensure_platform_int,
28 is_list_like,
29 is_scalar,
30)
31from pandas.core.dtypes.dtypes import CategoricalDtype
33from pandas.core import algorithms
34from pandas.core.arrays import (
35 Categorical,
36 ExtensionArray,
37)
38import pandas.core.common as com
39from pandas.core.frame import DataFrame
40from pandas.core.groupby import ops
41from pandas.core.groupby.categorical import recode_for_groupby
42from pandas.core.indexes.api import (
43 Index,
44 MultiIndex,
45 default_index,
46)
47from pandas.core.series import Series
49from pandas.io.formats.printing import (
50 PrettyDict,
51 pprint_thing,
52)
54if TYPE_CHECKING:
55 from collections.abc import (
56 Hashable,
57 Iterator,
58 )
60 from pandas._typing import (
61 ArrayLike,
62 NDFrameT,
63 npt,
64 )
66 from pandas.core.generic import NDFrame
69@set_module("pandas")
70class Grouper:
71 """
72 A Grouper allows the user to specify a groupby instruction for an object.
74 This specification will select a column via the key parameter, or if the
75 level parameter is given, a level of the index of the target
76 object.
78 If ``level`` is passed as a keyword to both `Grouper` and
79 `groupby`, the values passed to `Grouper` take precedence.
81 Parameters
82 ----------
83 *args
84 Currently unused, reserved for future use.
85 **kwargs
86 Dictionary of the keyword arguments to pass to Grouper.
88 Attributes
89 ----------
90 key : str, defaults to None
91 Groupby key, which selects the grouping column of the target.
92 level : name/number, defaults to None
93 The level for the target index.
94 freq : str / frequency object, defaults to None
95 This will groupby the specified frequency if the target selection
96 (via key or level) is a datetime-like object. For full specification
97 of available frequencies, please see :ref:`here<timeseries.offset_aliases>`.
98 sort : bool, default to False
99 Whether to sort the resulting labels.
100 closed : {'left' or 'right'}
101 Closed end of interval. Only when `freq` parameter is passed.
102 label : {'left' or 'right'}
103 Interval boundary to use for labeling.
104 Only when `freq` parameter is passed.
105 convention : {'start', 'end', 'e', 's'}
106 If grouper is PeriodIndex and `freq` parameter is passed.
108 origin : Timestamp or str, default 'start_day'
109 The timestamp on which to adjust the grouping. The timezone of origin must
110 match the timezone of the index.
111 If string, must be one of the following:
113 - 'epoch': `origin` is 1970-01-01
114 - 'start': `origin` is the first value of the timeseries
115 - 'start_day': `origin` is the first day at midnight of the timeseries
117 - 'end': `origin` is the last value of the timeseries
118 - 'end_day': `origin` is the ceiling midnight of the last day
120 offset : Timedelta or str, default is None
121 An offset timedelta added to the origin.
123 dropna : bool, default True
124 If True, and if group keys contain NA values, NA values together with
125 row/column will be dropped. If False, NA values will also be treated as
126 the key in groups.
128 Returns
129 -------
130 Grouper or pandas.api.typing.TimeGrouper
131 A TimeGrouper is returned if ``freq`` is not ``None``. Otherwise, a Grouper
132 is returned.
134 See Also
135 --------
136 Series.groupby : Apply a function groupby to a Series.
137 DataFrame.groupby : Apply a function groupby.
139 Examples
140 --------
141 ``df.groupby(pd.Grouper(key="Animal"))`` is equivalent to ``df.groupby('Animal')``
143 >>> df = pd.DataFrame(
144 ... {
145 ... "Animal": ["Falcon", "Parrot", "Falcon", "Falcon", "Parrot"],
146 ... "Speed": [100, 5, 200, 300, 15],
147 ... }
148 ... )
149 >>> df
150 Animal Speed
151 0 Falcon 100
152 1 Parrot 5
153 2 Falcon 200
154 3 Falcon 300
155 4 Parrot 15
156 >>> df.groupby(pd.Grouper(key="Animal")).mean()
157 Speed
158 Animal
159 Falcon 200.0
160 Parrot 10.0
162 Specify a resample operation on the column 'Publish date'
164 >>> df = pd.DataFrame(
165 ... {
166 ... "Publish date": [
167 ... pd.Timestamp("2000-01-02"),
168 ... pd.Timestamp("2000-01-02"),
169 ... pd.Timestamp("2000-01-09"),
170 ... pd.Timestamp("2000-01-16"),
171 ... ],
172 ... "ID": [0, 1, 2, 3],
173 ... "Price": [10, 20, 30, 40],
174 ... }
175 ... )
176 >>> df
177 Publish date ID Price
178 0 2000-01-02 0 10
179 1 2000-01-02 1 20
180 2 2000-01-09 2 30
181 3 2000-01-16 3 40
182 >>> df.groupby(pd.Grouper(key="Publish date", freq="1W")).mean()
183 ID Price
184 Publish date
185 2000-01-02 0.5 15.0
186 2000-01-09 2.0 30.0
187 2000-01-16 3.0 40.0
189 If you want to adjust the start of the bins based on a fixed timestamp:
191 >>> start, end = "2000-10-01 23:30:00", "2000-10-02 00:30:00"
192 >>> rng = pd.date_range(start, end, freq="7min")
193 >>> ts = pd.Series(np.arange(len(rng)) * 3, index=rng)
194 >>> ts
195 2000-10-01 23:30:00 0
196 2000-10-01 23:37:00 3
197 2000-10-01 23:44:00 6
198 2000-10-01 23:51:00 9
199 2000-10-01 23:58:00 12
200 2000-10-02 00:05:00 15
201 2000-10-02 00:12:00 18
202 2000-10-02 00:19:00 21
203 2000-10-02 00:26:00 24
204 Freq: 7min, dtype: int64
206 >>> ts.groupby(pd.Grouper(freq="17min")).sum()
207 2000-10-01 23:14:00 0
208 2000-10-01 23:31:00 9
209 2000-10-01 23:48:00 21
210 2000-10-02 00:05:00 54
211 2000-10-02 00:22:00 24
212 Freq: 17min, dtype: int64
214 >>> ts.groupby(pd.Grouper(freq="17min", origin="epoch")).sum()
215 2000-10-01 23:18:00 0
216 2000-10-01 23:35:00 18
217 2000-10-01 23:52:00 27
218 2000-10-02 00:09:00 39
219 2000-10-02 00:26:00 24
220 Freq: 17min, dtype: int64
222 >>> ts.groupby(pd.Grouper(freq="17min", origin="2000-01-01")).sum()
223 2000-10-01 23:24:00 3
224 2000-10-01 23:41:00 15
225 2000-10-01 23:58:00 45
226 2000-10-02 00:15:00 45
227 Freq: 17min, dtype: int64
229 If you want to adjust the start of the bins with an `offset` Timedelta, the two
230 following lines are equivalent:
232 >>> ts.groupby(pd.Grouper(freq="17min", origin="start")).sum()
233 2000-10-01 23:30:00 9
234 2000-10-01 23:47:00 21
235 2000-10-02 00:04:00 54
236 2000-10-02 00:21:00 24
237 Freq: 17min, dtype: int64
239 >>> ts.groupby(pd.Grouper(freq="17min", offset="23h30min")).sum()
240 2000-10-01 23:30:00 9
241 2000-10-01 23:47:00 21
242 2000-10-02 00:04:00 54
243 2000-10-02 00:21:00 24
244 Freq: 17min, dtype: int64
246 To replace the use of the deprecated `base` argument, you can now use `offset`,
247 in this example it is equivalent to have `base=2`:
249 >>> ts.groupby(pd.Grouper(freq="17min", offset="2min")).sum()
250 2000-10-01 23:16:00 0
251 2000-10-01 23:33:00 9
252 2000-10-01 23:50:00 36
253 2000-10-02 00:07:00 39
254 2000-10-02 00:24:00 24
255 Freq: 17min, dtype: int64
256 """
258 sort: bool
259 dropna: bool
260 _grouper: Index | None
262 _attributes: tuple[str, ...] = ("key", "level", "freq", "sort", "dropna")
264 def __new__(cls, *args, **kwargs):
265 if kwargs.get("freq") is not None:
266 from pandas.core.resample import TimeGrouper
268 cls = TimeGrouper
269 return super().__new__(cls)
271 def __init__(
272 self,
273 key=None,
274 level=None,
275 freq=None,
276 sort: bool = False,
277 dropna: bool = True,
278 ) -> None:
279 self.key = key
280 self.level = level
281 self.freq = freq
282 self.sort = sort
283 self.dropna = dropna
285 self._indexer_deprecated: npt.NDArray[np.intp] | None = None
286 self.binner = None
287 self._grouper = None
288 self._indexer: npt.NDArray[np.intp] | None = None
290 def _get_grouper(
291 self, obj: NDFrameT, validate: bool = True, observed: bool = True
292 ) -> tuple[ops.BaseGrouper, NDFrameT]:
293 """
294 Parameters
295 ----------
296 obj : Series or DataFrame
297 Object being grouped.
298 validate : bool, default True
299 If True, validate the grouper.
300 observed : bool, default True
301 Whether only observed groups should be in the result. Only
302 has an impact when grouping on categorical data.
304 Returns
305 -------
306 A tuple of grouper, obj (possibly sorted)
307 """
308 obj, _, _ = self._set_grouper(obj)
309 grouper, _, obj = get_grouper(
310 obj,
311 [self.key],
312 level=self.level,
313 sort=self.sort,
314 validate=validate,
315 dropna=self.dropna,
316 observed=observed,
317 )
319 return grouper, obj
321 def _set_grouper(
322 self, obj: NDFrameT, sort: bool = False, *, gpr_index: Index | None = None
323 ) -> tuple[NDFrameT, Index, npt.NDArray[np.intp] | None]:
324 """
325 given an object and the specifications, setup the internal grouper
326 for this particular specification
328 Parameters
329 ----------
330 obj : Series or DataFrame
331 sort : bool, default False
332 whether the resulting grouper should be sorted
333 gpr_index : Index or None, default None
335 Returns
336 -------
337 NDFrame
338 Index
339 np.ndarray[np.intp] | None
340 """
341 assert obj is not None
343 if self.key is not None and self.level is not None:
344 raise ValueError("The Grouper cannot specify both a key and a level!")
346 # Keep self._grouper value before overriding
347 if self._grouper is None:
348 # TODO: What are we assuming about subsequent calls?
349 self._grouper = gpr_index
350 self._indexer = self._indexer_deprecated
352 # the key must be a valid info item
353 if self.key is not None:
354 key = self.key
355 # The 'on' is already defined
356 if getattr(gpr_index, "name", None) == key and isinstance(obj, Series):
357 # Sometimes self._grouper will have been resorted while
358 # obj has not. In this case there is a mismatch when we
359 # call self._grouper.take(obj.index) so we need to undo the sorting
360 # before we call _grouper.take.
361 assert self._grouper is not None
362 if self._indexer is not None:
363 reverse_indexer = self._indexer.argsort()
364 unsorted_ax = self._grouper.take(reverse_indexer)
365 ax = unsorted_ax.take(obj.index)
366 else:
367 ax = self._grouper.take(obj.index)
368 else:
369 if key not in obj._info_axis:
370 raise KeyError(f"The grouper name {key} is not found")
371 ax = Index(obj[key], name=key)
373 else:
374 ax = obj.index
375 if self.level is not None:
376 level = self.level
378 # if a level is given it must be a mi level or
379 # equivalent to the axis name
380 if isinstance(ax, MultiIndex):
381 level = ax._get_level_number(level)
382 ax = Index(ax._get_level_values(level), name=ax.names[level])
384 elif level not in (0, ax.name):
385 raise ValueError(f"The level {level} is not valid")
387 # possibly sort
388 indexer: npt.NDArray[np.intp] | None = None
389 if (self.sort or sort) and not ax.is_monotonic_increasing:
390 # use stable sort to support first, last, nth
391 # TODO: why does putting na_position="first" fix datetimelike cases?
392 indexer = self._indexer_deprecated = ax.array.argsort(
393 kind="mergesort", na_position="first"
394 )
395 ax = ax.take(indexer)
396 obj = obj.take(indexer, axis=0)
398 return obj, ax, indexer
400 @final
401 def __repr__(self) -> str:
402 attrs_list = (
403 f"{attr_name}={getattr(self, attr_name)!r}"
404 for attr_name in self._attributes
405 if getattr(self, attr_name) is not None
406 )
407 attrs = ", ".join(attrs_list)
408 cls_name = type(self).__name__
409 return f"{cls_name}({attrs})"
412@final
413class Grouping:
414 """
415 Holds the grouping information for a single key
417 Parameters
418 ----------
419 index : Index
420 grouper :
421 obj : DataFrame or Series
422 name : Label
423 level :
424 observed : bool, default False
425 If we are a Categorical, use the observed values
426 in_axis : if the Grouping is a column in self.obj and hence among
427 Groupby.exclusions list
428 dropna : bool, default True
429 Whether to drop NA groups.
430 uniques : Array-like, optional
431 When specified, will be used for unique values. Enables including empty groups
432 in the result for a BinGrouper. Must not contain duplicates.
434 Attributes
435 -------
436 indices : dict
437 Mapping of {group -> index_list}
438 codes : ndarray
439 Group codes
440 group_index : Index or None
441 unique groups
442 groups : dict
443 Mapping of {group -> label_list}
444 """
446 _codes: npt.NDArray[np.signedinteger] | None = None
447 _orig_cats: Index | None
448 _index: Index
450 def __init__(
451 self,
452 index: Index,
453 grouper=None,
454 obj: NDFrame | None = None,
455 level=None,
456 sort: bool = True,
457 observed: bool = False,
458 in_axis: bool = False,
459 dropna: bool = True,
460 uniques: ArrayLike | None = None,
461 ) -> None:
462 if isinstance(grouper, Series):
463 grouper = grouper.copy(deep=False)
464 self.level = level
465 self._orig_grouper = grouper
466 grouping_vector = _convert_grouper(index, grouper)
467 self._orig_cats = None
468 self._index = index
469 self._sort = sort
470 self.obj = obj
471 self._observed = observed
472 self.in_axis = in_axis
473 self._dropna = dropna
474 self._uniques = uniques
476 # we have a single grouper which may be a myriad of things,
477 # some of which are dependent on the passing in level
479 ilevel = self._ilevel
480 if ilevel is not None:
481 # In extant tests, the new self.grouping_vector matches
482 # `index.get_level_values(ilevel)` whenever
483 # mapper is None and isinstance(index, MultiIndex)
484 if isinstance(index, MultiIndex):
485 index_level = index.get_level_values(ilevel)
486 else:
487 index_level = index
489 if grouping_vector is None:
490 grouping_vector = index_level
491 else:
492 mapper = grouping_vector
493 grouping_vector = index_level.map(mapper)
495 # a passed Grouper like, directly get the grouper in the same way
496 # as single grouper groupby, use the group_info to get codes
497 elif isinstance(grouping_vector, Grouper):
498 # get the new grouper; we already have disambiguated
499 # what key/level refer to exactly, don't need to
500 # check again as we have by this point converted these
501 # to an actual value (rather than a pd.Grouper)
502 assert self.obj is not None # for mypy
503 newgrouper, newobj = grouping_vector._get_grouper(self.obj, validate=False)
504 self.obj = newobj
506 if isinstance(newgrouper, ops.BinGrouper):
507 # TODO: can we unwrap this and get a tighter typing
508 # for self.grouping_vector?
509 grouping_vector = newgrouper
510 else:
511 # ops.BaseGrouper
512 # TODO: 2023-02-03 no test cases with len(newgrouper.groupings) > 1.
513 # If that were to occur, would we be throwing out information?
514 # error: Cannot determine type of "grouping_vector" [has-type]
515 ng = newgrouper.groupings[0].grouping_vector # type: ignore[has-type]
516 # use Index instead of ndarray so we can recover the name
517 grouping_vector = Index(
518 ng, name=newgrouper.result_index.name, copy=False
519 )
521 elif not isinstance(
522 grouping_vector, (Series, Index, ExtensionArray, np.ndarray)
523 ):
524 # no level passed
525 if getattr(grouping_vector, "ndim", 1) != 1:
526 t = str(type(grouping_vector))
527 raise ValueError(f"Grouper for '{t}' not 1-dimensional")
529 grouping_vector = index.map(grouping_vector)
531 if not (
532 hasattr(grouping_vector, "__len__")
533 and len(grouping_vector) == len(index)
534 ):
535 grper = pprint_thing(grouping_vector)
536 errmsg = (
537 f"Grouper result violates len(labels) == len(data)\nresult: {grper}"
538 )
539 raise AssertionError(errmsg)
541 if isinstance(grouping_vector, np.ndarray):
542 if grouping_vector.dtype.kind in "mM":
543 # if we have a date/time-like grouper, make sure that we have
544 # Timestamps like
545 # TODO 2022-10-08 we only have one test that gets here and
546 # values are already in nanoseconds in that case.
547 grouping_vector = Series(grouping_vector).to_numpy()
548 elif isinstance(getattr(grouping_vector, "dtype", None), CategoricalDtype):
549 # a passed Categorical
550 self._orig_cats = grouping_vector.categories
551 grouping_vector = recode_for_groupby(grouping_vector, sort, observed)
553 self.grouping_vector = grouping_vector
555 def __repr__(self) -> str:
556 return f"Grouping({self.name})"
558 def __iter__(self) -> Iterator:
559 return iter(self.indices)
561 @cache_readonly
562 def _passed_categorical(self) -> bool:
563 dtype = getattr(self.grouping_vector, "dtype", None)
564 return isinstance(dtype, CategoricalDtype)
566 @cache_readonly
567 def name(self) -> Hashable:
568 ilevel = self._ilevel
569 if ilevel is not None:
570 return self._index.names[ilevel]
572 if isinstance(self._orig_grouper, (Index, Series)):
573 return self._orig_grouper.name
575 elif isinstance(self.grouping_vector, ops.BaseGrouper):
576 return self.grouping_vector.result_index.name
578 elif isinstance(self.grouping_vector, Index):
579 return self.grouping_vector.name
581 # otherwise we have ndarray or ExtensionArray -> no name
582 return None
584 @cache_readonly
585 def _ilevel(self) -> int | None:
586 """
587 If necessary, converted index level name to index level position.
588 """
589 level = self.level
590 if level is None:
591 return None
592 if not isinstance(level, int):
593 index = self._index
594 if level not in index.names:
595 raise AssertionError(f"Level {level} not in index")
596 return index.names.index(level)
597 return level
599 @property
600 def ngroups(self) -> int:
601 return len(self.uniques)
603 @cache_readonly
604 def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
605 # we have a list of groupers
606 if isinstance(self.grouping_vector, ops.BaseGrouper):
607 return self.grouping_vector.indices
609 values = Categorical(self.grouping_vector)
610 return values._reverse_indexer()
612 @property
613 def codes(self) -> npt.NDArray[np.signedinteger]:
614 return self._codes_and_uniques[0]
616 @property
617 def uniques(self) -> ArrayLike:
618 return self._codes_and_uniques[1]
620 @cache_readonly
621 def _codes_and_uniques(self) -> tuple[npt.NDArray[np.signedinteger], ArrayLike]:
622 uniques: ArrayLike
623 if self._passed_categorical:
624 # we make a CategoricalIndex out of the cat grouper
625 # preserving the categories / ordered attributes;
626 # doesn't (yet - GH#46909) handle dropna=False
627 cat = self.grouping_vector
628 categories = cat.categories
630 if self._observed:
631 ucodes = algorithms.unique1d(cat.codes)
632 ucodes = ucodes[ucodes != -1]
633 if self._sort:
634 ucodes = np.sort(ucodes)
635 else:
636 ucodes = np.arange(len(categories))
638 has_dropped_na = False
639 if not self._dropna:
640 na_mask = cat.isna()
641 if np.any(na_mask):
642 has_dropped_na = True
643 if self._sort:
644 # NA goes at the end, gets `largest non-NA code + 1`
645 na_code = len(categories)
646 else:
647 # Insert NA in result based on first appearance, need
648 # the number of unique codes prior
649 na_idx = na_mask.argmax()
650 na_code = algorithms.nunique_ints(cat.codes[:na_idx])
651 ucodes = np.insert(ucodes, na_code, -1)
653 uniques = Categorical.from_codes(
654 codes=ucodes, categories=categories, ordered=cat.ordered, validate=False
655 )
656 codes = cat.codes
658 if has_dropped_na:
659 if not self._sort:
660 # NA code is based on first appearance, increment higher codes
661 codes = np.where(codes >= na_code, codes + 1, codes)
662 codes = np.where(na_mask, na_code, codes)
664 return codes, uniques
666 elif isinstance(self.grouping_vector, ops.BaseGrouper):
667 # we have a list of groupers
668 codes = self.grouping_vector.codes_info
669 uniques = self.grouping_vector.result_index._values
670 elif self._uniques is not None:
671 # GH#50486 Code grouping_vector using _uniques; allows
672 # including uniques that are not present in grouping_vector.
673 cat = Categorical(self.grouping_vector, categories=self._uniques)
674 codes = cat.codes
675 uniques = self._uniques
676 else:
677 # GH35667, replace dropna=False with use_na_sentinel=False
678 # error: Incompatible types in assignment (expression has type "Union[
679 # ndarray[Any, Any], Index]", variable has type "Categorical")
680 codes, uniques = algorithms.factorize( # type: ignore[assignment]
681 self.grouping_vector, sort=self._sort, use_na_sentinel=self._dropna
682 )
683 return codes, uniques
685 @cache_readonly
686 def groups(self) -> dict[Hashable, Index]:
687 codes, uniques = self._codes_and_uniques
688 uniques = Index._with_infer(uniques, name=self.name, copy=False)
690 r, counts = libalgos.groupsort_indexer(ensure_platform_int(codes), len(uniques))
691 counts = ensure_int64(counts).cumsum()
692 _result = (r[start:end] for start, end in zip(counts, counts[1:], strict=False))
693 # map to the label
694 result = {k: self._index.take(v) for k, v in zip(uniques, _result, strict=True)}
696 return PrettyDict(result)
698 @property
699 def observed_grouping(self) -> Grouping:
700 if self._observed:
701 return self
703 return self._observed_grouping
705 @cache_readonly
706 def _observed_grouping(self) -> Grouping:
707 grouping = Grouping(
708 self._index,
709 self._orig_grouper,
710 obj=self.obj,
711 level=self.level,
712 sort=self._sort,
713 observed=True,
714 in_axis=self.in_axis,
715 dropna=self._dropna,
716 uniques=self._uniques,
717 )
718 return grouping
721def get_grouper(
722 obj: NDFrameT,
723 key=None,
724 level=None,
725 sort: bool = True,
726 observed: bool = False,
727 validate: bool = True,
728 dropna: bool = True,
729) -> tuple[ops.BaseGrouper, frozenset[Hashable], NDFrameT]:
730 """
731 Create and return a BaseGrouper, which is an internal
732 mapping of how to create the grouper indexers.
733 This may be composed of multiple Grouping objects, indicating
734 multiple groupers
736 Groupers are ultimately index mappings. They can originate as:
737 index mappings, keys to columns, functions, or Groupers
739 Groupers enable local references to level,sort, while
740 the passed in level, and sort are 'global'.
742 This routine tries to figure out what the passing in references
743 are and then creates a Grouping for each one, combined into
744 a BaseGrouper.
746 If observed & we have a categorical grouper, only show the observed
747 values.
749 If validate, then check for key/level overlaps.
751 """
752 group_axis = obj.index
754 # validate that the passed single level is compatible with the passed
755 # index of the object
756 if level is not None:
757 # TODO: These if-block and else-block are almost same.
758 # MultiIndex instance check is removable, but it seems that there are
759 # some processes only for non-MultiIndex in else-block,
760 # eg. `obj.index.name != level`. We have to consider carefully whether
761 # these are applicable for MultiIndex. Even if these are applicable,
762 # we need to check if it makes no side effect to subsequent processes
763 # on the outside of this condition.
764 # (GH 17621)
765 if isinstance(group_axis, MultiIndex):
766 if is_list_like(level) and len(level) == 1:
767 level = level[0]
769 if key is None and is_scalar(level):
770 # Get the level values from group_axis
771 key = group_axis.get_level_values(level)
772 level = None
774 else:
775 # allow level to be a length-one list-like object
776 # (e.g., level=[0])
777 # GH 13901
778 if is_list_like(level):
779 nlevels = len(level)
780 if nlevels == 1:
781 level = level[0]
782 elif nlevels == 0:
783 raise ValueError("No group keys passed!")
784 else:
785 raise ValueError("multiple levels only valid with MultiIndex")
787 if isinstance(level, str):
788 if obj.index.name != level:
789 raise ValueError(f"level name {level} is not the name of the index")
790 elif level > 0 or level < -1:
791 raise ValueError("level > 0 or level < -1 only valid with MultiIndex")
793 # NOTE: `group_axis` and `group_axis.get_level_values(level)`
794 # are same in this section.
795 level = None
796 key = group_axis
798 # a passed-in Grouper, directly convert
799 if isinstance(key, Grouper):
800 grouper, obj = key._get_grouper(obj, validate=False, observed=observed)
801 if key.key is None:
802 return grouper, frozenset(), obj
803 else:
804 return grouper, frozenset({key.key}), obj
806 # already have a BaseGrouper, just return it
807 elif isinstance(key, ops.BaseGrouper):
808 return key, frozenset(), obj
810 if not isinstance(key, list):
811 keys = [key]
812 match_axis_length = False
813 else:
814 keys = key
815 match_axis_length = len(keys) == len(group_axis)
817 # what are we after, exactly?
818 any_callable = any(callable(g) or isinstance(g, dict) for g in keys)
819 any_groupers = any(isinstance(g, (Grouper, Grouping)) for g in keys)
820 any_arraylike = any(
821 isinstance(g, (list, tuple, Series, Index, np.ndarray)) for g in keys
822 )
824 # is this an index replacement?
825 if (
826 not any_callable
827 and not any_arraylike
828 and not any_groupers
829 and match_axis_length
830 and level is None
831 ):
832 if isinstance(obj, DataFrame):
833 all_in_columns_index = all(
834 g in obj.columns or g in obj.index.names for g in keys
835 )
836 else:
837 assert isinstance(obj, Series)
838 all_in_columns_index = all(g in obj.index.names for g in keys)
840 if not all_in_columns_index:
841 keys = [com.asarray_tuplesafe(keys)]
843 if isinstance(level, (tuple, list)):
844 if key is None:
845 keys = [None] * len(level)
846 levels = level
847 else:
848 levels = [level] * len(keys)
850 groupings: list[Grouping] = []
851 exclusions: set[Hashable] = set()
853 # if the actual grouper should be obj[key]
854 def is_in_axis(key) -> bool:
855 if not _is_label_like(key):
856 if obj.ndim == 1:
857 return False
859 # items -> .columns for DataFrame, .index for Series
860 items = obj.axes[-1]
861 try:
862 items.get_loc(key)
863 except (KeyError, TypeError, InvalidIndexError):
864 # TypeError shows up here if we pass e.g. an Index
865 return False
867 return True
869 # if the grouper is obj[name]
870 def is_in_obj(gpr) -> bool:
871 if not hasattr(gpr, "name"):
872 return False
873 # We check the references to determine if the
874 # series is part of the object
875 try:
876 obj_gpr_column = obj[gpr.name]
877 except (KeyError, IndexError, InvalidIndexError, OutOfBoundsDatetime):
878 return False
879 if isinstance(gpr, Series) and isinstance(obj_gpr_column, Series):
880 return gpr._mgr.references_same_values(obj_gpr_column._mgr, 0)
881 return False
883 for gpr, level in zip(keys, levels, strict=True):
884 if is_in_obj(gpr): # df.groupby(df['name'])
885 in_axis = True
886 exclusions.add(gpr.name)
888 elif is_in_axis(gpr): # df.groupby('name')
889 if obj.ndim != 1 and gpr in obj:
890 if validate:
891 obj._check_label_or_level_ambiguity(gpr, axis=0)
892 in_axis, name, gpr = True, gpr, obj[gpr]
893 if gpr.ndim != 1:
894 # non-unique columns; raise here to get the name in the
895 # exception message
896 raise ValueError(f"Grouper for '{name}' not 1-dimensional")
897 exclusions.add(name)
898 elif obj._is_level_reference(gpr, axis=0):
899 in_axis, level, gpr = False, gpr, None
900 else:
901 raise KeyError(gpr)
902 elif isinstance(gpr, Grouper) and gpr.key is not None:
903 # Add key to exclusions
904 exclusions.add(gpr.key)
905 in_axis = True
906 else:
907 in_axis = False
909 # create the Grouping
910 # allow us to passing the actual Grouping as the gpr
911 ping = (
912 Grouping(
913 group_axis,
914 gpr,
915 obj=obj,
916 level=level,
917 sort=sort,
918 observed=observed,
919 in_axis=in_axis,
920 dropna=dropna,
921 )
922 if not isinstance(gpr, Grouping)
923 else gpr
924 )
926 groupings.append(ping)
928 if len(groupings) == 0 and len(obj):
929 raise ValueError("No group keys passed!")
930 if len(groupings) == 0:
931 groupings.append(Grouping(default_index(0), np.array([], dtype=np.intp)))
933 # create the internals grouper
934 grouper = ops.BaseGrouper(group_axis, groupings, sort=sort, dropna=dropna)
935 return grouper, frozenset(exclusions), obj
938def _is_label_like(val) -> bool:
939 return isinstance(val, (str, tuple)) or (val is not None and is_scalar(val))
942def _convert_grouper(axis: Index, grouper):
943 if isinstance(grouper, dict):
944 return grouper.get
945 elif isinstance(grouper, Series):
946 if grouper.index.equals(axis):
947 return grouper._values
948 else:
949 return grouper.reindex(axis)._values
950 elif isinstance(grouper, MultiIndex):
951 return grouper._values
952 elif isinstance(grouper, (list, tuple, Index, Categorical, np.ndarray)):
953 if len(grouper) != len(axis):
954 raise ValueError("Grouper and axis must be same length")
956 if isinstance(grouper, (list, tuple)):
957 grouper = com.asarray_tuplesafe(grouper)
958 return grouper
959 else:
960 return grouper