Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Provide the groupby split-apply-combine paradigm. Define the GroupBy
3class providing the base-class of operations.
5The SeriesGroupBy and DataFrameGroupBy sub-class
6(defined in pandas.core.groupby.generic)
7expose these user-facing objects to provide specific functionality.
8"""
10from __future__ import annotations
12from collections.abc import (
13 Callable,
14 Hashable,
15 Iterable,
16 Iterator,
17 Mapping,
18 Sequence,
19)
20import datetime
21from functools import (
22 partial,
23 wraps,
24)
25from typing import (
26 TYPE_CHECKING,
27 Concatenate,
28 Literal,
29 Self,
30 TypeAlias,
31 TypeVar,
32 cast,
33 final,
34 overload,
35)
36import warnings
38import numpy as np
40from pandas._libs import (
41 Timestamp,
42 lib,
43)
44from pandas._libs.algos import rank_1d
45import pandas._libs.groupby as libgroupby
46from pandas._libs.missing import NA
47from pandas._typing import (
48 AnyArrayLike,
49 ArrayLike,
50 IndexLabel,
51 IntervalClosedType,
52 ListLike,
53 NDFrameT,
54 PositionalIndexer,
55 RandomState,
56 RankMethod,
57 RankNaOption,
58 npt,
59)
60from pandas.compat.numpy import function as nv
61from pandas.errors import (
62 AbstractMethodError,
63 DataError,
64 Pandas4Warning,
65)
66from pandas.util._decorators import cache_readonly
67from pandas.util._exceptions import find_stack_level
69from pandas.core.dtypes.cast import (
70 coerce_indexer_dtype,
71 ensure_dtype_can_hold_na,
72)
73from pandas.core.dtypes.common import (
74 is_bool,
75 is_bool_dtype,
76 is_hashable,
77 is_integer,
78 is_integer_dtype,
79 is_list_like,
80 is_numeric_dtype,
81 is_object_dtype,
82 is_scalar,
83 is_string_dtype,
84 pandas_dtype,
85)
86from pandas.core.dtypes.missing import (
87 isna,
88 na_value_for_dtype,
89 notna,
90)
92from pandas.core import (
93 algorithms,
94 sample,
95)
96from pandas.core._numba import executor
97from pandas.core.arrays import (
98 ArrowExtensionArray,
99 BaseMaskedArray,
100 ExtensionArray,
101 IntegerArray,
102)
103from pandas.core.arrays.string_ import StringDtype
104from pandas.core.arrays.string_arrow import ArrowStringArray
105from pandas.core.base import (
106 PandasObject,
107 SelectionMixin,
108)
109import pandas.core.common as com
110from pandas.core.frame import DataFrame
111from pandas.core.generic import NDFrame
112from pandas.core.groupby import (
113 base,
114 numba_,
115 ops,
116)
117from pandas.core.groupby.grouper import get_grouper
118from pandas.core.groupby.indexing import (
119 GroupByIndexingMixin,
120 GroupByNthSelector,
121)
122from pandas.core.indexes.api import (
123 Index,
124 MultiIndex,
125 default_index,
126)
127from pandas.core.internals.blocks import ensure_block_shape
128from pandas.core.series import Series
129from pandas.core.sorting import get_group_index_sorter
130from pandas.core.util.numba_ import (
131 get_jit_arguments,
132 maybe_use_numba,
133 prepare_function_arguments,
134)
136if TYPE_CHECKING:
137 from pandas._libs.tslibs import BaseOffset
138 from pandas._libs.tslibs.timedeltas import Timedelta
139 from pandas._typing import (
140 Any,
141 Level,
142 P,
143 T,
144 TimedeltaConvertibleTypes,
145 TimestampConvertibleTypes,
146 )
148 from pandas.core.indexers.objects import BaseIndexer
149 from pandas.core.resample import Resampler
150 from pandas.core.window import (
151 ExpandingGroupBy,
152 ExponentialMovingWindowGroupBy,
153 RollingGroupBy,
154 )
157@final
158class GroupByPlot(PandasObject):
159 """
160 Class implementing the .plot attribute for groupby objects.
161 """
163 def __init__(self, groupby: GroupBy) -> None:
164 self._groupby = groupby
166 def __call__(self, *args, **kwargs):
167 def f(self, key):
168 return self.plot(*args, **kwargs)
170 f.__name__ = "plot"
172 return self._groupby._python_apply_plot(f)
174 def __getattr__(self, name: str):
175 def attr(*args, **kwargs):
176 def f(self, key):
177 local_kwargs = kwargs.copy()
179 if name == "scatter" and kwargs.get("legend"):
180 local_kwargs.setdefault("label", key)
182 return getattr(self.plot, name)(*args, **local_kwargs)
184 return self._groupby._python_apply_plot(f)
186 return attr
189_KeysArgType: TypeAlias = (
190 Hashable
191 | list[Hashable]
192 | Callable[[Hashable], Hashable]
193 | list[Callable[[Hashable], Hashable]]
194 | Mapping[Hashable, Hashable]
195)
198class BaseGroupBy(PandasObject, SelectionMixin[NDFrameT], GroupByIndexingMixin):
199 _hidden_attrs = PandasObject._hidden_attrs | {
200 "as_index",
201 "dropna",
202 "exclusions",
203 "grouper",
204 "group_keys",
205 "keys",
206 "level",
207 "obj",
208 "observed",
209 "sort",
210 }
212 _grouper: ops.BaseGrouper
213 keys: _KeysArgType | None = None
214 level: IndexLabel | None = None
215 group_keys: bool
217 @final
218 def __len__(self) -> int:
219 return self._grouper.ngroups
221 @final
222 def __repr__(self) -> str:
223 # TODO: Better repr for GroupBy object
224 return object.__repr__(self)
226 @final
227 @property
228 def groups(self) -> dict[Hashable, Index]:
229 """
230 Dict {group name -> group labels}.
232 This property provides a dictionary representation of the groupings formed
233 during a groupby operation, where each key represents a unique group value from
234 the specified column(s), and each value is a list of index labels
235 that belong to that group.
237 See Also
238 --------
239 core.groupby.DataFrameGroupBy.get_group : Retrieve group from a
240 ``DataFrameGroupBy`` object with provided name.
241 core.groupby.SeriesGroupBy.get_group : Retrieve group from a
242 ``SeriesGroupBy`` object with provided name.
243 core.resample.Resampler.get_group : Retrieve group from a
244 ``Resampler`` object with provided name.
246 Examples
247 --------
249 For SeriesGroupBy:
251 >>> lst = ["a", "a", "b"]
252 >>> ser = pd.Series([1, 2, 3], index=lst)
253 >>> ser
254 a 1
255 a 2
256 b 3
257 dtype: int64
258 >>> ser.groupby(level=0).groups
259 {'a': ['a', 'a'], 'b': ['b']}
261 For DataFrameGroupBy:
263 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
264 >>> df = pd.DataFrame(data, columns=["a", "b", "c"])
265 >>> df
266 a b c
267 0 1 2 3
268 1 1 5 6
269 2 7 8 9
270 >>> df.groupby(by="a").groups
271 {1: [0, 1], 7: [2]}
273 For Resampler:
275 >>> ser = pd.Series(
276 ... [1, 2, 3, 4],
277 ... index=pd.DatetimeIndex(
278 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
279 ... ),
280 ... )
281 >>> ser
282 2023-01-01 1
283 2023-01-15 2
284 2023-02-01 3
285 2023-02-15 4
286 dtype: int64
287 >>> ser.resample("MS").groups
288 {Timestamp('2023-01-01 00:00:00'): np.int64(2),
289 Timestamp('2023-02-01 00:00:00'): np.int64(4)}
290 """
291 if isinstance(self.keys, list) and len(self.keys) == 1:
292 warnings.warn(
293 "In a future version, the keys of `groups` will be a "
294 f"tuple with a single element, e.g. ({self.keys[0]},) , "
295 f"instead of a scalar, e.g. {self.keys[0]}, when grouping "
296 "by a list with a single element. Use ``df.groupby(by='a').groups`` "
297 "instead of ``df.groupby(by=['a']).groups`` to avoid this warning",
298 Pandas4Warning,
299 stacklevel=find_stack_level(),
300 )
301 return self._grouper.groups
303 @final
304 @property
305 def ngroups(self) -> int:
306 return self._grouper.ngroups
308 @final
309 @property
310 def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
311 """
312 Dict {group name -> group indices}.
314 The dictionary keys represent the group labels (e.g., timestamps for a
315 time-based resampling operation), and the values are arrays of integer
316 positions indicating where the elements of each group are located in the
317 original data. This property is particularly useful when working with
318 resampled data, as it provides insight into how the original time-series data
319 has been grouped.
321 See Also
322 --------
323 core.groupby.DataFrameGroupBy.indices : Provides a mapping of group rows to
324 positions of the elements.
325 core.groupby.SeriesGroupBy.indices : Provides a mapping of group rows to
326 positions of the elements.
327 core.resample.Resampler.indices : Provides a mapping of group rows to
328 positions of the elements.
330 Examples
331 --------
333 For SeriesGroupBy:
335 >>> lst = ["a", "a", "b"]
336 >>> ser = pd.Series([1, 2, 3], index=lst)
337 >>> ser
338 a 1
339 a 2
340 b 3
341 dtype: int64
342 >>> ser.groupby(level=0).indices
343 {'a': array([0, 1]), 'b': array([2])}
345 For DataFrameGroupBy:
347 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
348 >>> df = pd.DataFrame(
349 ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
350 ... )
351 >>> df
352 a b c
353 owl 1 2 3
354 toucan 1 5 6
355 eagle 7 8 9
356 >>> df.groupby(by=["a"]).indices
357 {np.int64(1): array([0, 1]), np.int64(7): array([2])}
359 For Resampler:
361 >>> ser = pd.Series(
362 ... [1, 2, 3, 4],
363 ... index=pd.DatetimeIndex(
364 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
365 ... ),
366 ... )
367 >>> ser
368 2023-01-01 1
369 2023-01-15 2
370 2023-02-01 3
371 2023-02-15 4
372 dtype: int64
373 >>> ser.resample("MS").indices
374 defaultdict(<class 'list'>, {Timestamp('2023-01-01 00:00:00'): [0, 1],
375 Timestamp('2023-02-01 00:00:00'): [2, 3]})
376 """
377 return self._grouper.indices
379 @final
380 def _get_index(self, name):
381 """
382 Safe get multiple indices, translate keys for
383 datelike to underlying repr.
384 """
386 def get_converter(s):
387 # possibly convert to the actual key types
388 # in the indices, could be a Timestamp or an np.datetime64
389 if isinstance(s, datetime.datetime):
390 return lambda key: Timestamp(key)
391 elif isinstance(s, np.datetime64):
392 return lambda key: Timestamp(key).asm8
393 else:
394 return lambda key: key
396 if isna(name):
397 return self.indices.get(np.nan, [])
398 if isinstance(name, tuple):
399 name = tuple(np.nan if isna(comp) else comp for comp in name)
401 if len(self.indices) > 0:
402 index_sample = next(iter(self.indices))
403 else:
404 index_sample = None # Dummy sample
406 if isinstance(index_sample, tuple):
407 if not isinstance(name, tuple):
408 msg = "must supply a tuple to get_group with multiple grouping keys"
409 raise ValueError(msg)
410 if not len(name) == len(index_sample):
411 try:
412 # If the original grouper was a tuple
413 return self.indices[name]
414 except KeyError as err:
415 # turns out it wasn't a tuple
416 msg = (
417 "must supply a same-length tuple to get_group "
418 "with multiple grouping keys"
419 )
420 raise ValueError(msg) from err
422 converters = (get_converter(s) for s in index_sample)
423 name = tuple(f(n) for f, n in zip(converters, name, strict=True))
424 else:
425 converter = get_converter(index_sample)
426 name = converter(name)
428 return self.indices.get(name, [])
430 @final
431 @cache_readonly
432 def _selected_obj(self):
433 # Note: _selected_obj is always just `self.obj` for SeriesGroupBy
434 if isinstance(self.obj, Series):
435 return self.obj
437 if self._selection is not None:
438 if is_hashable(self._selection):
439 # i.e. a single key, so selecting it will return a Series.
440 # In this case, _obj_with_exclusions would wrap the key
441 # in a list and return a single-column DataFrame.
442 return self.obj[self._selection]
444 # Otherwise _selection is equivalent to _selection_list, so
445 # _selected_obj matches _obj_with_exclusions, so we can reuse
446 # that and avoid making a copy.
447 return self._obj_with_exclusions
449 return self.obj
451 @final
452 def _dir_additions(self) -> set[str]:
453 return self.obj._dir_additions()
455 @overload
456 def pipe(
457 self,
458 func: Callable[Concatenate[Self, P], T],
459 *args: P.args,
460 **kwargs: P.kwargs,
461 ) -> T: ...
463 @overload
464 def pipe(
465 self,
466 func: tuple[Callable[..., T], str],
467 *args: Any,
468 **kwargs: Any,
469 ) -> T: ...
471 def pipe(
472 self,
473 func: Callable[Concatenate[Self, P], T] | tuple[Callable[..., T], str],
474 *args: Any,
475 **kwargs: Any,
476 ) -> T:
477 """
478 Apply a ``func`` with arguments to this GroupBy object and return its result.
480 Use `.pipe` when you want to improve readability by chaining together
481 functions that expect Series, DataFrames, GroupBy or Resampler objects.
482 Instead of writing
484 >>> h = lambda x, arg2, arg3: x + 1 - arg2 * arg3
485 >>> g = lambda x, arg1: x * 5 / arg1
486 >>> f = lambda x: x**4
487 >>> df = pd.DataFrame([["a", 4], ["b", 5]], columns=["group", "value"])
488 >>> h(g(f(df.groupby("group")), arg1=1), arg2=2, arg3=3) # doctest: +SKIP
490 You can write
492 >>> (
493 ... df.groupby("group").pipe(f).pipe(g, arg1=1).pipe(h, arg2=2, arg3=3)
494 ... ) # doctest: +SKIP
496 which is much more readable.
498 Parameters
499 ----------
500 func : callable or tuple of (callable, str)
501 Function to apply to this GroupBy object or, alternatively,
502 a `(callable, data_keyword)` tuple where `data_keyword` is a
503 string indicating the keyword of `callable` that expects the
504 GroupBy object.
505 *args : iterable, optional
506 Positional arguments passed into `func`.
507 **kwargs : dict, optional
508 A dictionary of keyword arguments passed into `func`.
510 Returns
511 -------
512 GroupBy
513 The return type of `func`.
515 See Also
516 --------
517 Series.pipe : Apply a function with arguments to a series.
518 DataFrame.pipe : Apply a function with arguments to a dataframe.
519 apply : Apply function to each group instead of to the
520 full GroupBy object.
522 Notes
523 -----
524 See more `here
525 <https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html#piping-function-calls>`_
527 Examples
528 --------
529 >>> df = pd.DataFrame({"A": "a b a b".split(), "B": [1, 2, 3, 4]})
530 >>> df
531 A B
532 0 a 1
533 1 b 2
534 2 a 3
535 3 b 4
537 To get the difference between each groups maximum and minimum value in one
538 pass, you can do
540 >>> df.groupby("A").pipe(lambda x: x.max() - x.min())
541 B
542 A
543 a 2
544 b 2
545 """
546 return com.pipe(self, func, *args, **kwargs)
548 @final
549 def get_group(self, name) -> DataFrame | Series:
550 """
551 Construct DataFrame from group with provided name.
553 This method retrieves all rows belonging to a specific group identified
554 by the given name.
556 Parameters
557 ----------
558 name : object
559 The name of the group to get as a DataFrame.
561 Returns
562 -------
563 Series or DataFrame
564 Get the respective Series or DataFrame corresponding to the group provided.
566 See Also
567 --------
568 DataFrameGroupBy.groups: Dictionary representation of the groupings formed
569 during a groupby operation.
570 DataFrameGroupBy.indices: Provides a mapping of group rows to positions
571 of the elements.
572 SeriesGroupBy.groups: Dictionary representation of the groupings formed
573 during a groupby operation.
574 SeriesGroupBy.indices: Provides a mapping of group rows to positions
575 of the elements.
577 Examples
578 --------
580 For SeriesGroupBy:
582 >>> lst = ["a", "a", "b"]
583 >>> ser = pd.Series([1, 2, 3], index=lst)
584 >>> ser
585 a 1
586 a 2
587 b 3
588 dtype: int64
589 >>> ser.groupby(level=0).get_group("a")
590 a 1
591 a 2
592 dtype: int64
594 For DataFrameGroupBy:
596 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
597 >>> df = pd.DataFrame(
598 ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
599 ... )
600 >>> df
601 a b c
602 owl 1 2 3
603 toucan 1 5 6
604 eagle 7 8 9
605 >>> df.groupby(by=["a"]).get_group((1,))
606 a b c
607 owl 1 2 3
608 toucan 1 5 6
610 For Resampler:
612 >>> ser = pd.Series(
613 ... [1, 2, 3, 4],
614 ... index=pd.DatetimeIndex(
615 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
616 ... ),
617 ... )
618 >>> ser
619 2023-01-01 1
620 2023-01-15 2
621 2023-02-01 3
622 2023-02-15 4
623 dtype: int64
624 >>> ser.resample("MS").get_group("2023-01-01")
625 2023-01-01 1
626 2023-01-15 2
627 dtype: int64
628 """
629 keys = self.keys
630 level = self.level
631 # mypy doesn't recognize level/keys as being sized when passed to len
632 if (is_list_like(level) and len(level) == 1) or ( # type: ignore[arg-type]
633 is_list_like(keys) and len(keys) == 1 # type: ignore[arg-type]
634 ):
635 # GH#25971
636 if isinstance(name, tuple) and len(name) == 1:
637 name = name[0]
638 else:
639 raise KeyError(name)
641 inds = self._get_index(name)
642 if not len(inds):
643 raise KeyError(name)
644 return self._selected_obj.iloc[inds]
646 @final
647 def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
648 """
649 Groupby iterator.
651 This method provides an iterator over the groups created by the ``resample``
652 or ``groupby`` operation on the object. The method yields tuples where
653 the first element is the label (group key) corresponding to each group or
654 resampled bin, and the second element is the subset of the data that falls
655 within that group or bin.
657 Returns
658 -------
659 Iterator
660 Generator yielding a sequence of (name, subsetted object)
661 for each group.
663 See Also
664 --------
665 Series.groupby : Group data by a specific key or column.
666 DataFrame.groupby : Group DataFrame using mapper or by columns.
667 DataFrame.resample : Resample a DataFrame.
668 Series.resample : Resample a Series.
670 Examples
671 --------
673 For SeriesGroupBy:
675 >>> lst = ["a", "a", "b"]
676 >>> ser = pd.Series([1, 2, 3], index=lst)
677 >>> ser
678 a 1
679 a 2
680 b 3
681 dtype: int64
682 >>> for x, y in ser.groupby(level=0):
683 ... print(f"{x}\\n{y}\\n")
684 a
685 a 1
686 a 2
687 dtype: int64
688 b
689 b 3
690 dtype: int64
692 For DataFrameGroupBy:
694 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
695 >>> df = pd.DataFrame(data, columns=["a", "b", "c"])
696 >>> df
697 a b c
698 0 1 2 3
699 1 1 5 6
700 2 7 8 9
701 >>> for x, y in df.groupby(by=["a"]):
702 ... print(f"{x}\\n{y}\\n")
703 (1,)
704 a b c
705 0 1 2 3
706 1 1 5 6
707 (7,)
708 a b c
709 2 7 8 9
711 For Resampler:
713 >>> ser = pd.Series(
714 ... [1, 2, 3, 4],
715 ... index=pd.DatetimeIndex(
716 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
717 ... ),
718 ... )
719 >>> ser
720 2023-01-01 1
721 2023-01-15 2
722 2023-02-01 3
723 2023-02-15 4
724 dtype: int64
725 >>> for x, y in ser.resample("MS"):
726 ... print(f"{x}\\n{y}\\n")
727 2023-01-01 00:00:00
728 2023-01-01 1
729 2023-01-15 2
730 dtype: int64
731 2023-02-01 00:00:00
732 2023-02-01 3
733 2023-02-15 4
734 dtype: int64
735 """
736 keys = self.keys
737 level = self.level
738 result = self._grouper.get_iterator(self._selected_obj)
739 # mypy: Argument 1 to "len" has incompatible type "Hashable"; expected "Sized"
740 if (is_list_like(level) and len(level) == 1) or ( # type: ignore[arg-type]
741 isinstance(keys, list) and len(keys) == 1
742 ):
743 # GH#42795 - when keys is a list, return tuples even when length is 1
744 result = (((key,), group) for key, group in result)
745 return result
748# To track operations that expand dimensions, like ohlc
749OutputFrameOrSeries = TypeVar("OutputFrameOrSeries", bound=NDFrame)
752class GroupBy(BaseGroupBy[NDFrameT]):
753 """
754 Class for grouping and aggregating relational data.
756 See aggregate, transform, and apply functions on this object.
758 It's easiest to use obj.groupby(...) to use GroupBy, but you can also do:
760 ::
762 grouped = groupby(obj, ...)
764 Parameters
765 ----------
766 obj : pandas object
767 level : int, default None
768 Level of MultiIndex
769 groupings : list of Grouping objects
770 Most users should ignore this
771 exclusions : array-like, optional
772 List of columns to exclude
773 name : str
774 Most users should ignore this
776 Returns
777 -------
778 **Attributes**
779 groups : dict
780 {group name -> group labels}
781 len(grouped) : int
782 Number of groups
784 Notes
785 -----
786 After grouping, see aggregate, apply, and transform functions. Here are
787 some other brief notes about usage. When grouping by multiple groups, the
788 result index will be a MultiIndex (hierarchical) by default.
790 Iteration produces (key, group) tuples, i.e. chunking the data by group. So
791 you can write code like:
793 ::
795 grouped = obj.groupby(keys)
796 for key, group in grouped:
797 # do something with the data
799 Function calls on GroupBy, if not specially implemented, "dispatch" to the
800 grouped data. So if you group a DataFrame and wish to invoke the std()
801 method on each group, you can simply do:
803 ::
805 df.groupby(mapper).std()
807 rather than
809 ::
811 df.groupby(mapper).aggregate(np.std)
813 You can pass arguments to these "wrapped" functions, too.
815 See the online documentation for full exposition on these topics and much
816 more
817 """
819 _grouper: ops.BaseGrouper
820 as_index: bool
822 @final
823 def __init__(
824 self,
825 obj: NDFrameT,
826 keys: _KeysArgType | None = None,
827 level: IndexLabel | None = None,
828 grouper: ops.BaseGrouper | None = None,
829 exclusions: frozenset[Hashable] | None = None,
830 selection: IndexLabel | None = None,
831 as_index: bool = True,
832 sort: bool = True,
833 group_keys: bool = True,
834 observed: bool = True,
835 dropna: bool = True,
836 ) -> None:
837 self._selection = selection
839 assert isinstance(obj, NDFrame), type(obj)
841 self.level = level
842 self.as_index = as_index
843 self.keys = keys
844 self.sort = sort
845 self.group_keys = group_keys
846 self.dropna = dropna
848 if grouper is None:
849 grouper, exclusions, obj = get_grouper(
850 obj,
851 keys,
852 level=level,
853 sort=sort,
854 observed=observed,
855 dropna=self.dropna,
856 )
858 self.observed = observed
859 self.obj = obj
860 self._grouper = grouper
861 self.exclusions = frozenset(exclusions) if exclusions else frozenset()
863 def __getattr__(self, attr: str):
864 if attr in self._internal_names_set:
865 return object.__getattribute__(self, attr)
866 if attr in self.obj:
867 return self[attr]
869 raise AttributeError(
870 f"'{type(self).__name__}' object has no attribute '{attr}'"
871 )
873 @final
874 def _op_via_apply(self, name: str, *args, **kwargs):
875 """Compute the result of an operation by using GroupBy's apply."""
876 f = getattr(type(self._obj_with_exclusions), name)
878 def curried(x, key=None):
879 return f(x, *args, **kwargs)
881 # preserve the name so we can detect it when calling plot methods,
882 # to avoid duplicates
883 curried.__name__ = name
885 # special case otherwise extra plots are created when catching the
886 # exception below
887 if name in base.plotting_methods:
888 return self._python_apply_plot(curried)
890 is_transform = name in base.transformation_kernels
891 result = self._python_apply_general(
892 curried,
893 self._obj_with_exclusions,
894 is_transform=is_transform,
895 not_indexed_same=not is_transform,
896 )
898 if self._grouper.has_dropped_na and is_transform:
899 # result will have dropped rows due to nans, fill with null
900 # and ensure index is ordered same as the input
901 result = self._set_result_index_ordered(result)
902 return result
904 # -----------------------------------------------------------------
905 # Dispatch/Wrapping
907 @final
908 def _concat_objects(
909 self,
910 values,
911 not_indexed_same: bool = False,
912 is_transform: bool = False,
913 ):
914 from pandas.core.reshape.concat import concat
916 if self.group_keys and not is_transform:
917 if self.as_index:
918 # possible MI return case
919 group_keys = self._grouper.result_index
920 group_levels = self._grouper.levels
921 group_names = self._grouper.names
923 result = concat(
924 values,
925 axis=0,
926 keys=group_keys,
927 levels=group_levels,
928 names=group_names,
929 sort=False,
930 )
931 else:
932 result = concat(values, axis=0)
934 elif not not_indexed_same:
935 result = concat(values, axis=0)
937 ax = self._selected_obj.index
938 if self.dropna:
939 labels = self._grouper.ids
940 mask = labels != -1
941 ax = ax[mask]
943 # this is a very unfortunate situation
944 # we can't use reindex to restore the original order
945 # when the ax has duplicates
946 # so we resort to this
947 # GH 14776, 30667
948 # TODO: can we reuse e.g. _reindex_non_unique?
949 if ax.has_duplicates and not result.axes[0].equals(ax):
950 # e.g. test_category_order_transformer
951 target = algorithms.unique1d(ax._values)
952 indexer, _ = result.index.get_indexer_non_unique(target)
953 result = result.take(indexer, axis=0)
954 else:
955 result = result.reindex(ax, axis=0)
957 else:
958 result = concat(values, axis=0)
960 if self.obj.ndim == 1:
961 name = self.obj.name
962 elif is_hashable(self._selection):
963 name = self._selection
964 else:
965 name = None
967 if isinstance(result, Series) and name is not None:
968 result.name = name
970 return result.__finalize__(self.obj, method="groupby")
972 @final
973 def _set_result_index_ordered(
974 self, result: OutputFrameOrSeries
975 ) -> OutputFrameOrSeries:
976 # set the result index on the passed values object and
977 # return the new object, xref 8046
979 index = self.obj.index
981 if self._grouper.is_monotonic and not self._grouper.has_dropped_na:
982 # shortcut if we have an already ordered grouper
983 result = result.set_axis(index, axis=0)
984 return result
986 # row order is scrambled => sort the rows by position in original index
987 original_positions = Index(self._grouper.result_ilocs, copy=False)
988 result = result.set_axis(original_positions, axis=0)
989 result = result.sort_index(axis=0)
990 if self._grouper.has_dropped_na:
991 # Add back in any missing rows due to dropna - index here is integral
992 # with values referring to the row of the input so can use RangeIndex
993 result = result.reindex(default_index(len(index)), axis=0)
994 result = result.set_axis(index, axis=0)
996 return result
998 @final
999 def _insert_inaxis_grouper(
1000 self, result: Series | DataFrame, qs: npt.NDArray[np.float64] | None = None
1001 ) -> DataFrame:
1002 if isinstance(result, Series):
1003 result = result.to_frame()
1005 n_groupings = len(self._grouper.groupings)
1007 def full_key(name: Hashable) -> Hashable:
1008 # GH#17024 expanding MultiIndex columns with a partial key is
1009 # deprecated; pad the key to full length ourselves
1010 if isinstance(result.columns, MultiIndex) and not isinstance(name, tuple):
1011 return (name,) + ("",) * (result.columns.nlevels - 1)
1012 return name
1014 if qs is not None:
1015 result.insert(
1016 0, full_key(f"level_{n_groupings}"), np.tile(qs, len(result) // len(qs))
1017 )
1019 # zip in reverse so we can always insert at loc 0
1020 for level, (name, lev) in enumerate(
1021 zip(
1022 reversed(self._grouper.names),
1023 self._grouper.get_group_levels(),
1024 strict=True,
1025 )
1026 ):
1027 if name is None:
1028 # Behave the same as .reset_index() when a level is unnamed
1029 name = (
1030 "index"
1031 if n_groupings == 1 and qs is None
1032 else f"level_{n_groupings - level - 1}"
1033 )
1035 # GH #28549
1036 # When using .apply(-), name will be in columns already
1037 if name not in result.columns:
1038 # if in_axis:
1039 if qs is None:
1040 result.insert(0, full_key(name), lev)
1041 else:
1042 result.insert(
1043 0, full_key(name), Index(np.repeat(lev, len(qs)), copy=False)
1044 )
1046 return result
1048 @final
1049 def _wrap_aggregated_output(
1050 self,
1051 result: Series | DataFrame,
1052 qs: npt.NDArray[np.float64] | None = None,
1053 ):
1054 """
1055 Wraps the output of GroupBy aggregations into the expected result.
1057 Parameters
1058 ----------
1059 result : Series, DataFrame
1061 Returns
1062 -------
1063 Series or DataFrame
1064 """
1065 # ATM we do not get here for SeriesGroupBy; when we do, we will
1066 # need to require that result.name already match self.obj.name
1068 if not self.as_index:
1069 # `not self.as_index` is only relevant for DataFrameGroupBy,
1070 # enforced in __init__
1071 result = self._insert_inaxis_grouper(result, qs=qs)
1072 result = result._consolidate()
1073 result.index = default_index(len(result))
1075 else:
1076 index = self._grouper.result_index
1077 if qs is not None:
1078 # We get here with len(qs) != 1 and not self.as_index
1079 # in test_pass_args_kwargs
1080 index = _insert_quantile_level(index, qs)
1081 result.index = index
1083 return result
1085 def _wrap_applied_output(
1086 self,
1087 data,
1088 values: list,
1089 not_indexed_same: bool = False,
1090 is_transform: bool = False,
1091 ):
1092 raise AbstractMethodError(self)
1094 # -----------------------------------------------------------------
1095 # numba
1097 @final
1098 def _numba_prep(self, data: DataFrame):
1099 ngroups = self._grouper.ngroups
1100 sorted_index = self._grouper.result_ilocs
1101 sorted_ids = self._grouper._sorted_ids
1103 sorted_data = data.take(sorted_index, axis=0).to_numpy()
1104 # GH 46867
1105 index_data = data.index
1106 if isinstance(index_data, MultiIndex):
1107 if len(self._grouper.groupings) > 1:
1108 raise NotImplementedError(
1109 "Grouping with more than 1 grouping labels and "
1110 "a MultiIndex is not supported with engine='numba'"
1111 )
1112 group_key = self._grouper.groupings[0].name
1113 index_data = index_data.get_level_values(group_key)
1114 sorted_index_data = index_data.take(sorted_index).to_numpy()
1116 starts, ends = lib.generate_slices(sorted_ids, ngroups)
1117 return (
1118 starts,
1119 ends,
1120 sorted_index_data,
1121 sorted_data,
1122 )
1124 def _numba_agg_general(
1125 self,
1126 func: Callable,
1127 dtype_mapping: dict[np.dtype, Any],
1128 engine_kwargs: dict[str, bool] | None,
1129 **aggregator_kwargs,
1130 ):
1131 """
1132 Perform groupby with a standard numerical aggregation function (e.g. mean)
1133 with Numba.
1134 """
1135 if not self.as_index:
1136 raise NotImplementedError(
1137 "as_index=False is not supported. Use .reset_index() instead."
1138 )
1140 data = self._obj_with_exclusions
1141 df = data if data.ndim == 2 else data.to_frame()
1143 aggregator = executor.generate_shared_aggregator(
1144 func,
1145 dtype_mapping,
1146 True, # is_grouped_kernel
1147 **get_jit_arguments(engine_kwargs),
1148 )
1149 # Pass group ids to kernel directly if it can handle it
1150 # (This is faster since it doesn't require a sort)
1151 ids = self._grouper.ids
1152 ngroups = self._grouper.ngroups
1154 res_mgr = df._mgr.apply(
1155 aggregator, labels=ids, ngroups=ngroups, **aggregator_kwargs
1156 )
1157 res_mgr.axes[1] = self._grouper.result_index
1158 result = df._constructor_from_mgr(res_mgr, axes=res_mgr.axes)
1160 if data.ndim == 1:
1161 result = result.squeeze("columns")
1162 result.name = data.name
1163 else:
1164 result.columns = data.columns
1165 return result
1167 @final
1168 def _transform_with_numba(self, func, *args, engine_kwargs=None, **kwargs):
1169 """
1170 Perform groupby transform routine with the numba engine.
1172 This routine mimics the data splitting routine of the DataSplitter class
1173 to generate the indices of each group in the sorted data and then passes the
1174 data and indices into a Numba jitted function.
1175 """
1176 data = self._obj_with_exclusions
1177 index_sorting = self._grouper.result_ilocs
1178 df = data if data.ndim == 2 else data.to_frame()
1180 starts, ends, sorted_index, sorted_data = self._numba_prep(df)
1181 numba_.validate_udf(func)
1182 args, kwargs = prepare_function_arguments(
1183 func, args, kwargs, num_required_args=2
1184 )
1185 numba_transform_func = numba_.generate_numba_transform_func(
1186 func, **get_jit_arguments(engine_kwargs)
1187 )
1188 result = numba_transform_func(
1189 sorted_data,
1190 sorted_index,
1191 starts,
1192 ends,
1193 len(df.columns),
1194 *args,
1195 )
1196 # result values needs to be resorted to their original positions since we
1197 # evaluated the data sorted by group
1198 result = result.take(np.argsort(index_sorting), axis=0)
1199 index = data.index
1200 if data.ndim == 1:
1201 result_kwargs = {"name": data.name}
1202 result = result.ravel()
1203 else:
1204 result_kwargs = {"columns": data.columns}
1205 return data._constructor(result, index=index, **result_kwargs)
1207 @final
1208 def _aggregate_with_numba(self, func, *args, engine_kwargs=None, **kwargs):
1209 """
1210 Perform groupby aggregation routine with the numba engine.
1212 This routine mimics the data splitting routine of the DataSplitter class
1213 to generate the indices of each group in the sorted data and then passes the
1214 data and indices into a Numba jitted function.
1215 """
1216 data = self._obj_with_exclusions
1217 df = data if data.ndim == 2 else data.to_frame()
1219 starts, ends, sorted_index, sorted_data = self._numba_prep(df)
1220 numba_.validate_udf(func)
1221 args, kwargs = prepare_function_arguments(
1222 func, args, kwargs, num_required_args=2
1223 )
1224 numba_agg_func = numba_.generate_numba_agg_func(
1225 func, **get_jit_arguments(engine_kwargs)
1226 )
1227 result = numba_agg_func(
1228 sorted_data,
1229 sorted_index,
1230 starts,
1231 ends,
1232 len(df.columns),
1233 *args,
1234 )
1235 index = self._grouper.result_index
1236 if data.ndim == 1:
1237 result_kwargs = {"name": data.name}
1238 result = result.ravel()
1239 else:
1240 result_kwargs = {"columns": data.columns}
1241 res = data._constructor(result, index=index, **result_kwargs)
1242 if not self.as_index:
1243 res = self._insert_inaxis_grouper(res)
1244 res.index = default_index(len(res))
1245 return res
1247 # -----------------------------------------------------------------
1248 # apply/agg/transform
1250 def apply(self, func, *args, include_groups: bool = False, **kwargs) -> NDFrameT:
1251 """
1252 Apply function ``func`` group-wise and combine the results together.
1254 The function passed to ``apply`` must take a dataframe as its first
1255 argument and return a DataFrame, Series or scalar. ``apply`` will
1256 then take care of combining the results back together into a single
1257 dataframe or series. ``apply`` is therefore a highly flexible
1258 grouping method.
1260 While ``apply`` is a very flexible method, its downside is that
1261 using it can be quite a bit slower than using more specific methods
1262 like ``agg`` or ``transform``. Pandas offers a wide range of method that will
1263 be much faster than using ``apply`` for their specific purposes, so try to
1264 use them before reaching for ``apply``.
1266 Parameters
1267 ----------
1268 func : callable
1269 A callable that takes a dataframe as its first argument, and
1270 returns a dataframe, a series or a scalar. In addition the
1271 callable may take positional and keyword arguments.
1273 *args : tuple
1274 Optional positional arguments to pass to ``func``.
1276 include_groups : bool, default False
1277 When True, will attempt to apply ``func`` to the groupings in
1278 the case that they are columns of the DataFrame. If this raises a
1279 TypeError, the result will be computed with the groupings excluded.
1280 When False, the groupings will be excluded when applying ``func``.
1282 .. versionadded:: 2.2.0
1284 .. versionchanged:: 3.0.0
1286 The default changed from True to False, and True is no longer allowed.
1288 **kwargs : dict
1289 Optional keyword arguments to pass to ``func``.
1291 Returns
1292 -------
1293 Series or DataFrame
1294 A pandas object with the result of applying ``func`` to each group.
1296 See Also
1297 --------
1298 pipe : Apply function to the full GroupBy object instead of to each
1299 group.
1300 aggregate : Apply aggregate function to the GroupBy object.
1301 transform : Apply function column-by-column to the GroupBy object.
1302 Series.apply : Apply a function to a Series.
1303 DataFrame.apply : Apply a function to each row or column of a DataFrame.
1305 Notes
1306 -----
1307 See :ref:`groupby.apply` in the User Guide for more details and examples.
1309 The resulting dtype will reflect the return value of the passed ``func``,
1310 see the examples below.
1312 Functions that mutate the passed object can produce unexpected
1313 behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
1314 for more details.
1316 Each group passed to ``func`` has a ``name`` attribute set to the group
1317 key (the value of the grouping for that group). This is useful for
1318 identifying the current group, for example when grouping by an external
1319 grouper rather than by a column of the object.
1321 Examples
1322 --------
1323 >>> df = pd.DataFrame({"A": "a a b".split(), "B": [1, 2, 3], "C": [4, 6, 5]})
1324 >>> g1 = df.groupby("A", group_keys=False)
1325 >>> g2 = df.groupby("A", group_keys=True)
1327 Notice that ``g1`` and ``g2`` have two groups, ``a`` and ``b``, and only
1328 differ in their ``group_keys`` argument. Calling `apply` in various ways,
1329 we can get different grouping results:
1331 Example 1: below the function passed to `apply` takes a DataFrame as
1332 its argument and returns a DataFrame. `apply` combines the result for
1333 each group together into a new DataFrame:
1335 >>> g1[["B", "C"]].apply(lambda x: x / x.sum())
1336 B C
1337 0 0.333333 0.4
1338 1 0.666667 0.6
1339 2 1.000000 1.0
1341 In the above, the groups are not part of the index. We can have them included
1342 by using ``g2`` where ``group_keys=True``:
1344 >>> g2[["B", "C"]].apply(lambda x: x / x.sum())
1345 B C
1346 A
1347 a 0 0.333333 0.4
1348 1 0.666667 0.6
1349 b 2 1.000000 1.0
1351 Example 2: The function passed to `apply` takes a DataFrame as
1352 its argument and returns a Series. `apply` combines the result for
1353 each group together into a new DataFrame.
1355 The resulting dtype will reflect the return value of the passed ``func``.
1357 >>> g1[["B", "C"]].apply(lambda x: x.astype(float).max() - x.min())
1358 B C
1359 A
1360 a 1.0 2.0
1361 b 0.0 0.0
1363 >>> g2[["B", "C"]].apply(lambda x: x.astype(float).max() - x.min())
1364 B C
1365 A
1366 a 1.0 2.0
1367 b 0.0 0.0
1369 The ``group_keys`` argument has no effect here because the result is not
1370 like-indexed (i.e. :ref:`a transform <groupby.transform>`) when compared
1371 to the input.
1373 Example 3: The function passed to `apply` takes a DataFrame as
1374 its argument and returns a scalar. `apply` combines the result for
1375 each group together into a Series, including setting the index as
1376 appropriate:
1378 >>> g1.apply(lambda x: x.C.max() - x.B.min())
1379 A
1380 a 5
1381 b 2
1382 dtype: int64
1384 Example 4: The function passed to ``apply`` returns ``None`` for one of the
1385 group. This group is filtered from the result:
1387 >>> g1.apply(lambda x: None if x.iloc[0, 0] == 3 else x)
1388 B C
1389 0 1 4
1390 1 2 6
1391 """
1392 if include_groups:
1393 raise ValueError("include_groups=True is no longer allowed.")
1394 if isinstance(func, str):
1395 if hasattr(self, func):
1396 res = getattr(self, func)
1397 if callable(res):
1398 return res(*args, **kwargs)
1399 elif args or kwargs:
1400 raise ValueError(f"Cannot pass arguments to property {func}")
1401 return res
1403 else:
1404 raise TypeError(f"apply func should be callable, not '{func}'")
1406 elif args or kwargs:
1407 if callable(func):
1409 @wraps(func)
1410 def f(g):
1411 return func(g, *args, **kwargs)
1413 else:
1414 raise ValueError(
1415 "func must be a callable if args or kwargs are supplied"
1416 )
1417 else:
1418 f = func
1420 return self._python_apply_general(f, self._obj_with_exclusions)
1422 @final
1423 def _python_apply_general(
1424 self,
1425 f: Callable,
1426 data: DataFrame | Series,
1427 not_indexed_same: bool | None = None,
1428 is_transform: bool = False,
1429 ) -> NDFrameT:
1430 """
1431 Apply function f in python space
1433 Parameters
1434 ----------
1435 f : callable
1436 Function to apply
1437 data : Series or DataFrame
1438 Data to apply f to
1439 not_indexed_same: bool, optional
1440 When specified, overrides the value of not_indexed_same. Apply behaves
1441 differently when the result index is equal to the input index, but
1442 this can be coincidental leading to value-dependent behavior.
1443 is_transform : bool, default False
1444 Indicator for whether the function is actually a transform
1445 and should not have group keys prepended.
1447 Returns
1448 -------
1449 Series or DataFrame
1450 data after applying f
1451 """
1452 values, mutated = self._grouper.apply_groupwise(f, data)
1453 if not_indexed_same is None:
1454 not_indexed_same = mutated
1456 return self._wrap_applied_output(
1457 data,
1458 values,
1459 not_indexed_same,
1460 is_transform,
1461 )
1463 @final
1464 def _python_apply_plot(self, f: Callable) -> NDFrameT:
1465 """
1466 Apply a plotting function f group-wise.
1468 Each Series group gets its group key as the name, which plotting
1469 methods use for legend labels. Unlike the general apply path, this
1470 labels the group explicitly rather than relying on the name attribute
1471 being pinned to the group key as a side effect (GH#41090).
1472 """
1473 data = self._selected_obj
1474 values = []
1475 for key, group in self._grouper.get_iterator(data):
1476 if group.ndim == 1:
1477 group.name = key
1478 values.append(f(group, key))
1479 # plotting functions return matplotlib objects, never something
1480 # indexed like the group, so this is always not_indexed_same
1481 return self._wrap_applied_output(data, values, not_indexed_same=True)
1483 def _agg_py_fallback(
1484 self, how: str, values: ArrayLike, ndim: int, alt: Callable
1485 ) -> ArrayLike:
1486 """
1487 Fallback to pure-python aggregation if _cython_operation raises
1488 NotImplementedError.
1489 """
1490 # We get here with a) EADtypes and b) object dtype
1491 assert alt is not None
1493 if values.ndim == 1:
1494 # For DataFrameGroupBy we only get here with ExtensionArray
1495 ser = Series(values, copy=False)
1496 else:
1497 # We only get here with values.dtype == object
1498 df = DataFrame(values.T, dtype=values.dtype)
1499 # bc we split object blocks in grouped_reduce, we have only 1 col
1500 # otherwise we'd have to worry about block-splitting GH#39329
1501 assert df.shape[1] == 1
1502 # Avoid call to self.values that can occur in DataFrame
1503 # reductions; see GH#28949
1504 ser = df.iloc[:, 0]
1506 # We do not get here with UDFs, so we know that our dtype
1507 # should always be preserved by the implemented aggregations
1508 # TODO: Is this exactly right; see WrappedCythonOp get_result_dtype?
1509 try:
1510 res_values = self._grouper.agg_series(ser, alt)
1511 except Exception as err:
1512 msg = f"agg function failed [how->{how},dtype->{ser.dtype}]"
1513 # preserve the kind of exception that raised
1514 raise type(err)(msg) from err
1516 dtype = ser.dtype
1517 if dtype == object:
1518 res_values = res_values.astype(object, copy=False)
1519 elif is_string_dtype(dtype):
1520 # mypy doesn't infer dtype is an ExtensionDtype
1521 string_array_cls = dtype.construct_array_type() # type: ignore[union-attr]
1522 res_values = string_array_cls._from_sequence(res_values, dtype=dtype)
1524 # If we are DataFrameGroupBy and went through a SeriesGroupByPath
1525 # then we need to reshape
1526 # GH#32223 includes case with IntegerArray values, ndarray res_values
1527 # test_groupby_duplicate_columns with object dtype values
1528 return ensure_block_shape(res_values, ndim=ndim)
1530 @final
1531 def _cython_agg_general(
1532 self,
1533 how: str,
1534 alt: Callable | None = None,
1535 numeric_only: bool = False,
1536 min_count: int = -1,
1537 **kwargs,
1538 ):
1539 # Note: we never get here with how="ohlc" for DataFrameGroupBy;
1540 # that goes through SeriesGroupBy
1542 if not is_bool(numeric_only):
1543 raise ValueError("numeric_only accepts only Boolean values")
1545 data = self._get_data_to_aggregate(numeric_only=numeric_only, name=how)
1547 def array_func(values: ArrayLike) -> ArrayLike:
1548 # GH#37850 For numpy boolean data, any==max and all==min.
1549 # The min/max Cython path is faster (fused types, no mask overhead).
1550 # Excludes nullable BooleanDtype which needs Kleene logic.
1551 use_bool_fastpath = how in ["any", "all"] and values.dtype == np.dtype(
1552 "bool"
1553 )
1554 if use_bool_fastpath:
1555 _how = "max" if how == "any" else "min"
1556 else:
1557 _how = how
1558 try:
1559 result = self._grouper._cython_operation(
1560 values,
1561 _how,
1562 axis=data.ndim - 1,
1563 min_count=min_count,
1564 **kwargs,
1565 )
1566 except NotImplementedError as err:
1567 # generally if we have numeric_only=False
1568 # and non-applicable functions
1569 # try to python agg
1570 # TODO: shouldn't min_count matter?
1571 if alt is None or how in ["any", "all", "std", "sem"]:
1572 raise TypeError(
1573 f"{how} is not supported for {values.dtype} dtype"
1574 ) from err
1575 else:
1576 if use_bool_fastpath and result.dtype.kind == "f":
1577 fill = 0.0 if how == "any" else 1.0
1578 result = np.where(
1579 np.isnan(result), fill, np.asarray(result)
1580 ).astype(bool)
1581 return result
1583 assert alt is not None
1584 result = self._agg_py_fallback(how, values, ndim=data.ndim, alt=alt)
1585 return result
1587 new_mgr = data.grouped_reduce(array_func)
1588 res = self._wrap_agged_manager(new_mgr)
1589 if how in ["idxmin", "idxmax"]:
1590 # mypy expects how to be Literal["idxmin", "idxmax"].
1591 res = self._wrap_idxmax_idxmin(res, how=how, skipna=kwargs["skipna"]) # type: ignore[arg-type]
1592 out = self._wrap_aggregated_output(res)
1593 return out.__finalize__(self.obj, method="groupby")
1595 def _cython_transform(self, how: str, numeric_only: bool = False, **kwargs):
1596 raise AbstractMethodError(self)
1598 @final
1599 def _transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs):
1600 # optimized transforms
1601 if not isinstance(func, str):
1602 return self._transform_general(func, engine, engine_kwargs, *args, **kwargs)
1604 elif func not in base.transform_kernel_allowlist:
1605 msg = f"'{func}' is not a valid function name for transform(name)"
1606 raise ValueError(msg)
1607 elif func in base.cythonized_kernels or func in base.transformation_kernels:
1608 # cythonized transform or canned "agg+broadcast"
1609 if engine is not None:
1610 kwargs["engine"] = engine
1611 kwargs["engine_kwargs"] = engine_kwargs
1612 return getattr(self, func)(*args, **kwargs)
1614 else:
1615 # i.e. func in base.reduction_kernels
1616 if self.observed:
1617 return self._reduction_kernel_transform(
1618 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
1619 )
1621 with (
1622 com.temp_setattr(self, "observed", True),
1623 com.temp_setattr(self, "_grouper", self._grouper.observed_grouper),
1624 ):
1625 return self._reduction_kernel_transform(
1626 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
1627 )
1629 @final
1630 def _reduction_kernel_transform(
1631 self, func, *args, engine=None, engine_kwargs=None, **kwargs
1632 ):
1633 # GH#30918 Use _transform_fast only when we know func is an aggregation
1634 # If func is a reduction, we need to broadcast the
1635 # result to the whole group. Compute func result
1636 # and deal with possible broadcasting below.
1637 with com.temp_setattr(self, "as_index", True):
1638 # GH#49834 - result needs groups in the index for
1639 # _wrap_transform_fast_result
1640 if func in ["idxmin", "idxmax"]:
1641 func = cast("Literal['idxmin', 'idxmax']", func)
1642 result = self._idxmax_idxmin(func, True, *args, **kwargs)
1643 else:
1644 if engine is not None:
1645 kwargs["engine"] = engine
1646 kwargs["engine_kwargs"] = engine_kwargs
1647 result = getattr(self, func)(*args, **kwargs)
1649 return self._wrap_transform_fast_result(result)
1651 @final
1652 def _wrap_transform_fast_result(self, result: NDFrameT) -> NDFrameT:
1653 """
1654 Fast transform path for aggregations.
1655 """
1656 obj = self._obj_with_exclusions
1658 # for each col, reshape to size of original frame by take operation
1659 ids = self._grouper.ids
1660 result = result.reindex(self._grouper.result_index, axis=0)
1662 if self.obj.ndim == 1:
1663 # i.e. SeriesGroupBy
1664 out = algorithms.take_nd(
1665 result._values, ids, allow_fill=self._grouper.has_dropped_na
1666 )
1667 output = obj._constructor(out, index=obj.index, name=obj.name)
1668 else:
1669 # `.size()` gives Series output on DataFrame input, need axis 0
1670 # GH#46209
1671 # Don't convert indices: negative indices need to give rise
1672 # to null values in the result
1673 new_ax = result.index.take(ids)
1674 output = result._reindex_with_indexers({0: (new_ax, ids)}, allow_dups=True)
1675 output = output.set_axis(obj.index, axis=0)
1676 return output
1678 # -----------------------------------------------------------------
1679 # Utilities
1681 @final
1682 def _apply_filter(self, indices, dropna):
1683 if len(indices) == 0:
1684 indices = np.array([], dtype="int64")
1685 else:
1686 indices = np.sort(np.concatenate(indices))
1687 if dropna:
1688 filtered = self._selected_obj.take(indices, axis=0)
1689 else:
1690 mask = np.empty(len(self._selected_obj.index), dtype=bool)
1691 mask.fill(False)
1692 mask[indices.astype(int)] = True
1693 # mask fails to broadcast when passed to where; broadcast manually.
1694 mask = np.tile(mask, [*self._selected_obj.shape[1:], 1]).T
1695 filtered = self._selected_obj.where(mask) # Fill with NaNs.
1696 return filtered
1698 @final
1699 def _cumcount_array(self, ascending: bool = True) -> np.ndarray:
1700 """
1701 Parameters
1702 ----------
1703 ascending : bool, default True
1704 If False, number in reverse, from length of group - 1 to 0.
1706 Notes
1707 -----
1708 this is currently implementing sort=False
1709 (though the default is sort=True) for groupby in general
1710 """
1711 ids = self._grouper.ids
1712 ngroups = self._grouper.ngroups
1713 sorter = get_group_index_sorter(ids, ngroups)
1714 ids, count = ids[sorter], len(ids)
1716 if count == 0:
1717 return np.empty(0, dtype=np.int64)
1719 run = np.r_[True, ids[:-1] != ids[1:]]
1720 rep = np.diff(np.r_[np.nonzero(run)[0], count])
1721 out = (~run).cumsum()
1723 if ascending:
1724 out -= np.repeat(out[run], rep)
1725 else:
1726 out = np.repeat(out[np.r_[run[1:], True]], rep) - out
1728 if self._grouper.has_dropped_na:
1729 out = np.where(ids == -1, np.nan, out.astype(np.float64, copy=False))
1730 else:
1731 out = out.astype(np.int64, copy=False)
1733 rev = np.empty(count, dtype=np.intp)
1734 rev[sorter] = np.arange(count, dtype=np.intp)
1735 return out[rev]
1737 # -----------------------------------------------------------------
1739 @final
1740 @property
1741 def _obj_1d_constructor(self) -> Callable:
1742 # GH28330 preserve subclassed Series/DataFrames
1743 if isinstance(self.obj, DataFrame):
1744 return self.obj._constructor_sliced
1745 assert isinstance(self.obj, Series)
1746 return self.obj._constructor
1748 @final
1749 def any(self, skipna: bool = True) -> NDFrameT:
1750 """
1751 Return True if any value in the group is truthful, else False.
1753 This is equivalent to calling ``bool`` on each value in the group and
1754 returning ``True`` if at least one is truthful.
1756 Parameters
1757 ----------
1758 skipna : bool, default True
1759 Flag to ignore nan values during truth testing.
1761 Returns
1762 -------
1763 Series or DataFrame
1764 DataFrame or Series of boolean values, where a value is True if any element
1765 is True within its respective group, False otherwise.
1767 See Also
1768 --------
1769 Series.any : Apply function any to a Series.
1770 DataFrame.any : Apply function any to each row or column of a DataFrame.
1772 Examples
1773 --------
1774 For SeriesGroupBy:
1776 >>> lst = ["a", "a", "b"]
1777 >>> ser = pd.Series([1, 2, 0], index=lst)
1778 >>> ser
1779 a 1
1780 a 2
1781 b 0
1782 dtype: int64
1783 >>> ser.groupby(level=0).any()
1784 a True
1785 b False
1786 dtype: bool
1788 For DataFrameGroupBy:
1790 >>> data = [[1, 0, 3], [1, 0, 6], [7, 1, 9]]
1791 >>> df = pd.DataFrame(
1792 ... data, columns=["a", "b", "c"], index=["ostrich", "penguin", "parrot"]
1793 ... )
1794 >>> df
1795 a b c
1796 ostrich 1 0 3
1797 penguin 1 0 6
1798 parrot 7 1 9
1799 >>> df.groupby(by=["a"]).any()
1800 b c
1801 a
1802 1 False True
1803 7 True True
1804 """
1805 return self._cython_agg_general(
1806 "any",
1807 alt=lambda x: Series(x, copy=False).any(skipna=skipna),
1808 skipna=skipna,
1809 )
1811 @final
1812 def all(self, skipna: bool = True) -> NDFrameT:
1813 """
1814 Return True if all values in the group are truthful, else False.
1816 This is equivalent to calling ``bool`` on each value in the group and
1817 returning ``True`` only if all are truthful.
1819 Parameters
1820 ----------
1821 skipna : bool, default True
1822 Flag to ignore nan values during truth testing.
1824 Returns
1825 -------
1826 Series or DataFrame
1827 DataFrame or Series of boolean values, where a value is True if all elements
1828 are True within its respective group, False otherwise.
1830 See Also
1831 --------
1832 Series.all : Apply function all to a Series.
1833 DataFrame.all : Apply function all to each row or column of a DataFrame.
1835 Examples
1836 --------
1838 For SeriesGroupBy:
1840 >>> lst = ["a", "a", "b"]
1841 >>> ser = pd.Series([1, 2, 0], index=lst)
1842 >>> ser
1843 a 1
1844 a 2
1845 b 0
1846 dtype: int64
1847 >>> ser.groupby(level=0).all()
1848 a True
1849 b False
1850 dtype: bool
1852 For DataFrameGroupBy:
1854 >>> data = [[1, 0, 3], [1, 5, 6], [7, 8, 9]]
1855 >>> df = pd.DataFrame(
1856 ... data, columns=["a", "b", "c"], index=["ostrich", "penguin", "parrot"]
1857 ... )
1858 >>> df
1859 a b c
1860 ostrich 1 0 3
1861 penguin 1 5 6
1862 parrot 7 8 9
1863 >>> df.groupby(by=["a"]).all()
1864 b c
1865 a
1866 1 False True
1867 7 True True
1868 """
1869 return self._cython_agg_general(
1870 "all",
1871 alt=lambda x: Series(x, copy=False).all(skipna=skipna),
1872 skipna=skipna,
1873 )
1875 @final
1876 def count(self) -> NDFrameT:
1877 """
1878 Compute count of group, excluding missing values.
1880 Returns the number of non-NA/null observations per group.
1882 Returns
1883 -------
1884 Series or DataFrame
1885 Count of values within each group.
1887 See Also
1888 --------
1889 Series.count : Apply function count to a Series.
1890 DataFrame.count : Apply function count to each row or column of a DataFrame.
1892 Examples
1893 --------
1894 For SeriesGroupBy:
1896 >>> lst = ["a", "a", "b"]
1897 >>> ser = pd.Series([1, 2, np.nan], index=lst)
1898 >>> ser
1899 a 1.0
1900 a 2.0
1901 b NaN
1902 dtype: float64
1903 >>> ser.groupby(level=0).count()
1904 a 2
1905 b 0
1906 dtype: int64
1908 For DataFrameGroupBy:
1910 >>> data = [[1, np.nan, 3], [1, np.nan, 6], [7, 8, 9]]
1911 >>> df = pd.DataFrame(
1912 ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
1913 ... )
1914 >>> df
1915 a b c
1916 cow 1 NaN 3
1917 horse 1 NaN 6
1918 bull 7 8.0 9
1919 >>> df.groupby("a").count()
1920 b c
1921 a
1922 1 0 2
1923 7 1 1
1925 For Resampler:
1927 >>> ser = pd.Series(
1928 ... [1, 2, 3, 4],
1929 ... index=pd.DatetimeIndex(
1930 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
1931 ... ),
1932 ... )
1933 >>> ser
1934 2023-01-01 1
1935 2023-01-15 2
1936 2023-02-01 3
1937 2023-02-15 4
1938 dtype: int64
1939 >>> ser.resample("MS").count()
1940 2023-01-01 2
1941 2023-02-01 2
1942 Freq: MS, dtype: int64
1943 """
1944 data = self._get_data_to_aggregate()
1945 ids = self._grouper.ids
1946 ngroups = self._grouper.ngroups
1947 mask = ids != -1
1949 is_series = data.ndim == 1
1951 def hfunc(bvalues: ArrayLike) -> ArrayLike:
1952 # TODO(EA2D): reshape would not be necessary with 2D EAs
1953 if bvalues.ndim == 1:
1954 # EA
1955 masked = mask & ~isna(bvalues).reshape(1, -1)
1956 else:
1957 masked = mask & ~isna(bvalues)
1959 counted = lib.count_level_2d(masked, labels=ids, max_bin=ngroups)
1960 if isinstance(bvalues, BaseMaskedArray):
1961 return IntegerArray(
1962 counted[0], mask=np.zeros(counted.shape[1], dtype=np.bool_)
1963 )
1964 elif isinstance(bvalues, ArrowExtensionArray) and not isinstance(
1965 bvalues.dtype, StringDtype
1966 ):
1967 dtype = pandas_dtype("int64[pyarrow]")
1968 return type(bvalues)._from_sequence(counted[0], dtype=dtype)
1969 if is_series:
1970 assert counted.ndim == 2
1971 assert counted.shape[0] == 1
1972 return counted[0]
1973 return counted
1975 new_mgr = data.grouped_reduce(hfunc)
1976 new_obj = self._wrap_agged_manager(new_mgr)
1977 result = self._wrap_aggregated_output(new_obj)
1979 return result
1981 @final
1982 def mean(
1983 self,
1984 numeric_only: bool = False,
1985 skipna: bool = True,
1986 engine: Literal["cython", "numba"] | None = None,
1987 engine_kwargs: dict[str, bool] | None = None,
1988 ):
1989 """
1990 Compute mean of groups, excluding missing values.
1992 Returns the arithmetic mean for each group. Missing values are
1993 ignored unless the entire group is NA, in which case the result
1994 for that group is NA.
1996 Parameters
1997 ----------
1998 numeric_only : bool, default False
1999 Include only float, int, boolean columns.
2001 .. versionchanged:: 2.0.0
2003 numeric_only no longer accepts ``None`` and defaults to ``False``.
2005 skipna : bool, default True
2006 Exclude NA/null values. If an entire group is NA, the result will be NA.
2008 engine : str, default None
2009 * ``'cython'`` : Runs the operation through C-extensions from cython.
2010 * ``'numba'`` : Runs the operation through JIT compiled code from numba.
2011 * ``None`` : Defaults to ``'cython'`` or globally setting
2012 ``compute.use_numba``
2014 engine_kwargs : dict, default None
2015 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2016 * For ``'numba'`` engine, the engine can accept ``nogil``
2017 and ``parallel`` dictionary keys. The values must either be ``True`` or
2018 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2019 ``{'nogil': False, 'parallel': False}``
2021 Returns
2022 -------
2023 pandas.Series or pandas.DataFrame
2024 Mean of values within each group. Same object type as the caller.
2026 See Also
2027 --------
2028 Series.mean : Apply function mean to a Series.
2029 DataFrame.mean : Apply function mean to each row or column of a DataFrame.
2031 Examples
2032 --------
2033 >>> df = pd.DataFrame(
2034 ... {"A": [1, 1, 2, 1, 2], "B": [np.nan, 2, 3, 4, 5], "C": [1, 2, 1, 1, 2]},
2035 ... columns=["A", "B", "C"],
2036 ... )
2038 Groupby one column and return the mean of the remaining columns in
2039 each group.
2041 >>> df.groupby("A").mean()
2042 B C
2043 A
2044 1 3.0 1.333333
2045 2 4.0 1.500000
2047 Groupby two columns and return the mean of the remaining column.
2049 >>> df.groupby(["A", "B"]).mean()
2050 C
2051 A B
2052 1 2.0 2.0
2053 4.0 1.0
2054 2 3.0 1.0
2055 5.0 2.0
2057 Groupby one column and return the mean of only particular column in
2058 the group.
2060 >>> df.groupby("A")["B"].mean()
2061 A
2062 1 3.0
2063 2 4.0
2064 Name: B, dtype: float64
2065 """
2067 if maybe_use_numba(engine):
2068 from pandas.core._numba.kernels import grouped_mean
2070 return self._numba_agg_general(
2071 grouped_mean,
2072 executor.float_dtype_mapping,
2073 engine_kwargs,
2074 min_periods=0,
2075 skipna=skipna,
2076 )
2077 else:
2078 return self._cython_agg_general(
2079 "mean",
2080 alt=lambda x: Series(x, copy=False).mean(
2081 numeric_only=numeric_only, skipna=skipna
2082 ),
2083 numeric_only=numeric_only,
2084 skipna=skipna,
2085 )
2087 @final
2088 def median(self, numeric_only: bool = False, skipna: bool = True) -> NDFrameT:
2089 """
2090 Compute median of groups, excluding missing values.
2092 For multiple groupings, the result index will be a MultiIndex
2094 Parameters
2095 ----------
2096 numeric_only : bool, default False
2097 Include only float, int, boolean columns.
2099 .. versionchanged:: 2.0.0
2101 numeric_only no longer accepts ``None`` and defaults to False.
2103 skipna : bool, default True
2104 Exclude NA/null values. If an entire group is NA, the result will be NA.
2106 .. versionadded:: 3.0.0
2108 Returns
2109 -------
2110 Series or DataFrame
2111 Median of values within each group.
2113 See Also
2114 --------
2115 Series.median : Apply function median to a Series.
2116 DataFrame.median : Apply function median to each row or column of a DataFrame.
2118 Examples
2119 --------
2120 For SeriesGroupBy:
2122 >>> lst = ["a", "a", "a", "b", "b", "b"]
2123 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
2124 >>> ser
2125 a 7
2126 a 2
2127 a 8
2128 b 4
2129 b 3
2130 b 3
2131 dtype: int64
2132 >>> ser.groupby(level=0).median()
2133 a 7.0
2134 b 3.0
2135 dtype: float64
2137 For DataFrameGroupBy:
2139 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
2140 >>> df = pd.DataFrame(
2141 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
2142 ... )
2143 >>> df
2144 a b
2145 dog 1 1
2146 dog 3 4
2147 dog 5 8
2148 mouse 7 4
2149 mouse 7 4
2150 mouse 8 2
2151 mouse 3 1
2152 >>> df.groupby(level=0).median()
2153 a b
2154 dog 3.0 4.0
2155 mouse 7.0 3.0
2157 For Resampler:
2159 >>> ser = pd.Series(
2160 ... [1, 2, 3, 3, 4, 5],
2161 ... index=pd.DatetimeIndex(
2162 ... [
2163 ... "2023-01-01",
2164 ... "2023-01-10",
2165 ... "2023-01-15",
2166 ... "2023-02-01",
2167 ... "2023-02-10",
2168 ... "2023-02-15",
2169 ... ]
2170 ... ),
2171 ... )
2172 >>> ser.resample("MS").median()
2173 2023-01-01 2.0
2174 2023-02-01 4.0
2175 Freq: MS, dtype: float64
2176 """
2177 return self._cython_agg_general(
2178 "median",
2179 alt=lambda x: Series(x, copy=False).median(
2180 numeric_only=numeric_only, skipna=skipna
2181 ),
2182 numeric_only=numeric_only,
2183 skipna=skipna,
2184 )
2186 @final
2187 def std(
2188 self,
2189 ddof: int = 1,
2190 engine: Literal["cython", "numba"] | None = None,
2191 engine_kwargs: dict[str, bool] | None = None,
2192 numeric_only: bool = False,
2193 skipna: bool = True,
2194 ):
2195 """
2196 Compute standard deviation of groups, excluding missing values.
2198 For multiple groupings, the result index will be a MultiIndex.
2200 Parameters
2201 ----------
2202 ddof : int, default 1
2203 Delta Degrees of Freedom. The divisor used in calculations is ``N - ddof``,
2204 where ``N`` represents the number of elements.
2206 engine : str, default None
2207 * ``'cython'`` : Runs the operation through C-extensions from cython.
2208 * ``'numba'`` : Runs the operation through JIT compiled code from numba.
2209 * ``None`` : Defaults to ``'cython'`` or globally setting
2210 ``compute.use_numba``
2212 engine_kwargs : dict, default None
2213 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2214 * For ``'numba'`` engine, the engine can accept ``nogil``
2215 and ``parallel`` dictionary keys. The values must either be ``True`` or
2216 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2217 ``{'nogil': False, 'parallel': False}``
2219 numeric_only : bool, default False
2220 Include only `float`, `int` or `boolean` data.
2222 .. versionchanged:: 2.0.0
2224 numeric_only now defaults to ``False``.
2226 skipna : bool, default True
2227 Exclude NA/null values. If an entire group is NA, the result will be NA.
2229 .. versionadded:: 3.0.0
2231 Returns
2232 -------
2233 Series or DataFrame
2234 Standard deviation of values within each group.
2236 See Also
2237 --------
2238 Series.std : Apply function std to a Series.
2239 DataFrame.std : Apply function std to each row or column of a DataFrame.
2241 Notes
2242 -----
2243 To have the same behaviour as ``numpy.std``, use ``ddof=0`` (instead of
2244 the default ``ddof=1``) and ``skipna=False``.
2246 Examples
2247 --------
2248 For SeriesGroupBy:
2250 >>> lst = ["a", "a", "a", "b", "b", "b"]
2251 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
2252 >>> ser
2253 a 7
2254 a 2
2255 a 8
2256 b 4
2257 b 3
2258 b 3
2259 dtype: int64
2260 >>> ser.groupby(level=0).std()
2261 a 3.21455
2262 b 0.57735
2263 dtype: float64
2265 For DataFrameGroupBy:
2267 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
2268 >>> df = pd.DataFrame(
2269 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
2270 ... )
2271 >>> df
2272 a b
2273 dog 1 1
2274 dog 3 4
2275 dog 5 8
2276 mouse 7 4
2277 mouse 7 4
2278 mouse 8 2
2279 mouse 3 1
2280 >>> df.groupby(level=0).std()
2281 a b
2282 dog 2.000000 3.511885
2283 mouse 2.217356 1.500000
2284 """
2285 if maybe_use_numba(engine):
2286 from pandas.core._numba.kernels import grouped_var
2288 return np.sqrt(
2289 self._numba_agg_general(
2290 grouped_var,
2291 executor.float_dtype_mapping,
2292 engine_kwargs,
2293 min_periods=0,
2294 ddof=ddof,
2295 skipna=skipna,
2296 )
2297 )
2298 else:
2299 return self._cython_agg_general(
2300 "std",
2301 alt=lambda x: Series(x, copy=False).std(ddof=ddof, skipna=skipna),
2302 numeric_only=numeric_only,
2303 ddof=ddof,
2304 skipna=skipna,
2305 )
2307 @final
2308 def var(
2309 self,
2310 ddof: int = 1,
2311 engine: Literal["cython", "numba"] | None = None,
2312 engine_kwargs: dict[str, bool] | None = None,
2313 numeric_only: bool = False,
2314 skipna: bool = True,
2315 ):
2316 """
2317 Compute variance of groups, excluding missing values.
2319 For multiple groupings, the result index will be a MultiIndex.
2321 Parameters
2322 ----------
2323 ddof : int, default 1
2324 Degrees of freedom.
2326 engine : str, default None
2327 * ``'cython'`` : Runs the operation through C-extensions from cython.
2328 * ``'numba'`` : Runs the operation through JIT compiled code from numba.
2329 * ``None`` : Defaults to ``'cython'`` or globally setting
2330 ``compute.use_numba``
2332 engine_kwargs : dict, default None
2333 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2334 * For ``'numba'`` engine, the engine can accept ``nogil``
2335 and ``parallel`` dictionary keys. The values must either be ``True`` or
2336 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2337 ``{'nogil': False, 'parallel': False}``
2339 numeric_only : bool, default False
2340 Include only `float`, `int` or `boolean` data.
2342 .. versionchanged:: 2.0.0
2344 numeric_only now defaults to ``False``.
2346 skipna : bool, default True
2347 Exclude NA/null values. If an entire group is NA, the result will be NA.
2349 .. versionadded:: 3.0.0
2351 Returns
2352 -------
2353 Series or DataFrame
2354 Variance of values within each group.
2356 See Also
2357 --------
2358 Series.var : Apply function var to a Series.
2359 DataFrame.var : Apply function var to each row or column of a DataFrame.
2361 Examples
2362 --------
2363 For SeriesGroupBy:
2365 >>> lst = ["a", "a", "a", "b", "b", "b"]
2366 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
2367 >>> ser
2368 a 7
2369 a 2
2370 a 8
2371 b 4
2372 b 3
2373 b 3
2374 dtype: int64
2375 >>> ser.groupby(level=0).var()
2376 a 10.333333
2377 b 0.333333
2378 dtype: float64
2380 For DataFrameGroupBy:
2382 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
2383 >>> df = pd.DataFrame(
2384 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
2385 ... )
2386 >>> df
2387 a b
2388 dog 1 1
2389 dog 3 4
2390 dog 5 8
2391 mouse 7 4
2392 mouse 7 4
2393 mouse 8 2
2394 mouse 3 1
2395 >>> df.groupby(level=0).var()
2396 a b
2397 dog 4.000000 12.333333
2398 mouse 4.916667 2.250000
2399 """
2400 if maybe_use_numba(engine):
2401 from pandas.core._numba.kernels import grouped_var
2403 return self._numba_agg_general(
2404 grouped_var,
2405 executor.float_dtype_mapping,
2406 engine_kwargs,
2407 min_periods=0,
2408 ddof=ddof,
2409 skipna=skipna,
2410 )
2411 else:
2412 return self._cython_agg_general(
2413 "var",
2414 alt=lambda x: Series(x, copy=False).var(ddof=ddof, skipna=skipna),
2415 numeric_only=numeric_only,
2416 ddof=ddof,
2417 skipna=skipna,
2418 )
2420 @final
2421 def _value_counts(
2422 self,
2423 subset: Sequence[Hashable] | None = None,
2424 normalize: bool = False,
2425 sort: bool = True,
2426 ascending: bool = False,
2427 dropna: bool = True,
2428 ) -> DataFrame | Series:
2429 """
2430 Shared implementation of value_counts for SeriesGroupBy and DataFrameGroupBy.
2432 SeriesGroupBy additionally supports a bins argument. See the docstring of
2433 DataFrameGroupBy.value_counts for a description of arguments.
2434 """
2435 name = "proportion" if normalize else "count"
2437 df = self.obj
2438 obj = self._obj_with_exclusions
2440 in_axis_names = {
2441 grouping.name for grouping in self._grouper.groupings if grouping.in_axis
2442 }
2443 if isinstance(obj, Series):
2444 _name = obj.name
2445 keys: Iterable[Series] = [] if _name in in_axis_names else [obj]
2446 else:
2447 unique_cols = set(obj.columns)
2448 if subset is not None:
2449 subsetted = set(subset)
2450 clashing = subsetted & set(in_axis_names)
2451 if clashing:
2452 raise ValueError(
2453 f"Keys {clashing} in subset cannot be in "
2454 "the groupby column keys."
2455 )
2456 doesnt_exist = subsetted - unique_cols
2457 if doesnt_exist:
2458 raise ValueError(
2459 f"Keys {doesnt_exist} in subset do not exist in the DataFrame."
2460 )
2461 else:
2462 subsetted = unique_cols
2464 keys = (
2465 # Can't use .values because the column label needs to be preserved
2466 obj.iloc[:, idx]
2467 for idx, _name in enumerate(obj.columns)
2468 if _name not in in_axis_names and _name in subsetted
2469 )
2471 groupings = list(self._grouper.groupings)
2472 for key in keys:
2473 grouper, _, _ = get_grouper(
2474 df,
2475 key=key,
2476 sort=False,
2477 observed=False,
2478 dropna=dropna,
2479 )
2480 groupings += list(grouper.groupings)
2482 # Take the size of the overall columns
2483 gb = df.groupby(
2484 groupings,
2485 sort=False,
2486 observed=self.observed,
2487 dropna=self.dropna,
2488 )
2489 result_series = cast("Series", gb.size())
2490 result_series.name = name
2492 if sort:
2493 # Sort by the values
2494 result_series = result_series.sort_values(
2495 ascending=ascending, kind="stable"
2496 )
2497 if self.sort:
2498 # Sort by the groupings
2499 names = result_series.index.names
2500 # GH#55951 - Temporarily replace names in case they are integers
2501 result_series.index.names = range(len(names))
2502 index_level = range(len(self._grouper.groupings))
2503 result_series = result_series.sort_index(
2504 level=index_level, sort_remaining=False
2505 )
2506 result_series.index.names = names
2508 if normalize:
2509 # Normalize the results by dividing by the original group sizes.
2510 # We are guaranteed to have the first N levels be the
2511 # user-requested grouping.
2512 levels = list(
2513 range(len(self._grouper.groupings), result_series.index.nlevels)
2514 )
2515 indexed_group_size = result_series.groupby(
2516 result_series.index.droplevel(levels),
2517 sort=self.sort,
2518 dropna=self.dropna,
2519 # GH#43999 - deprecation of observed=False
2520 observed=False,
2521 ).transform("sum")
2522 result_series /= indexed_group_size
2524 # Handle groups of non-observed categories
2525 result_series = result_series.fillna(0.0)
2527 result: Series | DataFrame
2528 if self.as_index:
2529 result = result_series
2530 else:
2531 # Convert to frame
2532 index = result_series.index
2533 columns = com.fill_missing_names(index.names)
2534 if name in columns:
2535 raise ValueError(f"Column label '{name}' is duplicate of result column")
2536 result_series.name = name
2537 result_series.index = index.set_names(range(len(columns)))
2538 result_frame = result_series.reset_index()
2539 orig_dtype = self._grouper.groupings[0].obj.columns.dtype # type: ignore[union-attr]
2540 cols = Index(columns, dtype=orig_dtype).insert(len(columns), name)
2541 result_frame.columns = cols
2542 result = result_frame
2543 return result.__finalize__(self.obj, method="value_counts")
2545 @final
2546 def sem(
2547 self, ddof: int = 1, numeric_only: bool = False, skipna: bool = True
2548 ) -> NDFrameT:
2549 """
2550 Compute standard error of the mean of groups, excluding missing values.
2552 For multiple groupings, the result index will be a MultiIndex.
2554 Parameters
2555 ----------
2556 ddof : int, default 1
2557 Degrees of freedom.
2559 numeric_only : bool, default False
2560 Include only `float`, `int` or `boolean` data.
2562 .. versionchanged:: 2.0.0
2564 numeric_only now defaults to ``False``.
2566 skipna : bool, default True
2567 Exclude NA/null values. If an entire group is NA, the result will be NA.
2569 .. versionadded:: 3.0.0
2571 Returns
2572 -------
2573 Series or DataFrame
2574 Standard error of the mean of values within each group.
2576 See Also
2577 --------
2578 DataFrame.sem : Return unbiased standard error of the mean over requested axis.
2579 Series.sem : Return unbiased standard error of the mean over requested axis.
2581 Examples
2582 --------
2583 For SeriesGroupBy:
2585 >>> lst = ["a", "a", "b", "b"]
2586 >>> ser = pd.Series([5, 10, 8, 14], index=lst)
2587 >>> ser
2588 a 5
2589 a 10
2590 b 8
2591 b 14
2592 dtype: int64
2593 >>> ser.groupby(level=0).sem()
2594 a 2.5
2595 b 3.0
2596 dtype: float64
2598 For DataFrameGroupBy:
2600 >>> data = [[1, 12, 11], [1, 15, 2], [2, 5, 8], [2, 6, 12]]
2601 >>> df = pd.DataFrame(
2602 ... data,
2603 ... columns=["a", "b", "c"],
2604 ... index=["tuna", "salmon", "catfish", "goldfish"],
2605 ... )
2606 >>> df
2607 a b c
2608 tuna 1 12 11
2609 salmon 1 15 2
2610 catfish 2 5 8
2611 goldfish 2 6 12
2612 >>> df.groupby("a").sem()
2613 b c
2614 a
2615 1 1.5 4.5
2616 2 0.5 2.0
2618 For Resampler:
2620 >>> ser = pd.Series(
2621 ... [1, 3, 2, 4, 3, 8],
2622 ... index=pd.DatetimeIndex(
2623 ... [
2624 ... "2023-01-01",
2625 ... "2023-01-10",
2626 ... "2023-01-15",
2627 ... "2023-02-01",
2628 ... "2023-02-10",
2629 ... "2023-02-15",
2630 ... ]
2631 ... ),
2632 ... )
2633 >>> ser.resample("MS").sem()
2634 2023-01-01 0.577350
2635 2023-02-01 1.527525
2636 Freq: MS, dtype: float64
2637 """
2638 if numeric_only and self.obj.ndim == 1 and not is_numeric_dtype(self.obj.dtype):
2639 raise TypeError(
2640 f"{type(self).__name__}.sem called with "
2641 f"numeric_only={numeric_only} and dtype {self.obj.dtype}"
2642 )
2643 return self._cython_agg_general(
2644 "sem",
2645 alt=lambda x: Series(x, copy=False).sem(ddof=ddof, skipna=skipna),
2646 numeric_only=numeric_only,
2647 ddof=ddof,
2648 skipna=skipna,
2649 )
2651 @final
2652 def size(self) -> DataFrame | Series:
2653 """
2654 Compute group sizes.
2656 Returns the number of rows in each group. This is useful for
2657 understanding the distribution of data across groups.
2659 Returns
2660 -------
2661 DataFrame or Series
2662 Number of rows in each group as a Series if as_index is True
2663 or a DataFrame if as_index is False.
2665 See Also
2666 --------
2667 Series.size : Apply function size to a Series.
2668 DataFrame.size : Apply function size to each row or column of a DataFrame.
2670 Examples
2671 --------
2673 For SeriesGroupBy:
2675 >>> lst = ["a", "a", "b"]
2676 >>> ser = pd.Series([1, 2, 3], index=lst)
2677 >>> ser
2678 a 1
2679 a 2
2680 b 3
2681 dtype: int64
2682 >>> ser.groupby(level=0).size()
2683 a 2
2684 b 1
2685 dtype: int64
2687 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
2688 >>> df = pd.DataFrame(
2689 ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
2690 ... )
2691 >>> df
2692 a b c
2693 owl 1 2 3
2694 toucan 1 5 6
2695 eagle 7 8 9
2696 >>> df.groupby("a").size()
2697 a
2698 1 2
2699 7 1
2700 dtype: int64
2702 For Resampler:
2704 >>> ser = pd.Series(
2705 ... [1, 2, 3],
2706 ... index=pd.DatetimeIndex(["2023-01-01", "2023-01-15", "2023-02-01"]),
2707 ... )
2708 >>> ser
2709 2023-01-01 1
2710 2023-01-15 2
2711 2023-02-01 3
2712 dtype: int64
2713 >>> ser.resample("MS").size()
2714 2023-01-01 2
2715 2023-02-01 1
2716 Freq: MS, dtype: int64
2717 """
2718 result = self._grouper.size()
2719 dtype_backend: Literal["pyarrow", "numpy_nullable"] | None = None
2720 if isinstance(self.obj, Series):
2721 if isinstance(self.obj.array, ArrowExtensionArray):
2722 if isinstance(self.obj.array, ArrowStringArray):
2723 if self.obj.array.dtype.na_value is np.nan:
2724 dtype_backend = None
2725 else:
2726 dtype_backend = "numpy_nullable"
2727 else:
2728 dtype_backend = "pyarrow"
2729 elif isinstance(self.obj.array, BaseMaskedArray):
2730 dtype_backend = "numpy_nullable"
2731 # TODO: For DataFrames what if columns are mixed arrow/numpy/masked?
2733 # GH28330 preserve subclassed Series/DataFrames through calls
2734 if isinstance(self.obj, Series):
2735 result = self._obj_1d_constructor(result, name=self.obj.name)
2736 else:
2737 result = self._obj_1d_constructor(result)
2739 if dtype_backend is not None:
2740 result = result.convert_dtypes(dtype_backend=dtype_backend)
2742 if not self.as_index:
2743 result = result.rename("size").reset_index()
2744 return result
2746 @final
2747 def sum(
2748 self,
2749 numeric_only: bool = False,
2750 min_count: int = 0,
2751 skipna: bool = True,
2752 engine: Literal["cython", "numba"] | None = None,
2753 engine_kwargs: dict[str, bool] | None = None,
2754 ):
2755 """
2756 Compute sum of group values.
2758 Computes the sum for each group. Null values are excluded by
2759 default unless ``skipna`` is set to ``False``.
2761 Parameters
2762 ----------
2763 numeric_only : bool, default False
2764 Include only float, int, boolean columns.
2766 .. versionchanged:: 2.0.0
2768 numeric_only no longer accepts ``None``.
2770 min_count : int, default 0
2771 The required number of valid values to perform the operation. If fewer
2772 than ``min_count`` non-NA values are present the result will be NA.
2774 skipna : bool, default True
2775 Exclude NA/null values. If the entire group is NA and ``skipna`` is
2776 ``True``, the result will be NA.
2778 .. versionchanged:: 3.0.0
2780 engine : str, default None None
2781 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
2782 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
2783 Only available when ``raw`` is set to ``True``.
2784 * ``None`` : Defaults to ``'cython'`` or globally setting
2785 ``compute.use_numba``
2787 engine_kwargs : dict, default None None
2788 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2789 * For ``'numba'`` engine, the engine can accept ``nogil``
2790 and ``parallel`` dictionary keys. The values must either be ``True`` or
2791 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2792 ``{'nogil': False, 'parallel': False}`` and will be
2793 applied to both the ``func`` and the ``apply`` groupby aggregation.
2795 Returns
2796 -------
2797 Series or DataFrame
2798 Computed sum of values within each group.
2800 See Also
2801 --------
2802 Series.sum : Return the sum of the values over the requested axis.
2803 DataFrame.sum : Return the sum of the values over the requested axis.
2805 Examples
2806 --------
2807 For SeriesGroupBy:
2809 >>> lst = ["a", "a", "b", "b"]
2810 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
2811 >>> ser
2812 a 1
2813 a 2
2814 b 3
2815 b 4
2816 dtype: int64
2817 >>> ser.groupby(level=0).sum()
2818 a 3
2819 b 7
2820 dtype: int64
2822 For DataFrameGroupBy:
2824 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
2825 >>> df = pd.DataFrame(
2826 ... data,
2827 ... columns=["a", "b", "c"],
2828 ... index=["tiger", "leopard", "cheetah", "lion"],
2829 ... )
2830 >>> df
2831 a b c
2832 tiger 1 8 2
2833 leopard 1 2 5
2834 cheetah 2 5 8
2835 lion 2 6 9
2836 >>> df.groupby("a").sum()
2837 b c
2838 a
2839 1 10 7
2840 2 11 17
2841 """
2842 if maybe_use_numba(engine):
2843 from pandas.core._numba.kernels import grouped_sum
2845 return self._numba_agg_general(
2846 grouped_sum,
2847 executor.default_dtype_mapping,
2848 engine_kwargs,
2849 min_periods=min_count,
2850 skipna=skipna,
2851 )
2852 else:
2854 def sum_compat(obj: NDFrameT):
2855 # GH#18588: see min_compat below
2856 return obj.sum(skipna=skipna)
2858 return self._cython_agg_general(
2859 "sum",
2860 alt=sum_compat,
2861 numeric_only=numeric_only,
2862 min_count=min_count,
2863 skipna=skipna,
2864 )
2866 @final
2867 def prod(
2868 self, numeric_only: bool = False, min_count: int = 0, skipna: bool = True
2869 ) -> NDFrameT:
2870 """
2871 Compute prod of group values.
2873 This method computes the product of all values within each group,
2874 returning a result for each group.
2876 Parameters
2877 ----------
2878 numeric_only : bool, default False
2879 Include only float, int, boolean columns.
2881 .. versionchanged:: 2.0.0
2883 numeric_only no longer accepts ``None``.
2885 min_count : int, default 0
2886 The required number of valid values to perform the operation. If fewer
2887 than ``min_count`` non-NA values are present the result will be NA.
2889 skipna : bool, default True
2890 Exclude NA/null values. If an entire group is NA, the result will be NA.
2892 .. versionadded:: 3.0.0
2894 Returns
2895 -------
2896 Series or DataFrame
2897 Computed prod of values within each group.
2899 See Also
2900 --------
2901 Series.prod : Return the product of the values over the requested axis.
2902 DataFrame.prod : Return the product of the values over the requested axis.
2904 Examples
2905 --------
2906 For SeriesGroupBy:
2908 >>> lst = ["a", "a", "b", "b"]
2909 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
2910 >>> ser
2911 a 1
2912 a 2
2913 b 3
2914 b 4
2915 dtype: int64
2916 >>> ser.groupby(level=0).prod()
2917 a 2
2918 b 12
2919 dtype: int64
2921 For DataFrameGroupBy:
2923 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
2924 >>> df = pd.DataFrame(
2925 ... data,
2926 ... columns=["a", "b", "c"],
2927 ... index=["tiger", "leopard", "cheetah", "lion"],
2928 ... )
2929 >>> df
2930 a b c
2931 tiger 1 8 2
2932 leopard 1 2 5
2933 cheetah 2 5 8
2934 lion 2 6 9
2935 >>> df.groupby("a").prod()
2936 b c
2937 a
2938 1 16 10
2939 2 30 72
2940 """
2942 def prod_compat(obj: NDFrameT):
2943 # GH#18588: see min_compat below
2944 return obj.prod(skipna=skipna)
2946 return self._cython_agg_general(
2947 "prod",
2948 alt=prod_compat,
2949 numeric_only=numeric_only,
2950 min_count=min_count,
2951 skipna=skipna,
2952 )
2954 @final
2955 def min(
2956 self,
2957 numeric_only: bool = False,
2958 min_count: int = -1,
2959 skipna: bool = True,
2960 engine: Literal["cython", "numba"] | None = None,
2961 engine_kwargs: dict[str, bool] | None = None,
2962 ):
2963 """
2964 Compute min of group values.
2966 Returns the minimum value for each group. Missing values are
2967 excluded by default but this behavior can be controlled with
2968 the ``skipna`` parameter.
2970 Parameters
2971 ----------
2972 numeric_only : bool, default False
2973 Include only float, int, boolean columns.
2975 .. versionchanged:: 2.0.0
2977 numeric_only no longer accepts ``None``.
2979 min_count : int, default -1
2980 The required number of valid values to perform the operation. If fewer
2981 than ``min_count`` non-NA values are present the result will be NA.
2983 skipna : bool, default True
2984 Exclude NA/null values. If the entire group is NA and ``skipna`` is
2985 ``True``, the result will be NA.
2987 .. versionchanged:: 3.0.0
2989 engine : str, default None None
2990 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
2991 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
2992 Only available when ``raw`` is set to ``True``.
2993 * ``None`` : Defaults to ``'cython'`` or globally setting
2994 ``compute.use_numba``
2996 engine_kwargs : dict, default None None
2997 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2998 * For ``'numba'`` engine, the engine can accept ``nogil``
2999 and ``parallel`` dictionary keys. The values must either be ``True`` or
3000 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
3001 ``{'nogil': False, 'parallel': False}`` and will be
3002 applied to both the ``func`` and the ``apply`` groupby aggregation.
3004 Returns
3005 -------
3006 Series or DataFrame
3007 Computed min of values within each group.
3009 See Also
3010 --------
3011 Series.min : Return the minimum of the values over the requested axis.
3012 DataFrame.min : Return the minimum of the values over the requested axis.
3014 Examples
3015 --------
3016 For SeriesGroupBy:
3018 >>> lst = ["a", "a", "b", "b"]
3019 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
3020 >>> ser
3021 a 1
3022 a 2
3023 b 3
3024 b 4
3025 dtype: int64
3026 >>> ser.groupby(level=0).min()
3027 a 1
3028 b 3
3029 dtype: int64
3031 For DataFrameGroupBy:
3033 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
3034 >>> df = pd.DataFrame(
3035 ... data,
3036 ... columns=["a", "b", "c"],
3037 ... index=["tiger", "leopard", "cheetah", "lion"],
3038 ... )
3039 >>> df
3040 a b c
3041 tiger 1 8 2
3042 leopard 1 2 5
3043 cheetah 2 5 8
3044 lion 2 6 9
3045 >>> df.groupby("a").min()
3046 b c
3047 a
3048 1 2 2
3049 2 5 8
3050 """
3051 if maybe_use_numba(engine):
3052 from pandas.core._numba.kernels import grouped_min_max
3054 return self._numba_agg_general(
3055 grouped_min_max,
3056 executor.identity_dtype_mapping,
3057 engine_kwargs,
3058 min_periods=min_count,
3059 is_max=False,
3060 skipna=skipna,
3061 )
3062 else:
3064 def min_compat(obj: NDFrameT):
3065 # GH#18588: object/string dtypes have no cython group_min_max
3066 # and reduce through this alt instead, so it has to apply
3067 # skipna itself; np.min would always skip.
3068 return obj.min(skipna=skipna)
3070 return self._cython_agg_general(
3071 "min",
3072 alt=min_compat,
3073 numeric_only=numeric_only,
3074 min_count=min_count,
3075 skipna=skipna,
3076 )
3078 @final
3079 def max(
3080 self,
3081 numeric_only: bool = False,
3082 min_count: int = -1,
3083 skipna: bool = True,
3084 engine: Literal["cython", "numba"] | None = None,
3085 engine_kwargs: dict[str, bool] | None = None,
3086 ):
3087 """
3088 Compute max of group values.
3090 Returns the maximum value for each group. Missing values are
3091 excluded by default but this behavior can be controlled with
3092 the ``skipna`` parameter.
3094 Parameters
3095 ----------
3096 numeric_only : bool, default False
3097 Include only float, int, boolean columns.
3099 .. versionchanged:: 2.0.0
3101 numeric_only no longer accepts ``None``.
3103 min_count : int, default -1
3104 The required number of valid values to perform the operation. If fewer
3105 than ``min_count`` non-NA values are present the result will be NA.
3107 skipna : bool, default True
3108 Exclude NA/null values. If the entire group is NA and ``skipna`` is
3109 ``True``, the result will be NA.
3111 .. versionchanged:: 3.0.0
3113 engine : str, default None None
3114 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
3115 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
3116 Only available when ``raw`` is set to ``True``.
3117 * ``None`` : Defaults to ``'cython'`` or globally setting
3118 ``compute.use_numba``
3120 engine_kwargs : dict, default None None
3121 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
3122 * For ``'numba'`` engine, the engine can accept ``nogil``
3123 and ``parallel`` dictionary keys. The values must either be ``True`` or
3124 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
3125 ``{'nogil': False, 'parallel': False}`` and will be
3126 applied to both the ``func`` and the ``apply`` groupby aggregation.
3128 Returns
3129 -------
3130 Series or DataFrame
3131 Computed max of values within each group.
3133 See Also
3134 --------
3135 Series.max : Return the maximum of the values over the requested axis.
3136 DataFrame.max : Return the maximum of the values over the requested axis.
3138 Examples
3139 --------
3140 For SeriesGroupBy:
3142 >>> lst = ["a", "a", "b", "b"]
3143 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
3144 >>> ser
3145 a 1
3146 a 2
3147 b 3
3148 b 4
3149 dtype: int64
3150 >>> ser.groupby(level=0).max()
3151 a 2
3152 b 4
3153 dtype: int64
3155 For DataFrameGroupBy:
3157 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
3158 >>> df = pd.DataFrame(
3159 ... data,
3160 ... columns=["a", "b", "c"],
3161 ... index=["tiger", "leopard", "cheetah", "lion"],
3162 ... )
3163 >>> df
3164 a b c
3165 tiger 1 8 2
3166 leopard 1 2 5
3167 cheetah 2 5 8
3168 lion 2 6 9
3169 >>> df.groupby("a").max()
3170 b c
3171 a
3172 1 8 5
3173 2 6 9
3174 """
3175 if maybe_use_numba(engine):
3176 from pandas.core._numba.kernels import grouped_min_max
3178 return self._numba_agg_general(
3179 grouped_min_max,
3180 executor.identity_dtype_mapping,
3181 engine_kwargs,
3182 min_periods=min_count,
3183 is_max=True,
3184 skipna=skipna,
3185 )
3186 else:
3188 def max_compat(obj: NDFrameT):
3189 # GH#18588: see min_compat above
3190 return obj.max(skipna=skipna)
3192 return self._cython_agg_general(
3193 "max",
3194 alt=max_compat,
3195 numeric_only=numeric_only,
3196 min_count=min_count,
3197 skipna=skipna,
3198 )
3200 @final
3201 def first(
3202 self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True
3203 ) -> NDFrameT:
3204 """
3205 Compute the first entry of each column within each group.
3207 NA values are skipped by default, so the result is the first non-NA
3208 value per column — which may differ from the first row of the group.
3209 Pass ``skipna=False`` to keep NAs.
3211 Parameters
3212 ----------
3213 numeric_only : bool, default False
3214 Include only float, int, boolean columns.
3215 min_count : int, default -1
3216 The required number of valid values to perform the operation. If fewer
3217 than ``min_count`` valid values are present the result will be NA.
3218 skipna : bool, default True
3219 Exclude NA values. If an entire group is NA, the result will be NA.
3221 .. versionadded:: 2.2.1
3223 Returns
3224 -------
3225 Series or DataFrame
3226 First values within each group.
3228 See Also
3229 --------
3230 DataFrame.groupby : Apply a function groupby to each row or column of a
3231 DataFrame.
3232 api.typing.DataFrameGroupBy.last : Compute the last entry of each
3233 column within each group.
3234 api.typing.DataFrameGroupBy.nth : Take the nth row from each group.
3236 Examples
3237 --------
3238 >>> df = pd.DataFrame(
3239 ... dict(
3240 ... A=[1, 1, 3],
3241 ... B=[None, 5, 6],
3242 ... C=[1, 2, 3],
3243 ... D=["3/11/2000", "3/12/2000", "3/13/2000"],
3244 ... )
3245 ... )
3246 >>> df["D"] = pd.to_datetime(df["D"])
3247 >>> df.groupby("A").first()
3248 B C D
3249 A
3250 1 5.0 1 2000-03-11
3251 3 6.0 3 2000-03-13
3252 >>> df.groupby("A").first(min_count=2)
3253 B C D
3254 A
3255 1 NaN 1.0 2000-03-11
3256 3 NaN NaN NaT
3257 >>> df.groupby("A").first(numeric_only=True)
3258 B C
3259 A
3260 1 5.0 1
3261 3 6.0 3
3262 """
3264 def first_compat(obj: NDFrameT):
3265 def first(x: Series):
3266 """Helper function for first item that isn't NA."""
3267 arr = x.array[notna(x.array)]
3268 if not len(arr):
3269 return x.array.dtype.na_value
3270 return arr[0]
3272 if isinstance(obj, DataFrame):
3273 return obj.apply(first)
3274 elif isinstance(obj, Series):
3275 return first(obj)
3276 else: # pragma: no cover
3277 raise TypeError(type(obj))
3279 return self._cython_agg_general(
3280 "first",
3281 alt=first_compat,
3282 numeric_only=numeric_only,
3283 min_count=min_count,
3284 skipna=skipna,
3285 )
3287 @final
3288 def last(
3289 self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True
3290 ) -> NDFrameT:
3291 """
3292 Compute the last entry of each column within each group.
3294 NA values are skipped by default, so the result is the last non-NA
3295 value per column — which may differ from the last row of the group.
3296 Pass ``skipna=False`` to keep NAs.
3298 Parameters
3299 ----------
3300 numeric_only : bool, default False
3301 Include only float, int, boolean columns. If None, will attempt to use
3302 everything, then use only numeric data.
3303 min_count : int, default -1
3304 The required number of valid values to perform the operation. If fewer
3305 than ``min_count`` valid values are present the result will be NA.
3306 skipna : bool, default True
3307 Exclude NA values. If an entire group is NA, the result will be NA.
3309 .. versionadded:: 2.2.1
3311 Returns
3312 -------
3313 Series or DataFrame
3314 Last values within each group.
3316 See Also
3317 --------
3318 DataFrame.groupby : Apply a function groupby to each row or column of a
3319 DataFrame.
3320 api.typing.DataFrameGroupBy.first : Compute the first entry of each
3321 column within each group.
3322 api.typing.DataFrameGroupBy.nth : Take the nth row from each group.
3324 Examples
3325 --------
3326 >>> df = pd.DataFrame(dict(A=[1, 1, 3], B=[5, None, 6], C=[1, 2, 3]))
3327 >>> df.groupby("A").last()
3328 B C
3329 A
3330 1 5.0 2
3331 3 6.0 3
3332 """
3334 def last_compat(obj: NDFrameT):
3335 def last(x: Series):
3336 """Helper function for last item that isn't NA."""
3337 arr = x.array[notna(x.array)]
3338 if not len(arr):
3339 return x.array.dtype.na_value
3340 return arr[-1]
3342 if isinstance(obj, DataFrame):
3343 return obj.apply(last)
3344 elif isinstance(obj, Series):
3345 return last(obj)
3346 else: # pragma: no cover
3347 raise TypeError(type(obj))
3349 return self._cython_agg_general(
3350 "last",
3351 alt=last_compat,
3352 numeric_only=numeric_only,
3353 min_count=min_count,
3354 skipna=skipna,
3355 )
3357 @final
3358 def ohlc(self) -> DataFrame:
3359 """
3360 Compute open, high, low and close values of a group, excluding missing values.
3362 For multiple groupings, the result index will be a MultiIndex
3364 Returns
3365 -------
3366 DataFrame
3367 Open, high, low and close values within each group.
3369 See Also
3370 --------
3371 DataFrame.agg : Aggregate using one or more operations over the specified axis.
3372 DataFrame.resample : Resample time-series data.
3373 DataFrame.groupby : Group DataFrame using a mapper or by a Series of columns.
3375 Examples
3376 --------
3378 For SeriesGroupBy:
3380 >>> lst = [
3381 ... "SPX",
3382 ... "CAC",
3383 ... "SPX",
3384 ... "CAC",
3385 ... "SPX",
3386 ... "CAC",
3387 ... "SPX",
3388 ... "CAC",
3389 ... ]
3390 >>> ser = pd.Series([3.4, 9.0, 7.2, 5.2, 8.8, 9.4, 0.1, 0.5], index=lst)
3391 >>> ser
3392 SPX 3.4
3393 CAC 9.0
3394 SPX 7.2
3395 CAC 5.2
3396 SPX 8.8
3397 CAC 9.4
3398 SPX 0.1
3399 CAC 0.5
3400 dtype: float64
3401 >>> ser.groupby(level=0).ohlc()
3402 open high low close
3403 CAC 9.0 9.4 0.5 0.5
3404 SPX 3.4 8.8 0.1 0.1
3406 For DataFrameGroupBy:
3408 >>> data = {
3409 ... 2022: [1.2, 2.3, 8.9, 4.5, 4.4, 3, 2, 1],
3410 ... 2023: [3.4, 9.0, 7.2, 5.2, 8.8, 9.4, 8.2, 1.0],
3411 ... }
3412 >>> df = pd.DataFrame(
3413 ... data, index=["SPX", "CAC", "SPX", "CAC", "SPX", "CAC", "SPX", "CAC"]
3414 ... )
3415 >>> df
3416 2022 2023
3417 SPX 1.2 3.4
3418 CAC 2.3 9.0
3419 SPX 8.9 7.2
3420 CAC 4.5 5.2
3421 SPX 4.4 8.8
3422 CAC 3.0 9.4
3423 SPX 2.0 8.2
3424 CAC 1.0 1.0
3425 >>> df.groupby(level=0).ohlc()
3426 2022 2023
3427 open high low close open high low close
3428 CAC 2.3 4.5 1.0 1.0 9.0 9.4 1.0 1.0
3429 SPX 1.2 8.9 1.2 2.0 3.4 8.8 3.4 8.2
3431 For Resampler:
3433 >>> ser = pd.Series(
3434 ... [1, 3, 2, 4, 3, 5],
3435 ... index=pd.DatetimeIndex(
3436 ... [
3437 ... "2023-01-01",
3438 ... "2023-01-10",
3439 ... "2023-01-15",
3440 ... "2023-02-01",
3441 ... "2023-02-10",
3442 ... "2023-02-15",
3443 ... ]
3444 ... ),
3445 ... )
3446 >>> ser.resample("MS").ohlc()
3447 open high low close
3448 2023-01-01 1 3 1 2
3449 2023-02-01 4 5 3 5
3450 """
3451 if self.obj.ndim == 1:
3452 obj = self._selected_obj
3454 is_numeric = is_numeric_dtype(obj.dtype)
3455 if not is_numeric:
3456 raise DataError("No numeric types to aggregate")
3458 res_values = self._grouper._cython_operation(
3459 obj._values, "ohlc", axis=0, min_count=-1
3460 )
3462 agg_names = ["open", "high", "low", "close"]
3463 result = self.obj._constructor_expanddim(
3464 res_values, index=self._grouper.result_index, columns=agg_names
3465 )
3466 if not self.as_index:
3467 result = result.reset_index()
3468 return result
3470 result = self._apply_to_column_groupbys(lambda sgb: sgb.ohlc())
3471 return result
3473 def describe(
3474 self,
3475 percentiles=None,
3476 include=None,
3477 exclude=None,
3478 ) -> NDFrameT:
3479 obj = self._obj_with_exclusions
3481 if len(obj) == 0:
3482 described = obj.describe(
3483 percentiles=percentiles, include=include, exclude=exclude
3484 )
3485 if obj.ndim == 1:
3486 result = described
3487 else:
3488 result = described.unstack()
3489 return result.to_frame().T.iloc[:0]
3491 with com.temp_setattr(self, "as_index", True):
3492 result = self._python_apply_general(
3493 lambda x: x.describe(
3494 percentiles=percentiles, include=include, exclude=exclude
3495 ),
3496 obj,
3497 not_indexed_same=True,
3498 )
3500 # GH#49256 - properly handle the grouping column(s)
3501 result = result.unstack()
3502 if not self.as_index:
3503 result = self._insert_inaxis_grouper(result)
3504 result.index = default_index(len(result))
3506 return result
3508 @final
3509 def resample(
3510 self,
3511 rule,
3512 closed: Literal["right", "left"] | None = None,
3513 label: Literal["right", "left"] | None = None,
3514 convention: Literal["start", "end", "s", "e"] | None = None,
3515 on: Level | None = None,
3516 origin: str | TimestampConvertibleTypes = "start_day",
3517 offset: TimedeltaConvertibleTypes | None = None,
3518 group_keys: bool = False,
3519 *,
3520 include_groups: bool = False,
3521 **kwargs,
3522 ) -> Resampler:
3523 """
3524 Provide resampling within each group of a groupby.
3526 Given a grouper, the function resamples it according to a string
3527 "string" -> "frequency".
3529 See the :ref:`frequency aliases <timeseries.offset_aliases>`
3530 documentation for more details.
3532 Parameters
3533 ----------
3534 rule : str or DateOffset
3535 The offset string or object representing target grouper conversion.
3536 closed : {'right', 'left'}, optional
3537 Which side of bin interval is closed. See :meth:`Series.resample`.
3538 label : {'right', 'left'}, optional
3539 Which bin edge label to label bucket with. See :meth:`Series.resample`.
3540 convention : {'start', 'end', 's', 'e'}, optional
3541 For ``PeriodIndex`` only. See :meth:`Series.resample`.
3542 on : str, optional
3543 For a DataFrame, column to use instead of index for resampling.
3544 Column must be datetime-like.
3545 origin : Timestamp or str, default 'start_day'
3546 The timestamp on which to adjust the grouping. See
3547 :meth:`Series.resample` for accepted string values.
3548 offset : Timedelta or str, optional
3549 An offset timedelta added to the origin.
3550 group_keys : bool, default False
3551 Whether to include the group keys in the result index when using
3552 ``.apply()`` on the resampled object.
3553 include_groups : bool, default False
3554 When True, will attempt to include the groupings in the operation in
3555 the case that they are columns of the DataFrame. If this raises a
3556 TypeError, the result will be computed with the groupings excluded.
3557 When False, the groupings will be excluded when applying ``func``.
3559 .. versionadded:: 2.2.0
3561 .. versionchanged:: 3.0
3563 The default was changed to False, and True is no longer allowed.
3565 **kwargs
3566 Additional keyword arguments forwarded to the underlying
3567 :class:`Grouper`.
3569 Returns
3570 -------
3571 DatetimeIndexResampler, PeriodIndexResampler or TimedeltaResampler
3572 Resampler object for the type of the index.
3574 See Also
3575 --------
3576 Series.resample : Resample a Series.
3577 DataFrame.resample : Resample a DataFrame.
3578 Grouper : Specify a frequency to resample with when
3579 grouping by a key.
3580 DatetimeIndex.resample : Frequency conversion and resampling of
3581 time series.
3583 Examples
3584 --------
3585 >>> idx = pd.date_range("1/1/2000", periods=4, freq="min")
3586 >>> df = pd.DataFrame(data=4 * [range(2)], index=idx, columns=["a", "b"])
3587 >>> df.iloc[2, 0] = 5
3588 >>> df
3589 a b
3590 2000-01-01 00:00:00 0 1
3591 2000-01-01 00:01:00 0 1
3592 2000-01-01 00:02:00 5 1
3593 2000-01-01 00:03:00 0 1
3595 Downsample the DataFrame into 3 minute bins and sum the values of
3596 the timestamps falling into a bin.
3598 >>> df.groupby("a").resample("3min").sum()
3599 b
3600 a
3601 0 2000-01-01 00:00:00 2
3602 2000-01-01 00:03:00 1
3603 5 2000-01-01 00:00:00 1
3605 Upsample the series into 30 second bins.
3607 >>> df.groupby("a").resample("30s").sum()
3608 b
3609 a
3610 0 2000-01-01 00:00:00 1
3611 2000-01-01 00:00:30 0
3612 2000-01-01 00:01:00 1
3613 2000-01-01 00:01:30 0
3614 2000-01-01 00:02:00 0
3615 2000-01-01 00:02:30 0
3616 2000-01-01 00:03:00 1
3617 5 2000-01-01 00:02:00 1
3619 Resample by month. Values are assigned to the month of the period.
3621 >>> df.groupby("a").resample("ME").sum()
3622 b
3623 a
3624 0 2000-01-31 3
3625 5 2000-01-31 1
3627 Downsample the series into 3 minute bins as above, but close the right
3628 side of the bin interval.
3630 >>> (df.groupby("a").resample("3min", closed="right").sum())
3631 b
3632 a
3633 0 1999-12-31 23:57:00 1
3634 2000-01-01 00:00:00 2
3635 5 2000-01-01 00:00:00 1
3637 Downsample the series into 3 minute bins and close the right side of
3638 the bin interval, but label each bin using the right edge instead of
3639 the left.
3641 >>> (df.groupby("a").resample("3min", closed="right", label="right").sum())
3642 b
3643 a
3644 0 2000-01-01 00:00:00 1
3645 2000-01-01 00:03:00 2
3646 5 2000-01-01 00:03:00 1
3647 """
3648 from pandas.core.resample import get_resampler_for_grouping
3650 if include_groups:
3651 raise ValueError("include_groups=True is no longer allowed.")
3653 return get_resampler_for_grouping(
3654 self,
3655 rule,
3656 on=on,
3657 closed=closed,
3658 label=label,
3659 convention=convention,
3660 origin=origin,
3661 offset=offset,
3662 group_keys=group_keys,
3663 **kwargs,
3664 )
3666 @final
3667 def rolling(
3668 self,
3669 window: int | datetime.timedelta | str | BaseOffset | BaseIndexer,
3670 min_periods: int | None = None,
3671 center: bool = False,
3672 win_type: str | None = None,
3673 on: str | None = None,
3674 closed: IntervalClosedType | None = None,
3675 method: str = "single",
3676 ) -> RollingGroupBy:
3677 """
3678 Return a rolling grouper, providing rolling functionality per group.
3680 Allows the application of rolling window operations
3681 (e.g., moving averages) independently within each group defined
3682 by the groupby keys.
3684 Parameters
3685 ----------
3686 window : int, timedelta, str, offset, or BaseIndexer subclass
3687 Interval of the moving window.
3689 If an integer, the delta between the start and end of each window.
3690 The number of points in the window depends on the ``closed`` argument.
3692 If a timedelta, str, or offset, the time period of each window. Each
3693 window will be a variable sized based on the observations included in
3694 the time-period. This is only valid for datetimelike indexes.
3695 The offset must correspond to a fixed frequency (for example, ``'2D'``
3696 or ``'1h'``); non-fixed frequencies such as ``'B'`` (business day) or
3697 ``'ME'`` (month end) are not supported and raise ``ValueError``.
3698 To learn more about the offsets & frequency strings, please see
3699 :ref:`this link<timeseries.offset_aliases>`.
3701 If a BaseIndexer subclass, the window boundaries
3702 based on the defined ``get_window_bounds`` method. Additional rolling
3703 keyword arguments, namely ``min_periods``, ``center``, ``closed`` and
3704 ``step`` will be passed to ``get_window_bounds``.
3706 min_periods : int, default None
3707 Minimum number of observations in window required to have a value;
3708 otherwise, result is ``np.nan``.
3710 For a window that is specified by an offset,
3711 ``min_periods`` will default to 1.
3713 For a window that is specified by an integer, ``min_periods`` will default
3714 to the size of the window.
3716 center : bool, default False
3717 If False, set the window labels as the right edge of the window index.
3719 If True, set the window labels as the center of the window index.
3721 win_type : str, default None
3722 If ``None``, all points are evenly weighted.
3724 If a string, it must be a valid `scipy.signal window function
3725 <https://docs.scipy.org/doc/scipy/reference/signal.windows.html#module-scipy.signal.windows>`__.
3727 Certain Scipy window types require additional parameters to be passed
3728 in the aggregation function. The additional parameters must match
3729 the keywords specified in the Scipy window type method signature.
3731 on : str, optional
3732 For a DataFrame, a column label or Index level on which
3733 to calculate the rolling window, rather than the DataFrame's index.
3735 For integer ``window`` values, the window bounds are based on the number
3736 of observations and are not calculated using the values of the
3737 ``on`` column. The ``on`` column is excluded from the aggregation,
3738 but is included in the result when its values differ from the
3739 object's index.
3741 closed : str, default None
3742 Determines the inclusivity of points in the window
3744 If ``'right'``, uses the window (first, last] meaning the last point
3745 is included in the calculations.
3747 If ``'left'``, uses the window [first, last) meaning the first point
3748 is included in the calculations.
3750 If ``'both'``, uses the window [first, last] meaning all points in
3751 the window are included in the calculations.
3753 If ``'neither'``, uses the window (first, last) meaning the first
3754 and last points in the window are excluded from calculations.
3756 () and [] are referencing open and closed set
3757 notation respetively.
3759 Default ``None`` (``'right'``).
3761 method : str {'single', 'table'}, default 'single'
3762 Execute the rolling operation per single column or row (``'single'``)
3763 or over the entire object (``'table'``).
3765 This argument is only implemented when specifying ``engine='numba'``
3766 in the method call.
3768 Returns
3769 -------
3770 pandas.api.typing.RollingGroupBy
3771 Return a new grouper with our rolling appended.
3773 See Also
3774 --------
3775 Series.rolling : Calling object with Series data.
3776 DataFrame.rolling : Calling object with DataFrames.
3777 Series.groupby : Apply a function groupby to a Series.
3778 DataFrame.groupby : Apply a function groupby.
3780 Examples
3781 --------
3782 >>> df = pd.DataFrame(
3783 ... {
3784 ... "A": [1, 1, 2, 2],
3785 ... "B": [1, 2, 3, 4],
3786 ... "C": [0.362, 0.227, 1.267, -0.562],
3787 ... }
3788 ... )
3789 >>> df
3790 A B C
3791 0 1 1 0.362
3792 1 1 2 0.227
3793 2 2 3 1.267
3794 3 2 4 -0.562
3796 >>> df.groupby("A").rolling(2).sum()
3797 B C
3798 A
3799 1 0 NaN NaN
3800 1 3.0 0.589
3801 2 2 NaN NaN
3802 3 7.0 0.705
3804 >>> df.groupby("A").rolling(2, min_periods=1).sum()
3805 B C
3806 A
3807 1 0 1.0 0.362
3808 1 3.0 0.589
3809 2 2 3.0 1.267
3810 3 7.0 0.705
3812 >>> df.groupby("A").rolling(2, on="B").sum()
3813 B C
3814 A
3815 1 0 1 NaN
3816 1 2 0.589
3817 2 2 3 NaN
3818 3 4 0.705
3819 """
3820 from pandas.core.window import RollingGroupBy
3822 return RollingGroupBy(
3823 self._selected_obj,
3824 window=window,
3825 min_periods=min_periods,
3826 center=center,
3827 win_type=win_type,
3828 on=on,
3829 closed=closed,
3830 method=method,
3831 _grouper=self._grouper,
3832 _as_index=self.as_index,
3833 )
3835 @final
3836 def expanding(
3837 self,
3838 min_periods: int = 1,
3839 method: str = "single",
3840 ) -> ExpandingGroupBy:
3841 """
3842 Return an expanding grouper, providing expanding functionality per group.
3844 Each group's expanding window includes all prior rows within that
3845 group up to and including the current row.
3847 Parameters
3848 ----------
3849 min_periods : int, default 1
3850 Minimum number of observations in window required to have a value;
3851 otherwise, result is ``np.nan``.
3853 method : str {'single', 'table'}, default 'single'
3854 Execute the expanding operation per single column or row (``'single'``)
3855 or over the entire object (``'table'``).
3857 This argument is only implemented when specifying ``engine='numba'``
3858 in the method call.
3860 Returns
3861 -------
3862 pandas.api.typing.ExpandingGroupBy
3863 An object that supports expanding transformations over each group.
3865 See Also
3866 --------
3867 Series.expanding : Expanding transformations for Series.
3868 DataFrame.expanding : Expanding transformations for DataFrames.
3869 Series.groupby : Apply a function groupby to a Series.
3870 DataFrame.groupby : Apply a function groupby.
3872 Examples
3873 --------
3874 >>> df = pd.DataFrame(
3875 ... {
3876 ... "Class": ["A", "A", "A", "B", "B", "B"],
3877 ... "Value": [10, 20, 30, 40, 50, 60],
3878 ... }
3879 ... )
3880 >>> df
3881 Class Value
3882 0 A 10
3883 1 A 20
3884 2 A 30
3885 3 B 40
3886 4 B 50
3887 5 B 60
3889 >>> df.groupby("Class").expanding().mean()
3890 Value
3891 Class
3892 A 0 10.0
3893 1 15.0
3894 2 20.0
3895 B 3 40.0
3896 4 45.0
3897 5 50.0
3898 """
3899 from pandas.core.window import ExpandingGroupBy
3901 return ExpandingGroupBy(
3902 self._selected_obj,
3903 min_periods=min_periods,
3904 method=method,
3905 _grouper=self._grouper,
3906 )
3908 @final
3909 def ewm(
3910 self,
3911 com: float | None = None,
3912 span: float | None = None,
3913 halflife: float | str | Timedelta | None = None,
3914 alpha: float | None = None,
3915 min_periods: int | None = 0,
3916 adjust: bool = True,
3917 ignore_na: bool = False,
3918 times: np.ndarray | Series | None = None,
3919 method: str = "single",
3920 ) -> ExponentialMovingWindowGroupBy:
3921 """
3922 Return an ewm grouper, providing ewm functionality per group.
3924 The decay can be specified in terms of center of mass, span,
3925 half-life, or smoothing factor alpha.
3927 Parameters
3928 ----------
3929 com : float, optional
3930 Specify decay in terms of center of mass.
3931 Alternative to ``span``, ``halflife``, and ``alpha``.
3933 span : float, optional
3934 Specify decay in terms of span.
3936 halflife : float, str, or Timedelta, optional
3937 Specify decay in terms of half-life.
3939 alpha : float, optional
3940 Specify smoothing factor directly.
3942 min_periods : int, default 0
3943 Minimum number of observations in the window required to have a value;
3944 otherwise, result is ``np.nan``.
3946 adjust : bool, default True
3947 Divide by decaying adjustment factor to account for imbalance in
3948 relative weights.
3950 ignore_na : bool, default False
3951 Ignore missing values when calculating weights.
3953 times : str or array-like of datetime64, optional
3954 Times corresponding to the observations.
3956 method : {'single', 'table'}, default 'single'
3957 Execute the operation per group independently (``'single'``) or over the
3958 entire object before regrouping (``'table'``). Only applicable to
3959 ``mean()``, and only when using ``engine='numba'``.
3961 Returns
3962 -------
3963 pandas.api.typing.ExponentialMovingWindowGroupBy
3964 An object that supports exponentially weighted moving transformations over
3965 each group.
3967 See Also
3968 --------
3969 Series.ewm : EWM transformations for Series.
3970 DataFrame.ewm : EWM transformations for DataFrames.
3971 Series.groupby : Apply a function groupby to a Series.
3972 DataFrame.groupby : Apply a function groupby.
3974 Examples
3975 --------
3976 >>> df = pd.DataFrame(
3977 ... {
3978 ... "Class": ["A", "A", "A", "B", "B", "B"],
3979 ... "Value": [10, 20, 30, 40, 50, 60],
3980 ... }
3981 ... )
3982 >>> df
3983 Class Value
3984 0 A 10
3985 1 A 20
3986 2 A 30
3987 3 B 40
3988 4 B 50
3989 5 B 60
3991 >>> df.groupby("Class").ewm(com=0.5).mean()
3992 Value
3993 Class
3994 A 0 10.000000
3995 1 17.500000
3996 2 26.153846
3997 B 3 40.000000
3998 4 47.500000
3999 5 56.153846
4000 """
4001 from pandas.core.window import ExponentialMovingWindowGroupBy
4003 return ExponentialMovingWindowGroupBy(
4004 self._selected_obj,
4005 com=com,
4006 span=span,
4007 halflife=halflife,
4008 alpha=alpha,
4009 min_periods=min_periods,
4010 adjust=adjust,
4011 ignore_na=ignore_na,
4012 times=times,
4013 method=method,
4014 _grouper=self._grouper,
4015 )
4017 @final
4018 def _fill(self, direction: Literal["ffill", "bfill"], limit: int | None = None):
4019 """
4020 Shared function for `pad` and `backfill` to call Cython method.
4022 Parameters
4023 ----------
4024 direction : {'ffill', 'bfill'}
4025 Direction passed to underlying Cython function. `bfill` will cause
4026 values to be filled backwards. `ffill` and any other values will
4027 default to a forward fill
4028 limit : int, default None
4029 Maximum number of consecutive values to fill. If `None`, this
4030 method will convert to -1 prior to passing to Cython
4032 Returns
4033 -------
4034 `Series` or `DataFrame` with filled values
4036 See Also
4037 --------
4038 pad : Forward fill values within each group.
4039 backfill : Backward fill values within each group.
4040 """
4041 # Need int value for Cython
4042 if limit is None:
4043 limit = -1
4045 ids = self._grouper.ids
4046 ngroups = self._grouper.ngroups
4048 col_func = partial(
4049 libgroupby.group_fillna_indexer,
4050 labels=ids,
4051 limit=limit,
4052 compute_ffill=(direction == "ffill"),
4053 ngroups=ngroups,
4054 )
4056 def blk_func(values: ArrayLike) -> ArrayLike:
4057 mask = isna(values)
4058 if values.ndim == 1:
4059 indexer = np.empty(values.shape, dtype=np.intp)
4060 col_func(out=indexer, mask=mask) # type: ignore[arg-type]
4061 return algorithms.take_nd(values, indexer)
4063 else:
4064 # We broadcast algorithms.take_nd analogous to
4065 # np.take_along_axis
4066 if isinstance(values, np.ndarray):
4067 dtype = values.dtype
4068 if self._grouper.has_dropped_na:
4069 # dropped null groups give rise to nan in the result
4070 dtype = ensure_dtype_can_hold_na(values.dtype)
4071 out = np.empty(values.shape, dtype=dtype)
4072 else:
4073 # Note: we only get here with backfill/pad,
4074 # so if we have a dtype that cannot hold NAs,
4075 # then there will be no -1s in indexer, so we can use
4076 # the original dtype (no need to ensure_dtype_can_hold_na)
4077 out = type(values)._empty(values.shape, dtype=values.dtype)
4079 for i, value_element in enumerate(values):
4080 # call group_fillna_indexer column-wise
4081 indexer = np.empty(values.shape[1], dtype=np.intp)
4082 col_func(out=indexer, mask=mask[i])
4083 out[i, :] = algorithms.take_nd(value_element, indexer)
4084 return out
4086 mgr = self._get_data_to_aggregate()
4087 res_mgr = mgr.apply(blk_func)
4089 new_obj = self._wrap_agged_manager(res_mgr)
4090 new_obj.index = self.obj.index
4091 return new_obj
4093 @final
4094 def ffill(self, limit: int | None = None):
4095 """
4096 Forward fill the values.
4098 Propagates the last valid observation forward within each group
4099 to fill missing values.
4101 Parameters
4102 ----------
4103 limit : int, optional
4104 Limit of how many values to fill.
4106 Returns
4107 -------
4108 Series or DataFrame
4109 Object with missing values filled.
4111 See Also
4112 --------
4113 Series.ffill : Forward fill missing values in a Series.
4114 DataFrame.ffill : Forward fill missing values in a DataFrame.
4115 Series.fillna: Fill NaN values of a Series.
4116 DataFrame.fillna: Fill NaN values of a DataFrame.
4118 Examples
4119 --------
4121 For SeriesGroupBy:
4123 >>> key = [0, 0, 1, 1]
4124 >>> ser = pd.Series([np.nan, 2, 3, np.nan], index=key)
4125 >>> ser
4126 0 NaN
4127 0 2.0
4128 1 3.0
4129 1 NaN
4130 dtype: float64
4131 >>> ser.groupby(level=0).ffill()
4132 0 NaN
4133 0 2.0
4134 1 3.0
4135 1 3.0
4136 dtype: float64
4138 For DataFrameGroupBy:
4140 >>> df = pd.DataFrame(
4141 ... {
4142 ... "key": [0, 0, 1, 1, 1],
4143 ... "A": [np.nan, 2, np.nan, 3, np.nan],
4144 ... "B": [2, 3, np.nan, np.nan, np.nan],
4145 ... "C": [np.nan, np.nan, 2, np.nan, np.nan],
4146 ... }
4147 ... )
4148 >>> df
4149 key A B C
4150 0 0 NaN 2.0 NaN
4151 1 0 2.0 3.0 NaN
4152 2 1 NaN NaN 2.0
4153 3 1 3.0 NaN NaN
4154 4 1 NaN NaN NaN
4156 Propagate non-null values forward or backward within each group along columns.
4158 >>> df.groupby("key").ffill()
4159 A B C
4160 0 NaN 2.0 NaN
4161 1 2.0 3.0 NaN
4162 2 NaN NaN 2.0
4163 3 3.0 NaN 2.0
4164 4 3.0 NaN 2.0
4166 Propagate non-null values forward or backward within each group along rows.
4168 >>> df.T.groupby(np.array([0, 0, 1, 1])).ffill().T
4169 key A B C
4170 0 0.0 0.0 2.0 2.0
4171 1 0.0 2.0 3.0 3.0
4172 2 1.0 1.0 NaN 2.0
4173 3 1.0 3.0 NaN NaN
4174 4 1.0 1.0 NaN NaN
4176 Only replace the first NaN element within a group along columns.
4178 >>> df.groupby("key").ffill(limit=1)
4179 A B C
4180 0 NaN 2.0 NaN
4181 1 2.0 3.0 NaN
4182 2 NaN NaN 2.0
4183 3 3.0 NaN 2.0
4184 4 3.0 NaN NaN
4185 """
4186 return self._fill("ffill", limit=limit)
4188 @final
4189 def bfill(self, limit: int | None = None):
4190 """
4191 Backward fill the values.
4193 Fill missing values within each group by propagating the next
4194 valid observation backward.
4196 Parameters
4197 ----------
4198 limit : int, optional
4199 Limit of how many values to fill.
4201 Returns
4202 -------
4203 Series or DataFrame
4204 Object with missing values filled.
4206 See Also
4207 --------
4208 Series.bfill : Backward fill the missing values in the dataset.
4209 DataFrame.bfill: Backward fill the missing values in the dataset.
4210 Series.fillna: Fill NaN values of a Series.
4211 DataFrame.fillna: Fill NaN values of a DataFrame.
4213 Examples
4214 --------
4216 With Series:
4218 >>> index = ["Falcon", "Falcon", "Parrot", "Parrot", "Parrot"]
4219 >>> s = pd.Series([None, 1, None, None, 3], index=index)
4220 >>> s
4221 Falcon NaN
4222 Falcon 1.0
4223 Parrot NaN
4224 Parrot NaN
4225 Parrot 3.0
4226 dtype: float64
4227 >>> s.groupby(level=0).bfill()
4228 Falcon 1.0
4229 Falcon 1.0
4230 Parrot 3.0
4231 Parrot 3.0
4232 Parrot 3.0
4233 dtype: float64
4234 >>> s.groupby(level=0).bfill(limit=1)
4235 Falcon 1.0
4236 Falcon 1.0
4237 Parrot NaN
4238 Parrot 3.0
4239 Parrot 3.0
4240 dtype: float64
4242 With DataFrame:
4244 >>> df = pd.DataFrame(
4245 ... {"A": [1, None, None, None, 4], "B": [None, None, 5, None, 7]},
4246 ... index=index,
4247 ... )
4248 >>> df
4249 A B
4250 Falcon 1.0 NaN
4251 Falcon NaN NaN
4252 Parrot NaN 5.0
4253 Parrot NaN NaN
4254 Parrot 4.0 7.0
4255 >>> df.groupby(level=0).bfill()
4256 A B
4257 Falcon 1.0 NaN
4258 Falcon NaN NaN
4259 Parrot 4.0 5.0
4260 Parrot 4.0 7.0
4261 Parrot 4.0 7.0
4262 >>> df.groupby(level=0).bfill(limit=1)
4263 A B
4264 Falcon 1.0 NaN
4265 Falcon NaN NaN
4266 Parrot NaN 5.0
4267 Parrot 4.0 7.0
4268 Parrot 4.0 7.0
4269 """
4270 return self._fill("bfill", limit=limit)
4272 @final
4273 @property
4274 def nth(self) -> GroupByNthSelector:
4275 """
4276 Take the nth row from each group if n is an int, otherwise a subset of rows.
4278 .. deprecated:: 3.1.0
4280 Index notation (``g.nth[n]``) is deprecated in favor of calling
4281 ``g.nth(n)`` and will be removed in a future version of pandas.
4283 If dropna, will take the nth non-null row, dropna is either
4284 'all' or 'any'; this is equivalent to calling dropna(how=dropna)
4285 before the groupby.
4287 Returns
4288 -------
4289 Series or DataFrame
4290 N-th value within each group.
4292 See Also
4293 --------
4294 Series.nth : Apply function nth to a Series.
4295 DataFrame.nth : Apply function nth to each row or column of a DataFrame.
4297 Examples
4298 --------
4300 >>> df = pd.DataFrame(
4301 ... {"A": [1, 1, 2, 1, 2], "B": [np.nan, 2, 3, 4, 5]}, columns=["A", "B"]
4302 ... )
4303 >>> g = df.groupby("A")
4304 >>> g.nth(0)
4305 A B
4306 0 1 NaN
4307 2 2 3.0
4308 >>> g.nth(1)
4309 A B
4310 1 1 2.0
4311 4 2 5.0
4312 >>> g.nth(-1)
4313 A B
4314 3 1 4.0
4315 4 2 5.0
4316 >>> g.nth([0, 1])
4317 A B
4318 0 1 NaN
4319 1 1 2.0
4320 2 2 3.0
4321 4 2 5.0
4322 >>> g.nth(slice(None, -1))
4323 A B
4324 0 1 NaN
4325 1 1 2.0
4326 2 2 3.0
4328 Specifying `dropna` allows ignoring ``NaN`` values
4330 >>> g.nth(0, dropna="any")
4331 A B
4332 1 1 2.0
4333 2 2 3.0
4335 When the specified ``n`` is larger than any of the groups, an
4336 empty DataFrame is returned
4338 >>> g.nth(3, dropna="any")
4339 Empty DataFrame
4340 Columns: [A, B]
4341 Index: []
4342 """
4343 return GroupByNthSelector(self)
4345 def _nth(
4346 self,
4347 n: PositionalIndexer | tuple,
4348 dropna: Literal["any", "all"] | None = None,
4349 ) -> NDFrameT:
4350 if not dropna:
4351 mask = self._make_mask_from_positional_indexer(n)
4353 ids = self._grouper.ids
4355 # Drop NA values in grouping
4356 mask = mask & (ids != -1)
4358 out = self._mask_selected_obj(mask)
4359 return out
4361 # dropna is truthy
4362 if not is_integer(n):
4363 raise ValueError("dropna option only supported for an integer argument")
4365 if dropna not in ["any", "all"]:
4366 # Note: when agg-ing picker doesn't raise this, just returns NaN
4367 raise ValueError(
4368 "For a DataFrame or Series groupby.nth, dropna must be "
4369 "either None, 'any' or 'all', "
4370 f"(was passed {dropna})."
4371 )
4373 # old behaviour, but with all and any support for DataFrames.
4374 # modified in GH 7559 to have better perf
4375 n = cast("int", n)
4376 dropped = self._selected_obj.dropna(how=dropna, axis=0)
4378 # get a new grouper for our dropped obj
4379 grouper: np.ndarray | Index | ops.BaseGrouper
4380 if len(dropped) == len(self._selected_obj):
4381 # Nothing was dropped, can use the same grouper
4382 grouper = self._grouper
4383 else:
4384 # we don't have the grouper info available
4385 # (e.g. we have selected out
4386 # a column that is not in the current object)
4387 axis = self._grouper.axis
4388 grouper = self._grouper.codes_info[axis.isin(dropped.index)]
4389 if self._grouper.has_dropped_na:
4390 # Null groups need to still be encoded as -1 when passed to groupby
4391 nulls = grouper == -1
4392 # error: No overload variant of "where" matches argument types
4393 # "Any", "NAType", "Any"
4394 values = np.where(nulls, NA, grouper) # type: ignore[call-overload]
4395 grouper = Index(values, dtype="Int64", copy=False)
4397 grb = dropped.groupby(grouper, as_index=self.as_index, sort=self.sort)
4398 return grb.nth(n)
4400 @final
4401 def quantile(
4402 self,
4403 q: float | AnyArrayLike = 0.5,
4404 interpolation: Literal[
4405 "linear", "lower", "higher", "nearest", "midpoint"
4406 ] = "linear",
4407 numeric_only: bool = False,
4408 ):
4409 """
4410 Return group values at the given quantile, a la numpy.percentile.
4412 This method returns the value at the given quantile for each group,
4413 using the specified interpolation method when the desired quantile
4414 falls between two data points.
4416 Parameters
4417 ----------
4418 q : float or array-like, default 0.5 (50% quantile)
4419 Value(s) between 0 and 1 providing the quantile(s) to compute.
4420 interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
4421 Method to use when the desired quantile falls between two points.
4422 numeric_only : bool, default False
4423 Include only `float`, `int` or `boolean` data.
4425 .. versionchanged:: 2.0.0
4427 numeric_only now defaults to ``False``.
4429 Returns
4430 -------
4431 Series or DataFrame
4432 Return type determined by caller of GroupBy object.
4434 See Also
4435 --------
4436 Series.quantile : Similar method for Series.
4437 DataFrame.quantile : Similar method for DataFrame.
4438 numpy.percentile : NumPy method to compute qth percentile.
4440 Examples
4441 --------
4442 >>> df = pd.DataFrame(
4443 ... [["a", 1], ["a", 2], ["a", 3], ["b", 1], ["b", 3], ["b", 5]],
4444 ... columns=["key", "val"],
4445 ... )
4446 >>> df.groupby("key").quantile()
4447 val
4448 key
4449 a 2.0
4450 b 3.0
4451 """
4452 mgr = self._get_data_to_aggregate(numeric_only=numeric_only, name="quantile")
4453 obj = self._wrap_agged_manager(mgr)
4454 splitter = self._grouper._get_splitter(obj)
4455 sdata = splitter._sorted_data
4457 starts, ends = lib.generate_slices(splitter._slabels, splitter.ngroups)
4459 if is_scalar(q):
4460 qs = np.array([q], dtype=np.float64)
4461 pass_qs: np.ndarray | None = None
4462 else:
4463 qs = np.asarray(q, dtype=np.float64)
4464 pass_qs = qs
4466 ids = self._grouper.ids
4467 ngroups = self._grouper.ngroups
4468 if self.dropna:
4469 # splitter drops NA groups, we need to do the same
4470 ids = ids[ids >= 0]
4471 nqs = len(qs)
4473 func = partial(
4474 libgroupby.group_quantile,
4475 labels=ids,
4476 qs=qs,
4477 interpolation=interpolation,
4478 starts=starts,
4479 ends=ends,
4480 )
4482 def blk_func(values: ArrayLike) -> ArrayLike:
4483 if isinstance(values, ExtensionArray):
4484 return values._groupby_quantile(
4485 qs=qs,
4486 interpolation=interpolation,
4487 ids=ids,
4488 ngroups=ngroups,
4489 starts=starts,
4490 ends=ends,
4491 )
4493 # ndarray path
4494 if is_object_dtype(values.dtype):
4495 raise TypeError(
4496 f"dtype '{values.dtype}' does not support operation 'quantile'"
4497 )
4498 if is_bool_dtype(values.dtype):
4499 # GH#51424 remove to match Series/DataFrame behavior
4500 raise TypeError("Cannot use quantile with bool dtype")
4502 mask = isna(values)
4503 inference: np.dtype | None = (
4504 np.dtype(np.int64) if is_integer_dtype(values.dtype) else None
4505 )
4506 vals = np.asarray(values)
4508 ncols = 1
4509 if vals.ndim == 2:
4510 ncols = vals.shape[0]
4512 out = np.empty((ncols, ngroups, nqs), dtype=np.float64)
4514 if vals.ndim == 1:
4515 func(
4516 out[0],
4517 values=vals,
4518 mask=mask, # type: ignore[arg-type]
4519 result_mask=None,
4520 is_datetimelike=False,
4521 )
4522 else:
4523 for i in range(ncols):
4524 func(
4525 out[i],
4526 values=vals[i],
4527 mask=mask[i],
4528 result_mask=None,
4529 is_datetimelike=False,
4530 )
4532 if vals.ndim == 1:
4533 out = out.ravel("K") # type: ignore[assignment]
4534 else:
4535 out = out.reshape(ncols, ngroups * nqs) # type: ignore[assignment]
4537 if inference is not None and interpolation not in {
4538 "linear",
4539 "midpoint",
4540 }:
4541 out = out.astype(inference)
4543 return out
4545 res_mgr = sdata._mgr.grouped_reduce(blk_func)
4547 res = self._wrap_agged_manager(res_mgr)
4548 return self._wrap_aggregated_output(res, qs=pass_qs)
4550 @final
4551 def ngroup(self, ascending: bool = True):
4552 """
4553 Number each group from 0 to the number of groups - 1.
4555 This is the enumerative complement of cumcount. Note that the
4556 numbers given to the groups match the order in which the groups
4557 would be seen when iterating over the groupby object, not the
4558 order they are first observed.
4560 Groups with missing keys (where `pd.isna()` is True) will be labeled with `NaN`
4561 and will be skipped from the count.
4563 Parameters
4564 ----------
4565 ascending : bool, default True
4566 If False, number in reverse, from number of group - 1 to 0.
4568 Returns
4569 -------
4570 Series
4571 Unique numbers for each group.
4573 See Also
4574 --------
4575 .cumcount : Number the rows in each group.
4577 Examples
4578 --------
4579 >>> df = pd.DataFrame({"color": ["red", None, "red", "blue", "blue", "red"]})
4580 >>> df
4581 color
4582 0 red
4583 1 NaN
4584 2 red
4585 3 blue
4586 4 blue
4587 5 red
4588 >>> df.groupby("color").ngroup()
4589 0 1.0
4590 1 NaN
4591 2 1.0
4592 3 0.0
4593 4 0.0
4594 5 1.0
4595 dtype: float64
4596 >>> df.groupby("color", dropna=False).ngroup()
4597 0 1
4598 1 2
4599 2 1
4600 3 0
4601 4 0
4602 5 1
4603 dtype: int64
4604 >>> df.groupby("color", dropna=False).ngroup(ascending=False)
4605 0 1
4606 1 0
4607 2 1
4608 3 2
4609 4 2
4610 5 1
4611 dtype: int64
4612 """
4613 obj = self._obj_with_exclusions
4614 index = obj.index
4615 comp_ids = self._grouper.ids
4617 dtype: type
4618 if self._grouper.has_dropped_na:
4619 comp_ids = np.where(comp_ids == -1, np.nan, comp_ids)
4620 dtype = np.float64
4621 else:
4622 dtype = np.int64
4624 if any(ping._passed_categorical for ping in self._grouper.groupings):
4625 # comp_ids reflect non-observed groups, we need only observed
4626 comp_ids = rank_1d(comp_ids, ties_method="dense") - 1
4628 result = self._obj_1d_constructor(comp_ids, index, dtype=dtype)
4629 if not ascending:
4630 result = self.ngroups - 1 - result
4631 return result
4633 @final
4634 def cumcount(self, ascending: bool = True):
4635 """
4636 Number each item in each group from 0 to the length of that group - 1.
4638 Essentially this is equivalent to
4640 .. code-block:: python
4642 self.apply(lambda x: pd.Series(np.arange(len(x)), x.index))
4644 Parameters
4645 ----------
4646 ascending : bool, default True
4647 If False, number in reverse, from length of group - 1 to 0.
4649 Returns
4650 -------
4651 Series
4652 Sequence number of each element within each group.
4654 See Also
4655 --------
4656 .ngroup : Number the groups themselves.
4658 Examples
4659 --------
4660 >>> df = pd.DataFrame([["a"], ["a"], ["a"], ["b"], ["b"], ["a"]], columns=["A"])
4661 >>> df
4662 A
4663 0 a
4664 1 a
4665 2 a
4666 3 b
4667 4 b
4668 5 a
4669 >>> df.groupby("A").cumcount()
4670 0 0
4671 1 1
4672 2 2
4673 3 0
4674 4 1
4675 5 3
4676 dtype: int64
4677 >>> df.groupby("A").cumcount(ascending=False)
4678 0 3
4679 1 2
4680 2 1
4681 3 1
4682 4 0
4683 5 0
4684 dtype: int64
4685 """
4686 index = self._obj_with_exclusions.index
4687 cumcounts = self._cumcount_array(ascending=ascending)
4688 return self._obj_1d_constructor(cumcounts, index)
4690 @final
4691 def rank(
4692 self,
4693 method: RankMethod = "average",
4694 ascending: bool = True,
4695 na_option: RankNaOption = "keep",
4696 pct: bool = False,
4697 ) -> NDFrameT:
4698 """
4699 Provide the rank of values within each group.
4701 This method assigns ranks to values within each group, with options
4702 for handling duplicate values, NaN values, and computing percentage ranks.
4704 Parameters
4705 ----------
4706 method : {'average', 'min', 'max', 'first', 'dense'}, default 'average'
4707 * average: average rank of group.
4708 * min: lowest rank in group.
4709 * max: highest rank in group.
4710 * first: ranks assigned in order they appear in the array.
4711 * dense: like 'min', but rank always increases by 1 between groups.
4712 ascending : bool, default True
4713 False for ranks by high (1) to low (N).
4714 na_option : {'keep', 'top', 'bottom'}, default 'keep'
4715 * keep: leave NA values where they are.
4716 * top: smallest rank if ascending.
4717 * bottom: smallest rank if descending.
4718 pct : bool, default False
4719 Compute percentage rank of data within each group.
4721 Returns
4722 -------
4723 DataFrame
4724 The ranking of values within each group.
4726 See Also
4727 --------
4728 Series.rank : Apply function rank to a Series.
4729 DataFrame.rank : Apply function rank to each row or column of a DataFrame.
4731 Examples
4732 --------
4733 >>> df = pd.DataFrame(
4734 ... {
4735 ... "group": ["a", "a", "a", "a", "a", "b", "b", "b", "b", "b"],
4736 ... "value": [2, 4, 2, 3, 5, 1, 2, 4, 1, 5],
4737 ... }
4738 ... )
4739 >>> df
4740 group value
4741 0 a 2
4742 1 a 4
4743 2 a 2
4744 3 a 3
4745 4 a 5
4746 5 b 1
4747 6 b 2
4748 7 b 4
4749 8 b 1
4750 9 b 5
4751 >>> for method in ["average", "min", "max", "dense", "first"]:
4752 ... df[f"{method}_rank"] = df.groupby("group")["value"].rank(method)
4753 >>> df
4754 group value average_rank min_rank max_rank dense_rank first_rank
4755 0 a 2 1.5 1.0 2.0 1.0 1.0
4756 1 a 4 4.0 4.0 4.0 3.0 4.0
4757 2 a 2 1.5 1.0 2.0 1.0 2.0
4758 3 a 3 3.0 3.0 3.0 2.0 3.0
4759 4 a 5 5.0 5.0 5.0 4.0 5.0
4760 5 b 1 1.5 1.0 2.0 1.0 1.0
4761 6 b 2 3.0 3.0 3.0 2.0 3.0
4762 7 b 4 4.0 4.0 4.0 3.0 4.0
4763 8 b 1 1.5 1.0 2.0 1.0 2.0
4764 9 b 5 5.0 5.0 5.0 4.0 5.0
4765 """
4766 if na_option not in {"keep", "top", "bottom"}:
4767 msg = "na_option must be one of 'keep', 'top', or 'bottom'"
4768 raise ValueError(msg)
4770 kwargs = {
4771 "ties_method": method,
4772 "ascending": ascending,
4773 "na_option": na_option,
4774 "pct": pct,
4775 }
4777 return self._cython_transform(
4778 "rank",
4779 numeric_only=False,
4780 **kwargs,
4781 )
4783 @final
4784 def cumprod(
4785 self,
4786 numeric_only: bool = False,
4787 skipna: bool = True,
4788 *args,
4789 **kwargs,
4790 ) -> NDFrameT:
4791 """
4792 Cumulative product for each group.
4794 Returns a Series or DataFrame of the same size with the cumulative
4795 product computed within each group.
4797 Parameters
4798 ----------
4799 numeric_only : bool, default False
4800 Include only float, int, boolean columns.
4801 skipna : bool, default True
4802 Exclude NA/null values. If an entire row/column is NA, the result
4803 will be NA.
4804 *args : tuple
4805 Positional arguments to be passed to `func`.
4807 .. deprecated:: 3.1.0
4808 Passing ``*args`` to GroupBy.cumprod is deprecated
4809 and will be removed in a future version of pandas.
4811 **kwargs : dict
4812 Additional keyword arguments to be passed to the function.
4814 .. deprecated:: 3.1.0
4815 Passing ``**kwargs`` to GroupBy.cumprod is deprecated
4816 and will be removed in a future version of pandas.
4818 Returns
4819 -------
4820 Series or DataFrame
4821 Cumulative product for each group. Same object type as the caller.
4823 See Also
4824 --------
4825 Series.cumprod : Apply function cumprod to a Series.
4826 DataFrame.cumprod : Apply function cumprod to each row or column of a DataFrame.
4828 Examples
4829 --------
4830 For SeriesGroupBy:
4832 >>> lst = ["a", "a", "b"]
4833 >>> ser = pd.Series([6, 2, 0], index=lst)
4834 >>> ser
4835 a 6
4836 a 2
4837 b 0
4838 dtype: int64
4839 >>> ser.groupby(level=0).cumprod()
4840 a 6
4841 a 12
4842 b 0
4843 dtype: int64
4845 For DataFrameGroupBy:
4847 >>> data = [[1, 8, 2], [1, 2, 5], [2, 6, 9]]
4848 >>> df = pd.DataFrame(
4849 ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
4850 ... )
4851 >>> df
4852 a b c
4853 cow 1 8 2
4854 horse 1 2 5
4855 bull 2 6 9
4856 >>> df.groupby("a").groups
4857 {1: ['cow', 'horse'], 2: ['bull']}
4858 >>> df.groupby("a").cumprod()
4859 b c
4860 cow 8 2
4861 horse 16 10
4862 bull 6 9
4863 """
4864 nv.validate_groupby_func("cumprod", args, kwargs, ["skipna"])
4865 if args or kwargs:
4866 warnings.warn(
4867 "Passing additional arguments to GroupBy.cumprod is deprecated "
4868 "and will be removed in a future version of pandas.",
4869 Pandas4Warning,
4870 stacklevel=find_stack_level(),
4871 )
4872 return self._cython_transform("cumprod", numeric_only, skipna=skipna)
4874 @final
4875 def cumsum(
4876 self,
4877 numeric_only: bool = False,
4878 skipna: bool = True,
4879 *args,
4880 **kwargs,
4881 ) -> NDFrameT:
4882 """
4883 Cumulative sum for each group.
4885 Returns a Series or DataFrame of the same size with the cumulative
4886 sum computed within each group.
4888 Parameters
4889 ----------
4890 numeric_only : bool, default False
4891 Include only float, int, boolean columns.
4892 skipna : bool, default True
4893 Exclude NA/null values. If an entire row/column is NA, the result
4894 will be NA.
4895 *args : tuple
4896 Positional arguments to be passed to `func`.
4898 .. deprecated:: 3.1.0
4899 Passing ``*args`` to GroupBy.cumsum is deprecated
4900 and will be removed in a future version of pandas.
4902 **kwargs : dict
4903 Additional keyword arguments to be passed to the function.
4905 .. deprecated:: 3.1.0
4906 Passing ``**kwargs`` to GroupBy.cumsum is deprecated
4907 and will be removed in a future version of pandas.
4909 Returns
4910 -------
4911 Series or DataFrame
4912 Cumulative sum for each group. Same object type as the caller.
4914 See Also
4915 --------
4916 Series.cumsum : Apply function cumsum to a Series.
4917 DataFrame.cumsum : Apply function cumsum to each row or column of a DataFrame.
4919 Examples
4920 --------
4921 For SeriesGroupBy:
4923 >>> lst = ["a", "a", "b"]
4924 >>> ser = pd.Series([6, 2, 0], index=lst)
4925 >>> ser
4926 a 6
4927 a 2
4928 b 0
4929 dtype: int64
4930 >>> ser.groupby(level=0).cumsum()
4931 a 6
4932 a 8
4933 b 0
4934 dtype: int64
4936 For DataFrameGroupBy:
4938 >>> data = [[1, 8, 2], [1, 2, 5], [2, 6, 9]]
4939 >>> df = pd.DataFrame(
4940 ... data, columns=["a", "b", "c"], index=["fox", "gorilla", "lion"]
4941 ... )
4942 >>> df
4943 a b c
4944 fox 1 8 2
4945 gorilla 1 2 5
4946 lion 2 6 9
4947 >>> df.groupby("a").groups
4948 {1: ['fox', 'gorilla'], 2: ['lion']}
4949 >>> df.groupby("a").cumsum()
4950 b c
4951 fox 8 2
4952 gorilla 10 7
4953 lion 6 9
4954 """
4955 nv.validate_groupby_func("cumsum", args, kwargs, ["skipna"])
4956 if args or kwargs:
4957 warnings.warn(
4958 "Passing additional arguments to GroupBy.cumsum is deprecated "
4959 "and will be removed in a future version of pandas.",
4960 Pandas4Warning,
4961 stacklevel=find_stack_level(),
4962 )
4963 return self._cython_transform("cumsum", numeric_only, skipna=skipna)
4965 @final
4966 def cummin(
4967 self,
4968 numeric_only: bool = False,
4969 skipna: bool = True,
4970 **kwargs,
4971 ) -> NDFrameT:
4972 """
4973 Cumulative min for each group.
4975 Returns a same-sized object where each value is replaced by the
4976 minimum of all preceding values in its group.
4978 Parameters
4979 ----------
4980 numeric_only : bool, default False
4981 Include only `float`, `int` or `boolean` data.
4982 skipna : bool, default True
4983 Exclude NA/null values. If an entire row/column is NA, the result
4984 will be NA.
4985 **kwargs : dict, optional
4986 Additional keyword arguments to be passed to the function.
4988 .. deprecated:: 3.1.0
4989 Passing ``**kwargs`` to GroupBy.cummin is deprecated
4990 and will be removed in a future version of pandas.
4992 Returns
4993 -------
4994 Series or DataFrame
4995 Cumulative min for each group. Same object type as the caller.
4997 See Also
4998 --------
4999 Series.cummin : Apply function cummin to a Series.
5000 DataFrame.cummin : Apply function cummin to each row or column of a DataFrame.
5002 Examples
5003 --------
5004 For SeriesGroupBy:
5006 >>> lst = ["a", "a", "a", "b", "b", "b"]
5007 >>> ser = pd.Series([1, 6, 2, 3, 0, 4], index=lst)
5008 >>> ser
5009 a 1
5010 a 6
5011 a 2
5012 b 3
5013 b 0
5014 b 4
5015 dtype: int64
5016 >>> ser.groupby(level=0).cummin()
5017 a 1
5018 a 1
5019 a 1
5020 b 3
5021 b 0
5022 b 0
5023 dtype: int64
5025 For DataFrameGroupBy:
5027 >>> data = [[1, 0, 2], [1, 1, 5], [6, 6, 9]]
5028 >>> df = pd.DataFrame(
5029 ... data, columns=["a", "b", "c"], index=["snake", "rabbit", "turtle"]
5030 ... )
5031 >>> df
5032 a b c
5033 snake 1 0 2
5034 rabbit 1 1 5
5035 turtle 6 6 9
5036 >>> df.groupby("a").groups
5037 {1: ['snake', 'rabbit'], 6: ['turtle']}
5038 >>> df.groupby("a").cummin()
5039 b c
5040 snake 0 2
5041 rabbit 0 2
5042 turtle 6 9
5043 """
5044 if kwargs:
5045 warnings.warn(
5046 "Passing additional arguments to GroupBy.cummin is deprecated "
5047 "and will be removed in a future version of pandas.",
5048 Pandas4Warning,
5049 stacklevel=find_stack_level(),
5050 )
5051 return self._cython_transform(
5052 "cummin", numeric_only=numeric_only, skipna=skipna
5053 )
5055 @final
5056 def cummax(
5057 self,
5058 numeric_only: bool = False,
5059 skipna: bool = True,
5060 **kwargs,
5061 ) -> NDFrameT:
5062 """
5063 Cumulative max for each group.
5065 Returns the cumulative maximum of values within each group. The result
5066 has the same size as the input, with each element representing the
5067 maximum of all preceding elements (including itself) within its group.
5069 Parameters
5070 ----------
5071 numeric_only : bool, default False
5072 Include only `float`, `int` or `boolean` data.
5073 skipna : bool, default True
5074 Exclude NA/null values. If an entire row/column is NA, the result
5075 will be NA.
5076 **kwargs : dict, optional
5077 Additional keyword arguments to be passed to the function.
5079 .. deprecated:: 3.1.0
5080 Passing ``**kwargs`` to GroupBy.cummax is deprecated
5081 and will be removed in a future version of pandas.
5083 Returns
5084 -------
5085 Series or DataFrame
5086 Cumulative max for each group. Same object type as the caller.
5088 See Also
5089 --------
5090 Series.cummax : Apply function cummax to a Series.
5091 DataFrame.cummax : Apply function cummax to each row or column of a DataFrame.
5093 Examples
5094 --------
5095 For SeriesGroupBy:
5097 >>> lst = ["a", "a", "a", "b", "b", "b"]
5098 >>> ser = pd.Series([1, 6, 2, 3, 1, 4], index=lst)
5099 >>> ser
5100 a 1
5101 a 6
5102 a 2
5103 b 3
5104 b 1
5105 b 4
5106 dtype: int64
5107 >>> ser.groupby(level=0).cummax()
5108 a 1
5109 a 6
5110 a 6
5111 b 3
5112 b 3
5113 b 4
5114 dtype: int64
5116 For DataFrameGroupBy:
5118 >>> data = [[1, 8, 2], [1, 1, 0], [2, 6, 9]]
5119 >>> df = pd.DataFrame(
5120 ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
5121 ... )
5122 >>> df
5123 a b c
5124 cow 1 8 2
5125 horse 1 1 0
5126 bull 2 6 9
5127 >>> df.groupby("a").groups
5128 {1: ['cow', 'horse'], 2: ['bull']}
5129 >>> df.groupby("a").cummax()
5130 b c
5131 cow 8 2
5132 horse 8 2
5133 bull 6 9
5134 """
5135 if kwargs:
5136 warnings.warn(
5137 "Passing additional arguments to GroupBy.cummax is deprecated "
5138 "and will be removed in a future version of pandas.",
5139 Pandas4Warning,
5140 stacklevel=find_stack_level(),
5141 )
5142 return self._cython_transform(
5143 "cummax", numeric_only=numeric_only, skipna=skipna
5144 )
5146 @final
5147 def shift(
5148 self,
5149 periods: int | Sequence[int] = 1,
5150 freq=None,
5151 fill_value=lib.no_default,
5152 suffix: str | None = None,
5153 ):
5154 """
5155 Shift each group by periods observations.
5157 If freq is passed, the index will be increased using the periods and the freq.
5159 Parameters
5160 ----------
5161 periods : int | Sequence[int], default 1
5162 Number of periods to shift. If a list of values, shift each group by
5163 each period.
5164 freq : str, optional
5165 Frequency string.
5166 fill_value : optional
5167 The scalar value to use for newly introduced missing values.
5169 .. versionchanged:: 2.1.0
5170 Will raise a ``ValueError`` if ``freq`` is provided too.
5172 suffix : str, optional
5173 A string to add to each shifted column if there are multiple periods.
5174 Ignored otherwise.
5176 Returns
5177 -------
5178 Series or DataFrame
5179 Object shifted within each group.
5181 See Also
5182 --------
5183 Index.shift : Shift values of Index.
5185 Examples
5186 --------
5188 For SeriesGroupBy:
5190 >>> lst = ["a", "a", "b", "b"]
5191 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
5192 >>> ser
5193 a 1
5194 a 2
5195 b 3
5196 b 4
5197 dtype: int64
5198 >>> ser.groupby(level=0).shift(1)
5199 a NaN
5200 a 1.0
5201 b NaN
5202 b 3.0
5203 dtype: float64
5205 For DataFrameGroupBy:
5207 >>> data = [[1, 2, 3], [1, 5, 6], [2, 5, 8], [2, 6, 9]]
5208 >>> df = pd.DataFrame(
5209 ... data,
5210 ... columns=["a", "b", "c"],
5211 ... index=["tuna", "salmon", "catfish", "goldfish"],
5212 ... )
5213 >>> df
5214 a b c
5215 tuna 1 2 3
5216 salmon 1 5 6
5217 catfish 2 5 8
5218 goldfish 2 6 9
5219 >>> df.groupby("a").shift(1)
5220 b c
5221 tuna NaN NaN
5222 salmon 2.0 3.0
5223 catfish NaN NaN
5224 goldfish 5.0 8.0
5225 """
5226 if is_list_like(periods):
5227 periods = cast("Sequence", periods)
5228 if len(periods) == 0:
5229 raise ValueError("If `periods` is an iterable, it cannot be empty.")
5230 from pandas.core.reshape.concat import concat
5232 add_suffix = True
5233 else:
5234 if not is_integer(periods):
5235 raise TypeError(
5236 f"Periods must be integer, but {periods} is {type(periods)}."
5237 )
5238 if suffix:
5239 raise ValueError("Cannot specify `suffix` if `periods` is an int.")
5240 periods = [cast("int", periods)]
5241 add_suffix = False
5243 shifted_dataframes = []
5244 for period in periods:
5245 if not is_integer(period):
5246 raise TypeError(
5247 f"Periods must be integer, but {period} is {type(period)}."
5248 )
5249 if freq is not None:
5250 f = lambda x: x.shift(
5251 period,
5252 freq,
5253 0, # axis
5254 fill_value,
5255 )
5256 shifted = self._python_apply_general(
5257 f, self._selected_obj, is_transform=True
5258 )
5259 else:
5260 if fill_value is lib.no_default:
5261 fill_value = None
5262 ids = self._grouper.ids
5263 ngroups = self._grouper.ngroups
5264 res_indexer = np.zeros(len(ids), dtype=np.int64)
5266 libgroupby.group_shift_indexer(res_indexer, ids, ngroups, period)
5268 obj = self._obj_with_exclusions
5270 shifted = obj._reindex_with_indexers(
5271 {0: (obj.index, res_indexer)},
5272 fill_value=fill_value,
5273 allow_dups=True,
5274 )
5276 if add_suffix:
5277 if isinstance(shifted, Series):
5278 shifted = cast("NDFrameT", shifted.to_frame())
5279 shifted = shifted.add_suffix(
5280 f"{suffix}_{period}" if suffix else f"_{period}"
5281 )
5282 shifted_dataframes.append(cast("Series | DataFrame", shifted))
5284 return (
5285 shifted_dataframes[0]
5286 if len(shifted_dataframes) == 1
5287 else concat(shifted_dataframes, axis=1, sort=False)
5288 )
5290 @final
5291 def diff(
5292 self,
5293 periods: int = 1,
5294 ) -> NDFrameT:
5295 """
5296 First discrete difference of element.
5298 Calculates the difference of each element compared with another
5299 element in the group (default is element in previous row).
5301 Parameters
5302 ----------
5303 periods : int, default 1
5304 Periods to shift for calculating difference, accepts negative values.
5306 Returns
5307 -------
5308 Series or DataFrame
5309 First differences.
5311 See Also
5312 --------
5313 Series.diff : Apply function diff to a Series.
5314 DataFrame.diff : Apply function diff to each row or column of a DataFrame.
5316 Examples
5317 --------
5318 For SeriesGroupBy:
5320 >>> lst = ["a", "a", "a", "b", "b", "b"]
5321 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
5322 >>> ser
5323 a 7
5324 a 2
5325 a 8
5326 b 4
5327 b 3
5328 b 3
5329 dtype: int64
5330 >>> ser.groupby(level=0).diff()
5331 a NaN
5332 a -5.0
5333 a 6.0
5334 b NaN
5335 b -1.0
5336 b 0.0
5337 dtype: float64
5339 For DataFrameGroupBy:
5341 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
5342 >>> df = pd.DataFrame(
5343 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
5344 ... )
5345 >>> df
5346 a b
5347 dog 1 1
5348 dog 3 4
5349 dog 5 8
5350 mouse 7 4
5351 mouse 7 4
5352 mouse 8 2
5353 mouse 3 1
5354 >>> df.groupby(level=0).diff()
5355 a b
5356 dog NaN NaN
5357 dog 2.0 3.0
5358 dog 2.0 4.0
5359 mouse NaN NaN
5360 mouse 0.0 0.0
5361 mouse 1.0 -2.0
5362 mouse -5.0 -1.0
5363 """
5364 obj = self._obj_with_exclusions
5365 shifted = self.shift(periods=periods)
5367 # GH45562 - to retain existing behavior and match behavior of Series.diff(),
5368 # int8 and int16 are coerced to float32 rather than float64.
5369 dtypes_to_f32 = ["int8", "int16"]
5370 if obj.ndim == 1:
5371 if obj.dtype in dtypes_to_f32:
5372 shifted = shifted.astype("float32")
5373 else:
5374 to_coerce = [c for c, dtype in obj.dtypes.items() if dtype in dtypes_to_f32]
5375 if to_coerce:
5376 shifted = shifted.astype(dict.fromkeys(to_coerce, "float32"))
5378 return obj - shifted
5380 @final
5381 def pct_change(
5382 self,
5383 periods: int = 1,
5384 fill_method: None = None,
5385 freq=None,
5386 ):
5387 """
5388 Calculate pct_change of each value to previous entry in group.
5390 This method calculates the percentage change between the current and
5391 a prior element within each group, useful for computing growth rates.
5393 Parameters
5394 ----------
5395 periods : int, default 1
5396 Periods to shift for calculating percentage change. Comparing with
5397 a period of 1 means adjacent elements are compared, whereas a period
5398 of 2 compares every other element.
5400 fill_method : None
5401 Must be None. This argument will be removed in a future version of pandas.
5403 freq : str, pandas offset object, or None, default None
5404 The frequency increment for time series data (e.g., 'M' for month-end).
5405 If None, the frequency is inferred from the index. Relevant for time
5406 series data only.
5408 Returns
5409 -------
5410 Series or DataFrame
5411 Percentage changes within each group.
5413 See Also
5414 --------
5415 Series.pct_change : Apply function pct_change to a Series.
5416 DataFrame.pct_change : Apply function pct_change to each row or column of
5417 a DataFrame.
5419 Examples
5420 --------
5422 For SeriesGroupBy:
5424 >>> lst = ["a", "a", "b", "b"]
5425 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
5426 >>> ser
5427 a 1
5428 a 2
5429 b 3
5430 b 4
5431 dtype: int64
5432 >>> ser.groupby(level=0).pct_change()
5433 a NaN
5434 a 1.000000
5435 b NaN
5436 b 0.333333
5437 dtype: float64
5439 For DataFrameGroupBy:
5441 >>> data = [[1, 2, 3], [1, 5, 6], [2, 5, 8], [2, 6, 9]]
5442 >>> df = pd.DataFrame(
5443 ... data,
5444 ... columns=["a", "b", "c"],
5445 ... index=["tuna", "salmon", "catfish", "goldfish"],
5446 ... )
5447 >>> df
5448 a b c
5449 tuna 1 2 3
5450 salmon 1 5 6
5451 catfish 2 5 8
5452 goldfish 2 6 9
5453 >>> df.groupby("a").pct_change()
5454 b c
5455 tuna NaN NaN
5456 salmon 1.5 1.000
5457 catfish NaN NaN
5458 goldfish 0.2 0.125
5459 """
5460 # GH#53491
5461 if fill_method is not None:
5462 raise ValueError(f"fill_method must be None; got {fill_method=}.")
5464 # TODO(GH#23918): Remove this conditional for SeriesGroupBy when
5465 # GH#23918 is fixed
5466 if freq is not None:
5467 f = lambda x: x.pct_change(
5468 periods=periods,
5469 freq=freq,
5470 axis=0,
5471 )
5472 return self._python_apply_general(f, self._selected_obj, is_transform=True)
5474 # fill_method is None (validated above); GH30463
5475 filled = self.ffill(limit=0)
5476 fill_grp = filled.groupby(self._grouper.codes, group_keys=self.group_keys)
5477 shifted = fill_grp.shift(periods=periods, freq=freq)
5478 return (filled / shifted) - 1
5480 @final
5481 def head(self, n: int = 5) -> NDFrameT:
5482 """
5483 Return first n rows of each group.
5485 Similar to ``.apply(lambda x: x.head(n))``, but it returns a subset of rows
5486 from the original DataFrame with original index and order preserved
5487 (``as_index`` flag is ignored).
5489 Parameters
5490 ----------
5491 n : int
5492 If positive: number of entries to include from start of each group.
5493 If negative: number of entries to exclude from end of each group.
5495 Returns
5496 -------
5497 Series or DataFrame
5498 Subset of original Series or DataFrame as determined by n.
5500 See Also
5501 --------
5502 Series.head : Apply function head to a Series.
5503 DataFrame.head : Apply function head to each row or column of a DataFrame.
5505 Examples
5506 --------
5508 >>> df = pd.DataFrame([[1, 2], [1, 4], [5, 6]], columns=["A", "B"])
5509 >>> df.groupby("A").head(1)
5510 A B
5511 0 1 2
5512 2 5 6
5513 >>> df.groupby("A").head(-1)
5514 A B
5515 0 1 2
5516 """
5517 mask = self._make_mask_from_positional_indexer(slice(None, n))
5518 return self._mask_selected_obj(mask)
5520 @final
5521 def tail(self, n: int = 5) -> NDFrameT:
5522 """
5523 Return last n rows of each group.
5525 Similar to ``.apply(lambda x: x.tail(n))``, but it returns a subset of rows
5526 from the original DataFrame with original index and order preserved
5527 (``as_index`` flag is ignored).
5529 Parameters
5530 ----------
5531 n : int
5532 If positive: number of entries to include from end of each group.
5533 If negative: number of entries to exclude from start of each group.
5535 Returns
5536 -------
5537 Series or DataFrame
5538 Subset of original Series or DataFrame as determined by n.
5540 See Also
5541 --------
5542 Series.tail : Apply function tail to a Series.
5543 DataFrame.tail : Apply function tail to each row or column of a DataFrame.
5545 Examples
5546 --------
5548 >>> df = pd.DataFrame(
5549 ... [["a", 1], ["a", 2], ["b", 1], ["b", 2]], columns=["A", "B"]
5550 ... )
5551 >>> df.groupby("A").tail(1)
5552 A B
5553 1 a 2
5554 3 b 2
5555 >>> df.groupby("A").tail(-1)
5556 A B
5557 1 a 2
5558 3 b 2
5559 """
5560 if n:
5561 mask = self._make_mask_from_positional_indexer(slice(-n, None))
5562 else:
5563 mask = self._make_mask_from_positional_indexer([])
5565 return self._mask_selected_obj(mask)
5567 @final
5568 def _mask_selected_obj(self, mask: npt.NDArray[np.bool_]) -> NDFrameT:
5569 """
5570 Return _selected_obj with mask applied.
5572 Parameters
5573 ----------
5574 mask : np.ndarray[bool]
5575 Boolean mask to apply.
5577 Returns
5578 -------
5579 Series or DataFrame
5580 Filtered _selected_obj.
5581 """
5582 ids = self._grouper.ids
5583 mask = mask & (ids != -1)
5584 return self._selected_obj[mask]
5586 @final
5587 def sample(
5588 self,
5589 n: int | None = None,
5590 frac: float | None = None,
5591 replace: bool = False,
5592 weights: ListLike | None = None,
5593 random_state: RandomState | None = None,
5594 ):
5595 """
5596 Return a random sample of items from each group.
5598 You can use `random_state` for reproducibility.
5600 Parameters
5601 ----------
5602 n : int, optional
5603 Number of items to return for each group. Cannot be used with
5604 `frac` and must be no larger than the smallest group unless
5605 `replace` is True. Default is one if `frac` is None.
5606 frac : float, optional
5607 Fraction of items to return. Cannot be used with `n`.
5608 replace : bool, default False
5609 Allow or disallow sampling of the same row more than once.
5610 weights : list-like, optional
5611 Default None results in equal probability weighting.
5612 If passed a list-like then values must have the same length as
5613 the underlying DataFrame or Series object and will be used as
5614 sampling probabilities after normalization within each group.
5615 Values must be non-negative with at least one positive element
5616 within each group.
5617 random_state : int, array-like, BitGenerator, np.random.RandomState, np.random.Generator, optional
5618 If int, array-like, or BitGenerator, seed for random number generator.
5619 If np.random.RandomState or np.random.Generator, use as given.
5620 Default ``None`` results in sampling with the current state of np.random.
5622 Returns
5623 -------
5624 Series or DataFrame
5625 A new object of same type as caller containing items randomly
5626 sampled within each group from the caller object.
5628 See Also
5629 --------
5630 DataFrame.sample: Generate random samples from a DataFrame object.
5631 Series.sample: Generate random samples from a Series object.
5632 numpy.random.choice: Generate a random sample from a given 1-D numpy
5633 array.
5635 Examples
5636 --------
5637 >>> df = pd.DataFrame(
5638 ... {"a": ["red"] * 2 + ["blue"] * 2 + ["black"] * 2, "b": range(6)}
5639 ... )
5640 >>> df
5641 a b
5642 0 red 0
5643 1 red 1
5644 2 blue 2
5645 3 blue 3
5646 4 black 4
5647 5 black 5
5649 Select one row at random for each distinct value in column a. The
5650 `random_state` argument can be used to guarantee reproducibility:
5652 >>> df.groupby("a").sample(n=1, random_state=1)
5653 a b
5654 4 black 4
5655 2 blue 2
5656 1 red 1
5658 Set `frac` to sample fixed proportions rather than counts:
5660 >>> df.groupby("a")["b"].sample(frac=0.5, random_state=2)
5661 5 5
5662 2 2
5663 0 0
5664 Name: b, dtype: int64
5666 Control sample probabilities within groups by setting weights:
5668 >>> df.groupby("a").sample(
5669 ... n=1,
5670 ... weights=[1, 1, 1, 0, 0, 1],
5671 ... random_state=1,
5672 ... )
5673 a b
5674 5 black 5
5675 2 blue 2
5676 0 red 0
5677 """ # noqa: E501
5678 if self._selected_obj.empty:
5679 # GH48459 prevent ValueError when object is empty
5680 return self._selected_obj
5681 size = sample.process_sampling_size(n, frac, replace)
5682 if weights is not None:
5683 weights_arr = sample.preprocess_weights(self._selected_obj, weights, axis=0)
5685 random_state = com.random_state(random_state)
5687 group_iterator = self._grouper.get_iterator(self._selected_obj)
5689 sampled_indices = []
5690 for labels, obj in group_iterator:
5691 grp_indices = self.indices[labels]
5692 group_size = len(grp_indices)
5693 if size is not None:
5694 sample_size = size
5695 else:
5696 assert frac is not None
5697 sample_size = round(frac * group_size)
5699 grp_sample = sample.sample(
5700 group_size,
5701 size=sample_size,
5702 replace=replace,
5703 weights=None if weights is None else weights_arr[grp_indices],
5704 random_state=random_state,
5705 )
5706 sampled_indices.append(grp_indices[grp_sample])
5708 concatenated_sampled_indices = np.concatenate(sampled_indices)
5709 return self._selected_obj.take(concatenated_sampled_indices, axis=0)
5711 def _idxmax_idxmin(
5712 self,
5713 how: Literal["idxmax", "idxmin"],
5714 ignore_unobserved: bool = False,
5715 skipna: bool = True,
5716 numeric_only: bool = False,
5717 ) -> NDFrameT:
5718 """Compute idxmax/idxmin.
5720 Parameters
5721 ----------
5722 how : {'idxmin', 'idxmax'}
5723 Whether to compute idxmin or idxmax.
5724 numeric_only : bool, default False
5725 Include only float, int, boolean columns.
5726 skipna : bool, default True
5727 Exclude NA/null values. If an entire group is NA, the result will be NA.
5728 ignore_unobserved : bool, default False
5729 When True and an unobserved group is encountered, do not raise. This used
5730 for transform where unobserved groups do not play an impact on the result.
5732 Returns
5733 -------
5734 Series or DataFrame
5735 idxmax or idxmin for the groupby operation.
5736 """
5737 if not self.observed and any(
5738 ping._passed_categorical for ping in self._grouper.groupings
5739 ):
5740 expected_len = len(self._grouper.result_index)
5741 # TODO: Better way to find # of observed groups?
5742 group_sizes = self._grouper.size()
5743 result_len = group_sizes[group_sizes > 0].shape[0]
5744 assert result_len <= expected_len
5745 has_unobserved = result_len < expected_len
5747 raise_err: bool | np.bool_ = not ignore_unobserved and has_unobserved
5748 # Only raise an error if there are columns to compute; otherwise we return
5749 # an empty DataFrame with an index (possibly including unobserved) but no
5750 # columns
5751 data = self._obj_with_exclusions
5752 if raise_err and isinstance(data, DataFrame):
5753 if numeric_only:
5754 data = data._get_numeric_data()
5755 raise_err = len(data.columns) > 0
5757 if raise_err:
5758 raise ValueError(
5759 f"Can't get {how} of an empty group due to unobserved categories. "
5760 "Specify observed=True in groupby instead."
5761 )
5762 elif not skipna and self._obj_with_exclusions.isna().any(axis=None):
5763 raise ValueError(f"{how} with skipna=False encountered an NA value.")
5765 result = self._cython_agg_general(
5766 how,
5767 numeric_only=numeric_only,
5768 min_count=1,
5769 skipna=skipna,
5770 )
5771 return result
5773 def _wrap_idxmax_idxmin(
5774 self, res: NDFrameT, how: Literal["idxmax", "idxmin"], skipna: bool
5775 ) -> NDFrameT:
5776 index = self.obj.index
5777 if res.size == 0:
5778 result = res.astype(index.dtype)
5779 elif skipna and res.lt(0).any(axis=None):
5780 raise ValueError(
5781 f"{how} with skipna=True encountered all NA values in a group."
5782 )
5783 else:
5784 if isinstance(index, MultiIndex):
5785 index = index.to_flat_index()
5786 values = res._values
5787 assert isinstance(values, np.ndarray)
5788 na_value = na_value_for_dtype(index.dtype, compat=False)
5789 if isinstance(res, Series):
5790 # mypy: expression has type "Series", variable has type "NDFrameT"
5791 result = res._constructor( # type: ignore[assignment]
5792 index.array.take(values, allow_fill=True, fill_value=na_value),
5793 index=res.index,
5794 name=res.name,
5795 )
5796 else:
5797 data = {}
5798 for k, column_values in enumerate(values.T):
5799 data[k] = index.array.take(
5800 column_values, allow_fill=True, fill_value=na_value
5801 )
5802 result = self.obj._constructor(data, index=res.index)
5803 result.columns = res.columns
5804 return result
5807def get_groupby(
5808 obj: NDFrame,
5809 by: _KeysArgType | None = None,
5810 grouper: ops.BaseGrouper | None = None,
5811 group_keys: bool = True,
5812) -> GroupBy:
5813 """
5814 Class for grouping and aggregating relational data.
5816 See aggregate, transform, and apply functions on this object.
5818 It's easiest to use obj.groupby(...) to use GroupBy, but you can also do:
5820 ::
5822 grouped = groupby(obj, ...)
5824 Parameters
5825 ----------
5826 obj : pandas object
5828 Returns
5829 -------
5830 **Attributes**
5831 groups : dict
5832 {group name -> group labels}
5833 len(grouped) : int
5834 Number of groups
5836 Notes
5837 -----
5838 After grouping, see aggregate, apply, and transform functions. Here are
5839 some other brief notes about usage. When grouping by multiple groups, the
5840 result index will be a MultiIndex (hierarchical) by default.
5842 Iteration produces (key, group) tuples, i.e. chunking the data by group. So
5843 you can write code like:
5845 ::
5847 grouped = obj.groupby(keys)
5848 for key, group in grouped:
5849 # do something with the data
5851 Function calls on GroupBy, if not specially implemented, "dispatch" to the
5852 grouped data. So if you group a DataFrame and wish to invoke the std()
5853 method on each group, you can simply do:
5855 ::
5857 df.groupby(mapper).std()
5859 rather than
5861 ::
5863 df.groupby(mapper).aggregate(np.std)
5865 You can pass arguments to these "wrapped" functions, too.
5867 See the online documentation for full exposition on these topics and much
5868 more
5869 """
5870 if isinstance(obj, Series):
5871 from pandas.core.groupby.generic import SeriesGroupBy
5873 return SeriesGroupBy(
5874 obj=obj,
5875 keys=by,
5876 grouper=grouper,
5877 group_keys=group_keys,
5878 )
5879 elif isinstance(obj, DataFrame):
5880 from pandas.core.groupby.generic import DataFrameGroupBy
5882 return DataFrameGroupBy(
5883 obj=obj,
5884 keys=by,
5885 grouper=grouper,
5886 group_keys=group_keys,
5887 )
5888 else: # pragma: no cover
5889 raise TypeError(f"invalid type: {obj}")
5892def _insert_quantile_level(idx: Index, qs: npt.NDArray[np.float64]) -> MultiIndex:
5893 """
5894 Insert the sequence 'qs' of quantiles as the inner-most level of a MultiIndex.
5896 The quantile level in the MultiIndex is a repeated copy of 'qs'.
5898 Parameters
5899 ----------
5900 idx : Index
5901 qs : np.ndarray[float64]
5903 Returns
5904 -------
5905 MultiIndex
5906 """
5907 nqs = len(qs)
5908 lev_codes, lev = Index(qs, copy=False).factorize()
5909 lev_codes = coerce_indexer_dtype(lev_codes, lev)
5911 if idx._is_multi:
5912 idx = cast("MultiIndex", idx)
5913 levels = [*idx.levels, lev]
5914 codes = [np.repeat(x, nqs) for x in idx.codes] + [np.tile(lev_codes, len(idx))]
5915 mi = MultiIndex(levels=levels, codes=codes, names=[*idx.names, None])
5916 else:
5917 nidx = len(idx)
5918 idx_codes = coerce_indexer_dtype(np.arange(nidx), idx)
5919 levels = [idx, lev]
5920 codes = [np.repeat(idx_codes, nqs), np.tile(lev_codes, nidx)]
5921 mi = MultiIndex(levels=levels, codes=codes, names=[idx.name, None])
5923 return mi