Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Provide the groupby split-apply-combine paradigm. Define the GroupBy
3class providing the base-class of operations.
5The SeriesGroupBy and DataFrameGroupBy sub-class
6(defined in pandas.core.groupby.generic)
7expose these user-facing objects to provide specific functionality.
8"""
10from __future__ import annotations
12from collections.abc import (
13 Callable,
14 Hashable,
15 Iterable,
16 Iterator,
17 Mapping,
18 Sequence,
19)
20import datetime
21from functools import (
22 partial,
23 wraps,
24)
25from typing import (
26 TYPE_CHECKING,
27 Concatenate,
28 Literal,
29 Self,
30 TypeAlias,
31 TypeVar,
32 Union,
33 cast,
34 final,
35 overload,
36)
37import warnings
39import numpy as np
41from pandas._libs import (
42 Timestamp,
43 lib,
44)
45from pandas._libs.algos import rank_1d
46import pandas._libs.groupby as libgroupby
47from pandas._libs.missing import NA
48from pandas._typing import (
49 AnyArrayLike,
50 ArrayLike,
51 DtypeObj,
52 IndexLabel,
53 IntervalClosedType,
54 NDFrameT,
55 PositionalIndexer,
56 RandomState,
57 npt,
58)
59from pandas.compat.numpy import function as nv
60from pandas.errors import (
61 AbstractMethodError,
62 DataError,
63 Pandas4Warning,
64)
65from pandas.util._decorators import cache_readonly
66from pandas.util._exceptions import find_stack_level
68from pandas.core.dtypes.cast import (
69 coerce_indexer_dtype,
70 ensure_dtype_can_hold_na,
71)
72from pandas.core.dtypes.common import (
73 is_bool,
74 is_bool_dtype,
75 is_float_dtype,
76 is_hashable,
77 is_integer,
78 is_integer_dtype,
79 is_list_like,
80 is_numeric_dtype,
81 is_object_dtype,
82 is_scalar,
83 is_string_dtype,
84 needs_i8_conversion,
85 pandas_dtype,
86)
87from pandas.core.dtypes.missing import (
88 isna,
89 na_value_for_dtype,
90 notna,
91)
93from pandas.core import (
94 algorithms,
95 sample,
96)
97from pandas.core._numba import executor
98from pandas.core.arrays import (
99 ArrowExtensionArray,
100 BaseMaskedArray,
101 ExtensionArray,
102 FloatingArray,
103 IntegerArray,
104 SparseArray,
105)
106from pandas.core.arrays.string_ import StringDtype
107from pandas.core.arrays.string_arrow import ArrowStringArray
108from pandas.core.base import (
109 PandasObject,
110 SelectionMixin,
111)
112import pandas.core.common as com
113from pandas.core.frame import DataFrame
114from pandas.core.generic import NDFrame
115from pandas.core.groupby import (
116 base,
117 numba_,
118 ops,
119)
120from pandas.core.groupby.grouper import get_grouper
121from pandas.core.groupby.indexing import (
122 GroupByIndexingMixin,
123 GroupByNthSelector,
124)
125from pandas.core.indexes.api import (
126 Index,
127 MultiIndex,
128 default_index,
129)
130from pandas.core.internals.blocks import ensure_block_shape
131from pandas.core.series import Series
132from pandas.core.sorting import get_group_index_sorter
133from pandas.core.util.numba_ import (
134 get_jit_arguments,
135 maybe_use_numba,
136 prepare_function_arguments,
137)
139if TYPE_CHECKING:
140 from pandas._libs.tslibs import BaseOffset
141 from pandas._libs.tslibs.timedeltas import Timedelta
142 from pandas._typing import (
143 Any,
144 P,
145 T,
146 )
148 from pandas.core.indexers.objects import BaseIndexer
149 from pandas.core.resample import Resampler
150 from pandas.core.window import (
151 ExpandingGroupby,
152 ExponentialMovingWindowGroupby,
153 RollingGroupby,
154 )
156_groupby_agg_method_engine_template = """
157Compute {fname} of group values.
159Parameters
160----------
161numeric_only : bool, default {no}
162 Include only float, int, boolean columns.
164 .. versionchanged:: 2.0.0
166 numeric_only no longer accepts ``None``.
168min_count : int, default {mc}
169 The required number of valid values to perform the operation. If fewer
170 than ``min_count`` non-NA values are present the result will be NA.
172engine : str, default None {e}
173 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
174 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
175 Only available when ``raw`` is set to ``True``.
176 * ``None`` : Defaults to ``'cython'`` or globally setting
177 ``compute.use_numba``
179engine_kwargs : dict, default None {ek}
180 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
181 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
182 and ``parallel`` dictionary keys. The values must either be ``True`` or
183 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
184 ``{{'nopython': True, 'nogil': False, 'parallel': False}}`` and will be
185 applied to both the ``func`` and the ``apply`` groupby aggregation.
187Returns
188-------
189Series or DataFrame
190 Computed {fname} of values within each group.
192See Also
193--------
194SeriesGroupBy.min : Return the min of the group values.
195DataFrameGroupBy.min : Return the min of the group values.
196SeriesGroupBy.max : Return the max of the group values.
197DataFrameGroupBy.max : Return the max of the group values.
198SeriesGroupBy.sum : Return the sum of the group values.
199DataFrameGroupBy.sum : Return the sum of the group values.
201Examples
202--------
203{example}
204"""
206_groupby_agg_method_skipna_engine_template = """
207Compute {fname} of group values.
209Parameters
210----------
211numeric_only : bool, default {no}
212 Include only float, int, boolean columns.
214 .. versionchanged:: 2.0.0
216 numeric_only no longer accepts ``None``.
218min_count : int, default {mc}
219 The required number of valid values to perform the operation. If fewer
220 than ``min_count`` non-NA values are present the result will be NA.
222skipna : bool, default {s}
223 Exclude NA/null values. If the entire group is NA and ``skipna`` is
224 ``True``, the result will be NA.
226 .. versionchanged:: 3.0.0
228engine : str, default None {e}
229 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
230 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
231 Only available when ``raw`` is set to ``True``.
232 * ``None`` : Defaults to ``'cython'`` or globally setting
233 ``compute.use_numba``
235engine_kwargs : dict, default None {ek}
236 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
237 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
238 and ``parallel`` dictionary keys. The values must either be ``True`` or
239 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
240 ``{{'nopython': True, 'nogil': False, 'parallel': False}}`` and will be
241 applied to both the ``func`` and the ``apply`` groupby aggregation.
243Returns
244-------
245Series or DataFrame
246 Computed {fname} of values within each group.
248See Also
249--------
250SeriesGroupBy.min : Return the min of the group values.
251DataFrameGroupBy.min : Return the min of the group values.
252SeriesGroupBy.max : Return the max of the group values.
253DataFrameGroupBy.max : Return the max of the group values.
254SeriesGroupBy.sum : Return the sum of the group values.
255DataFrameGroupBy.sum : Return the sum of the group values.
257Examples
258--------
259{example}
260"""
262_pipe_template = """
263Apply a ``func`` with arguments to this %(klass)s object and return its result.
265Use `.pipe` when you want to improve readability by chaining together
266functions that expect Series, DataFrames, GroupBy or Resampler objects.
267Instead of writing
269>>> h = lambda x, arg2, arg3: x + 1 - arg2 * arg3
270>>> g = lambda x, arg1: x * 5 / arg1
271>>> f = lambda x: x ** 4
272>>> df = pd.DataFrame([["a", 4], ["b", 5]], columns=["group", "value"])
273>>> h(g(f(df.groupby('group')), arg1=1), arg2=2, arg3=3) # doctest: +SKIP
275You can write
277>>> (df.groupby('group')
278... .pipe(f)
279... .pipe(g, arg1=1)
280... .pipe(h, arg2=2, arg3=3)) # doctest: +SKIP
282which is much more readable.
284Parameters
285----------
286func : callable or tuple of (callable, str)
287 Function to apply to this %(klass)s object or, alternatively,
288 a `(callable, data_keyword)` tuple where `data_keyword` is a
289 string indicating the keyword of `callable` that expects the
290 %(klass)s object.
291*args : iterable, optional
292 Positional arguments passed into `func`.
293**kwargs : dict, optional
294 A dictionary of keyword arguments passed into `func`.
296Returns
297-------
298%(klass)s
299 The original object with the function `func` applied.
301See Also
302--------
303Series.pipe : Apply a function with arguments to a series.
304DataFrame.pipe: Apply a function with arguments to a dataframe.
305apply : Apply function to each group instead of to the
306 full %(klass)s object.
308Notes
309-----
310See more `here
311<https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html#piping-function-calls>`_
313Examples
314--------
315%(examples)s
316"""
318_transform_template = """
319Call function producing a same-indexed %(klass)s on each group.
321Returns a %(klass)s having the same indexes as the original object
322filled with the transformed values.
324Parameters
325----------
326func : function, str
327 Function to apply to each group. See the Notes section below for requirements.
329 Accepted inputs are:
331 - String
332 - Python function
333 - Numba JIT function with ``engine='numba'`` specified.
335 Only passing a single function is supported with this engine.
336 If the ``'numba'`` engine is chosen, the function must be
337 a user defined function with ``values`` and ``index`` as the
338 first and second arguments respectively in the function signature.
339 Each group's index will be passed to the user defined function
340 and optionally available for use.
342 If a string is chosen, then it needs to be the name
343 of the groupby method you want to use.
344*args
345 Positional arguments to pass to func.
346engine : str, default None
347 * ``'cython'`` : Runs the function through C-extensions from cython.
348 * ``'numba'`` : Runs the function through JIT compiled code from numba.
349 * ``None`` : Defaults to ``'cython'`` or the global setting ``compute.use_numba``
351engine_kwargs : dict, default None
352 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
353 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
354 and ``parallel`` dictionary keys. The values must either be ``True`` or
355 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
356 ``{'nopython': True, 'nogil': False, 'parallel': False}`` and will be
357 applied to the function
359**kwargs
360 Keyword arguments to be passed into func.
362Returns
363-------
364%(klass)s
365 %(klass)s with the same indexes as the original object filled
366 with transformed values.
368See Also
369--------
370%(klass)s.groupby.apply : Apply function ``func`` group-wise and combine
371 the results together.
372%(klass)s.groupby.aggregate : Aggregate using one or more operations.
373%(klass)s.transform : Call ``func`` on self producing a %(klass)s with the
374 same axis shape as self.
376Notes
377-----
378Each group is endowed the attribute 'name' in case you need to know
379which group you are working on.
381The current implementation imposes three requirements on f:
383* f must return a value that either has the same shape as the input
384 subframe or can be broadcast to the shape of the input subframe.
385 For example, if `f` returns a scalar it will be broadcast to have the
386 same shape as the input subframe.
387* if this is a DataFrame, f must support application column-by-column
388 in the subframe. If f also supports application to the entire subframe,
389 then a fast path is used starting from the second chunk.
390* f must not mutate groups. Mutation is not supported and may
391 produce unexpected results. See :ref:`gotchas.udf-mutation` for more details.
393When using ``engine='numba'``, there will be no "fall back" behavior internally.
394The group data and group index will be passed as numpy arrays to the JITed
395user defined function, and no alternative execution attempts will be tried.
397The resulting dtype will reflect the return value of the passed ``func``,
398see the examples below.
400.. versionchanged:: 2.0.0
402 When using ``.transform`` on a grouped DataFrame and the transformation function
403 returns a DataFrame, pandas now aligns the result's index
404 with the input's index. You can call ``.to_numpy()`` on the
405 result of the transformation function to avoid alignment.
407Examples
408--------
409%(example)s"""
412@final
413class GroupByPlot(PandasObject):
414 """
415 Class implementing the .plot attribute for groupby objects.
416 """
418 def __init__(self, groupby: GroupBy) -> None:
419 self._groupby = groupby
421 def __call__(self, *args, **kwargs):
422 def f(self):
423 return self.plot(*args, **kwargs)
425 f.__name__ = "plot"
426 return self._groupby._python_apply_general(f, self._groupby._selected_obj)
428 def __getattr__(self, name: str):
429 def attr(*args, **kwargs):
430 def f(self):
431 return getattr(self.plot, name)(*args, **kwargs)
433 return self._groupby._python_apply_general(f, self._groupby._selected_obj)
435 return attr
438_KeysArgType: TypeAlias = (
439 Hashable
440 | list[Hashable]
441 | Callable[[Hashable], Hashable]
442 | list[Callable[[Hashable], Hashable]]
443 | Mapping[Hashable, Hashable]
444)
447class BaseGroupBy(PandasObject, SelectionMixin[NDFrameT], GroupByIndexingMixin):
448 _hidden_attrs = PandasObject._hidden_attrs | {
449 "as_index",
450 "dropna",
451 "exclusions",
452 "grouper",
453 "group_keys",
454 "keys",
455 "level",
456 "obj",
457 "observed",
458 "sort",
459 }
461 _grouper: ops.BaseGrouper
462 keys: _KeysArgType | None = None
463 level: IndexLabel | None = None
464 group_keys: bool
466 @final
467 def __len__(self) -> int:
468 return self._grouper.ngroups
470 @final
471 def __repr__(self) -> str:
472 # TODO: Better repr for GroupBy object
473 return object.__repr__(self)
475 @final
476 @property
477 def groups(self) -> dict[Hashable, Index]:
478 """
479 Dict {group name -> group labels}.
481 This property provides a dictionary representation of the groupings formed
482 during a groupby operation, where each key represents a unique group value from
483 the specified column(s), and each value is a list of index labels
484 that belong to that group.
486 See Also
487 --------
488 core.groupby.DataFrameGroupBy.get_group : Retrieve group from a
489 ``DataFrameGroupBy`` object with provided name.
490 core.groupby.SeriesGroupBy.get_group : Retrieve group from a
491 ``SeriesGroupBy`` object with provided name.
492 core.resample.Resampler.get_group : Retrieve group from a
493 ``Resampler`` object with provided name.
495 Examples
496 --------
498 For SeriesGroupBy:
500 >>> lst = ["a", "a", "b"]
501 >>> ser = pd.Series([1, 2, 3], index=lst)
502 >>> ser
503 a 1
504 a 2
505 b 3
506 dtype: int64
507 >>> ser.groupby(level=0).groups
508 {'a': ['a', 'a'], 'b': ['b']}
510 For DataFrameGroupBy:
512 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
513 >>> df = pd.DataFrame(data, columns=["a", "b", "c"])
514 >>> df
515 a b c
516 0 1 2 3
517 1 1 5 6
518 2 7 8 9
519 >>> df.groupby(by="a").groups
520 {1: [0, 1], 7: [2]}
522 For Resampler:
524 >>> ser = pd.Series(
525 ... [1, 2, 3, 4],
526 ... index=pd.DatetimeIndex(
527 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
528 ... ),
529 ... )
530 >>> ser
531 2023-01-01 1
532 2023-01-15 2
533 2023-02-01 3
534 2023-02-15 4
535 dtype: int64
536 >>> ser.resample("MS").groups
537 {Timestamp('2023-01-01 00:00:00'): np.int64(2),
538 Timestamp('2023-02-01 00:00:00'): np.int64(4)}
539 """
540 if isinstance(self.keys, list) and len(self.keys) == 1:
541 warnings.warn(
542 "In a future version, the keys of `groups` will be a "
543 f"tuple with a single element, e.g. ({self.keys[0]},) , "
544 f"instead of a scalar, e.g. {self.keys[0]}, when grouping "
545 "by a list with a single element. Use ``df.groupby(by='a').groups`` "
546 "instead of ``df.groupby(by=['a']).groups`` to avoid this warning",
547 Pandas4Warning,
548 stacklevel=find_stack_level(),
549 )
550 return self._grouper.groups
552 @final
553 @property
554 def ngroups(self) -> int:
555 return self._grouper.ngroups
557 @final
558 @property
559 def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
560 """
561 Dict {group name -> group indices}.
563 The dictionary keys represent the group labels (e.g., timestamps for a
564 time-based resampling operation), and the values are arrays of integer
565 positions indicating where the elements of each group are located in the
566 original data. This property is particularly useful when working with
567 resampled data, as it provides insight into how the original time-series data
568 has been grouped.
570 See Also
571 --------
572 core.groupby.DataFrameGroupBy.indices : Provides a mapping of group rows to
573 positions of the elements.
574 core.groupby.SeriesGroupBy.indices : Provides a mapping of group rows to
575 positions of the elements.
576 core.resample.Resampler.indices : Provides a mapping of group rows to
577 positions of the elements.
579 Examples
580 --------
582 For SeriesGroupBy:
584 >>> lst = ["a", "a", "b"]
585 >>> ser = pd.Series([1, 2, 3], index=lst)
586 >>> ser
587 a 1
588 a 2
589 b 3
590 dtype: int64
591 >>> ser.groupby(level=0).indices
592 {'a': array([0, 1]), 'b': array([2])}
594 For DataFrameGroupBy:
596 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
597 >>> df = pd.DataFrame(
598 ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
599 ... )
600 >>> df
601 a b c
602 owl 1 2 3
603 toucan 1 5 6
604 eagle 7 8 9
605 >>> df.groupby(by=["a"]).indices
606 {np.int64(1): array([0, 1]), np.int64(7): array([2])}
608 For Resampler:
610 >>> ser = pd.Series(
611 ... [1, 2, 3, 4],
612 ... index=pd.DatetimeIndex(
613 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
614 ... ),
615 ... )
616 >>> ser
617 2023-01-01 1
618 2023-01-15 2
619 2023-02-01 3
620 2023-02-15 4
621 dtype: int64
622 >>> ser.resample("MS").indices
623 defaultdict(<class 'list'>, {Timestamp('2023-01-01 00:00:00'): [0, 1],
624 Timestamp('2023-02-01 00:00:00'): [2, 3]})
625 """
626 return self._grouper.indices
628 @final
629 def _get_index(self, name):
630 """
631 Safe get multiple indices, translate keys for
632 datelike to underlying repr.
633 """
635 def get_converter(s):
636 # possibly convert to the actual key types
637 # in the indices, could be a Timestamp or an np.datetime64
638 if isinstance(s, datetime.datetime):
639 return lambda key: Timestamp(key)
640 elif isinstance(s, np.datetime64):
641 return lambda key: Timestamp(key).asm8
642 else:
643 return lambda key: key
645 if isna(name):
646 return self.indices.get(np.nan, [])
647 if isinstance(name, tuple):
648 name = tuple(np.nan if isna(comp) else comp for comp in name)
650 if len(self.indices) > 0:
651 index_sample = next(iter(self.indices))
652 else:
653 index_sample = None # Dummy sample
655 if isinstance(index_sample, tuple):
656 if not isinstance(name, tuple):
657 msg = "must supply a tuple to get_group with multiple grouping keys"
658 raise ValueError(msg)
659 if not len(name) == len(index_sample):
660 try:
661 # If the original grouper was a tuple
662 return self.indices[name]
663 except KeyError as err:
664 # turns out it wasn't a tuple
665 msg = (
666 "must supply a same-length tuple to get_group "
667 "with multiple grouping keys"
668 )
669 raise ValueError(msg) from err
671 converters = (get_converter(s) for s in index_sample)
672 name = tuple(f(n) for f, n in zip(converters, name, strict=True))
673 else:
674 converter = get_converter(index_sample)
675 name = converter(name)
677 return self.indices.get(name, [])
679 @final
680 @cache_readonly
681 def _selected_obj(self):
682 # Note: _selected_obj is always just `self.obj` for SeriesGroupBy
683 if isinstance(self.obj, Series):
684 return self.obj
686 if self._selection is not None:
687 if is_hashable(self._selection):
688 # i.e. a single key, so selecting it will return a Series.
689 # In this case, _obj_with_exclusions would wrap the key
690 # in a list and return a single-column DataFrame.
691 return self.obj[self._selection]
693 # Otherwise _selection is equivalent to _selection_list, so
694 # _selected_obj matches _obj_with_exclusions, so we can reuse
695 # that and avoid making a copy.
696 return self._obj_with_exclusions
698 return self.obj
700 @final
701 def _dir_additions(self) -> set[str]:
702 return self.obj._dir_additions()
704 @overload
705 def pipe(
706 self,
707 func: Callable[Concatenate[Self, P], T],
708 *args: P.args,
709 **kwargs: P.kwargs,
710 ) -> T: ...
712 @overload
713 def pipe(
714 self,
715 func: tuple[Callable[..., T], str],
716 *args: Any,
717 **kwargs: Any,
718 ) -> T: ...
720 def pipe(
721 self,
722 func: Callable[Concatenate[Self, P], T] | tuple[Callable[..., T], str],
723 *args: Any,
724 **kwargs: Any,
725 ) -> T:
726 """
727 Apply a ``func`` with arguments to this GroupBy object and return its result.
729 Use `.pipe` when you want to improve readability by chaining together
730 functions that expect Series, DataFrames, GroupBy or Resampler objects.
731 Instead of writing
733 >>> h = lambda x, arg2, arg3: x + 1 - arg2 * arg3
734 >>> g = lambda x, arg1: x * 5 / arg1
735 >>> f = lambda x: x**4
736 >>> df = pd.DataFrame([["a", 4], ["b", 5]], columns=["group", "value"])
737 >>> h(g(f(df.groupby("group")), arg1=1), arg2=2, arg3=3) # doctest: +SKIP
739 You can write
741 >>> (
742 ... df.groupby("group").pipe(f).pipe(g, arg1=1).pipe(h, arg2=2, arg3=3)
743 ... ) # doctest: +SKIP
745 which is much more readable.
747 Parameters
748 ----------
749 func : callable or tuple of (callable, str)
750 Function to apply to this GroupBy object or, alternatively,
751 a `(callable, data_keyword)` tuple where `data_keyword` is a
752 string indicating the keyword of `callable` that expects the
753 GroupBy object.
754 *args : iterable, optional
755 Positional arguments passed into `func`.
756 **kwargs : dict, optional
757 A dictionary of keyword arguments passed into `func`.
759 Returns
760 -------
761 GroupBy
762 The return type of `func`.
764 See Also
765 --------
766 Series.pipe : Apply a function with arguments to a series.
767 DataFrame.pipe : Apply a function with arguments to a dataframe.
768 apply : Apply function to each group instead of to the
769 full GroupBy object.
771 Notes
772 -----
773 See more `here
774 <https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html#piping-function-calls>`_
776 Examples
777 --------
778 >>> df = pd.DataFrame({"A": "a b a b".split(), "B": [1, 2, 3, 4]})
779 >>> df
780 A B
781 0 a 1
782 1 b 2
783 2 a 3
784 3 b 4
786 To get the difference between each groups maximum and minimum value in one
787 pass, you can do
789 >>> df.groupby("A").pipe(lambda x: x.max() - x.min())
790 B
791 A
792 a 2
793 b 2
794 """
795 return com.pipe(self, func, *args, **kwargs)
797 @final
798 def get_group(self, name) -> DataFrame | Series:
799 """
800 Construct DataFrame from group with provided name.
802 Parameters
803 ----------
804 name : object
805 The name of the group to get as a DataFrame.
807 Returns
808 -------
809 Series or DataFrame
810 Get the respective Series or DataFrame corresponding to the group provided.
812 See Also
813 --------
814 DataFrameGroupBy.groups: Dictionary representation of the groupings formed
815 during a groupby operation.
816 DataFrameGroupBy.indices: Provides a mapping of group rows to positions
817 of the elements.
818 SeriesGroupBy.groups: Dictionary representation of the groupings formed
819 during a groupby operation.
820 SeriesGroupBy.indices: Provides a mapping of group rows to positions
821 of the elements.
823 Examples
824 --------
826 For SeriesGroupBy:
828 >>> lst = ["a", "a", "b"]
829 >>> ser = pd.Series([1, 2, 3], index=lst)
830 >>> ser
831 a 1
832 a 2
833 b 3
834 dtype: int64
835 >>> ser.groupby(level=0).get_group("a")
836 a 1
837 a 2
838 dtype: int64
840 For DataFrameGroupBy:
842 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
843 >>> df = pd.DataFrame(
844 ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
845 ... )
846 >>> df
847 a b c
848 owl 1 2 3
849 toucan 1 5 6
850 eagle 7 8 9
851 >>> df.groupby(by=["a"]).get_group((1,))
852 a b c
853 owl 1 2 3
854 toucan 1 5 6
856 For Resampler:
858 >>> ser = pd.Series(
859 ... [1, 2, 3, 4],
860 ... index=pd.DatetimeIndex(
861 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
862 ... ),
863 ... )
864 >>> ser
865 2023-01-01 1
866 2023-01-15 2
867 2023-02-01 3
868 2023-02-15 4
869 dtype: int64
870 >>> ser.resample("MS").get_group("2023-01-01")
871 2023-01-01 1
872 2023-01-15 2
873 dtype: int64
874 """
875 keys = self.keys
876 level = self.level
877 # mypy doesn't recognize level/keys as being sized when passed to len
878 if (is_list_like(level) and len(level) == 1) or ( # type: ignore[arg-type]
879 is_list_like(keys) and len(keys) == 1 # type: ignore[arg-type]
880 ):
881 # GH#25971
882 if isinstance(name, tuple) and len(name) == 1:
883 name = name[0]
884 else:
885 raise KeyError(name)
887 inds = self._get_index(name)
888 if not len(inds):
889 raise KeyError(name)
890 return self._selected_obj.iloc[inds]
892 @final
893 def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
894 """
895 Groupby iterator.
897 This method provides an iterator over the groups created by the ``resample``
898 or ``groupby`` operation on the object. The method yields tuples where
899 the first element is the label (group key) corresponding to each group or
900 resampled bin, and the second element is the subset of the data that falls
901 within that group or bin.
903 Returns
904 -------
905 Iterator
906 Generator yielding a sequence of (name, subsetted object)
907 for each group.
909 See Also
910 --------
911 Series.groupby : Group data by a specific key or column.
912 DataFrame.groupby : Group DataFrame using mapper or by columns.
913 DataFrame.resample : Resample a DataFrame.
914 Series.resample : Resample a Series.
916 Examples
917 --------
919 For SeriesGroupBy:
921 >>> lst = ["a", "a", "b"]
922 >>> ser = pd.Series([1, 2, 3], index=lst)
923 >>> ser
924 a 1
925 a 2
926 b 3
927 dtype: int64
928 >>> for x, y in ser.groupby(level=0):
929 ... print(f"{x}\\n{y}\\n")
930 a
931 a 1
932 a 2
933 dtype: int64
934 b
935 b 3
936 dtype: int64
938 For DataFrameGroupBy:
940 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
941 >>> df = pd.DataFrame(data, columns=["a", "b", "c"])
942 >>> df
943 a b c
944 0 1 2 3
945 1 1 5 6
946 2 7 8 9
947 >>> for x, y in df.groupby(by=["a"]):
948 ... print(f"{x}\\n{y}\\n")
949 (1,)
950 a b c
951 0 1 2 3
952 1 1 5 6
953 (7,)
954 a b c
955 2 7 8 9
957 For Resampler:
959 >>> ser = pd.Series(
960 ... [1, 2, 3, 4],
961 ... index=pd.DatetimeIndex(
962 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
963 ... ),
964 ... )
965 >>> ser
966 2023-01-01 1
967 2023-01-15 2
968 2023-02-01 3
969 2023-02-15 4
970 dtype: int64
971 >>> for x, y in ser.resample("MS"):
972 ... print(f"{x}\\n{y}\\n")
973 2023-01-01 00:00:00
974 2023-01-01 1
975 2023-01-15 2
976 dtype: int64
977 2023-02-01 00:00:00
978 2023-02-01 3
979 2023-02-15 4
980 dtype: int64
981 """
982 keys = self.keys
983 level = self.level
984 result = self._grouper.get_iterator(self._selected_obj)
985 # mypy: Argument 1 to "len" has incompatible type "Hashable"; expected "Sized"
986 if (is_list_like(level) and len(level) == 1) or ( # type: ignore[arg-type]
987 isinstance(keys, list) and len(keys) == 1
988 ):
989 # GH#42795 - when keys is a list, return tuples even when length is 1
990 result = (((key,), group) for key, group in result)
991 return result
994# To track operations that expand dimensions, like ohlc
995OutputFrameOrSeries = TypeVar("OutputFrameOrSeries", bound=NDFrame)
998class GroupBy(BaseGroupBy[NDFrameT]):
999 """
1000 Class for grouping and aggregating relational data.
1002 See aggregate, transform, and apply functions on this object.
1004 It's easiest to use obj.groupby(...) to use GroupBy, but you can also do:
1006 ::
1008 grouped = groupby(obj, ...)
1010 Parameters
1011 ----------
1012 obj : pandas object
1013 level : int, default None
1014 Level of MultiIndex
1015 groupings : list of Grouping objects
1016 Most users should ignore this
1017 exclusions : array-like, optional
1018 List of columns to exclude
1019 name : str
1020 Most users should ignore this
1022 Returns
1023 -------
1024 **Attributes**
1025 groups : dict
1026 {group name -> group labels}
1027 len(grouped) : int
1028 Number of groups
1030 Notes
1031 -----
1032 After grouping, see aggregate, apply, and transform functions. Here are
1033 some other brief notes about usage. When grouping by multiple groups, the
1034 result index will be a MultiIndex (hierarchical) by default.
1036 Iteration produces (key, group) tuples, i.e. chunking the data by group. So
1037 you can write code like:
1039 ::
1041 grouped = obj.groupby(keys)
1042 for key, group in grouped:
1043 # do something with the data
1045 Function calls on GroupBy, if not specially implemented, "dispatch" to the
1046 grouped data. So if you group a DataFrame and wish to invoke the std()
1047 method on each group, you can simply do:
1049 ::
1051 df.groupby(mapper).std()
1053 rather than
1055 ::
1057 df.groupby(mapper).aggregate(np.std)
1059 You can pass arguments to these "wrapped" functions, too.
1061 See the online documentation for full exposition on these topics and much
1062 more
1063 """
1065 _grouper: ops.BaseGrouper
1066 as_index: bool
1068 @final
1069 def __init__(
1070 self,
1071 obj: NDFrameT,
1072 keys: _KeysArgType | None = None,
1073 level: IndexLabel | None = None,
1074 grouper: ops.BaseGrouper | None = None,
1075 exclusions: frozenset[Hashable] | None = None,
1076 selection: IndexLabel | None = None,
1077 as_index: bool = True,
1078 sort: bool = True,
1079 group_keys: bool = True,
1080 observed: bool = False,
1081 dropna: bool = True,
1082 ) -> None:
1083 self._selection = selection
1085 assert isinstance(obj, NDFrame), type(obj)
1087 self.level = level
1088 self.as_index = as_index
1089 self.keys = keys
1090 self.sort = sort
1091 self.group_keys = group_keys
1092 self.dropna = dropna
1094 if grouper is None:
1095 grouper, exclusions, obj = get_grouper(
1096 obj,
1097 keys,
1098 level=level,
1099 sort=sort,
1100 observed=observed,
1101 dropna=self.dropna,
1102 )
1104 self.observed = observed
1105 self.obj = obj
1106 self._grouper = grouper
1107 self.exclusions = frozenset(exclusions) if exclusions else frozenset()
1109 def __getattr__(self, attr: str):
1110 if attr in self._internal_names_set:
1111 return object.__getattribute__(self, attr)
1112 if attr in self.obj:
1113 return self[attr]
1115 raise AttributeError(
1116 f"'{type(self).__name__}' object has no attribute '{attr}'"
1117 )
1119 @final
1120 def _op_via_apply(self, name: str, *args, **kwargs):
1121 """Compute the result of an operation by using GroupBy's apply."""
1122 f = getattr(type(self._obj_with_exclusions), name)
1124 def curried(x):
1125 return f(x, *args, **kwargs)
1127 # preserve the name so we can detect it when calling plot methods,
1128 # to avoid duplicates
1129 curried.__name__ = name
1131 # special case otherwise extra plots are created when catching the
1132 # exception below
1133 if name in base.plotting_methods:
1134 return self._python_apply_general(curried, self._selected_obj)
1136 is_transform = name in base.transformation_kernels
1137 result = self._python_apply_general(
1138 curried,
1139 self._obj_with_exclusions,
1140 is_transform=is_transform,
1141 not_indexed_same=not is_transform,
1142 )
1144 if self._grouper.has_dropped_na and is_transform:
1145 # result will have dropped rows due to nans, fill with null
1146 # and ensure index is ordered same as the input
1147 result = self._set_result_index_ordered(result)
1148 return result
1150 # -----------------------------------------------------------------
1151 # Dispatch/Wrapping
1153 @final
1154 def _concat_objects(
1155 self,
1156 values,
1157 not_indexed_same: bool = False,
1158 is_transform: bool = False,
1159 ):
1160 from pandas.core.reshape.concat import concat
1162 if self.group_keys and not is_transform:
1163 if self.as_index:
1164 # possible MI return case
1165 group_keys = self._grouper.result_index
1166 group_levels = self._grouper.levels
1167 group_names = self._grouper.names
1169 result = concat(
1170 values,
1171 axis=0,
1172 keys=group_keys,
1173 levels=group_levels,
1174 names=group_names,
1175 sort=False,
1176 )
1177 else:
1178 result = concat(values, axis=0)
1180 elif not not_indexed_same:
1181 result = concat(values, axis=0)
1183 ax = self._selected_obj.index
1184 if self.dropna:
1185 labels = self._grouper.ids
1186 mask = labels != -1
1187 ax = ax[mask]
1189 # this is a very unfortunate situation
1190 # we can't use reindex to restore the original order
1191 # when the ax has duplicates
1192 # so we resort to this
1193 # GH 14776, 30667
1194 # TODO: can we reuse e.g. _reindex_non_unique?
1195 if ax.has_duplicates and not result.axes[0].equals(ax):
1196 # e.g. test_category_order_transformer
1197 target = algorithms.unique1d(ax._values)
1198 indexer, _ = result.index.get_indexer_non_unique(target)
1199 result = result.take(indexer, axis=0)
1200 else:
1201 result = result.reindex(ax, axis=0)
1203 else:
1204 result = concat(values, axis=0)
1206 if self.obj.ndim == 1:
1207 name = self.obj.name
1208 elif is_hashable(self._selection):
1209 name = self._selection
1210 else:
1211 name = None
1213 if isinstance(result, Series) and name is not None:
1214 result.name = name
1216 return result.__finalize__(self.obj, method="groupby")
1218 @final
1219 def _set_result_index_ordered(
1220 self, result: OutputFrameOrSeries
1221 ) -> OutputFrameOrSeries:
1222 # set the result index on the passed values object and
1223 # return the new object, xref 8046
1225 index = self.obj.index
1227 if self._grouper.is_monotonic and not self._grouper.has_dropped_na:
1228 # shortcut if we have an already ordered grouper
1229 result = result.set_axis(index, axis=0)
1230 return result
1232 # row order is scrambled => sort the rows by position in original index
1233 original_positions = Index(self._grouper.result_ilocs, copy=False)
1234 result = result.set_axis(original_positions, axis=0)
1235 result = result.sort_index(axis=0)
1236 if self._grouper.has_dropped_na:
1237 # Add back in any missing rows due to dropna - index here is integral
1238 # with values referring to the row of the input so can use RangeIndex
1239 result = result.reindex(default_index(len(index)), axis=0)
1240 result = result.set_axis(index, axis=0)
1242 return result
1244 @final
1245 def _insert_inaxis_grouper(
1246 self, result: Series | DataFrame, qs: npt.NDArray[np.float64] | None = None
1247 ) -> DataFrame:
1248 if isinstance(result, Series):
1249 result = result.to_frame()
1251 n_groupings = len(self._grouper.groupings)
1253 if qs is not None:
1254 result.insert(
1255 0, f"level_{n_groupings}", np.tile(qs, len(result) // len(qs))
1256 )
1258 # zip in reverse so we can always insert at loc 0
1259 for level, (name, lev) in enumerate(
1260 zip(
1261 reversed(self._grouper.names),
1262 self._grouper.get_group_levels(),
1263 strict=True,
1264 )
1265 ):
1266 if name is None:
1267 # Behave the same as .reset_index() when a level is unnamed
1268 name = (
1269 "index"
1270 if n_groupings == 1 and qs is None
1271 else f"level_{n_groupings - level - 1}"
1272 )
1274 # GH #28549
1275 # When using .apply(-), name will be in columns already
1276 if name not in result.columns:
1277 # if in_axis:
1278 if qs is None:
1279 result.insert(0, name, lev)
1280 else:
1281 result.insert(0, name, Index(np.repeat(lev, len(qs)), copy=False))
1283 return result
1285 @final
1286 def _wrap_aggregated_output(
1287 self,
1288 result: Series | DataFrame,
1289 qs: npt.NDArray[np.float64] | None = None,
1290 ):
1291 """
1292 Wraps the output of GroupBy aggregations into the expected result.
1294 Parameters
1295 ----------
1296 result : Series, DataFrame
1298 Returns
1299 -------
1300 Series or DataFrame
1301 """
1302 # ATM we do not get here for SeriesGroupBy; when we do, we will
1303 # need to require that result.name already match self.obj.name
1305 if not self.as_index:
1306 # `not self.as_index` is only relevant for DataFrameGroupBy,
1307 # enforced in __init__
1308 result = self._insert_inaxis_grouper(result, qs=qs)
1309 result = result._consolidate()
1310 result.index = default_index(len(result))
1312 else:
1313 index = self._grouper.result_index
1314 if qs is not None:
1315 # We get here with len(qs) != 1 and not self.as_index
1316 # in test_pass_args_kwargs
1317 index = _insert_quantile_level(index, qs)
1318 result.index = index
1320 return result
1322 def _wrap_applied_output(
1323 self,
1324 data,
1325 values: list,
1326 not_indexed_same: bool = False,
1327 is_transform: bool = False,
1328 ):
1329 raise AbstractMethodError(self)
1331 # -----------------------------------------------------------------
1332 # numba
1334 @final
1335 def _numba_prep(self, data: DataFrame):
1336 ngroups = self._grouper.ngroups
1337 sorted_index = self._grouper.result_ilocs
1338 sorted_ids = self._grouper._sorted_ids
1340 sorted_data = data.take(sorted_index, axis=0).to_numpy()
1341 # GH 46867
1342 index_data = data.index
1343 if isinstance(index_data, MultiIndex):
1344 if len(self._grouper.groupings) > 1:
1345 raise NotImplementedError(
1346 "Grouping with more than 1 grouping labels and "
1347 "a MultiIndex is not supported with engine='numba'"
1348 )
1349 group_key = self._grouper.groupings[0].name
1350 index_data = index_data.get_level_values(group_key)
1351 sorted_index_data = index_data.take(sorted_index).to_numpy()
1353 starts, ends = lib.generate_slices(sorted_ids, ngroups)
1354 return (
1355 starts,
1356 ends,
1357 sorted_index_data,
1358 sorted_data,
1359 )
1361 def _numba_agg_general(
1362 self,
1363 func: Callable,
1364 dtype_mapping: dict[np.dtype, Any],
1365 engine_kwargs: dict[str, bool] | None,
1366 **aggregator_kwargs,
1367 ):
1368 """
1369 Perform groupby with a standard numerical aggregation function (e.g. mean)
1370 with Numba.
1371 """
1372 if not self.as_index:
1373 raise NotImplementedError(
1374 "as_index=False is not supported. Use .reset_index() instead."
1375 )
1377 data = self._obj_with_exclusions
1378 df = data if data.ndim == 2 else data.to_frame()
1380 aggregator = executor.generate_shared_aggregator(
1381 func,
1382 dtype_mapping,
1383 True, # is_grouped_kernel
1384 **get_jit_arguments(engine_kwargs),
1385 )
1386 # Pass group ids to kernel directly if it can handle it
1387 # (This is faster since it doesn't require a sort)
1388 ids = self._grouper.ids
1389 ngroups = self._grouper.ngroups
1391 res_mgr = df._mgr.apply(
1392 aggregator, labels=ids, ngroups=ngroups, **aggregator_kwargs
1393 )
1394 res_mgr.axes[1] = self._grouper.result_index
1395 result = df._constructor_from_mgr(res_mgr, axes=res_mgr.axes)
1397 if data.ndim == 1:
1398 result = result.squeeze("columns")
1399 result.name = data.name
1400 else:
1401 result.columns = data.columns
1402 return result
1404 @final
1405 def _transform_with_numba(self, func, *args, engine_kwargs=None, **kwargs):
1406 """
1407 Perform groupby transform routine with the numba engine.
1409 This routine mimics the data splitting routine of the DataSplitter class
1410 to generate the indices of each group in the sorted data and then passes the
1411 data and indices into a Numba jitted function.
1412 """
1413 data = self._obj_with_exclusions
1414 index_sorting = self._grouper.result_ilocs
1415 df = data if data.ndim == 2 else data.to_frame()
1417 starts, ends, sorted_index, sorted_data = self._numba_prep(df)
1418 numba_.validate_udf(func)
1419 args, kwargs = prepare_function_arguments(
1420 func, args, kwargs, num_required_args=2
1421 )
1422 numba_transform_func = numba_.generate_numba_transform_func(
1423 func, **get_jit_arguments(engine_kwargs)
1424 )
1425 result = numba_transform_func(
1426 sorted_data,
1427 sorted_index,
1428 starts,
1429 ends,
1430 len(df.columns),
1431 *args,
1432 )
1433 # result values needs to be resorted to their original positions since we
1434 # evaluated the data sorted by group
1435 result = result.take(np.argsort(index_sorting), axis=0)
1436 index = data.index
1437 if data.ndim == 1:
1438 result_kwargs = {"name": data.name}
1439 result = result.ravel()
1440 else:
1441 result_kwargs = {"columns": data.columns}
1442 return data._constructor(result, index=index, **result_kwargs)
1444 @final
1445 def _aggregate_with_numba(self, func, *args, engine_kwargs=None, **kwargs):
1446 """
1447 Perform groupby aggregation routine with the numba engine.
1449 This routine mimics the data splitting routine of the DataSplitter class
1450 to generate the indices of each group in the sorted data and then passes the
1451 data and indices into a Numba jitted function.
1452 """
1453 data = self._obj_with_exclusions
1454 df = data if data.ndim == 2 else data.to_frame()
1456 starts, ends, sorted_index, sorted_data = self._numba_prep(df)
1457 numba_.validate_udf(func)
1458 args, kwargs = prepare_function_arguments(
1459 func, args, kwargs, num_required_args=2
1460 )
1461 numba_agg_func = numba_.generate_numba_agg_func(
1462 func, **get_jit_arguments(engine_kwargs)
1463 )
1464 result = numba_agg_func(
1465 sorted_data,
1466 sorted_index,
1467 starts,
1468 ends,
1469 len(df.columns),
1470 *args,
1471 )
1472 index = self._grouper.result_index
1473 if data.ndim == 1:
1474 result_kwargs = {"name": data.name}
1475 result = result.ravel()
1476 else:
1477 result_kwargs = {"columns": data.columns}
1478 res = data._constructor(result, index=index, **result_kwargs)
1479 if not self.as_index:
1480 res = self._insert_inaxis_grouper(res)
1481 res.index = default_index(len(res))
1482 return res
1484 # -----------------------------------------------------------------
1485 # apply/agg/transform
1487 def apply(self, func, *args, include_groups: bool = False, **kwargs) -> NDFrameT:
1488 """
1489 Apply function ``func`` group-wise and combine the results together.
1491 The function passed to ``apply`` must take a dataframe as its first
1492 argument and return a DataFrame, Series or scalar. ``apply`` will
1493 then take care of combining the results back together into a single
1494 dataframe or series. ``apply`` is therefore a highly flexible
1495 grouping method.
1497 While ``apply`` is a very flexible method, its downside is that
1498 using it can be quite a bit slower than using more specific methods
1499 like ``agg`` or ``transform``. Pandas offers a wide range of method that will
1500 be much faster than using ``apply`` for their specific purposes, so try to
1501 use them before reaching for ``apply``.
1503 Parameters
1504 ----------
1505 func : callable
1506 A callable that takes a dataframe as its first argument, and
1507 returns a dataframe, a series or a scalar. In addition the
1508 callable may take positional and keyword arguments.
1510 *args : tuple
1511 Optional positional arguments to pass to ``func``.
1513 include_groups : bool, default False
1514 When True, will attempt to apply ``func`` to the groupings in
1515 the case that they are columns of the DataFrame. If this raises a
1516 TypeError, the result will be computed with the groupings excluded.
1517 When False, the groupings will be excluded when applying ``func``.
1519 .. versionadded:: 2.2.0
1521 .. versionchanged:: 3.0.0
1523 The default changed from True to False, and True is no longer allowed.
1525 **kwargs : dict
1526 Optional keyword arguments to pass to ``func``.
1528 Returns
1529 -------
1530 Series or DataFrame
1531 A pandas object with the result of applying ``func`` to each group.
1533 See Also
1534 --------
1535 pipe : Apply function to the full GroupBy object instead of to each
1536 group.
1537 aggregate : Apply aggregate function to the GroupBy object.
1538 transform : Apply function column-by-column to the GroupBy object.
1539 Series.apply : Apply a function to a Series.
1540 DataFrame.apply : Apply a function to each row or column of a DataFrame.
1542 Notes
1543 -----
1544 The resulting dtype will reflect the return value of the passed ``func``,
1545 see the examples below.
1547 Functions that mutate the passed object can produce unexpected
1548 behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
1549 for more details.
1551 Examples
1552 --------
1553 >>> df = pd.DataFrame({"A": "a a b".split(), "B": [1, 2, 3], "C": [4, 6, 5]})
1554 >>> g1 = df.groupby("A", group_keys=False)
1555 >>> g2 = df.groupby("A", group_keys=True)
1557 Notice that ``g1`` and ``g2`` have two groups, ``a`` and ``b``, and only
1558 differ in their ``group_keys`` argument. Calling `apply` in various ways,
1559 we can get different grouping results:
1561 Example 1: below the function passed to `apply` takes a DataFrame as
1562 its argument and returns a DataFrame. `apply` combines the result for
1563 each group together into a new DataFrame:
1565 >>> g1[["B", "C"]].apply(lambda x: x / x.sum())
1566 B C
1567 0 0.333333 0.4
1568 1 0.666667 0.6
1569 2 1.000000 1.0
1571 In the above, the groups are not part of the index. We can have them included
1572 by using ``g2`` where ``group_keys=True``:
1574 >>> g2[["B", "C"]].apply(lambda x: x / x.sum())
1575 B C
1576 A
1577 a 0 0.333333 0.4
1578 1 0.666667 0.6
1579 b 2 1.000000 1.0
1581 Example 2: The function passed to `apply` takes a DataFrame as
1582 its argument and returns a Series. `apply` combines the result for
1583 each group together into a new DataFrame.
1585 The resulting dtype will reflect the return value of the passed ``func``.
1587 >>> g1[["B", "C"]].apply(lambda x: x.astype(float).max() - x.min())
1588 B C
1589 A
1590 a 1.0 2.0
1591 b 0.0 0.0
1593 >>> g2[["B", "C"]].apply(lambda x: x.astype(float).max() - x.min())
1594 B C
1595 A
1596 a 1.0 2.0
1597 b 0.0 0.0
1599 The ``group_keys`` argument has no effect here because the result is not
1600 like-indexed (i.e. :ref:`a transform <groupby.transform>`) when compared
1601 to the input.
1603 Example 3: The function passed to `apply` takes a DataFrame as
1604 its argument and returns a scalar. `apply` combines the result for
1605 each group together into a Series, including setting the index as
1606 appropriate:
1608 >>> g1.apply(lambda x: x.C.max() - x.B.min())
1609 A
1610 a 5
1611 b 2
1612 dtype: int64
1614 Example 4: The function passed to ``apply`` returns ``None`` for one of the
1615 group. This group is filtered from the result:
1617 >>> g1.apply(lambda x: None if x.iloc[0, 0] == 3 else x)
1618 B C
1619 0 1 4
1620 1 2 6
1621 """
1622 if include_groups:
1623 raise ValueError("include_groups=True is no longer allowed.")
1624 if isinstance(func, str):
1625 if hasattr(self, func):
1626 res = getattr(self, func)
1627 if callable(res):
1628 return res(*args, **kwargs)
1629 elif args or kwargs:
1630 raise ValueError(f"Cannot pass arguments to property {func}")
1631 return res
1633 else:
1634 raise TypeError(f"apply func should be callable, not '{func}'")
1636 elif args or kwargs:
1637 if callable(func):
1639 @wraps(func)
1640 def f(g):
1641 return func(g, *args, **kwargs)
1643 else:
1644 raise ValueError(
1645 "func must be a callable if args or kwargs are supplied"
1646 )
1647 else:
1648 f = func
1650 return self._python_apply_general(f, self._obj_with_exclusions)
1652 @final
1653 def _python_apply_general(
1654 self,
1655 f: Callable,
1656 data: DataFrame | Series,
1657 not_indexed_same: bool | None = None,
1658 is_transform: bool = False,
1659 is_agg: bool = False,
1660 ) -> NDFrameT:
1661 """
1662 Apply function f in python space
1664 Parameters
1665 ----------
1666 f : callable
1667 Function to apply
1668 data : Series or DataFrame
1669 Data to apply f to
1670 not_indexed_same: bool, optional
1671 When specified, overrides the value of not_indexed_same. Apply behaves
1672 differently when the result index is equal to the input index, but
1673 this can be coincidental leading to value-dependent behavior.
1674 is_transform : bool, default False
1675 Indicator for whether the function is actually a transform
1676 and should not have group keys prepended.
1677 is_agg : bool, default False
1678 Indicator for whether the function is an aggregation. When the
1679 result is empty, we don't want to warn for this case.
1680 See _GroupBy._python_agg_general.
1682 Returns
1683 -------
1684 Series or DataFrame
1685 data after applying f
1686 """
1687 values, mutated = self._grouper.apply_groupwise(f, data)
1688 if not_indexed_same is None:
1689 not_indexed_same = mutated
1691 return self._wrap_applied_output(
1692 data,
1693 values,
1694 not_indexed_same,
1695 is_transform,
1696 )
1698 @final
1699 def _agg_general(
1700 self,
1701 numeric_only: bool = False,
1702 min_count: int = -1,
1703 *,
1704 alias: str,
1705 npfunc: Callable | None = None,
1706 **kwargs,
1707 ):
1708 result = self._cython_agg_general(
1709 how=alias,
1710 alt=npfunc,
1711 numeric_only=numeric_only,
1712 min_count=min_count,
1713 **kwargs,
1714 )
1715 return result.__finalize__(self.obj, method="groupby")
1717 def _agg_py_fallback(
1718 self, how: str, values: ArrayLike, ndim: int, alt: Callable
1719 ) -> ArrayLike:
1720 """
1721 Fallback to pure-python aggregation if _cython_operation raises
1722 NotImplementedError.
1723 """
1724 # We get here with a) EADtypes and b) object dtype
1725 assert alt is not None
1727 if values.ndim == 1:
1728 # For DataFrameGroupBy we only get here with ExtensionArray
1729 ser = Series(values, copy=False)
1730 else:
1731 # We only get here with values.dtype == object
1732 df = DataFrame(values.T, dtype=values.dtype)
1733 # bc we split object blocks in grouped_reduce, we have only 1 col
1734 # otherwise we'd have to worry about block-splitting GH#39329
1735 assert df.shape[1] == 1
1736 # Avoid call to self.values that can occur in DataFrame
1737 # reductions; see GH#28949
1738 ser = df.iloc[:, 0]
1740 # We do not get here with UDFs, so we know that our dtype
1741 # should always be preserved by the implemented aggregations
1742 # TODO: Is this exactly right; see WrappedCythonOp get_result_dtype?
1743 try:
1744 res_values = self._grouper.agg_series(ser, alt, preserve_dtype=True)
1745 except Exception as err:
1746 msg = f"agg function failed [how->{how},dtype->{ser.dtype}]"
1747 # preserve the kind of exception that raised
1748 raise type(err)(msg) from err
1750 dtype = ser.dtype
1751 if dtype == object:
1752 res_values = res_values.astype(object, copy=False)
1753 elif is_string_dtype(dtype):
1754 # mypy doesn't infer dtype is an ExtensionDtype
1755 string_array_cls = dtype.construct_array_type() # type: ignore[union-attr]
1756 res_values = string_array_cls._from_sequence(res_values, dtype=dtype)
1758 # If we are DataFrameGroupBy and went through a SeriesGroupByPath
1759 # then we need to reshape
1760 # GH#32223 includes case with IntegerArray values, ndarray res_values
1761 # test_groupby_duplicate_columns with object dtype values
1762 return ensure_block_shape(res_values, ndim=ndim)
1764 @final
1765 def _cython_agg_general(
1766 self,
1767 how: str,
1768 alt: Callable | None = None,
1769 numeric_only: bool = False,
1770 min_count: int = -1,
1771 **kwargs,
1772 ):
1773 # Note: we never get here with how="ohlc" for DataFrameGroupBy;
1774 # that goes through SeriesGroupBy
1776 if not is_bool(numeric_only):
1777 raise ValueError("numeric_only accepts only Boolean values")
1779 data = self._get_data_to_aggregate(numeric_only=numeric_only, name=how)
1781 def array_func(values: ArrayLike) -> ArrayLike:
1782 try:
1783 result = self._grouper._cython_operation(
1784 "aggregate",
1785 values,
1786 how,
1787 axis=data.ndim - 1,
1788 min_count=min_count,
1789 **kwargs,
1790 )
1791 except NotImplementedError:
1792 # generally if we have numeric_only=False
1793 # and non-applicable functions
1794 # try to python agg
1795 # TODO: shouldn't min_count matter?
1796 # TODO: avoid special casing SparseArray here
1797 if how in ["any", "all"] and isinstance(values, SparseArray):
1798 pass
1799 elif alt is None or how in ["any", "all", "std", "sem"]:
1800 raise # TODO: re-raise as TypeError? should not be reached
1801 else:
1802 return result
1804 assert alt is not None
1805 result = self._agg_py_fallback(how, values, ndim=data.ndim, alt=alt)
1806 return result
1808 new_mgr = data.grouped_reduce(array_func)
1809 res = self._wrap_agged_manager(new_mgr)
1810 if how in ["idxmin", "idxmax"]:
1811 # mypy expects how to be Literal["idxmin", "idxmax"].
1812 res = self._wrap_idxmax_idxmin(res, how=how, skipna=kwargs["skipna"]) # type: ignore[arg-type]
1813 out = self._wrap_aggregated_output(res)
1814 return out
1816 def _cython_transform(self, how: str, numeric_only: bool = False, **kwargs):
1817 raise AbstractMethodError(self)
1819 @final
1820 def _transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs):
1821 # optimized transforms
1822 if not isinstance(func, str):
1823 return self._transform_general(func, engine, engine_kwargs, *args, **kwargs)
1825 elif func not in base.transform_kernel_allowlist:
1826 msg = f"'{func}' is not a valid function name for transform(name)"
1827 raise ValueError(msg)
1828 elif func in base.cythonized_kernels or func in base.transformation_kernels:
1829 # cythonized transform or canned "agg+broadcast"
1830 if engine is not None:
1831 kwargs["engine"] = engine
1832 kwargs["engine_kwargs"] = engine_kwargs
1833 return getattr(self, func)(*args, **kwargs)
1835 else:
1836 # i.e. func in base.reduction_kernels
1837 if self.observed:
1838 return self._reduction_kernel_transform(
1839 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
1840 )
1842 with (
1843 com.temp_setattr(self, "observed", True),
1844 com.temp_setattr(self, "_grouper", self._grouper.observed_grouper),
1845 ):
1846 return self._reduction_kernel_transform(
1847 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
1848 )
1850 @final
1851 def _reduction_kernel_transform(
1852 self, func, *args, engine=None, engine_kwargs=None, **kwargs
1853 ):
1854 # GH#30918 Use _transform_fast only when we know func is an aggregation
1855 # If func is a reduction, we need to broadcast the
1856 # result to the whole group. Compute func result
1857 # and deal with possible broadcasting below.
1858 with com.temp_setattr(self, "as_index", True):
1859 # GH#49834 - result needs groups in the index for
1860 # _wrap_transform_fast_result
1861 if func in ["idxmin", "idxmax"]:
1862 func = cast(Literal["idxmin", "idxmax"], func)
1863 result = self._idxmax_idxmin(func, True, *args, **kwargs)
1864 else:
1865 if engine is not None:
1866 kwargs["engine"] = engine
1867 kwargs["engine_kwargs"] = engine_kwargs
1868 result = getattr(self, func)(*args, **kwargs)
1870 return self._wrap_transform_fast_result(result)
1872 @final
1873 def _wrap_transform_fast_result(self, result: NDFrameT) -> NDFrameT:
1874 """
1875 Fast transform path for aggregations.
1876 """
1877 obj = self._obj_with_exclusions
1879 # for each col, reshape to size of original frame by take operation
1880 ids = self._grouper.ids
1881 result = result.reindex(self._grouper.result_index, axis=0)
1883 if self.obj.ndim == 1:
1884 # i.e. SeriesGroupBy
1885 out = algorithms.take_nd(result._values, ids)
1886 output = obj._constructor(out, index=obj.index, name=obj.name)
1887 else:
1888 # `.size()` gives Series output on DataFrame input, need axis 0
1889 # GH#46209
1890 # Don't convert indices: negative indices need to give rise
1891 # to null values in the result
1892 new_ax = result.index.take(ids)
1893 output = result._reindex_with_indexers({0: (new_ax, ids)}, allow_dups=True)
1894 output = output.set_axis(obj.index, axis=0)
1895 return output
1897 # -----------------------------------------------------------------
1898 # Utilities
1900 @final
1901 def _apply_filter(self, indices, dropna):
1902 if len(indices) == 0:
1903 indices = np.array([], dtype="int64")
1904 else:
1905 indices = np.sort(np.concatenate(indices))
1906 if dropna:
1907 filtered = self._selected_obj.take(indices, axis=0)
1908 else:
1909 mask = np.empty(len(self._selected_obj.index), dtype=bool)
1910 mask.fill(False)
1911 mask[indices.astype(int)] = True
1912 # mask fails to broadcast when passed to where; broadcast manually.
1913 mask = np.tile(mask, [*self._selected_obj.shape[1:], 1]).T
1914 filtered = self._selected_obj.where(mask) # Fill with NaNs.
1915 return filtered
1917 @final
1918 def _cumcount_array(self, ascending: bool = True) -> np.ndarray:
1919 """
1920 Parameters
1921 ----------
1922 ascending : bool, default True
1923 If False, number in reverse, from length of group - 1 to 0.
1925 Notes
1926 -----
1927 this is currently implementing sort=False
1928 (though the default is sort=True) for groupby in general
1929 """
1930 ids = self._grouper.ids
1931 ngroups = self._grouper.ngroups
1932 sorter = get_group_index_sorter(ids, ngroups)
1933 ids, count = ids[sorter], len(ids)
1935 if count == 0:
1936 return np.empty(0, dtype=np.int64)
1938 run = np.r_[True, ids[:-1] != ids[1:]]
1939 rep = np.diff(np.r_[np.nonzero(run)[0], count])
1940 out = (~run).cumsum()
1942 if ascending:
1943 out -= np.repeat(out[run], rep)
1944 else:
1945 out = np.repeat(out[np.r_[run[1:], True]], rep) - out
1947 if self._grouper.has_dropped_na:
1948 out = np.where(ids == -1, np.nan, out.astype(np.float64, copy=False))
1949 else:
1950 out = out.astype(np.int64, copy=False)
1952 rev = np.empty(count, dtype=np.intp)
1953 rev[sorter] = np.arange(count, dtype=np.intp)
1954 return out[rev]
1956 # -----------------------------------------------------------------
1958 @final
1959 @property
1960 def _obj_1d_constructor(self) -> Callable:
1961 # GH28330 preserve subclassed Series/DataFrames
1962 if isinstance(self.obj, DataFrame):
1963 return self.obj._constructor_sliced
1964 assert isinstance(self.obj, Series)
1965 return self.obj._constructor
1967 @final
1968 def any(self, skipna: bool = True) -> NDFrameT:
1969 """
1970 Return True if any value in the group is truthful, else False.
1972 Parameters
1973 ----------
1974 skipna : bool, default True
1975 Flag to ignore nan values during truth testing.
1977 Returns
1978 -------
1979 Series or DataFrame
1980 DataFrame or Series of boolean values, where a value is True if any element
1981 is True within its respective group, False otherwise.
1983 See Also
1984 --------
1985 Series.any : Apply function any to a Series.
1986 DataFrame.any : Apply function any to each row or column of a DataFrame.
1988 Examples
1989 --------
1990 For SeriesGroupBy:
1992 >>> lst = ["a", "a", "b"]
1993 >>> ser = pd.Series([1, 2, 0], index=lst)
1994 >>> ser
1995 a 1
1996 a 2
1997 b 0
1998 dtype: int64
1999 >>> ser.groupby(level=0).any()
2000 a True
2001 b False
2002 dtype: bool
2004 For DataFrameGroupBy:
2006 >>> data = [[1, 0, 3], [1, 0, 6], [7, 1, 9]]
2007 >>> df = pd.DataFrame(
2008 ... data, columns=["a", "b", "c"], index=["ostrich", "penguin", "parrot"]
2009 ... )
2010 >>> df
2011 a b c
2012 ostrich 1 0 3
2013 penguin 1 0 6
2014 parrot 7 1 9
2015 >>> df.groupby(by=["a"]).any()
2016 b c
2017 a
2018 1 False True
2019 7 True True
2020 """
2021 return self._cython_agg_general(
2022 "any",
2023 alt=lambda x: Series(x, copy=False).any(skipna=skipna),
2024 skipna=skipna,
2025 )
2027 @final
2028 def all(self, skipna: bool = True) -> NDFrameT:
2029 """
2030 Return True if all values in the group are truthful, else False.
2032 Parameters
2033 ----------
2034 skipna : bool, default True
2035 Flag to ignore nan values during truth testing.
2037 Returns
2038 -------
2039 Series or DataFrame
2040 DataFrame or Series of boolean values, where a value is True if all elements
2041 are True within its respective group, False otherwise.
2043 See Also
2044 --------
2045 Series.all : Apply function all to a Series.
2046 DataFrame.all : Apply function all to each row or column of a DataFrame.
2048 Examples
2049 --------
2051 For SeriesGroupBy:
2053 >>> lst = ["a", "a", "b"]
2054 >>> ser = pd.Series([1, 2, 0], index=lst)
2055 >>> ser
2056 a 1
2057 a 2
2058 b 0
2059 dtype: int64
2060 >>> ser.groupby(level=0).all()
2061 a True
2062 b False
2063 dtype: bool
2065 For DataFrameGroupBy:
2067 >>> data = [[1, 0, 3], [1, 5, 6], [7, 8, 9]]
2068 >>> df = pd.DataFrame(
2069 ... data, columns=["a", "b", "c"], index=["ostrich", "penguin", "parrot"]
2070 ... )
2071 >>> df
2072 a b c
2073 ostrich 1 0 3
2074 penguin 1 5 6
2075 parrot 7 8 9
2076 >>> df.groupby(by=["a"]).all()
2077 b c
2078 a
2079 1 False True
2080 7 True True
2081 """
2082 return self._cython_agg_general(
2083 "all",
2084 alt=lambda x: Series(x, copy=False).all(skipna=skipna),
2085 skipna=skipna,
2086 )
2088 @final
2089 def count(self) -> NDFrameT:
2090 """
2091 Compute count of group, excluding missing values.
2093 Returns
2094 -------
2095 Series or DataFrame
2096 Count of values within each group.
2098 See Also
2099 --------
2100 Series.count : Apply function count to a Series.
2101 DataFrame.count : Apply function count to each row or column of a DataFrame.
2103 Examples
2104 --------
2105 For SeriesGroupBy:
2107 >>> lst = ["a", "a", "b"]
2108 >>> ser = pd.Series([1, 2, np.nan], index=lst)
2109 >>> ser
2110 a 1.0
2111 a 2.0
2112 b NaN
2113 dtype: float64
2114 >>> ser.groupby(level=0).count()
2115 a 2
2116 b 0
2117 dtype: int64
2119 For DataFrameGroupBy:
2121 >>> data = [[1, np.nan, 3], [1, np.nan, 6], [7, 8, 9]]
2122 >>> df = pd.DataFrame(
2123 ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
2124 ... )
2125 >>> df
2126 a b c
2127 cow 1 NaN 3
2128 horse 1 NaN 6
2129 bull 7 8.0 9
2130 >>> df.groupby("a").count()
2131 b c
2132 a
2133 1 0 2
2134 7 1 1
2136 For Resampler:
2138 >>> ser = pd.Series(
2139 ... [1, 2, 3, 4],
2140 ... index=pd.DatetimeIndex(
2141 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
2142 ... ),
2143 ... )
2144 >>> ser
2145 2023-01-01 1
2146 2023-01-15 2
2147 2023-02-01 3
2148 2023-02-15 4
2149 dtype: int64
2150 >>> ser.resample("MS").count()
2151 2023-01-01 2
2152 2023-02-01 2
2153 Freq: MS, dtype: int64
2154 """
2155 data = self._get_data_to_aggregate()
2156 ids = self._grouper.ids
2157 ngroups = self._grouper.ngroups
2158 mask = ids != -1
2160 is_series = data.ndim == 1
2162 def hfunc(bvalues: ArrayLike) -> ArrayLike:
2163 # TODO(EA2D): reshape would not be necessary with 2D EAs
2164 if bvalues.ndim == 1:
2165 # EA
2166 masked = mask & ~isna(bvalues).reshape(1, -1)
2167 else:
2168 masked = mask & ~isna(bvalues)
2170 counted = lib.count_level_2d(masked, labels=ids, max_bin=ngroups)
2171 if isinstance(bvalues, BaseMaskedArray):
2172 return IntegerArray(
2173 counted[0], mask=np.zeros(counted.shape[1], dtype=np.bool_)
2174 )
2175 elif isinstance(bvalues, ArrowExtensionArray) and not isinstance(
2176 bvalues.dtype, StringDtype
2177 ):
2178 dtype = pandas_dtype("int64[pyarrow]")
2179 return type(bvalues)._from_sequence(counted[0], dtype=dtype)
2180 if is_series:
2181 assert counted.ndim == 2
2182 assert counted.shape[0] == 1
2183 return counted[0]
2184 return counted
2186 new_mgr = data.grouped_reduce(hfunc)
2187 new_obj = self._wrap_agged_manager(new_mgr)
2188 result = self._wrap_aggregated_output(new_obj)
2190 return result
2192 @final
2193 def mean(
2194 self,
2195 numeric_only: bool = False,
2196 skipna: bool = True,
2197 engine: Literal["cython", "numba"] | None = None,
2198 engine_kwargs: dict[str, bool] | None = None,
2199 ):
2200 """
2201 Compute mean of groups, excluding missing values.
2203 Parameters
2204 ----------
2205 numeric_only : bool, default False
2206 Include only float, int, boolean columns.
2208 .. versionchanged:: 2.0.0
2210 numeric_only no longer accepts ``None`` and defaults to ``False``.
2212 skipna : bool, default True
2213 Exclude NA/null values. If an entire group is NA, the result will be NA.
2215 engine : str, default None
2216 * ``'cython'`` : Runs the operation through C-extensions from cython.
2217 * ``'numba'`` : Runs the operation through JIT compiled code from numba.
2218 * ``None`` : Defaults to ``'cython'`` or globally setting
2219 ``compute.use_numba``
2221 engine_kwargs : dict, default None
2222 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2223 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
2224 and ``parallel`` dictionary keys. The values must either be ``True`` or
2225 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2226 ``{'nopython': True, 'nogil': False, 'parallel': False}``
2228 Returns
2229 -------
2230 pandas.Series or pandas.DataFrame
2231 Mean of values within each group. Same object type as the caller.
2233 See Also
2234 --------
2235 Series.mean : Apply function mean to a Series.
2236 DataFrame.mean : Apply function mean to each row or column of a DataFrame.
2238 Examples
2239 --------
2240 >>> df = pd.DataFrame(
2241 ... {"A": [1, 1, 2, 1, 2], "B": [np.nan, 2, 3, 4, 5], "C": [1, 2, 1, 1, 2]},
2242 ... columns=["A", "B", "C"],
2243 ... )
2245 Groupby one column and return the mean of the remaining columns in
2246 each group.
2248 >>> df.groupby("A").mean()
2249 B C
2250 A
2251 1 3.0 1.333333
2252 2 4.0 1.500000
2254 Groupby two columns and return the mean of the remaining column.
2256 >>> df.groupby(["A", "B"]).mean()
2257 C
2258 A B
2259 1 2.0 2.0
2260 4.0 1.0
2261 2 3.0 1.0
2262 5.0 2.0
2264 Groupby one column and return the mean of only particular column in
2265 the group.
2267 >>> df.groupby("A")["B"].mean()
2268 A
2269 1 3.0
2270 2 4.0
2271 Name: B, dtype: float64
2272 """
2274 if maybe_use_numba(engine):
2275 from pandas.core._numba.kernels import grouped_mean
2277 return self._numba_agg_general(
2278 grouped_mean,
2279 executor.float_dtype_mapping,
2280 engine_kwargs,
2281 min_periods=0,
2282 skipna=skipna,
2283 )
2284 else:
2285 result = self._cython_agg_general(
2286 "mean",
2287 alt=lambda x: Series(x, copy=False).mean(
2288 numeric_only=numeric_only, skipna=skipna
2289 ),
2290 numeric_only=numeric_only,
2291 skipna=skipna,
2292 )
2293 return result.__finalize__(self.obj, method="groupby")
2295 @final
2296 def median(self, numeric_only: bool = False, skipna: bool = True) -> NDFrameT:
2297 """
2298 Compute median of groups, excluding missing values.
2300 For multiple groupings, the result index will be a MultiIndex
2302 Parameters
2303 ----------
2304 numeric_only : bool, default False
2305 Include only float, int, boolean columns.
2307 .. versionchanged:: 2.0.0
2309 numeric_only no longer accepts ``None`` and defaults to False.
2311 skipna : bool, default True
2312 Exclude NA/null values. If an entire group is NA, the result will be NA.
2314 .. versionadded:: 3.0.0
2316 Returns
2317 -------
2318 Series or DataFrame
2319 Median of values within each group.
2321 See Also
2322 --------
2323 Series.median : Apply function median to a Series.
2324 DataFrame.median : Apply function median to each row or column of a DataFrame.
2326 Examples
2327 --------
2328 For SeriesGroupBy:
2330 >>> lst = ["a", "a", "a", "b", "b", "b"]
2331 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
2332 >>> ser
2333 a 7
2334 a 2
2335 a 8
2336 b 4
2337 b 3
2338 b 3
2339 dtype: int64
2340 >>> ser.groupby(level=0).median()
2341 a 7.0
2342 b 3.0
2343 dtype: float64
2345 For DataFrameGroupBy:
2347 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
2348 >>> df = pd.DataFrame(
2349 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
2350 ... )
2351 >>> df
2352 a b
2353 dog 1 1
2354 dog 3 4
2355 dog 5 8
2356 mouse 7 4
2357 mouse 7 4
2358 mouse 8 2
2359 mouse 3 1
2360 >>> df.groupby(level=0).median()
2361 a b
2362 dog 3.0 4.0
2363 mouse 7.0 3.0
2365 For Resampler:
2367 >>> ser = pd.Series(
2368 ... [1, 2, 3, 3, 4, 5],
2369 ... index=pd.DatetimeIndex(
2370 ... [
2371 ... "2023-01-01",
2372 ... "2023-01-10",
2373 ... "2023-01-15",
2374 ... "2023-02-01",
2375 ... "2023-02-10",
2376 ... "2023-02-15",
2377 ... ]
2378 ... ),
2379 ... )
2380 >>> ser.resample("MS").median()
2381 2023-01-01 2.0
2382 2023-02-01 4.0
2383 Freq: MS, dtype: float64
2384 """
2385 result = self._cython_agg_general(
2386 "median",
2387 alt=lambda x: Series(x, copy=False).median(
2388 numeric_only=numeric_only, skipna=skipna
2389 ),
2390 numeric_only=numeric_only,
2391 skipna=skipna,
2392 )
2393 return result.__finalize__(self.obj, method="groupby")
2395 @final
2396 def std(
2397 self,
2398 ddof: int = 1,
2399 engine: Literal["cython", "numba"] | None = None,
2400 engine_kwargs: dict[str, bool] | None = None,
2401 numeric_only: bool = False,
2402 skipna: bool = True,
2403 ):
2404 """
2405 Compute standard deviation of groups, excluding missing values.
2407 For multiple groupings, the result index will be a MultiIndex.
2409 Parameters
2410 ----------
2411 ddof : int, default 1
2412 Delta Degrees of Freedom. The divisor used in calculations is ``N - ddof``,
2413 where ``N`` represents the number of elements.
2415 engine : str, default None
2416 * ``'cython'`` : Runs the operation through C-extensions from cython.
2417 * ``'numba'`` : Runs the operation through JIT compiled code from numba.
2418 * ``None`` : Defaults to ``'cython'`` or globally setting
2419 ``compute.use_numba``
2421 engine_kwargs : dict, default None
2422 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2423 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
2424 and ``parallel`` dictionary keys. The values must either be ``True`` or
2425 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2426 ``{'nopython': True, 'nogil': False, 'parallel': False}``
2428 numeric_only : bool, default False
2429 Include only `float`, `int` or `boolean` data.
2431 .. versionchanged:: 2.0.0
2433 numeric_only now defaults to ``False``.
2435 skipna : bool, default True
2436 Exclude NA/null values. If an entire group is NA, the result will be NA.
2438 .. versionadded:: 3.0.0
2440 Returns
2441 -------
2442 Series or DataFrame
2443 Standard deviation of values within each group.
2445 See Also
2446 --------
2447 Series.std : Apply function std to a Series.
2448 DataFrame.std : Apply function std to each row or column of a DataFrame.
2450 Examples
2451 --------
2452 For SeriesGroupBy:
2454 >>> lst = ["a", "a", "a", "b", "b", "b"]
2455 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
2456 >>> ser
2457 a 7
2458 a 2
2459 a 8
2460 b 4
2461 b 3
2462 b 3
2463 dtype: int64
2464 >>> ser.groupby(level=0).std()
2465 a 3.21455
2466 b 0.57735
2467 dtype: float64
2469 For DataFrameGroupBy:
2471 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
2472 >>> df = pd.DataFrame(
2473 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
2474 ... )
2475 >>> df
2476 a b
2477 dog 1 1
2478 dog 3 4
2479 dog 5 8
2480 mouse 7 4
2481 mouse 7 4
2482 mouse 8 2
2483 mouse 3 1
2484 >>> df.groupby(level=0).std()
2485 a b
2486 dog 2.000000 3.511885
2487 mouse 2.217356 1.500000
2488 """
2489 if maybe_use_numba(engine):
2490 from pandas.core._numba.kernels import grouped_var
2492 return np.sqrt(
2493 self._numba_agg_general(
2494 grouped_var,
2495 executor.float_dtype_mapping,
2496 engine_kwargs,
2497 min_periods=0,
2498 ddof=ddof,
2499 skipna=skipna,
2500 )
2501 )
2502 else:
2503 return self._cython_agg_general(
2504 "std",
2505 alt=lambda x: Series(x, copy=False).std(ddof=ddof, skipna=skipna),
2506 numeric_only=numeric_only,
2507 ddof=ddof,
2508 skipna=skipna,
2509 )
2511 @final
2512 def var(
2513 self,
2514 ddof: int = 1,
2515 engine: Literal["cython", "numba"] | None = None,
2516 engine_kwargs: dict[str, bool] | None = None,
2517 numeric_only: bool = False,
2518 skipna: bool = True,
2519 ):
2520 """
2521 Compute variance of groups, excluding missing values.
2523 For multiple groupings, the result index will be a MultiIndex.
2525 Parameters
2526 ----------
2527 ddof : int, default 1
2528 Degrees of freedom.
2530 engine : str, default None
2531 * ``'cython'`` : Runs the operation through C-extensions from cython.
2532 * ``'numba'`` : Runs the operation through JIT compiled code from numba.
2533 * ``None`` : Defaults to ``'cython'`` or globally setting
2534 ``compute.use_numba``
2536 engine_kwargs : dict, default None
2537 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2538 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
2539 and ``parallel`` dictionary keys. The values must either be ``True`` or
2540 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2541 ``{'nopython': True, 'nogil': False, 'parallel': False}``
2543 numeric_only : bool, default False
2544 Include only `float`, `int` or `boolean` data.
2546 .. versionchanged:: 2.0.0
2548 numeric_only now defaults to ``False``.
2550 skipna : bool, default True
2551 Exclude NA/null values. If an entire group is NA, the result will be NA.
2553 .. versionadded:: 3.0.0
2555 Returns
2556 -------
2557 Series or DataFrame
2558 Variance of values within each group.
2560 See Also
2561 --------
2562 Series.var : Apply function var to a Series.
2563 DataFrame.var : Apply function var to each row or column of a DataFrame.
2565 Examples
2566 --------
2567 For SeriesGroupBy:
2569 >>> lst = ["a", "a", "a", "b", "b", "b"]
2570 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
2571 >>> ser
2572 a 7
2573 a 2
2574 a 8
2575 b 4
2576 b 3
2577 b 3
2578 dtype: int64
2579 >>> ser.groupby(level=0).var()
2580 a 10.333333
2581 b 0.333333
2582 dtype: float64
2584 For DataFrameGroupBy:
2586 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
2587 >>> df = pd.DataFrame(
2588 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
2589 ... )
2590 >>> df
2591 a b
2592 dog 1 1
2593 dog 3 4
2594 dog 5 8
2595 mouse 7 4
2596 mouse 7 4
2597 mouse 8 2
2598 mouse 3 1
2599 >>> df.groupby(level=0).var()
2600 a b
2601 dog 4.000000 12.333333
2602 mouse 4.916667 2.250000
2603 """
2604 if maybe_use_numba(engine):
2605 from pandas.core._numba.kernels import grouped_var
2607 return self._numba_agg_general(
2608 grouped_var,
2609 executor.float_dtype_mapping,
2610 engine_kwargs,
2611 min_periods=0,
2612 ddof=ddof,
2613 skipna=skipna,
2614 )
2615 else:
2616 return self._cython_agg_general(
2617 "var",
2618 alt=lambda x: Series(x, copy=False).var(ddof=ddof, skipna=skipna),
2619 numeric_only=numeric_only,
2620 ddof=ddof,
2621 skipna=skipna,
2622 )
2624 @final
2625 def _value_counts(
2626 self,
2627 subset: Sequence[Hashable] | None = None,
2628 normalize: bool = False,
2629 sort: bool = True,
2630 ascending: bool = False,
2631 dropna: bool = True,
2632 ) -> DataFrame | Series:
2633 """
2634 Shared implementation of value_counts for SeriesGroupBy and DataFrameGroupBy.
2636 SeriesGroupBy additionally supports a bins argument. See the docstring of
2637 DataFrameGroupBy.value_counts for a description of arguments.
2638 """
2639 name = "proportion" if normalize else "count"
2641 df = self.obj
2642 obj = self._obj_with_exclusions
2644 in_axis_names = {
2645 grouping.name for grouping in self._grouper.groupings if grouping.in_axis
2646 }
2647 if isinstance(obj, Series):
2648 _name = obj.name
2649 keys: Iterable[Series] = [] if _name in in_axis_names else [obj]
2650 else:
2651 unique_cols = set(obj.columns)
2652 if subset is not None:
2653 subsetted = set(subset)
2654 clashing = subsetted & set(in_axis_names)
2655 if clashing:
2656 raise ValueError(
2657 f"Keys {clashing} in subset cannot be in "
2658 "the groupby column keys."
2659 )
2660 doesnt_exist = subsetted - unique_cols
2661 if doesnt_exist:
2662 raise ValueError(
2663 f"Keys {doesnt_exist} in subset do not exist in the DataFrame."
2664 )
2665 else:
2666 subsetted = unique_cols
2668 keys = (
2669 # Can't use .values because the column label needs to be preserved
2670 obj.iloc[:, idx]
2671 for idx, _name in enumerate(obj.columns)
2672 if _name not in in_axis_names and _name in subsetted
2673 )
2675 groupings = list(self._grouper.groupings)
2676 for key in keys:
2677 grouper, _, _ = get_grouper(
2678 df,
2679 key=key,
2680 sort=False,
2681 observed=False,
2682 dropna=dropna,
2683 )
2684 groupings += list(grouper.groupings)
2686 # Take the size of the overall columns
2687 gb = df.groupby(
2688 groupings,
2689 sort=False,
2690 observed=self.observed,
2691 dropna=self.dropna,
2692 )
2693 result_series = cast(Series, gb.size())
2694 result_series.name = name
2696 if sort:
2697 # Sort by the values
2698 result_series = result_series.sort_values(
2699 ascending=ascending, kind="stable"
2700 )
2701 if self.sort:
2702 # Sort by the groupings
2703 names = result_series.index.names
2704 # GH#55951 - Temporarily replace names in case they are integers
2705 result_series.index.names = range(len(names))
2706 index_level = range(len(self._grouper.groupings))
2707 result_series = result_series.sort_index(
2708 level=index_level, sort_remaining=False
2709 )
2710 result_series.index.names = names
2712 if normalize:
2713 # Normalize the results by dividing by the original group sizes.
2714 # We are guaranteed to have the first N levels be the
2715 # user-requested grouping.
2716 levels = list(
2717 range(len(self._grouper.groupings), result_series.index.nlevels)
2718 )
2719 indexed_group_size = result_series.groupby(
2720 result_series.index.droplevel(levels),
2721 sort=self.sort,
2722 dropna=self.dropna,
2723 # GH#43999 - deprecation of observed=False
2724 observed=False,
2725 ).transform("sum")
2726 result_series /= indexed_group_size
2728 # Handle groups of non-observed categories
2729 result_series = result_series.fillna(0.0)
2731 result: Series | DataFrame
2732 if self.as_index:
2733 result = result_series
2734 else:
2735 # Convert to frame
2736 index = result_series.index
2737 columns = com.fill_missing_names(index.names)
2738 if name in columns:
2739 raise ValueError(f"Column label '{name}' is duplicate of result column")
2740 result_series.name = name
2741 result_series.index = index.set_names(range(len(columns)))
2742 result_frame = result_series.reset_index()
2743 orig_dtype = self._grouper.groupings[0].obj.columns.dtype # type: ignore[union-attr]
2744 cols = Index(columns, dtype=orig_dtype).insert(len(columns), name)
2745 result_frame.columns = cols
2746 result = result_frame
2747 return result.__finalize__(self.obj, method="value_counts")
2749 @final
2750 def sem(
2751 self, ddof: int = 1, numeric_only: bool = False, skipna: bool = True
2752 ) -> NDFrameT:
2753 """
2754 Compute standard error of the mean of groups, excluding missing values.
2756 For multiple groupings, the result index will be a MultiIndex.
2758 Parameters
2759 ----------
2760 ddof : int, default 1
2761 Degrees of freedom.
2763 numeric_only : bool, default False
2764 Include only `float`, `int` or `boolean` data.
2766 .. versionchanged:: 2.0.0
2768 numeric_only now defaults to ``False``.
2770 skipna : bool, default True
2771 Exclude NA/null values. If an entire group is NA, the result will be NA.
2773 .. versionadded:: 3.0.0
2775 Returns
2776 -------
2777 Series or DataFrame
2778 Standard error of the mean of values within each group.
2780 See Also
2781 --------
2782 DataFrame.sem : Return unbiased standard error of the mean over requested axis.
2783 Series.sem : Return unbiased standard error of the mean over requested axis.
2785 Examples
2786 --------
2787 For SeriesGroupBy:
2789 >>> lst = ["a", "a", "b", "b"]
2790 >>> ser = pd.Series([5, 10, 8, 14], index=lst)
2791 >>> ser
2792 a 5
2793 a 10
2794 b 8
2795 b 14
2796 dtype: int64
2797 >>> ser.groupby(level=0).sem()
2798 a 2.5
2799 b 3.0
2800 dtype: float64
2802 For DataFrameGroupBy:
2804 >>> data = [[1, 12, 11], [1, 15, 2], [2, 5, 8], [2, 6, 12]]
2805 >>> df = pd.DataFrame(
2806 ... data,
2807 ... columns=["a", "b", "c"],
2808 ... index=["tuna", "salmon", "catfish", "goldfish"],
2809 ... )
2810 >>> df
2811 a b c
2812 tuna 1 12 11
2813 salmon 1 15 2
2814 catfish 2 5 8
2815 goldfish 2 6 12
2816 >>> df.groupby("a").sem()
2817 b c
2818 a
2819 1 1.5 4.5
2820 2 0.5 2.0
2822 For Resampler:
2824 >>> ser = pd.Series(
2825 ... [1, 3, 2, 4, 3, 8],
2826 ... index=pd.DatetimeIndex(
2827 ... [
2828 ... "2023-01-01",
2829 ... "2023-01-10",
2830 ... "2023-01-15",
2831 ... "2023-02-01",
2832 ... "2023-02-10",
2833 ... "2023-02-15",
2834 ... ]
2835 ... ),
2836 ... )
2837 >>> ser.resample("MS").sem()
2838 2023-01-01 0.577350
2839 2023-02-01 1.527525
2840 Freq: MS, dtype: float64
2841 """
2842 if numeric_only and self.obj.ndim == 1 and not is_numeric_dtype(self.obj.dtype):
2843 raise TypeError(
2844 f"{type(self).__name__}.sem called with "
2845 f"numeric_only={numeric_only} and dtype {self.obj.dtype}"
2846 )
2847 return self._cython_agg_general(
2848 "sem",
2849 alt=lambda x: Series(x, copy=False).sem(ddof=ddof, skipna=skipna),
2850 numeric_only=numeric_only,
2851 ddof=ddof,
2852 skipna=skipna,
2853 )
2855 @final
2856 def size(self) -> DataFrame | Series:
2857 """
2858 Compute group sizes.
2860 Returns
2861 -------
2862 DataFrame or Series
2863 Number of rows in each group as a Series if as_index is True
2864 or a DataFrame if as_index is False.
2866 See Also
2867 --------
2868 Series.size : Apply function size to a Series.
2869 DataFrame.size : Apply function size to each row or column of a DataFrame.
2871 Examples
2872 --------
2874 For SeriesGroupBy:
2876 >>> lst = ["a", "a", "b"]
2877 >>> ser = pd.Series([1, 2, 3], index=lst)
2878 >>> ser
2879 a 1
2880 a 2
2881 b 3
2882 dtype: int64
2883 >>> ser.groupby(level=0).size()
2884 a 2
2885 b 1
2886 dtype: int64
2888 >>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
2889 >>> df = pd.DataFrame(
2890 ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
2891 ... )
2892 >>> df
2893 a b c
2894 owl 1 2 3
2895 toucan 1 5 6
2896 eagle 7 8 9
2897 >>> df.groupby("a").size()
2898 a
2899 1 2
2900 7 1
2901 dtype: int64
2903 For Resampler:
2905 >>> ser = pd.Series(
2906 ... [1, 2, 3],
2907 ... index=pd.DatetimeIndex(["2023-01-01", "2023-01-15", "2023-02-01"]),
2908 ... )
2909 >>> ser
2910 2023-01-01 1
2911 2023-01-15 2
2912 2023-02-01 3
2913 dtype: int64
2914 >>> ser.resample("MS").size()
2915 2023-01-01 2
2916 2023-02-01 1
2917 Freq: MS, dtype: int64
2918 """
2919 result = self._grouper.size()
2920 dtype_backend: None | Literal["pyarrow", "numpy_nullable"] = None
2921 if isinstance(self.obj, Series):
2922 if isinstance(self.obj.array, ArrowExtensionArray):
2923 if isinstance(self.obj.array, ArrowStringArray):
2924 if self.obj.array.dtype.na_value is np.nan:
2925 dtype_backend = None
2926 else:
2927 dtype_backend = "numpy_nullable"
2928 else:
2929 dtype_backend = "pyarrow"
2930 elif isinstance(self.obj.array, BaseMaskedArray):
2931 dtype_backend = "numpy_nullable"
2932 # TODO: For DataFrames what if columns are mixed arrow/numpy/masked?
2934 # GH28330 preserve subclassed Series/DataFrames through calls
2935 if isinstance(self.obj, Series):
2936 result = self._obj_1d_constructor(result, name=self.obj.name)
2937 else:
2938 result = self._obj_1d_constructor(result)
2940 if dtype_backend is not None:
2941 result = result.convert_dtypes(
2942 infer_objects=False,
2943 convert_string=False,
2944 convert_boolean=False,
2945 convert_floating=False,
2946 dtype_backend=dtype_backend,
2947 )
2949 if not self.as_index:
2950 result = result.rename("size").reset_index()
2951 return result
2953 @final
2954 def sum(
2955 self,
2956 numeric_only: bool = False,
2957 min_count: int = 0,
2958 skipna: bool = True,
2959 engine: Literal["cython", "numba"] | None = None,
2960 engine_kwargs: dict[str, bool] | None = None,
2961 ):
2962 """
2963 Compute sum of group values.
2965 Parameters
2966 ----------
2967 numeric_only : bool, default False
2968 Include only float, int, boolean columns.
2970 .. versionchanged:: 2.0.0
2972 numeric_only no longer accepts ``None``.
2974 min_count : int, default 0
2975 The required number of valid values to perform the operation. If fewer
2976 than ``min_count`` non-NA values are present the result will be NA.
2978 skipna : bool, default True
2979 Exclude NA/null values. If the entire group is NA and ``skipna`` is
2980 ``True``, the result will be NA.
2982 .. versionchanged:: 3.0.0
2984 engine : str, default None None
2985 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
2986 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
2987 Only available when ``raw`` is set to ``True``.
2988 * ``None`` : Defaults to ``'cython'`` or globally setting
2989 ``compute.use_numba``
2991 engine_kwargs : dict, default None None
2992 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2993 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
2994 and ``parallel`` dictionary keys. The values must either be ``True`` or
2995 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2996 ``{'nopython': True, 'nogil': False, 'parallel': False}`` and will be
2997 applied to both the ``func`` and the ``apply`` groupby aggregation.
2999 Returns
3000 -------
3001 Series or DataFrame
3002 Computed sum of values within each group.
3004 See Also
3005 --------
3006 SeriesGroupBy.min : Return the min of the group values.
3007 DataFrameGroupBy.min : Return the min of the group values.
3008 SeriesGroupBy.max : Return the max of the group values.
3009 DataFrameGroupBy.max : Return the max of the group values.
3010 SeriesGroupBy.sum : Return the sum of the group values.
3011 DataFrameGroupBy.sum : Return the sum of the group values.
3013 Examples
3014 --------
3015 For SeriesGroupBy:
3017 >>> lst = ["a", "a", "b", "b"]
3018 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
3019 >>> ser
3020 a 1
3021 a 2
3022 b 3
3023 b 4
3024 dtype: int64
3025 >>> ser.groupby(level=0).sum()
3026 a 3
3027 b 7
3028 dtype: int64
3030 For DataFrameGroupBy:
3032 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
3033 >>> df = pd.DataFrame(
3034 ... data,
3035 ... columns=["a", "b", "c"],
3036 ... index=["tiger", "leopard", "cheetah", "lion"],
3037 ... )
3038 >>> df
3039 a b c
3040 tiger 1 8 2
3041 leopard 1 2 5
3042 cheetah 2 5 8
3043 lion 2 6 9
3044 >>> df.groupby("a").sum()
3045 b c
3046 a
3047 1 10 7
3048 2 11 17
3049 """
3050 if maybe_use_numba(engine):
3051 from pandas.core._numba.kernels import grouped_sum
3053 return self._numba_agg_general(
3054 grouped_sum,
3055 executor.default_dtype_mapping,
3056 engine_kwargs,
3057 min_periods=min_count,
3058 skipna=skipna,
3059 )
3060 else:
3061 # If we are grouping on categoricals we want unobserved categories to
3062 # return zero, rather than the default of NaN which the reindexing in
3063 # _agg_general() returns. GH #31422
3064 with com.temp_setattr(self, "observed", True):
3065 result = self._agg_general(
3066 numeric_only=numeric_only,
3067 min_count=min_count,
3068 alias="sum",
3069 npfunc=np.sum,
3070 skipna=skipna,
3071 )
3073 return result
3075 @final
3076 def prod(
3077 self, numeric_only: bool = False, min_count: int = 0, skipna: bool = True
3078 ) -> NDFrameT:
3079 """
3080 Compute prod of group values.
3082 Parameters
3083 ----------
3084 numeric_only : bool, default False
3085 Include only float, int, boolean columns.
3087 .. versionchanged:: 2.0.0
3089 numeric_only no longer accepts ``None``.
3091 min_count : int, default 0
3092 The required number of valid values to perform the operation. If fewer
3093 than ``min_count`` non-NA values are present the result will be NA.
3095 skipna : bool, default True
3096 Exclude NA/null values. If an entire group is NA, the result will be NA.
3098 .. versionadded:: 3.0.0
3100 Returns
3101 -------
3102 Series or DataFrame
3103 Computed prod of values within each group.
3105 See Also
3106 --------
3107 Series.prod : Return the product of the values over the requested axis.
3108 DataFrame.prod : Return the product of the values over the requested axis.
3110 Examples
3111 --------
3112 For SeriesGroupBy:
3114 >>> lst = ["a", "a", "b", "b"]
3115 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
3116 >>> ser
3117 a 1
3118 a 2
3119 b 3
3120 b 4
3121 dtype: int64
3122 >>> ser.groupby(level=0).prod()
3123 a 2
3124 b 12
3125 dtype: int64
3127 For DataFrameGroupBy:
3129 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
3130 >>> df = pd.DataFrame(
3131 ... data,
3132 ... columns=["a", "b", "c"],
3133 ... index=["tiger", "leopard", "cheetah", "lion"],
3134 ... )
3135 >>> df
3136 a b c
3137 tiger 1 8 2
3138 leopard 1 2 5
3139 cheetah 2 5 8
3140 lion 2 6 9
3141 >>> df.groupby("a").prod()
3142 b c
3143 a
3144 1 16 10
3145 2 30 72
3146 """
3147 return self._agg_general(
3148 numeric_only=numeric_only,
3149 min_count=min_count,
3150 skipna=skipna,
3151 alias="prod",
3152 npfunc=np.prod,
3153 )
3155 @final
3156 def min(
3157 self,
3158 numeric_only: bool = False,
3159 min_count: int = -1,
3160 skipna: bool = True,
3161 engine: Literal["cython", "numba"] | None = None,
3162 engine_kwargs: dict[str, bool] | None = None,
3163 ):
3164 """
3165 Compute min of group values.
3167 Parameters
3168 ----------
3169 numeric_only : bool, default False
3170 Include only float, int, boolean columns.
3172 .. versionchanged:: 2.0.0
3174 numeric_only no longer accepts ``None``.
3176 min_count : int, default -1
3177 The required number of valid values to perform the operation. If fewer
3178 than ``min_count`` non-NA values are present the result will be NA.
3180 skipna : bool, default True
3181 Exclude NA/null values. If the entire group is NA and ``skipna`` is
3182 ``True``, the result will be NA.
3184 .. versionchanged:: 3.0.0
3186 engine : str, default None None
3187 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
3188 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
3189 Only available when ``raw`` is set to ``True``.
3190 * ``None`` : Defaults to ``'cython'`` or globally setting
3191 ``compute.use_numba``
3193 engine_kwargs : dict, default None None
3194 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
3195 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
3196 and ``parallel`` dictionary keys. The values must either be ``True`` or
3197 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
3198 ``{'nopython': True, 'nogil': False, 'parallel': False}`` and will be
3199 applied to both the ``func`` and the ``apply`` groupby aggregation.
3201 Returns
3202 -------
3203 Series or DataFrame
3204 Computed min of values within each group.
3206 See Also
3207 --------
3208 SeriesGroupBy.min : Return the min of the group values.
3209 DataFrameGroupBy.min : Return the min of the group values.
3210 SeriesGroupBy.max : Return the max of the group values.
3211 DataFrameGroupBy.max : Return the max of the group values.
3212 SeriesGroupBy.sum : Return the sum of the group values.
3213 DataFrameGroupBy.sum : Return the sum of the group values.
3215 Examples
3216 --------
3217 For SeriesGroupBy:
3219 >>> lst = ["a", "a", "b", "b"]
3220 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
3221 >>> ser
3222 a 1
3223 a 2
3224 b 3
3225 b 4
3226 dtype: int64
3227 >>> ser.groupby(level=0).min()
3228 a 1
3229 b 3
3230 dtype: int64
3232 For DataFrameGroupBy:
3234 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
3235 >>> df = pd.DataFrame(
3236 ... data,
3237 ... columns=["a", "b", "c"],
3238 ... index=["tiger", "leopard", "cheetah", "lion"],
3239 ... )
3240 >>> df
3241 a b c
3242 tiger 1 8 2
3243 leopard 1 2 5
3244 cheetah 2 5 8
3245 lion 2 6 9
3246 >>> df.groupby("a").min()
3247 b c
3248 a
3249 1 2 2
3250 2 5 8
3251 """
3252 if maybe_use_numba(engine):
3253 from pandas.core._numba.kernels import grouped_min_max
3255 return self._numba_agg_general(
3256 grouped_min_max,
3257 executor.identity_dtype_mapping,
3258 engine_kwargs,
3259 min_periods=min_count,
3260 is_max=False,
3261 skipna=skipna,
3262 )
3263 else:
3264 return self._agg_general(
3265 numeric_only=numeric_only,
3266 min_count=min_count,
3267 skipna=skipna,
3268 alias="min",
3269 npfunc=np.min,
3270 )
3272 @final
3273 def max(
3274 self,
3275 numeric_only: bool = False,
3276 min_count: int = -1,
3277 skipna: bool = True,
3278 engine: Literal["cython", "numba"] | None = None,
3279 engine_kwargs: dict[str, bool] | None = None,
3280 ):
3281 """
3282 Compute max of group values.
3284 Parameters
3285 ----------
3286 numeric_only : bool, default False
3287 Include only float, int, boolean columns.
3289 .. versionchanged:: 2.0.0
3291 numeric_only no longer accepts ``None``.
3293 min_count : int, default -1
3294 The required number of valid values to perform the operation. If fewer
3295 than ``min_count`` non-NA values are present the result will be NA.
3297 skipna : bool, default True
3298 Exclude NA/null values. If the entire group is NA and ``skipna`` is
3299 ``True``, the result will be NA.
3301 .. versionchanged:: 3.0.0
3303 engine : str, default None None
3304 * ``'cython'`` : Runs rolling apply through C-extensions from cython.
3305 * ``'numba'`` : Runs rolling apply through JIT compiled code from numba.
3306 Only available when ``raw`` is set to ``True``.
3307 * ``None`` : Defaults to ``'cython'`` or globally setting
3308 ``compute.use_numba``
3310 engine_kwargs : dict, default None None
3311 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
3312 * For ``'numba'`` engine, the engine can accept ``nopython``, ``nogil``
3313 and ``parallel`` dictionary keys. The values must either be ``True`` or
3314 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
3315 ``{'nopython': True, 'nogil': False, 'parallel': False}`` and will be
3316 applied to both the ``func`` and the ``apply`` groupby aggregation.
3318 Returns
3319 -------
3320 Series or DataFrame
3321 Computed max of values within each group.
3323 See Also
3324 --------
3325 SeriesGroupBy.min : Return the min of the group values.
3326 DataFrameGroupBy.min : Return the min of the group values.
3327 SeriesGroupBy.max : Return the max of the group values.
3328 DataFrameGroupBy.max : Return the max of the group values.
3329 SeriesGroupBy.sum : Return the sum of the group values.
3330 DataFrameGroupBy.sum : Return the sum of the group values.
3332 Examples
3333 --------
3334 For SeriesGroupBy:
3336 >>> lst = ["a", "a", "b", "b"]
3337 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
3338 >>> ser
3339 a 1
3340 a 2
3341 b 3
3342 b 4
3343 dtype: int64
3344 >>> ser.groupby(level=0).max()
3345 a 2
3346 b 4
3347 dtype: int64
3349 For DataFrameGroupBy:
3351 >>> data = [[1, 8, 2], [1, 2, 5], [2, 5, 8], [2, 6, 9]]
3352 >>> df = pd.DataFrame(
3353 ... data,
3354 ... columns=["a", "b", "c"],
3355 ... index=["tiger", "leopard", "cheetah", "lion"],
3356 ... )
3357 >>> df
3358 a b c
3359 tiger 1 8 2
3360 leopard 1 2 5
3361 cheetah 2 5 8
3362 lion 2 6 9
3363 >>> df.groupby("a").max()
3364 b c
3365 a
3366 1 8 5
3367 2 6 9
3368 """
3369 if maybe_use_numba(engine):
3370 from pandas.core._numba.kernels import grouped_min_max
3372 return self._numba_agg_general(
3373 grouped_min_max,
3374 executor.identity_dtype_mapping,
3375 engine_kwargs,
3376 min_periods=min_count,
3377 is_max=True,
3378 skipna=skipna,
3379 )
3380 else:
3381 return self._agg_general(
3382 numeric_only=numeric_only,
3383 min_count=min_count,
3384 skipna=skipna,
3385 alias="max",
3386 npfunc=np.max,
3387 )
3389 @final
3390 def first(
3391 self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True
3392 ) -> NDFrameT:
3393 """
3394 Compute the first entry of each column within each group.
3396 Defaults to skipping NA elements.
3398 Parameters
3399 ----------
3400 numeric_only : bool, default False
3401 Include only float, int, boolean columns.
3402 min_count : int, default -1
3403 The required number of valid values to perform the operation. If fewer
3404 than ``min_count`` valid values are present the result will be NA.
3405 skipna : bool, default True
3406 Exclude NA/null values. If an entire group is NA, the result will be NA.
3408 .. versionadded:: 2.2.1
3410 Returns
3411 -------
3412 Series or DataFrame
3413 First values within each group.
3415 See Also
3416 --------
3417 DataFrame.groupby : Apply a function groupby to each row or column of a
3418 DataFrame.
3419 core.groupby.DataFrameGroupBy.last : Compute the last non-null entry
3420 of each column.
3421 core.groupby.DataFrameGroupBy.nth : Take the nth row from each group.
3423 Examples
3424 --------
3425 >>> df = pd.DataFrame(
3426 ... dict(
3427 ... A=[1, 1, 3],
3428 ... B=[None, 5, 6],
3429 ... C=[1, 2, 3],
3430 ... D=["3/11/2000", "3/12/2000", "3/13/2000"],
3431 ... )
3432 ... )
3433 >>> df["D"] = pd.to_datetime(df["D"])
3434 >>> df.groupby("A").first()
3435 B C D
3436 A
3437 1 5.0 1 2000-03-11
3438 3 6.0 3 2000-03-13
3439 >>> df.groupby("A").first(min_count=2)
3440 B C D
3441 A
3442 1 NaN 1.0 2000-03-11
3443 3 NaN NaN NaT
3444 >>> df.groupby("A").first(numeric_only=True)
3445 B C
3446 A
3447 1 5.0 1
3448 3 6.0 3
3449 """
3451 def first_compat(obj: NDFrameT):
3452 def first(x: Series):
3453 """Helper function for first item that isn't NA."""
3454 arr = x.array[notna(x.array)]
3455 if not len(arr):
3456 return x.array.dtype.na_value
3457 return arr[0]
3459 if isinstance(obj, DataFrame):
3460 return obj.apply(first)
3461 elif isinstance(obj, Series):
3462 return first(obj)
3463 else: # pragma: no cover
3464 raise TypeError(type(obj))
3466 return self._agg_general(
3467 numeric_only=numeric_only,
3468 min_count=min_count,
3469 alias="first",
3470 npfunc=first_compat,
3471 skipna=skipna,
3472 )
3474 @final
3475 def last(
3476 self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True
3477 ) -> NDFrameT:
3478 """
3479 Compute the last entry of each column within each group.
3481 Defaults to skipping NA elements.
3483 Parameters
3484 ----------
3485 numeric_only : bool, default False
3486 Include only float, int, boolean columns. If None, will attempt to use
3487 everything, then use only numeric data.
3488 min_count : int, default -1
3489 The required number of valid values to perform the operation. If fewer
3490 than ``min_count`` valid values are present the result will be NA.
3491 skipna : bool, default True
3492 Exclude NA/null values. If an entire group is NA, the result will be NA.
3494 .. versionadded:: 2.2.1
3496 Returns
3497 -------
3498 Series or DataFrame
3499 Last of values within each group.
3501 See Also
3502 --------
3503 DataFrame.groupby : Apply a function groupby to each row or column of a
3504 DataFrame.
3505 core.groupby.DataFrameGroupBy.first : Compute the first non-null entry
3506 of each column.
3507 core.groupby.DataFrameGroupBy.nth : Take the nth row from each group.
3509 Examples
3510 --------
3511 >>> df = pd.DataFrame(dict(A=[1, 1, 3], B=[5, None, 6], C=[1, 2, 3]))
3512 >>> df.groupby("A").last()
3513 B C
3514 A
3515 1 5.0 2
3516 3 6.0 3
3517 """
3519 def last_compat(obj: NDFrameT):
3520 def last(x: Series):
3521 """Helper function for last item that isn't NA."""
3522 arr = x.array[notna(x.array)]
3523 if not len(arr):
3524 return x.array.dtype.na_value
3525 return arr[-1]
3527 if isinstance(obj, DataFrame):
3528 return obj.apply(last)
3529 elif isinstance(obj, Series):
3530 return last(obj)
3531 else: # pragma: no cover
3532 raise TypeError(type(obj))
3534 return self._agg_general(
3535 numeric_only=numeric_only,
3536 min_count=min_count,
3537 alias="last",
3538 npfunc=last_compat,
3539 skipna=skipna,
3540 )
3542 @final
3543 def ohlc(self) -> DataFrame:
3544 """
3545 Compute open, high, low and close values of a group, excluding missing values.
3547 For multiple groupings, the result index will be a MultiIndex
3549 Returns
3550 -------
3551 DataFrame
3552 Open, high, low and close values within each group.
3554 See Also
3555 --------
3556 DataFrame.agg : Aggregate using one or more operations over the specified axis.
3557 DataFrame.resample : Resample time-series data.
3558 DataFrame.groupby : Group DataFrame using a mapper or by a Series of columns.
3560 Examples
3561 --------
3563 For SeriesGroupBy:
3565 >>> lst = [
3566 ... "SPX",
3567 ... "CAC",
3568 ... "SPX",
3569 ... "CAC",
3570 ... "SPX",
3571 ... "CAC",
3572 ... "SPX",
3573 ... "CAC",
3574 ... ]
3575 >>> ser = pd.Series([3.4, 9.0, 7.2, 5.2, 8.8, 9.4, 0.1, 0.5], index=lst)
3576 >>> ser
3577 SPX 3.4
3578 CAC 9.0
3579 SPX 7.2
3580 CAC 5.2
3581 SPX 8.8
3582 CAC 9.4
3583 SPX 0.1
3584 CAC 0.5
3585 dtype: float64
3586 >>> ser.groupby(level=0).ohlc()
3587 open high low close
3588 CAC 9.0 9.4 0.5 0.5
3589 SPX 3.4 8.8 0.1 0.1
3591 For DataFrameGroupBy:
3593 >>> data = {
3594 ... 2022: [1.2, 2.3, 8.9, 4.5, 4.4, 3, 2, 1],
3595 ... 2023: [3.4, 9.0, 7.2, 5.2, 8.8, 9.4, 8.2, 1.0],
3596 ... }
3597 >>> df = pd.DataFrame(
3598 ... data, index=["SPX", "CAC", "SPX", "CAC", "SPX", "CAC", "SPX", "CAC"]
3599 ... )
3600 >>> df
3601 2022 2023
3602 SPX 1.2 3.4
3603 CAC 2.3 9.0
3604 SPX 8.9 7.2
3605 CAC 4.5 5.2
3606 SPX 4.4 8.8
3607 CAC 3.0 9.4
3608 SPX 2.0 8.2
3609 CAC 1.0 1.0
3610 >>> df.groupby(level=0).ohlc()
3611 2022 2023
3612 open high low close open high low close
3613 CAC 2.3 4.5 1.0 1.0 9.0 9.4 1.0 1.0
3614 SPX 1.2 8.9 1.2 2.0 3.4 8.8 3.4 8.2
3616 For Resampler:
3618 >>> ser = pd.Series(
3619 ... [1, 3, 2, 4, 3, 5],
3620 ... index=pd.DatetimeIndex(
3621 ... [
3622 ... "2023-01-01",
3623 ... "2023-01-10",
3624 ... "2023-01-15",
3625 ... "2023-02-01",
3626 ... "2023-02-10",
3627 ... "2023-02-15",
3628 ... ]
3629 ... ),
3630 ... )
3631 >>> ser.resample("MS").ohlc()
3632 open high low close
3633 2023-01-01 1 3 1 2
3634 2023-02-01 4 5 3 5
3635 """
3636 if self.obj.ndim == 1:
3637 obj = self._selected_obj
3639 is_numeric = is_numeric_dtype(obj.dtype)
3640 if not is_numeric:
3641 raise DataError("No numeric types to aggregate")
3643 res_values = self._grouper._cython_operation(
3644 "aggregate", obj._values, "ohlc", axis=0, min_count=-1
3645 )
3647 agg_names = ["open", "high", "low", "close"]
3648 result = self.obj._constructor_expanddim(
3649 res_values, index=self._grouper.result_index, columns=agg_names
3650 )
3651 return result
3653 result = self._apply_to_column_groupbys(lambda sgb: sgb.ohlc())
3654 return result
3656 def describe(
3657 self,
3658 percentiles=None,
3659 include=None,
3660 exclude=None,
3661 ) -> NDFrameT:
3662 """
3663 Generate descriptive statistics.
3665 Descriptive statistics include those that summarize the central
3666 tendency, dispersion and shape of a
3667 dataset's distribution, excluding ``NaN`` values.
3669 Analyzes both numeric and object series, as well
3670 as ``DataFrame`` column sets of mixed data types. The output
3671 will vary depending on what is provided. Refer to the notes
3672 below for more detail.
3674 Parameters
3675 ----------
3676 percentiles : list-like of numbers, optional
3677 The percentiles to include in the output. All should
3678 fall between 0 and 1. The default, ``None``, will automatically
3679 return the 25th, 50th, and 75th percentiles.
3680 include : 'all', list-like of dtypes or None (default), optional
3681 A white list of data types to include in the result. Ignored
3682 for ``Series``. Here are the options:
3684 - 'all' : All columns of the input will be included in the output.
3685 - A list-like of dtypes : Limits the results to the
3686 provided data types.
3687 To limit the result to numeric types submit
3688 ``numpy.number``. To limit it instead to object columns submit
3689 the ``numpy.object`` data type. Strings
3690 can also be used in the style of
3691 ``select_dtypes`` (e.g. ``df.describe(include=['O'])``). To
3692 select pandas categorical columns, use ``'category'``
3693 - None (default) : The result will include all numeric columns.
3694 exclude : list-like of dtypes or None (default), optional,
3695 A black list of data types to omit from the result. Ignored
3696 for ``Series``. Here are the options:
3698 - A list-like of dtypes : Excludes the provided data types
3699 from the result. To exclude numeric types submit
3700 ``numpy.number``. To exclude object columns submit the data
3701 type ``numpy.object``. Strings can also be used in the style of
3702 ``select_dtypes`` (e.g. ``df.describe(exclude=['O'])``). To
3703 exclude pandas categorical columns, use ``'category'``
3704 - None (default) : The result will exclude nothing.
3706 Returns
3707 -------
3708 Series or DataFrame
3709 Summary statistics of the Series or Dataframe provided.
3711 See Also
3712 --------
3713 DataFrame.count: Count number of non-NA/null observations.
3714 DataFrame.max: Maximum of the values in the object.
3715 DataFrame.min: Minimum of the values in the object.
3716 DataFrame.mean: Mean of the values.
3717 DataFrame.std: Standard deviation of the observations.
3718 DataFrame.select_dtypes: Subset of a DataFrame including/excluding
3719 columns based on their dtype.
3721 Notes
3722 -----
3723 For numeric data, the result's index will include ``count``,
3724 ``mean``, ``std``, ``min``, ``max`` as well as lower, ``50`` and
3725 upper percentiles. By default the lower percentile is ``25`` and the
3726 upper percentile is ``75``. The ``50`` percentile is the
3727 same as the median.
3729 For object data (e.g. strings), the result's index
3730 will include ``count``, ``unique``, ``top``, and ``freq``. The ``top``
3731 is the most common value. The ``freq`` is the most common value's
3732 frequency.
3734 If multiple object values have the highest count, then the
3735 ``count`` and ``top`` results will be arbitrarily chosen from
3736 among those with the highest count.
3738 For mixed data types provided via a ``DataFrame``, the default is to
3739 return only an analysis of numeric columns. If the DataFrame consists
3740 only of object and categorical data without any numeric columns, the
3741 default is to return an analysis of both the object and categorical
3742 columns. If ``include='all'`` is provided as an option, the result
3743 will include a union of attributes of each type.
3745 The `include` and `exclude` parameters can be used to limit
3746 which columns in a ``DataFrame`` are analyzed for the output.
3747 The parameters are ignored when analyzing a ``Series``.
3749 Examples
3750 --------
3751 Describing a numeric ``Series``.
3753 >>> s = pd.Series([1, 2, 3])
3754 >>> s.describe()
3755 count 3.0
3756 mean 2.0
3757 std 1.0
3758 min 1.0
3759 25% 1.5
3760 50% 2.0
3761 75% 2.5
3762 max 3.0
3763 dtype: float64
3765 Describing a categorical ``Series``.
3767 >>> s = pd.Series(["a", "a", "b", "c"])
3768 >>> s.describe()
3769 count 4
3770 unique 3
3771 top a
3772 freq 2
3773 dtype: object
3775 Describing a timestamp ``Series``.
3777 >>> s = pd.Series(
3778 ... [
3779 ... np.datetime64("2000-01-01"),
3780 ... np.datetime64("2010-01-01"),
3781 ... np.datetime64("2010-01-01"),
3782 ... ]
3783 ... )
3784 >>> s.describe()
3785 count 3
3786 mean 2006-09-01 08:00:00
3787 min 2000-01-01 00:00:00
3788 25% 2004-12-31 12:00:00
3789 50% 2010-01-01 00:00:00
3790 75% 2010-01-01 00:00:00
3791 max 2010-01-01 00:00:00
3792 dtype: object
3794 Describing a ``DataFrame``. By default only numeric fields
3795 are returned.
3797 >>> df = pd.DataFrame(
3798 ... {
3799 ... "categorical": pd.Categorical(["d", "e", "f"]),
3800 ... "numeric": [1, 2, 3],
3801 ... "object": ["a", "b", "c"],
3802 ... }
3803 ... )
3804 >>> df.describe()
3805 numeric
3806 count 3.0
3807 mean 2.0
3808 std 1.0
3809 min 1.0
3810 25% 1.5
3811 50% 2.0
3812 75% 2.5
3813 max 3.0
3815 Describing all columns of a ``DataFrame`` regardless of data type.
3817 >>> df.describe(include="all") # doctest: +SKIP
3818 categorical numeric object
3819 count 3 3.0 3
3820 unique 3 NaN 3
3821 top f NaN a
3822 freq 1 NaN 1
3823 mean NaN 2.0 NaN
3824 std NaN 1.0 NaN
3825 min NaN 1.0 NaN
3826 25% NaN 1.5 NaN
3827 50% NaN 2.0 NaN
3828 75% NaN 2.5 NaN
3829 max NaN 3.0 NaN
3831 Describing a column from a ``DataFrame`` by accessing it as
3832 an attribute.
3834 >>> df.numeric.describe()
3835 count 3.0
3836 mean 2.0
3837 std 1.0
3838 min 1.0
3839 25% 1.5
3840 50% 2.0
3841 75% 2.5
3842 max 3.0
3843 Name: numeric, dtype: float64
3845 Including only numeric columns in a ``DataFrame`` description.
3847 >>> df.describe(include=[np.number])
3848 numeric
3849 count 3.0
3850 mean 2.0
3851 std 1.0
3852 min 1.0
3853 25% 1.5
3854 50% 2.0
3855 75% 2.5
3856 max 3.0
3858 Including only string columns in a ``DataFrame`` description.
3860 >>> df.describe(include=[object]) # doctest: +SKIP
3861 object
3862 count 3
3863 unique 3
3864 top a
3865 freq 1
3867 Including only categorical columns from a ``DataFrame`` description.
3869 >>> df.describe(include=["category"])
3870 categorical
3871 count 3
3872 unique 3
3873 top d
3874 freq 1
3876 Excluding numeric columns from a ``DataFrame`` description.
3878 >>> df.describe(exclude=[np.number]) # doctest: +SKIP
3879 categorical object
3880 count 3 3
3881 unique 3 3
3882 top f a
3883 freq 1 1
3885 Excluding object columns from a ``DataFrame`` description.
3887 >>> df.describe(exclude=[object]) # doctest: +SKIP
3888 categorical numeric
3889 count 3 3.0
3890 unique 3 NaN
3891 top f NaN
3892 freq 1 NaN
3893 mean NaN 2.0
3894 std NaN 1.0
3895 min NaN 1.0
3896 25% NaN 1.5
3897 50% NaN 2.0
3898 75% NaN 2.5
3899 max NaN 3.0
3900 """
3901 obj = self._obj_with_exclusions
3903 if len(obj) == 0:
3904 described = obj.describe(
3905 percentiles=percentiles, include=include, exclude=exclude
3906 )
3907 if obj.ndim == 1:
3908 result = described
3909 else:
3910 result = described.unstack()
3911 return result.to_frame().T.iloc[:0]
3913 with com.temp_setattr(self, "as_index", True):
3914 result = self._python_apply_general(
3915 lambda x: x.describe(
3916 percentiles=percentiles, include=include, exclude=exclude
3917 ),
3918 obj,
3919 not_indexed_same=True,
3920 )
3922 # GH#49256 - properly handle the grouping column(s)
3923 result = result.unstack()
3924 if not self.as_index:
3925 result = self._insert_inaxis_grouper(result)
3926 result.index = default_index(len(result))
3928 return result
3930 @final
3931 def resample(
3932 self, rule, *args, include_groups: bool = False, **kwargs
3933 ) -> Resampler:
3934 """
3935 Provide resampling when using a TimeGrouper.
3937 Given a grouper, the function resamples it according to a string
3938 "string" -> "frequency".
3940 See the :ref:`frequency aliases <timeseries.offset_aliases>`
3941 documentation for more details.
3943 Parameters
3944 ----------
3945 rule : str or DateOffset
3946 The offset string or object representing target grouper conversion.
3947 *args
3948 Possible arguments are `how`, `fill_method`, `limit`, `kind` and
3949 `on`, and other arguments of `TimeGrouper`.
3950 include_groups : bool, default True
3951 When True, will attempt to include the groupings in the operation in
3952 the case that they are columns of the DataFrame. If this raises a
3953 TypeError, the result will be computed with the groupings excluded.
3954 When False, the groupings will be excluded when applying ``func``.
3956 .. versionadded:: 2.2.0
3958 .. versionchanged:: 3.0
3960 The default was changed to False, and True is no longer allowed.
3962 **kwargs
3963 Possible arguments are `how`, `fill_method`, `limit`, `kind` and
3964 `on`, and other arguments of `TimeGrouper`.
3966 Returns
3967 -------
3968 DatetimeIndexResampler, PeriodIndexResampler or TimdeltaResampler
3969 Resampler object for the type of the index.
3971 See Also
3972 --------
3973 Grouper : Specify a frequency to resample with when
3974 grouping by a key.
3975 DatetimeIndex.resample : Frequency conversion and resampling of
3976 time series.
3978 Examples
3979 --------
3980 >>> idx = pd.date_range("1/1/2000", periods=4, freq="min")
3981 >>> df = pd.DataFrame(data=4 * [range(2)], index=idx, columns=["a", "b"])
3982 >>> df.iloc[2, 0] = 5
3983 >>> df
3984 a b
3985 2000-01-01 00:00:00 0 1
3986 2000-01-01 00:01:00 0 1
3987 2000-01-01 00:02:00 5 1
3988 2000-01-01 00:03:00 0 1
3990 Downsample the DataFrame into 3 minute bins and sum the values of
3991 the timestamps falling into a bin.
3993 >>> df.groupby("a").resample("3min").sum()
3994 b
3995 a
3996 0 2000-01-01 00:00:00 2
3997 2000-01-01 00:03:00 1
3998 5 2000-01-01 00:00:00 1
4000 Upsample the series into 30 second bins.
4002 >>> df.groupby("a").resample("30s").sum()
4003 b
4004 a
4005 0 2000-01-01 00:00:00 1
4006 2000-01-01 00:00:30 0
4007 2000-01-01 00:01:00 1
4008 2000-01-01 00:01:30 0
4009 2000-01-01 00:02:00 0
4010 2000-01-01 00:02:30 0
4011 2000-01-01 00:03:00 1
4012 5 2000-01-01 00:02:00 1
4014 Resample by month. Values are assigned to the month of the period.
4016 >>> df.groupby("a").resample("ME").sum()
4017 b
4018 a
4019 0 2000-01-31 3
4020 5 2000-01-31 1
4022 Downsample the series into 3 minute bins as above, but close the right
4023 side of the bin interval.
4025 >>> (df.groupby("a").resample("3min", closed="right").sum())
4026 b
4027 a
4028 0 1999-12-31 23:57:00 1
4029 2000-01-01 00:00:00 2
4030 5 2000-01-01 00:00:00 1
4032 Downsample the series into 3 minute bins and close the right side of
4033 the bin interval, but label each bin using the right edge instead of
4034 the left.
4036 >>> (df.groupby("a").resample("3min", closed="right", label="right").sum())
4037 b
4038 a
4039 0 2000-01-01 00:00:00 1
4040 2000-01-01 00:03:00 2
4041 5 2000-01-01 00:03:00 1
4042 """
4043 from pandas.core.resample import get_resampler_for_grouping
4045 if include_groups:
4046 raise ValueError("include_groups=True is no longer allowed.")
4048 return get_resampler_for_grouping(self, rule, *args, **kwargs)
4050 @final
4051 def rolling(
4052 self,
4053 window: int | datetime.timedelta | str | BaseOffset | BaseIndexer,
4054 min_periods: int | None = None,
4055 center: bool = False,
4056 win_type: str | None = None,
4057 on: str | None = None,
4058 closed: IntervalClosedType | None = None,
4059 method: str = "single",
4060 ) -> RollingGroupby:
4061 """
4062 Return a rolling grouper, providing rolling functionality per group.
4064 Parameters
4065 ----------
4066 window : int, timedelta, str, offset, or BaseIndexer subclass
4067 Interval of the moving window.
4069 If an integer, the delta between the start and end of each window.
4070 The number of points in the window depends on the ``closed`` argument.
4072 If a timedelta, str, or offset, the time period of each window. Each
4073 window will be a variable sized based on the observations included in
4074 the time-period. This is only valid for datetimelike indexes.
4075 To learn more about the offsets & frequency strings, please see
4076 :ref:`this link<timeseries.offset_aliases>`.
4078 If a BaseIndexer subclass, the window boundaries
4079 based on the defined ``get_window_bounds`` method. Additional rolling
4080 keyword arguments, namely ``min_periods``, ``center``, ``closed`` and
4081 ``step`` will be passed to ``get_window_bounds``.
4083 min_periods : int, default None
4084 Minimum number of observations in window required to have a value;
4085 otherwise, result is ``np.nan``.
4087 For a window that is specified by an offset,
4088 ``min_periods`` will default to 1.
4090 For a window that is specified by an integer, ``min_periods`` will default
4091 to the size of the window.
4093 center : bool, default False
4094 If False, set the window labels as the right edge of the window index.
4096 If True, set the window labels as the center of the window index.
4098 win_type : str, default None
4099 If ``None``, all points are evenly weighted.
4101 If a string, it must be a valid `scipy.signal window function
4102 <https://docs.scipy.org/doc/scipy/reference/signal.windows.html#module-scipy.signal.windows>`__.
4104 Certain Scipy window types require additional parameters to be passed
4105 in the aggregation function. The additional parameters must match
4106 the keywords specified in the Scipy window type method signature.
4108 on : str, optional
4109 For a DataFrame, a column label or Index level on which
4110 to calculate the rolling window, rather than the DataFrame's index.
4112 Provided integer column is ignored and excluded from result since
4113 an integer index is not used to calculate the rolling window.
4115 closed : str, default None
4116 Determines the inclusivity of points in the window
4118 If ``'right'``, uses the window (first, last] meaning the last point
4119 is included in the calculations.
4121 If ``'left'``, uses the window [first, last) meaning the first point
4122 is included in the calculations.
4124 If ``'both'``, uses the window [first, last] meaning all points in
4125 the window are included in the calculations.
4127 If ``'neither'``, uses the window (first, last) meaning the first
4128 and last points in the window are excluded from calculations.
4130 () and [] are referencing open and closed set
4131 notation respetively.
4133 Default ``None`` (``'right'``).
4135 method : str {'single', 'table'}, default 'single'
4136 Execute the rolling operation per single column or row (``'single'``)
4137 or over the entire object (``'table'``).
4139 This argument is only implemented when specifying ``engine='numba'``
4140 in the method call.
4142 Returns
4143 -------
4144 pandas.api.typing.RollingGroupby
4145 Return a new grouper with our rolling appended.
4147 See Also
4148 --------
4149 Series.rolling : Calling object with Series data.
4150 DataFrame.rolling : Calling object with DataFrames.
4151 Series.groupby : Apply a function groupby to a Series.
4152 DataFrame.groupby : Apply a function groupby.
4154 Examples
4155 --------
4156 >>> df = pd.DataFrame(
4157 ... {
4158 ... "A": [1, 1, 2, 2],
4159 ... "B": [1, 2, 3, 4],
4160 ... "C": [0.362, 0.227, 1.267, -0.562],
4161 ... }
4162 ... )
4163 >>> df
4164 A B C
4165 0 1 1 0.362
4166 1 1 2 0.227
4167 2 2 3 1.267
4168 3 2 4 -0.562
4170 >>> df.groupby("A").rolling(2).sum()
4171 B C
4172 A
4173 1 0 NaN NaN
4174 1 3.0 0.589
4175 2 2 NaN NaN
4176 3 7.0 0.705
4178 >>> df.groupby("A").rolling(2, min_periods=1).sum()
4179 B C
4180 A
4181 1 0 1.0 0.362
4182 1 3.0 0.589
4183 2 2 3.0 1.267
4184 3 7.0 0.705
4186 >>> df.groupby("A").rolling(2, on="B").sum()
4187 B C
4188 A
4189 1 0 1 NaN
4190 1 2 0.589
4191 2 2 3 NaN
4192 3 4 0.705
4193 """
4194 from pandas.core.window import RollingGroupby
4196 return RollingGroupby(
4197 self._selected_obj,
4198 window=window,
4199 min_periods=min_periods,
4200 center=center,
4201 win_type=win_type,
4202 on=on,
4203 closed=closed,
4204 method=method,
4205 _grouper=self._grouper,
4206 _as_index=self.as_index,
4207 )
4209 @final
4210 def expanding(
4211 self,
4212 min_periods: int = 1,
4213 method: str = "single",
4214 ) -> ExpandingGroupby:
4215 """
4216 Return an expanding grouper, providing expanding functionality per group.
4218 Parameters
4219 ----------
4220 min_periods : int, default 1
4221 Minimum number of observations in window required to have a value;
4222 otherwise, result is ``np.nan``.
4224 method : str {'single', 'table'}, default 'single'
4225 Execute the expanding operation per single column or row (``'single'``)
4226 or over the entire object (``'table'``).
4228 This argument is only implemented when specifying ``engine='numba'``
4229 in the method call.
4231 Returns
4232 -------
4233 pandas.api.typing.ExpandingGroupby
4234 An object that supports expanding transformations over each group.
4236 See Also
4237 --------
4238 Series.expanding : Expanding transformations for Series.
4239 DataFrame.expanding : Expanding transformations for DataFrames.
4240 Series.groupby : Apply a function groupby to a Series.
4241 DataFrame.groupby : Apply a function groupby.
4243 Examples
4244 --------
4245 >>> df = pd.DataFrame(
4246 ... {
4247 ... "Class": ["A", "A", "A", "B", "B", "B"],
4248 ... "Value": [10, 20, 30, 40, 50, 60],
4249 ... }
4250 ... )
4251 >>> df
4252 Class Value
4253 0 A 10
4254 1 A 20
4255 2 A 30
4256 3 B 40
4257 4 B 50
4258 5 B 60
4260 >>> df.groupby("Class").expanding().mean()
4261 Value
4262 Class
4263 A 0 10.0
4264 1 15.0
4265 2 20.0
4266 B 3 40.0
4267 4 45.0
4268 5 50.0
4269 """
4270 from pandas.core.window import ExpandingGroupby
4272 return ExpandingGroupby(
4273 self._selected_obj,
4274 min_periods=min_periods,
4275 method=method,
4276 _grouper=self._grouper,
4277 )
4279 @final
4280 def ewm(
4281 self,
4282 com: float | None = None,
4283 span: float | None = None,
4284 halflife: float | str | Timedelta | None = None,
4285 alpha: float | None = None,
4286 min_periods: int | None = 0,
4287 adjust: bool = True,
4288 ignore_na: bool = False,
4289 times: np.ndarray | Series | None = None,
4290 method: str = "single",
4291 ) -> ExponentialMovingWindowGroupby:
4292 """
4293 Return an ewm grouper, providing ewm functionality per group.
4295 Parameters
4296 ----------
4297 com : float, optional
4298 Specify decay in terms of center of mass.
4299 Alternative to ``span``, ``halflife``, and ``alpha``.
4301 span : float, optional
4302 Specify decay in terms of span.
4304 halflife : float, str, or Timedelta, optional
4305 Specify decay in terms of half-life.
4307 alpha : float, optional
4308 Specify smoothing factor directly.
4310 min_periods : int, default 0
4311 Minimum number of observations in the window required to have a value;
4312 otherwise, result is ``np.nan``.
4314 adjust : bool, default True
4315 Divide by decaying adjustment factor to account for imbalance in
4316 relative weights.
4318 ignore_na : bool, default False
4319 Ignore missing values when calculating weights.
4321 times : str or array-like of datetime64, optional
4322 Times corresponding to the observations.
4324 method : {'single', 'table'}, default 'single'
4325 Execute the operation per group independently (``'single'``) or over the
4326 entire object before regrouping (``'table'``). Only applicable to
4327 ``mean()``, and only when using ``engine='numba'``.
4329 Returns
4330 -------
4331 pandas.api.typing.ExponentialMovingWindowGroupby
4332 An object that supports exponentially weighted moving transformations over
4333 each group.
4335 See Also
4336 --------
4337 Series.ewm : EWM transformations for Series.
4338 DataFrame.ewm : EWM transformations for DataFrames.
4339 Series.groupby : Apply a function groupby to a Series.
4340 DataFrame.groupby : Apply a function groupby.
4342 Examples
4343 --------
4344 >>> df = pd.DataFrame(
4345 ... {
4346 ... "Class": ["A", "A", "A", "B", "B", "B"],
4347 ... "Value": [10, 20, 30, 40, 50, 60],
4348 ... }
4349 ... )
4350 >>> df
4351 Class Value
4352 0 A 10
4353 1 A 20
4354 2 A 30
4355 3 B 40
4356 4 B 50
4357 5 B 60
4359 >>> df.groupby("Class").ewm(com=0.5).mean()
4360 Value
4361 Class
4362 A 0 10.000000
4363 1 17.500000
4364 2 26.153846
4365 B 3 40.000000
4366 4 47.500000
4367 5 56.153846
4368 """
4369 from pandas.core.window import ExponentialMovingWindowGroupby
4371 return ExponentialMovingWindowGroupby(
4372 self._selected_obj,
4373 com=com,
4374 span=span,
4375 halflife=halflife,
4376 alpha=alpha,
4377 min_periods=min_periods,
4378 adjust=adjust,
4379 ignore_na=ignore_na,
4380 times=times,
4381 method=method,
4382 _grouper=self._grouper,
4383 )
4385 @final
4386 def _fill(self, direction: Literal["ffill", "bfill"], limit: int | None = None):
4387 """
4388 Shared function for `pad` and `backfill` to call Cython method.
4390 Parameters
4391 ----------
4392 direction : {'ffill', 'bfill'}
4393 Direction passed to underlying Cython function. `bfill` will cause
4394 values to be filled backwards. `ffill` and any other values will
4395 default to a forward fill
4396 limit : int, default None
4397 Maximum number of consecutive values to fill. If `None`, this
4398 method will convert to -1 prior to passing to Cython
4400 Returns
4401 -------
4402 `Series` or `DataFrame` with filled values
4404 See Also
4405 --------
4406 pad : Returns Series with minimum number of char in object.
4407 backfill : Backward fill the missing values in the dataset.
4408 """
4409 # Need int value for Cython
4410 if limit is None:
4411 limit = -1
4413 ids = self._grouper.ids
4414 ngroups = self._grouper.ngroups
4416 col_func = partial(
4417 libgroupby.group_fillna_indexer,
4418 labels=ids,
4419 limit=limit,
4420 compute_ffill=(direction == "ffill"),
4421 ngroups=ngroups,
4422 )
4424 def blk_func(values: ArrayLike) -> ArrayLike:
4425 mask = isna(values)
4426 if values.ndim == 1:
4427 indexer = np.empty(values.shape, dtype=np.intp)
4428 col_func(out=indexer, mask=mask) # type: ignore[arg-type]
4429 return algorithms.take_nd(values, indexer)
4431 else:
4432 # We broadcast algorithms.take_nd analogous to
4433 # np.take_along_axis
4434 if isinstance(values, np.ndarray):
4435 dtype = values.dtype
4436 if self._grouper.has_dropped_na:
4437 # dropped null groups give rise to nan in the result
4438 dtype = ensure_dtype_can_hold_na(values.dtype)
4439 out = np.empty(values.shape, dtype=dtype)
4440 else:
4441 # Note: we only get here with backfill/pad,
4442 # so if we have a dtype that cannot hold NAs,
4443 # then there will be no -1s in indexer, so we can use
4444 # the original dtype (no need to ensure_dtype_can_hold_na)
4445 out = type(values)._empty(values.shape, dtype=values.dtype)
4447 for i, value_element in enumerate(values):
4448 # call group_fillna_indexer column-wise
4449 indexer = np.empty(values.shape[1], dtype=np.intp)
4450 col_func(out=indexer, mask=mask[i])
4451 out[i, :] = algorithms.take_nd(value_element, indexer)
4452 return out
4454 mgr = self._get_data_to_aggregate()
4455 res_mgr = mgr.apply(blk_func)
4457 new_obj = self._wrap_agged_manager(res_mgr)
4458 new_obj.index = self.obj.index
4459 return new_obj
4461 @final
4462 def ffill(self, limit: int | None = None):
4463 """
4464 Forward fill the values.
4466 Parameters
4467 ----------
4468 limit : int, optional
4469 Limit of how many values to fill.
4471 Returns
4472 -------
4473 Series or DataFrame
4474 Object with missing values filled.
4476 See Also
4477 --------
4478 Series.ffill: Returns Series with minimum number of char in object.
4479 DataFrame.ffill: Object with missing values filled or None if inplace=True.
4480 Series.fillna: Fill NaN values of a Series.
4481 DataFrame.fillna: Fill NaN values of a DataFrame.
4483 Examples
4484 --------
4486 For SeriesGroupBy:
4488 >>> key = [0, 0, 1, 1]
4489 >>> ser = pd.Series([np.nan, 2, 3, np.nan], index=key)
4490 >>> ser
4491 0 NaN
4492 0 2.0
4493 1 3.0
4494 1 NaN
4495 dtype: float64
4496 >>> ser.groupby(level=0).ffill()
4497 0 NaN
4498 0 2.0
4499 1 3.0
4500 1 3.0
4501 dtype: float64
4503 For DataFrameGroupBy:
4505 >>> df = pd.DataFrame(
4506 ... {
4507 ... "key": [0, 0, 1, 1, 1],
4508 ... "A": [np.nan, 2, np.nan, 3, np.nan],
4509 ... "B": [2, 3, np.nan, np.nan, np.nan],
4510 ... "C": [np.nan, np.nan, 2, np.nan, np.nan],
4511 ... }
4512 ... )
4513 >>> df
4514 key A B C
4515 0 0 NaN 2.0 NaN
4516 1 0 2.0 3.0 NaN
4517 2 1 NaN NaN 2.0
4518 3 1 3.0 NaN NaN
4519 4 1 NaN NaN NaN
4521 Propagate non-null values forward or backward within each group along columns.
4523 >>> df.groupby("key").ffill()
4524 A B C
4525 0 NaN 2.0 NaN
4526 1 2.0 3.0 NaN
4527 2 NaN NaN 2.0
4528 3 3.0 NaN 2.0
4529 4 3.0 NaN 2.0
4531 Propagate non-null values forward or backward within each group along rows.
4533 >>> df.T.groupby(np.array([0, 0, 1, 1])).ffill().T
4534 key A B C
4535 0 0.0 0.0 2.0 2.0
4536 1 0.0 2.0 3.0 3.0
4537 2 1.0 1.0 NaN 2.0
4538 3 1.0 3.0 NaN NaN
4539 4 1.0 1.0 NaN NaN
4541 Only replace the first NaN element within a group along columns.
4543 >>> df.groupby("key").ffill(limit=1)
4544 A B C
4545 0 NaN 2.0 NaN
4546 1 2.0 3.0 NaN
4547 2 NaN NaN 2.0
4548 3 3.0 NaN 2.0
4549 4 3.0 NaN NaN
4550 """
4551 return self._fill("ffill", limit=limit)
4553 @final
4554 def bfill(self, limit: int | None = None):
4555 """
4556 Backward fill the values.
4558 Parameters
4559 ----------
4560 limit : int, optional
4561 Limit of how many values to fill.
4563 Returns
4564 -------
4565 Series or DataFrame
4566 Object with missing values filled.
4568 See Also
4569 --------
4570 Series.bfill : Backward fill the missing values in the dataset.
4571 DataFrame.bfill: Backward fill the missing values in the dataset.
4572 Series.fillna: Fill NaN values of a Series.
4573 DataFrame.fillna: Fill NaN values of a DataFrame.
4575 Examples
4576 --------
4578 With Series:
4580 >>> index = ["Falcon", "Falcon", "Parrot", "Parrot", "Parrot"]
4581 >>> s = pd.Series([None, 1, None, None, 3], index=index)
4582 >>> s
4583 Falcon NaN
4584 Falcon 1.0
4585 Parrot NaN
4586 Parrot NaN
4587 Parrot 3.0
4588 dtype: float64
4589 >>> s.groupby(level=0).bfill()
4590 Falcon 1.0
4591 Falcon 1.0
4592 Parrot 3.0
4593 Parrot 3.0
4594 Parrot 3.0
4595 dtype: float64
4596 >>> s.groupby(level=0).bfill(limit=1)
4597 Falcon 1.0
4598 Falcon 1.0
4599 Parrot NaN
4600 Parrot 3.0
4601 Parrot 3.0
4602 dtype: float64
4604 With DataFrame:
4606 >>> df = pd.DataFrame(
4607 ... {"A": [1, None, None, None, 4], "B": [None, None, 5, None, 7]},
4608 ... index=index,
4609 ... )
4610 >>> df
4611 A B
4612 Falcon 1.0 NaN
4613 Falcon NaN NaN
4614 Parrot NaN 5.0
4615 Parrot NaN NaN
4616 Parrot 4.0 7.0
4617 >>> df.groupby(level=0).bfill()
4618 A B
4619 Falcon 1.0 NaN
4620 Falcon NaN NaN
4621 Parrot 4.0 5.0
4622 Parrot 4.0 7.0
4623 Parrot 4.0 7.0
4624 >>> df.groupby(level=0).bfill(limit=1)
4625 A B
4626 Falcon 1.0 NaN
4627 Falcon NaN NaN
4628 Parrot NaN 5.0
4629 Parrot 4.0 7.0
4630 Parrot 4.0 7.0
4631 """
4632 return self._fill("bfill", limit=limit)
4634 @final
4635 @property
4636 def nth(self) -> GroupByNthSelector:
4637 """
4638 Take the nth row from each group if n is an int, otherwise a subset of rows.
4640 Can be either a call or an index. dropna is not available with index notation.
4641 Index notation accepts a comma separated list of integers and slices.
4643 If dropna, will take the nth non-null row, dropna is either
4644 'all' or 'any'; this is equivalent to calling dropna(how=dropna)
4645 before the groupby.
4647 Returns
4648 -------
4649 Series or DataFrame
4650 N-th value within each group.
4652 See Also
4653 --------
4654 Series.nth : Apply function nth to a Series.
4655 DataFrame.nth : Apply function nth to each row or column of a DataFrame.
4657 Examples
4658 --------
4660 >>> df = pd.DataFrame(
4661 ... {"A": [1, 1, 2, 1, 2], "B": [np.nan, 2, 3, 4, 5]}, columns=["A", "B"]
4662 ... )
4663 >>> g = df.groupby("A")
4664 >>> g.nth(0)
4665 A B
4666 0 1 NaN
4667 2 2 3.0
4668 >>> g.nth(1)
4669 A B
4670 1 1 2.0
4671 4 2 5.0
4672 >>> g.nth(-1)
4673 A B
4674 3 1 4.0
4675 4 2 5.0
4676 >>> g.nth([0, 1])
4677 A B
4678 0 1 NaN
4679 1 1 2.0
4680 2 2 3.0
4681 4 2 5.0
4682 >>> g.nth(slice(None, -1))
4683 A B
4684 0 1 NaN
4685 1 1 2.0
4686 2 2 3.0
4688 Index notation may also be used
4690 >>> g.nth[0, 1]
4691 A B
4692 0 1 NaN
4693 1 1 2.0
4694 2 2 3.0
4695 4 2 5.0
4696 >>> g.nth[:-1]
4697 A B
4698 0 1 NaN
4699 1 1 2.0
4700 2 2 3.0
4702 Specifying `dropna` allows ignoring ``NaN`` values
4704 >>> g.nth(0, dropna="any")
4705 A B
4706 1 1 2.0
4707 2 2 3.0
4709 When the specified ``n`` is larger than any of the groups, an
4710 empty DataFrame is returned
4712 >>> g.nth(3, dropna="any")
4713 Empty DataFrame
4714 Columns: [A, B]
4715 Index: []
4716 """
4717 return GroupByNthSelector(self)
4719 def _nth(
4720 self,
4721 n: PositionalIndexer | tuple,
4722 dropna: Literal["any", "all"] | None = None,
4723 ) -> NDFrameT:
4724 if not dropna:
4725 mask = self._make_mask_from_positional_indexer(n)
4727 ids = self._grouper.ids
4729 # Drop NA values in grouping
4730 mask = mask & (ids != -1)
4732 out = self._mask_selected_obj(mask)
4733 return out
4735 # dropna is truthy
4736 if not is_integer(n):
4737 raise ValueError("dropna option only supported for an integer argument")
4739 if dropna not in ["any", "all"]:
4740 # Note: when agg-ing picker doesn't raise this, just returns NaN
4741 raise ValueError(
4742 "For a DataFrame or Series groupby.nth, dropna must be "
4743 "either None, 'any' or 'all', "
4744 f"(was passed {dropna})."
4745 )
4747 # old behaviour, but with all and any support for DataFrames.
4748 # modified in GH 7559 to have better perf
4749 n = cast(int, n)
4750 dropped = self._selected_obj.dropna(how=dropna, axis=0)
4752 # get a new grouper for our dropped obj
4753 grouper: np.ndarray | Index | ops.BaseGrouper
4754 if len(dropped) == len(self._selected_obj):
4755 # Nothing was dropped, can use the same grouper
4756 grouper = self._grouper
4757 else:
4758 # we don't have the grouper info available
4759 # (e.g. we have selected out
4760 # a column that is not in the current object)
4761 axis = self._grouper.axis
4762 grouper = self._grouper.codes_info[axis.isin(dropped.index)]
4763 if self._grouper.has_dropped_na:
4764 # Null groups need to still be encoded as -1 when passed to groupby
4765 nulls = grouper == -1
4766 # error: No overload variant of "where" matches argument types
4767 # "Any", "NAType", "Any"
4768 values = np.where(nulls, NA, grouper) # type: ignore[call-overload]
4769 grouper = Index(values, dtype="Int64", copy=False)
4771 grb = dropped.groupby(grouper, as_index=self.as_index, sort=self.sort)
4772 return grb.nth(n)
4774 @final
4775 def quantile(
4776 self,
4777 q: float | AnyArrayLike = 0.5,
4778 interpolation: Literal[
4779 "linear", "lower", "higher", "nearest", "midpoint"
4780 ] = "linear",
4781 numeric_only: bool = False,
4782 ):
4783 """
4784 Return group values at the given quantile, a la numpy.percentile.
4786 Parameters
4787 ----------
4788 q : float or array-like, default 0.5 (50% quantile)
4789 Value(s) between 0 and 1 providing the quantile(s) to compute.
4790 interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
4791 Method to use when the desired quantile falls between two points.
4792 numeric_only : bool, default False
4793 Include only `float`, `int` or `boolean` data.
4795 .. versionchanged:: 2.0.0
4797 numeric_only now defaults to ``False``.
4799 Returns
4800 -------
4801 Series or DataFrame
4802 Return type determined by caller of GroupBy object.
4804 See Also
4805 --------
4806 Series.quantile : Similar method for Series.
4807 DataFrame.quantile : Similar method for DataFrame.
4808 numpy.percentile : NumPy method to compute qth percentile.
4810 Examples
4811 --------
4812 >>> df = pd.DataFrame(
4813 ... [["a", 1], ["a", 2], ["a", 3], ["b", 1], ["b", 3], ["b", 5]],
4814 ... columns=["key", "val"],
4815 ... )
4816 >>> df.groupby("key").quantile()
4817 val
4818 key
4819 a 2.0
4820 b 3.0
4821 """
4822 mgr = self._get_data_to_aggregate(numeric_only=numeric_only, name="quantile")
4823 obj = self._wrap_agged_manager(mgr)
4824 splitter = self._grouper._get_splitter(obj)
4825 sdata = splitter._sorted_data
4827 starts, ends = lib.generate_slices(splitter._slabels, splitter.ngroups)
4829 def pre_processor(vals: ArrayLike) -> tuple[np.ndarray, DtypeObj | None]:
4830 if isinstance(vals.dtype, StringDtype) or is_object_dtype(vals.dtype):
4831 raise TypeError(
4832 f"dtype '{vals.dtype}' does not support operation 'quantile'"
4833 )
4835 inference: DtypeObj | None = None
4836 if isinstance(vals, BaseMaskedArray) and is_numeric_dtype(vals.dtype):
4837 out = vals.to_numpy(dtype=float, na_value=np.nan)
4838 inference = vals.dtype
4839 elif is_integer_dtype(vals.dtype):
4840 if isinstance(vals, ExtensionArray):
4841 out = vals.to_numpy(dtype=float, na_value=np.nan)
4842 else:
4843 out = vals
4844 inference = np.dtype(np.int64)
4845 elif is_bool_dtype(vals.dtype) and isinstance(vals, ExtensionArray):
4846 out = vals.to_numpy(dtype=float, na_value=np.nan)
4847 elif is_bool_dtype(vals.dtype):
4848 # GH#51424 remove to match Series/DataFrame behavior
4849 raise TypeError("Cannot use quantile with bool dtype")
4850 elif needs_i8_conversion(vals.dtype):
4851 inference = vals.dtype
4852 # In this case we need to delay the casting until after the
4853 # np.lexsort below.
4854 # error: Incompatible return value type (got
4855 # "Tuple[Union[ExtensionArray, ndarray[Any, Any]], Union[Any,
4856 # ExtensionDtype]]", expected "Tuple[ndarray[Any, Any],
4857 # Optional[Union[dtype[Any], ExtensionDtype]]]")
4858 return vals, inference # type: ignore[return-value]
4859 elif isinstance(vals, ExtensionArray) and is_float_dtype(vals.dtype):
4860 inference = np.dtype(np.float64)
4861 out = vals.to_numpy(dtype=float, na_value=np.nan)
4862 else:
4863 out = np.asarray(vals)
4865 return out, inference
4867 def post_processor(
4868 vals: np.ndarray,
4869 inference: DtypeObj | None,
4870 result_mask: np.ndarray | None,
4871 orig_vals: ArrayLike,
4872 ) -> ArrayLike:
4873 if inference:
4874 # Check for edge case
4875 if isinstance(orig_vals, BaseMaskedArray):
4876 assert result_mask is not None # for mypy
4878 if interpolation in {"linear", "midpoint"} and not is_float_dtype(
4879 orig_vals
4880 ):
4881 return FloatingArray(vals, result_mask)
4882 else:
4883 # Item "ExtensionDtype" of "Union[ExtensionDtype, str,
4884 # dtype[Any], Type[object]]" has no attribute "numpy_dtype"
4885 # [union-attr]
4886 with warnings.catch_warnings():
4887 # vals.astype with nan can warn with numpy >1.24
4888 warnings.filterwarnings("ignore", category=RuntimeWarning)
4889 return type(orig_vals)(
4890 vals.astype(
4891 inference.numpy_dtype # type: ignore[union-attr]
4892 ),
4893 result_mask,
4894 )
4896 elif not (
4897 is_integer_dtype(inference)
4898 and interpolation in {"linear", "midpoint"}
4899 ):
4900 if needs_i8_conversion(inference):
4901 # error: Item "ExtensionArray" of "Union[ExtensionArray,
4902 # ndarray[Any, Any]]" has no attribute "_ndarray"
4903 vals = vals.astype("i8").view(
4904 orig_vals._ndarray.dtype # type: ignore[union-attr]
4905 )
4906 # error: Item "ExtensionArray" of "Union[ExtensionArray,
4907 # ndarray[Any, Any]]" has no attribute "_from_backing_data"
4908 return orig_vals._from_backing_data( # type: ignore[union-attr]
4909 vals
4910 )
4912 assert isinstance(inference, np.dtype) # for mypy
4913 return vals.astype(inference)
4915 return vals
4917 if is_scalar(q):
4918 qs = np.array([q], dtype=np.float64)
4919 pass_qs: None | np.ndarray = None
4920 else:
4921 qs = np.asarray(q, dtype=np.float64)
4922 pass_qs = qs
4924 ids = self._grouper.ids
4925 ngroups = self._grouper.ngroups
4926 if self.dropna:
4927 # splitter drops NA groups, we need to do the same
4928 ids = ids[ids >= 0]
4929 nqs = len(qs)
4931 func = partial(
4932 libgroupby.group_quantile,
4933 labels=ids,
4934 qs=qs,
4935 interpolation=interpolation,
4936 starts=starts,
4937 ends=ends,
4938 )
4940 def blk_func(values: ArrayLike) -> ArrayLike:
4941 orig_vals = values
4942 if isinstance(values, BaseMaskedArray):
4943 mask = values._mask
4944 result_mask = np.zeros((ngroups, nqs), dtype=np.bool_)
4945 else:
4946 mask = isna(values)
4947 result_mask = None
4949 is_datetimelike = needs_i8_conversion(values.dtype)
4951 vals, inference = pre_processor(values)
4953 ncols = 1
4954 if vals.ndim == 2:
4955 ncols = vals.shape[0]
4957 out = np.empty((ncols, ngroups, nqs), dtype=np.float64)
4959 if is_datetimelike:
4960 vals = vals.view("i8")
4962 if vals.ndim == 1:
4963 # EA is always 1d
4964 func(
4965 out[0],
4966 values=vals,
4967 mask=mask, # type: ignore[arg-type]
4968 result_mask=result_mask,
4969 is_datetimelike=is_datetimelike,
4970 )
4971 else:
4972 for i in range(ncols):
4973 func(
4974 out[i],
4975 values=vals[i],
4976 mask=mask[i],
4977 result_mask=None,
4978 is_datetimelike=is_datetimelike,
4979 )
4981 if vals.ndim == 1:
4982 out = out.ravel("K") # type: ignore[assignment]
4983 if result_mask is not None:
4984 result_mask = result_mask.ravel("K") # type: ignore[assignment]
4985 else:
4986 out = out.reshape(ncols, ngroups * nqs) # type: ignore[assignment]
4988 return post_processor(out, inference, result_mask, orig_vals)
4990 res_mgr = sdata._mgr.grouped_reduce(blk_func)
4992 res = self._wrap_agged_manager(res_mgr)
4993 return self._wrap_aggregated_output(res, qs=pass_qs)
4995 @final
4996 def ngroup(self, ascending: bool = True):
4997 """
4998 Number each group from 0 to the number of groups - 1.
5000 This is the enumerative complement of cumcount. Note that the
5001 numbers given to the groups match the order in which the groups
5002 would be seen when iterating over the groupby object, not the
5003 order they are first observed.
5005 Groups with missing keys (where `pd.isna()` is True) will be labeled with `NaN`
5006 and will be skipped from the count.
5008 Parameters
5009 ----------
5010 ascending : bool, default True
5011 If False, number in reverse, from number of group - 1 to 0.
5013 Returns
5014 -------
5015 Series
5016 Unique numbers for each group.
5018 See Also
5019 --------
5020 .cumcount : Number the rows in each group.
5022 Examples
5023 --------
5024 >>> df = pd.DataFrame({"color": ["red", None, "red", "blue", "blue", "red"]})
5025 >>> df
5026 color
5027 0 red
5028 1 NaN
5029 2 red
5030 3 blue
5031 4 blue
5032 5 red
5033 >>> df.groupby("color").ngroup()
5034 0 1.0
5035 1 NaN
5036 2 1.0
5037 3 0.0
5038 4 0.0
5039 5 1.0
5040 dtype: float64
5041 >>> df.groupby("color", dropna=False).ngroup()
5042 0 1
5043 1 2
5044 2 1
5045 3 0
5046 4 0
5047 5 1
5048 dtype: int64
5049 >>> df.groupby("color", dropna=False).ngroup(ascending=False)
5050 0 1
5051 1 0
5052 2 1
5053 3 2
5054 4 2
5055 5 1
5056 dtype: int64
5057 """
5058 obj = self._obj_with_exclusions
5059 index = obj.index
5060 comp_ids = self._grouper.ids
5062 dtype: type
5063 if self._grouper.has_dropped_na:
5064 comp_ids = np.where(comp_ids == -1, np.nan, comp_ids)
5065 dtype = np.float64
5066 else:
5067 dtype = np.int64
5069 if any(ping._passed_categorical for ping in self._grouper.groupings):
5070 # comp_ids reflect non-observed groups, we need only observed
5071 comp_ids = rank_1d(comp_ids, ties_method="dense") - 1
5073 result = self._obj_1d_constructor(comp_ids, index, dtype=dtype)
5074 if not ascending:
5075 result = self.ngroups - 1 - result
5076 return result
5078 @final
5079 def cumcount(self, ascending: bool = True):
5080 """
5081 Number each item in each group from 0 to the length of that group - 1.
5083 Essentially this is equivalent to
5085 .. code-block:: python
5087 self.apply(lambda x: pd.Series(np.arange(len(x)), x.index))
5089 Parameters
5090 ----------
5091 ascending : bool, default True
5092 If False, number in reverse, from length of group - 1 to 0.
5094 Returns
5095 -------
5096 Series
5097 Sequence number of each element within each group.
5099 See Also
5100 --------
5101 .ngroup : Number the groups themselves.
5103 Examples
5104 --------
5105 >>> df = pd.DataFrame([["a"], ["a"], ["a"], ["b"], ["b"], ["a"]], columns=["A"])
5106 >>> df
5107 A
5108 0 a
5109 1 a
5110 2 a
5111 3 b
5112 4 b
5113 5 a
5114 >>> df.groupby("A").cumcount()
5115 0 0
5116 1 1
5117 2 2
5118 3 0
5119 4 1
5120 5 3
5121 dtype: int64
5122 >>> df.groupby("A").cumcount(ascending=False)
5123 0 3
5124 1 2
5125 2 1
5126 3 1
5127 4 0
5128 5 0
5129 dtype: int64
5130 """
5131 index = self._obj_with_exclusions.index
5132 cumcounts = self._cumcount_array(ascending=ascending)
5133 return self._obj_1d_constructor(cumcounts, index)
5135 @final
5136 def rank(
5137 self,
5138 method: str = "average",
5139 ascending: bool = True,
5140 na_option: str = "keep",
5141 pct: bool = False,
5142 ) -> NDFrameT:
5143 """
5144 Provide the rank of values within each group.
5146 Parameters
5147 ----------
5148 method : {'average', 'min', 'max', 'first', 'dense'}, default 'average'
5149 * average: average rank of group.
5150 * min: lowest rank in group.
5151 * max: highest rank in group.
5152 * first: ranks assigned in order they appear in the array.
5153 * dense: like 'min', but rank always increases by 1 between groups.
5154 ascending : bool, default True
5155 False for ranks by high (1) to low (N).
5156 na_option : {'keep', 'top', 'bottom'}, default 'keep'
5157 * keep: leave NA values where they are.
5158 * top: smallest rank if ascending.
5159 * bottom: smallest rank if descending.
5160 pct : bool, default False
5161 Compute percentage rank of data within each group.
5163 Returns
5164 -------
5165 DataFrame
5166 The ranking of values within each group.
5168 See Also
5169 --------
5170 Series.rank : Apply function rank to a Series.
5171 DataFrame.rank : Apply function rank to each row or column of a DataFrame.
5173 Examples
5174 --------
5175 >>> df = pd.DataFrame(
5176 ... {
5177 ... "group": ["a", "a", "a", "a", "a", "b", "b", "b", "b", "b"],
5178 ... "value": [2, 4, 2, 3, 5, 1, 2, 4, 1, 5],
5179 ... }
5180 ... )
5181 >>> df
5182 group value
5183 0 a 2
5184 1 a 4
5185 2 a 2
5186 3 a 3
5187 4 a 5
5188 5 b 1
5189 6 b 2
5190 7 b 4
5191 8 b 1
5192 9 b 5
5193 >>> for method in ["average", "min", "max", "dense", "first"]:
5194 ... df[f"{method}_rank"] = df.groupby("group")["value"].rank(method)
5195 >>> df
5196 group value average_rank min_rank max_rank dense_rank first_rank
5197 0 a 2 1.5 1.0 2.0 1.0 1.0
5198 1 a 4 4.0 4.0 4.0 3.0 4.0
5199 2 a 2 1.5 1.0 2.0 1.0 2.0
5200 3 a 3 3.0 3.0 3.0 2.0 3.0
5201 4 a 5 5.0 5.0 5.0 4.0 5.0
5202 5 b 1 1.5 1.0 2.0 1.0 1.0
5203 6 b 2 3.0 3.0 3.0 2.0 3.0
5204 7 b 4 4.0 4.0 4.0 3.0 4.0
5205 8 b 1 1.5 1.0 2.0 1.0 2.0
5206 9 b 5 5.0 5.0 5.0 4.0 5.0
5207 """
5208 if na_option not in {"keep", "top", "bottom"}:
5209 msg = "na_option must be one of 'keep', 'top', or 'bottom'"
5210 raise ValueError(msg)
5212 kwargs = {
5213 "ties_method": method,
5214 "ascending": ascending,
5215 "na_option": na_option,
5216 "pct": pct,
5217 }
5219 return self._cython_transform(
5220 "rank",
5221 numeric_only=False,
5222 **kwargs,
5223 )
5225 @final
5226 def cumprod(self, numeric_only: bool = False, *args, **kwargs) -> NDFrameT:
5227 """
5228 Cumulative product for each group.
5230 Parameters
5231 ----------
5232 numeric_only : bool, default False
5233 Include only float, int, boolean columns.
5234 *args : tuple
5235 Positional arguments to be passed to `func`.
5236 **kwargs : dict
5237 Additional/specific keyword arguments to be passed to the function,
5238 such as `numeric_only` and `skipna`.
5240 Returns
5241 -------
5242 Series or DataFrame
5243 Cumulative product for each group. Same object type as the caller.
5245 See Also
5246 --------
5247 Series.cumprod : Apply function cumprod to a Series.
5248 DataFrame.cumprod : Apply function cumprod to each row or column of a DataFrame.
5250 Examples
5251 --------
5252 For SeriesGroupBy:
5254 >>> lst = ["a", "a", "b"]
5255 >>> ser = pd.Series([6, 2, 0], index=lst)
5256 >>> ser
5257 a 6
5258 a 2
5259 b 0
5260 dtype: int64
5261 >>> ser.groupby(level=0).cumprod()
5262 a 6
5263 a 12
5264 b 0
5265 dtype: int64
5267 For DataFrameGroupBy:
5269 >>> data = [[1, 8, 2], [1, 2, 5], [2, 6, 9]]
5270 >>> df = pd.DataFrame(
5271 ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
5272 ... )
5273 >>> df
5274 a b c
5275 cow 1 8 2
5276 horse 1 2 5
5277 bull 2 6 9
5278 >>> df.groupby("a").groups
5279 {1: ['cow', 'horse'], 2: ['bull']}
5280 >>> df.groupby("a").cumprod()
5281 b c
5282 cow 8 2
5283 horse 16 10
5284 bull 6 9
5285 """
5286 nv.validate_groupby_func("cumprod", args, kwargs, ["skipna"])
5287 return self._cython_transform("cumprod", numeric_only, **kwargs)
5289 @final
5290 def cumsum(self, numeric_only: bool = False, *args, **kwargs) -> NDFrameT:
5291 """
5292 Cumulative sum for each group.
5294 Parameters
5295 ----------
5296 numeric_only : bool, default False
5297 Include only float, int, boolean columns.
5298 *args : tuple
5299 Positional arguments to be passed to `func`.
5300 **kwargs : dict
5301 Additional/specific keyword arguments to be passed to the function,
5302 such as `numeric_only` and `skipna`.
5304 Returns
5305 -------
5306 Series or DataFrame
5307 Cumulative sum for each group. Same object type as the caller.
5309 See Also
5310 --------
5311 Series.cumsum : Apply function cumsum to a Series.
5312 DataFrame.cumsum : Apply function cumsum to each row or column of a DataFrame.
5314 Examples
5315 --------
5316 For SeriesGroupBy:
5318 >>> lst = ["a", "a", "b"]
5319 >>> ser = pd.Series([6, 2, 0], index=lst)
5320 >>> ser
5321 a 6
5322 a 2
5323 b 0
5324 dtype: int64
5325 >>> ser.groupby(level=0).cumsum()
5326 a 6
5327 a 8
5328 b 0
5329 dtype: int64
5331 For DataFrameGroupBy:
5333 >>> data = [[1, 8, 2], [1, 2, 5], [2, 6, 9]]
5334 >>> df = pd.DataFrame(
5335 ... data, columns=["a", "b", "c"], index=["fox", "gorilla", "lion"]
5336 ... )
5337 >>> df
5338 a b c
5339 fox 1 8 2
5340 gorilla 1 2 5
5341 lion 2 6 9
5342 >>> df.groupby("a").groups
5343 {1: ['fox', 'gorilla'], 2: ['lion']}
5344 >>> df.groupby("a").cumsum()
5345 b c
5346 fox 8 2
5347 gorilla 10 7
5348 lion 6 9
5349 """
5350 nv.validate_groupby_func("cumsum", args, kwargs, ["skipna"])
5351 return self._cython_transform("cumsum", numeric_only, **kwargs)
5353 @final
5354 def cummin(
5355 self,
5356 numeric_only: bool = False,
5357 **kwargs,
5358 ) -> NDFrameT:
5359 """
5360 Cumulative min for each group.
5362 Parameters
5363 ----------
5364 numeric_only : bool, default False
5365 Include only `float`, `int` or `boolean` data.
5366 **kwargs : dict, optional
5367 Additional keyword arguments to be passed to the function, such as `skipna`,
5368 to control whether NA/null values are ignored.
5370 Returns
5371 -------
5372 Series or DataFrame
5373 Cumulative min for each group. Same object type as the caller.
5375 See Also
5376 --------
5377 Series.cummin : Apply function cummin to a Series.
5378 DataFrame.cummin : Apply function cummin to each row or column of a DataFrame.
5380 Examples
5381 --------
5382 For SeriesGroupBy:
5384 >>> lst = ["a", "a", "a", "b", "b", "b"]
5385 >>> ser = pd.Series([1, 6, 2, 3, 0, 4], index=lst)
5386 >>> ser
5387 a 1
5388 a 6
5389 a 2
5390 b 3
5391 b 0
5392 b 4
5393 dtype: int64
5394 >>> ser.groupby(level=0).cummin()
5395 a 1
5396 a 1
5397 a 1
5398 b 3
5399 b 0
5400 b 0
5401 dtype: int64
5403 For DataFrameGroupBy:
5405 >>> data = [[1, 0, 2], [1, 1, 5], [6, 6, 9]]
5406 >>> df = pd.DataFrame(
5407 ... data, columns=["a", "b", "c"], index=["snake", "rabbit", "turtle"]
5408 ... )
5409 >>> df
5410 a b c
5411 snake 1 0 2
5412 rabbit 1 1 5
5413 turtle 6 6 9
5414 >>> df.groupby("a").groups
5415 {1: ['snake', 'rabbit'], 6: ['turtle']}
5416 >>> df.groupby("a").cummin()
5417 b c
5418 snake 0 2
5419 rabbit 0 2
5420 turtle 6 9
5421 """
5422 skipna = kwargs.get("skipna", True)
5423 return self._cython_transform(
5424 "cummin", numeric_only=numeric_only, skipna=skipna
5425 )
5427 @final
5428 def cummax(
5429 self,
5430 numeric_only: bool = False,
5431 **kwargs,
5432 ) -> NDFrameT:
5433 """
5434 Cumulative max for each group.
5436 Returns the cumulative maximum of values within each group. The result
5437 has the same size as the input, with each element representing the
5438 maximum of all preceding elements (including itself) within its group.
5440 Parameters
5441 ----------
5442 numeric_only : bool, default False
5443 Include only `float`, `int` or `boolean` data.
5444 **kwargs : dict, optional
5445 Additional keyword arguments to be passed to the function, such as `skipna`,
5446 to control whether NA/null values are ignored.
5448 Returns
5449 -------
5450 Series or DataFrame
5451 Cumulative max for each group. Same object type as the caller.
5453 See Also
5454 --------
5455 Series.cummax : Apply function cummax to a Series.
5456 DataFrame.cummax : Apply function cummax to each row or column of a DataFrame.
5458 Examples
5459 --------
5460 For SeriesGroupBy:
5462 >>> lst = ["a", "a", "a", "b", "b", "b"]
5463 >>> ser = pd.Series([1, 6, 2, 3, 1, 4], index=lst)
5464 >>> ser
5465 a 1
5466 a 6
5467 a 2
5468 b 3
5469 b 1
5470 b 4
5471 dtype: int64
5472 >>> ser.groupby(level=0).cummax()
5473 a 1
5474 a 6
5475 a 6
5476 b 3
5477 b 3
5478 b 4
5479 dtype: int64
5481 For DataFrameGroupBy:
5483 >>> data = [[1, 8, 2], [1, 1, 0], [2, 6, 9]]
5484 >>> df = pd.DataFrame(
5485 ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
5486 ... )
5487 >>> df
5488 a b c
5489 cow 1 8 2
5490 horse 1 1 0
5491 bull 2 6 9
5492 >>> df.groupby("a").groups
5493 {1: ['cow', 'horse'], 2: ['bull']}
5494 >>> df.groupby("a").cummax()
5495 b c
5496 cow 8 2
5497 horse 8 2
5498 bull 6 9
5499 """
5500 skipna = kwargs.get("skipna", True)
5501 return self._cython_transform(
5502 "cummax", numeric_only=numeric_only, skipna=skipna
5503 )
5505 @final
5506 def shift(
5507 self,
5508 periods: int | Sequence[int] = 1,
5509 freq=None,
5510 fill_value=lib.no_default,
5511 suffix: str | None = None,
5512 ):
5513 """
5514 Shift each group by periods observations.
5516 If freq is passed, the index will be increased using the periods and the freq.
5518 Parameters
5519 ----------
5520 periods : int | Sequence[int], default 1
5521 Number of periods to shift. If a list of values, shift each group by
5522 each period.
5523 freq : str, optional
5524 Frequency string.
5525 fill_value : optional
5526 The scalar value to use for newly introduced missing values.
5528 .. versionchanged:: 2.1.0
5529 Will raise a ``ValueError`` if ``freq`` is provided too.
5531 suffix : str, optional
5532 A string to add to each shifted column if there are multiple periods.
5533 Ignored otherwise.
5535 Returns
5536 -------
5537 Series or DataFrame
5538 Object shifted within each group.
5540 See Also
5541 --------
5542 Index.shift : Shift values of Index.
5544 Examples
5545 --------
5547 For SeriesGroupBy:
5549 >>> lst = ["a", "a", "b", "b"]
5550 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
5551 >>> ser
5552 a 1
5553 a 2
5554 b 3
5555 b 4
5556 dtype: int64
5557 >>> ser.groupby(level=0).shift(1)
5558 a NaN
5559 a 1.0
5560 b NaN
5561 b 3.0
5562 dtype: float64
5564 For DataFrameGroupBy:
5566 >>> data = [[1, 2, 3], [1, 5, 6], [2, 5, 8], [2, 6, 9]]
5567 >>> df = pd.DataFrame(
5568 ... data,
5569 ... columns=["a", "b", "c"],
5570 ... index=["tuna", "salmon", "catfish", "goldfish"],
5571 ... )
5572 >>> df
5573 a b c
5574 tuna 1 2 3
5575 salmon 1 5 6
5576 catfish 2 5 8
5577 goldfish 2 6 9
5578 >>> df.groupby("a").shift(1)
5579 b c
5580 tuna NaN NaN
5581 salmon 2.0 3.0
5582 catfish NaN NaN
5583 goldfish 5.0 8.0
5584 """
5585 if is_list_like(periods):
5586 periods = cast(Sequence, periods)
5587 if len(periods) == 0:
5588 raise ValueError("If `periods` is an iterable, it cannot be empty.")
5589 from pandas.core.reshape.concat import concat
5591 add_suffix = True
5592 else:
5593 if not is_integer(periods):
5594 raise TypeError(
5595 f"Periods must be integer, but {periods} is {type(periods)}."
5596 )
5597 if suffix:
5598 raise ValueError("Cannot specify `suffix` if `periods` is an int.")
5599 periods = [cast(int, periods)]
5600 add_suffix = False
5602 shifted_dataframes = []
5603 for period in periods:
5604 if not is_integer(period):
5605 raise TypeError(
5606 f"Periods must be integer, but {period} is {type(period)}."
5607 )
5608 period = cast(int, period)
5609 if freq is not None:
5610 f = lambda x: x.shift(
5611 period,
5612 freq,
5613 0, # axis
5614 fill_value,
5615 )
5616 shifted = self._python_apply_general(
5617 f, self._selected_obj, is_transform=True
5618 )
5619 else:
5620 if fill_value is lib.no_default:
5621 fill_value = None
5622 ids = self._grouper.ids
5623 ngroups = self._grouper.ngroups
5624 res_indexer = np.zeros(len(ids), dtype=np.int64)
5626 libgroupby.group_shift_indexer(res_indexer, ids, ngroups, period)
5628 obj = self._obj_with_exclusions
5630 shifted = obj._reindex_with_indexers(
5631 {0: (obj.index, res_indexer)},
5632 fill_value=fill_value,
5633 allow_dups=True,
5634 )
5636 if add_suffix:
5637 if isinstance(shifted, Series):
5638 shifted = cast(NDFrameT, shifted.to_frame())
5639 shifted = shifted.add_suffix(
5640 f"{suffix}_{period}" if suffix else f"_{period}"
5641 )
5642 shifted_dataframes.append(cast(Union[Series, DataFrame], shifted))
5644 return (
5645 shifted_dataframes[0]
5646 if len(shifted_dataframes) == 1
5647 else concat(shifted_dataframes, axis=1, sort=False)
5648 )
5650 @final
5651 def diff(
5652 self,
5653 periods: int = 1,
5654 ) -> NDFrameT:
5655 """
5656 First discrete difference of element.
5658 Calculates the difference of each element compared with another
5659 element in the group (default is element in previous row).
5661 Parameters
5662 ----------
5663 periods : int, default 1
5664 Periods to shift for calculating difference, accepts negative values.
5666 Returns
5667 -------
5668 Series or DataFrame
5669 First differences.
5671 See Also
5672 --------
5673 Series.diff : Apply function diff to a Series.
5674 DataFrame.diff : Apply function diff to each row or column of a DataFrame.
5676 Examples
5677 --------
5678 For SeriesGroupBy:
5680 >>> lst = ["a", "a", "a", "b", "b", "b"]
5681 >>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
5682 >>> ser
5683 a 7
5684 a 2
5685 a 8
5686 b 4
5687 b 3
5688 b 3
5689 dtype: int64
5690 >>> ser.groupby(level=0).diff()
5691 a NaN
5692 a -5.0
5693 a 6.0
5694 b NaN
5695 b -1.0
5696 b 0.0
5697 dtype: float64
5699 For DataFrameGroupBy:
5701 >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
5702 >>> df = pd.DataFrame(
5703 ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
5704 ... )
5705 >>> df
5706 a b
5707 dog 1 1
5708 dog 3 4
5709 dog 5 8
5710 mouse 7 4
5711 mouse 7 4
5712 mouse 8 2
5713 mouse 3 1
5714 >>> df.groupby(level=0).diff()
5715 a b
5716 dog NaN NaN
5717 dog 2.0 3.0
5718 dog 2.0 4.0
5719 mouse NaN NaN
5720 mouse 0.0 0.0
5721 mouse 1.0 -2.0
5722 mouse -5.0 -1.0
5723 """
5724 obj = self._obj_with_exclusions
5725 shifted = self.shift(periods=periods)
5727 # GH45562 - to retain existing behavior and match behavior of Series.diff(),
5728 # int8 and int16 are coerced to float32 rather than float64.
5729 dtypes_to_f32 = ["int8", "int16"]
5730 if obj.ndim == 1:
5731 if obj.dtype in dtypes_to_f32:
5732 shifted = shifted.astype("float32")
5733 else:
5734 to_coerce = [c for c, dtype in obj.dtypes.items() if dtype in dtypes_to_f32]
5735 if to_coerce:
5736 shifted = shifted.astype(dict.fromkeys(to_coerce, "float32"))
5738 return obj - shifted
5740 @final
5741 def pct_change(
5742 self,
5743 periods: int = 1,
5744 fill_method: None = None,
5745 freq=None,
5746 ):
5747 """
5748 Calculate pct_change of each value to previous entry in group.
5750 Parameters
5751 ----------
5752 periods : int, default 1
5753 Periods to shift for calculating percentage change. Comparing with
5754 a period of 1 means adjacent elements are compared, whereas a period
5755 of 2 compares every other element.
5757 fill_method : None
5758 Must be None. This argument will be removed in a future version of pandas.
5760 freq : str, pandas offset object, or None, default None
5761 The frequency increment for time series data (e.g., 'M' for month-end).
5762 If None, the frequency is inferred from the index. Relevant for time
5763 series data only.
5765 Returns
5766 -------
5767 Series or DataFrame
5768 Percentage changes within each group.
5770 See Also
5771 --------
5772 Series.pct_change : Apply function pct_change to a Series.
5773 DataFrame.pct_change : Apply function pct_change to each row or column of
5774 a DataFrame.
5776 Examples
5777 --------
5779 For SeriesGroupBy:
5781 >>> lst = ["a", "a", "b", "b"]
5782 >>> ser = pd.Series([1, 2, 3, 4], index=lst)
5783 >>> ser
5784 a 1
5785 a 2
5786 b 3
5787 b 4
5788 dtype: int64
5789 >>> ser.groupby(level=0).pct_change()
5790 a NaN
5791 a 1.000000
5792 b NaN
5793 b 0.333333
5794 dtype: float64
5796 For DataFrameGroupBy:
5798 >>> data = [[1, 2, 3], [1, 5, 6], [2, 5, 8], [2, 6, 9]]
5799 >>> df = pd.DataFrame(
5800 ... data,
5801 ... columns=["a", "b", "c"],
5802 ... index=["tuna", "salmon", "catfish", "goldfish"],
5803 ... )
5804 >>> df
5805 a b c
5806 tuna 1 2 3
5807 salmon 1 5 6
5808 catfish 2 5 8
5809 goldfish 2 6 9
5810 >>> df.groupby("a").pct_change()
5811 b c
5812 tuna NaN NaN
5813 salmon 1.5 1.000
5814 catfish NaN NaN
5815 goldfish 0.2 0.125
5816 """
5817 # GH#53491
5818 if fill_method is not None:
5819 raise ValueError(f"fill_method must be None; got {fill_method=}.")
5821 # TODO(GH#23918): Remove this conditional for SeriesGroupBy when
5822 # GH#23918 is fixed
5823 if freq is not None:
5824 f = lambda x: x.pct_change(
5825 periods=periods,
5826 freq=freq,
5827 axis=0,
5828 )
5829 return self._python_apply_general(f, self._selected_obj, is_transform=True)
5831 if fill_method is None: # GH30463
5832 op = "ffill"
5833 else:
5834 op = fill_method
5835 filled = getattr(self, op)(limit=0)
5836 fill_grp = filled.groupby(self._grouper.codes, group_keys=self.group_keys)
5837 shifted = fill_grp.shift(periods=periods, freq=freq)
5838 return (filled / shifted) - 1
5840 @final
5841 def head(self, n: int = 5) -> NDFrameT:
5842 """
5843 Return first n rows of each group.
5845 Similar to ``.apply(lambda x: x.head(n))``, but it returns a subset of rows
5846 from the original DataFrame with original index and order preserved
5847 (``as_index`` flag is ignored).
5849 Parameters
5850 ----------
5851 n : int
5852 If positive: number of entries to include from start of each group.
5853 If negative: number of entries to exclude from end of each group.
5855 Returns
5856 -------
5857 Series or DataFrame
5858 Subset of original Series or DataFrame as determined by n.
5860 See Also
5861 --------
5862 Series.head : Apply function head to a Series.
5863 DataFrame.head : Apply function head to each row or column of a DataFrame.
5865 Examples
5866 --------
5868 >>> df = pd.DataFrame([[1, 2], [1, 4], [5, 6]], columns=["A", "B"])
5869 >>> df.groupby("A").head(1)
5870 A B
5871 0 1 2
5872 2 5 6
5873 >>> df.groupby("A").head(-1)
5874 A B
5875 0 1 2
5876 """
5877 mask = self._make_mask_from_positional_indexer(slice(None, n))
5878 return self._mask_selected_obj(mask)
5880 @final
5881 def tail(self, n: int = 5) -> NDFrameT:
5882 """
5883 Return last n rows of each group.
5885 Similar to ``.apply(lambda x: x.tail(n))``, but it returns a subset of rows
5886 from the original DataFrame with original index and order preserved
5887 (``as_index`` flag is ignored).
5889 Parameters
5890 ----------
5891 n : int
5892 If positive: number of entries to include from end of each group.
5893 If negative: number of entries to exclude from start of each group.
5895 Returns
5896 -------
5897 Series or DataFrame
5898 Subset of original Series or DataFrame as determined by n.
5900 See Also
5901 --------
5902 Series.tail : Apply function tail to a Series.
5903 DataFrame.tail : Apply function tail to each row or column of a DataFrame.
5905 Examples
5906 --------
5908 >>> df = pd.DataFrame(
5909 ... [["a", 1], ["a", 2], ["b", 1], ["b", 2]], columns=["A", "B"]
5910 ... )
5911 >>> df.groupby("A").tail(1)
5912 A B
5913 1 a 2
5914 3 b 2
5915 >>> df.groupby("A").tail(-1)
5916 A B
5917 1 a 2
5918 3 b 2
5919 """
5920 if n:
5921 mask = self._make_mask_from_positional_indexer(slice(-n, None))
5922 else:
5923 mask = self._make_mask_from_positional_indexer([])
5925 return self._mask_selected_obj(mask)
5927 @final
5928 def _mask_selected_obj(self, mask: npt.NDArray[np.bool_]) -> NDFrameT:
5929 """
5930 Return _selected_obj with mask applied.
5932 Parameters
5933 ----------
5934 mask : np.ndarray[bool]
5935 Boolean mask to apply.
5937 Returns
5938 -------
5939 Series or DataFrame
5940 Filtered _selected_obj.
5941 """
5942 ids = self._grouper.ids
5943 mask = mask & (ids != -1)
5944 return self._selected_obj[mask]
5946 @final
5947 def sample(
5948 self,
5949 n: int | None = None,
5950 frac: float | None = None,
5951 replace: bool = False,
5952 weights: Sequence | Series | None = None,
5953 random_state: RandomState | None = None,
5954 ):
5955 """
5956 Return a random sample of items from each group.
5958 You can use `random_state` for reproducibility.
5960 Parameters
5961 ----------
5962 n : int, optional
5963 Number of items to return for each group. Cannot be used with
5964 `frac` and must be no larger than the smallest group unless
5965 `replace` is True. Default is one if `frac` is None.
5966 frac : float, optional
5967 Fraction of items to return. Cannot be used with `n`.
5968 replace : bool, default False
5969 Allow or disallow sampling of the same row more than once.
5970 weights : list-like, optional
5971 Default None results in equal probability weighting.
5972 If passed a list-like then values must have the same length as
5973 the underlying DataFrame or Series object and will be used as
5974 sampling probabilities after normalization within each group.
5975 Values must be non-negative with at least one positive element
5976 within each group.
5977 random_state : int, array-like, BitGenerator, np.random.RandomState, np.random.Generator, optional
5978 If int, array-like, or BitGenerator, seed for random number generator.
5979 If np.random.RandomState or np.random.Generator, use as given.
5980 Default ``None`` results in sampling with the current state of np.random.
5982 Returns
5983 -------
5984 Series or DataFrame
5985 A new object of same type as caller containing items randomly
5986 sampled within each group from the caller object.
5988 See Also
5989 --------
5990 DataFrame.sample: Generate random samples from a DataFrame object.
5991 Series.sample: Generate random samples from a Series object.
5992 numpy.random.choice: Generate a random sample from a given 1-D numpy
5993 array.
5995 Examples
5996 --------
5997 >>> df = pd.DataFrame(
5998 ... {"a": ["red"] * 2 + ["blue"] * 2 + ["black"] * 2, "b": range(6)}
5999 ... )
6000 >>> df
6001 a b
6002 0 red 0
6003 1 red 1
6004 2 blue 2
6005 3 blue 3
6006 4 black 4
6007 5 black 5
6009 Select one row at random for each distinct value in column a. The
6010 `random_state` argument can be used to guarantee reproducibility:
6012 >>> df.groupby("a").sample(n=1, random_state=1)
6013 a b
6014 4 black 4
6015 2 blue 2
6016 1 red 1
6018 Set `frac` to sample fixed proportions rather than counts:
6020 >>> df.groupby("a")["b"].sample(frac=0.5, random_state=2)
6021 5 5
6022 2 2
6023 0 0
6024 Name: b, dtype: int64
6026 Control sample probabilities within groups by setting weights:
6028 >>> df.groupby("a").sample(
6029 ... n=1,
6030 ... weights=[1, 1, 1, 0, 0, 1],
6031 ... random_state=1,
6032 ... )
6033 a b
6034 5 black 5
6035 2 blue 2
6036 0 red 0
6037 """ # noqa: E501
6038 if self._selected_obj.empty:
6039 # GH48459 prevent ValueError when object is empty
6040 return self._selected_obj
6041 size = sample.process_sampling_size(n, frac, replace)
6042 if weights is not None:
6043 weights_arr = sample.preprocess_weights(self._selected_obj, weights, axis=0)
6045 random_state = com.random_state(random_state)
6047 group_iterator = self._grouper.get_iterator(self._selected_obj)
6049 sampled_indices = []
6050 for labels, obj in group_iterator:
6051 grp_indices = self.indices[labels]
6052 group_size = len(grp_indices)
6053 if size is not None:
6054 sample_size = size
6055 else:
6056 assert frac is not None
6057 sample_size = round(frac * group_size)
6059 grp_sample = sample.sample(
6060 group_size,
6061 size=sample_size,
6062 replace=replace,
6063 weights=None if weights is None else weights_arr[grp_indices],
6064 random_state=random_state,
6065 )
6066 sampled_indices.append(grp_indices[grp_sample])
6068 concatenated_sampled_indices = np.concatenate(sampled_indices)
6069 return self._selected_obj.take(concatenated_sampled_indices, axis=0)
6071 def _idxmax_idxmin(
6072 self,
6073 how: Literal["idxmax", "idxmin"],
6074 ignore_unobserved: bool = False,
6075 skipna: bool = True,
6076 numeric_only: bool = False,
6077 ) -> NDFrameT:
6078 """Compute idxmax/idxmin.
6080 Parameters
6081 ----------
6082 how : {'idxmin', 'idxmax'}
6083 Whether to compute idxmin or idxmax.
6084 numeric_only : bool, default False
6085 Include only float, int, boolean columns.
6086 skipna : bool, default True
6087 Exclude NA/null values. If an entire group is NA, the result will be NA.
6088 ignore_unobserved : bool, default False
6089 When True and an unobserved group is encountered, do not raise. This used
6090 for transform where unobserved groups do not play an impact on the result.
6092 Returns
6093 -------
6094 Series or DataFrame
6095 idxmax or idxmin for the groupby operation.
6096 """
6097 if not self.observed and any(
6098 ping._passed_categorical for ping in self._grouper.groupings
6099 ):
6100 expected_len = len(self._grouper.result_index)
6101 # TODO: Better way to find # of observed groups?
6102 group_sizes = self._grouper.size()
6103 result_len = group_sizes[group_sizes > 0].shape[0]
6104 assert result_len <= expected_len
6105 has_unobserved = result_len < expected_len
6107 raise_err: bool | np.bool_ = not ignore_unobserved and has_unobserved
6108 # Only raise an error if there are columns to compute; otherwise we return
6109 # an empty DataFrame with an index (possibly including unobserved) but no
6110 # columns
6111 data = self._obj_with_exclusions
6112 if raise_err and isinstance(data, DataFrame):
6113 if numeric_only:
6114 data = data._get_numeric_data()
6115 raise_err = len(data.columns) > 0
6117 if raise_err:
6118 raise ValueError(
6119 f"Can't get {how} of an empty group due to unobserved categories. "
6120 "Specify observed=True in groupby instead."
6121 )
6122 elif not skipna and self._obj_with_exclusions.isna().any(axis=None):
6123 raise ValueError(f"{how} with skipna=False encountered an NA value.")
6125 result = self._agg_general(
6126 numeric_only=numeric_only,
6127 min_count=1,
6128 alias=how,
6129 skipna=skipna,
6130 )
6131 return result
6133 def _wrap_idxmax_idxmin(
6134 self, res: NDFrameT, how: Literal["idxmax", "idxmin"], skipna: bool
6135 ) -> NDFrameT:
6136 index = self.obj.index
6137 if res.size == 0:
6138 result = res.astype(index.dtype)
6139 elif skipna and res.lt(0).any(axis=None):
6140 raise ValueError(
6141 f"{how} with skipna=True encountered all NA values in a group."
6142 )
6143 else:
6144 if isinstance(index, MultiIndex):
6145 index = index.to_flat_index()
6146 values = res._values
6147 assert isinstance(values, np.ndarray)
6148 na_value = na_value_for_dtype(index.dtype, compat=False)
6149 if isinstance(res, Series):
6150 # mypy: expression has type "Series", variable has type "NDFrameT"
6151 result = res._constructor( # type: ignore[assignment]
6152 index.array.take(values, allow_fill=True, fill_value=na_value),
6153 index=res.index,
6154 name=res.name,
6155 )
6156 else:
6157 data = {}
6158 for k, column_values in enumerate(values.T):
6159 data[k] = index.array.take(
6160 column_values, allow_fill=True, fill_value=na_value
6161 )
6162 result = self.obj._constructor(data, index=res.index)
6163 result.columns = res.columns
6164 return result
6167def get_groupby(
6168 obj: NDFrame,
6169 by: _KeysArgType | None = None,
6170 grouper: ops.BaseGrouper | None = None,
6171 group_keys: bool = True,
6172) -> GroupBy:
6173 """
6174 Class for grouping and aggregating relational data.
6176 See aggregate, transform, and apply functions on this object.
6178 It's easiest to use obj.groupby(...) to use GroupBy, but you can also do:
6180 ::
6182 grouped = groupby(obj, ...)
6184 Parameters
6185 ----------
6186 obj : pandas object
6187 level : int, default None
6188 Level of MultiIndex
6189 groupings : list of Grouping objects
6190 Most users should ignore this
6191 exclusions : array-like, optional
6192 List of columns to exclude
6193 name : str
6194 Most users should ignore this
6196 Returns
6197 -------
6198 **Attributes**
6199 groups : dict
6200 {group name -> group labels}
6201 len(grouped) : int
6202 Number of groups
6204 Notes
6205 -----
6206 After grouping, see aggregate, apply, and transform functions. Here are
6207 some other brief notes about usage. When grouping by multiple groups, the
6208 result index will be a MultiIndex (hierarchical) by default.
6210 Iteration produces (key, group) tuples, i.e. chunking the data by group. So
6211 you can write code like:
6213 ::
6215 grouped = obj.groupby(keys)
6216 for key, group in grouped:
6217 # do something with the data
6219 Function calls on GroupBy, if not specially implemented, "dispatch" to the
6220 grouped data. So if you group a DataFrame and wish to invoke the std()
6221 method on each group, you can simply do:
6223 ::
6225 df.groupby(mapper).std()
6227 rather than
6229 ::
6231 df.groupby(mapper).aggregate(np.std)
6233 You can pass arguments to these "wrapped" functions, too.
6235 See the online documentation for full exposition on these topics and much
6236 more
6237 """
6238 if isinstance(obj, Series):
6239 from pandas.core.groupby.generic import SeriesGroupBy
6241 return SeriesGroupBy(
6242 obj=obj,
6243 keys=by,
6244 grouper=grouper,
6245 group_keys=group_keys,
6246 )
6247 elif isinstance(obj, DataFrame):
6248 from pandas.core.groupby.generic import DataFrameGroupBy
6250 return DataFrameGroupBy(
6251 obj=obj,
6252 keys=by,
6253 grouper=grouper,
6254 group_keys=group_keys,
6255 )
6256 else: # pragma: no cover
6257 raise TypeError(f"invalid type: {obj}")
6260def _insert_quantile_level(idx: Index, qs: npt.NDArray[np.float64]) -> MultiIndex:
6261 """
6262 Insert the sequence 'qs' of quantiles as the inner-most level of a MultiIndex.
6264 The quantile level in the MultiIndex is a repeated copy of 'qs'.
6266 Parameters
6267 ----------
6268 idx : Index
6269 qs : np.ndarray[float64]
6271 Returns
6272 -------
6273 MultiIndex
6274 """
6275 nqs = len(qs)
6276 lev_codes, lev = Index(qs, copy=False).factorize()
6277 lev_codes = coerce_indexer_dtype(lev_codes, lev)
6279 if idx._is_multi:
6280 idx = cast(MultiIndex, idx)
6281 levels = [*idx.levels, lev]
6282 codes = [np.repeat(x, nqs) for x in idx.codes] + [np.tile(lev_codes, len(idx))]
6283 mi = MultiIndex(levels=levels, codes=codes, names=[*idx.names, None])
6284 else:
6285 nidx = len(idx)
6286 idx_codes = coerce_indexer_dtype(np.arange(nidx), idx)
6287 levels = [idx, lev]
6288 codes = [np.repeat(idx_codes, nqs), np.tile(lev_codes, nidx)]
6289 mi = MultiIndex(levels=levels, codes=codes, names=[idx.name, None])
6291 return mi