Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Define the SeriesGroupBy and DataFrameGroupBy
3classes that hold the groupby interfaces (and some implementations).
5These are user facing as the result of the ``df.groupby(...)`` operations,
6which here returns a DataFrameGroupBy object.
7"""
9from __future__ import annotations
11from collections import abc
12from collections.abc import Callable
13import dataclasses
14from functools import partial
15from typing import (
16 TYPE_CHECKING,
17 Any,
18 Literal,
19 TypeAlias,
20 TypeVar,
21 cast,
22)
23import warnings
25import numpy as np
27from pandas._libs.hashtable import duplicated
28from pandas.errors import (
29 Pandas4Warning,
30 SpecificationError,
31)
32from pandas.util._decorators import set_module
33from pandas.util._exceptions import find_stack_level
35from pandas.core.dtypes.common import (
36 ensure_int64,
37 is_bool,
38 is_dict_like,
39 is_integer_dtype,
40 is_list_like,
41 is_numeric_dtype,
42 is_scalar,
43)
44from pandas.core.dtypes.dtypes import (
45 CategoricalDtype,
46 IntervalDtype,
47)
48from pandas.core.dtypes.inference import is_hashable
49from pandas.core.dtypes.missing import (
50 isna,
51 notna,
52)
54from pandas.core import algorithms
55from pandas.core.apply import (
56 GroupByApply,
57 maybe_mangle_lambdas,
58 reconstruct_func,
59 validate_func_kwargs,
60)
61import pandas.core.common as com
62from pandas.core.frame import DataFrame
63from pandas.core.groupby import base
64from pandas.core.groupby.groupby import (
65 GroupBy,
66 GroupByPlot,
67)
68from pandas.core.indexes.api import (
69 Index,
70 MultiIndex,
71 all_indexes_same,
72 default_index,
73)
74from pandas.core.series import Series
75from pandas.core.sorting import get_group_index
76from pandas.core.util.numba_ import maybe_use_numba
78from pandas.plotting import boxplot_frame_groupby
80if TYPE_CHECKING:
81 from collections.abc import (
82 Hashable,
83 Sequence,
84 )
86 from pandas._libs import Interval
87 from pandas._typing import (
88 ArrayLike,
89 BlockManager,
90 CorrelationMethod,
91 IndexLabel,
92 Manager,
93 SingleBlockManager,
94 TakeIndexer,
95 )
97 from pandas import Categorical
99# TODO(typing) the return value on this callable should be any *scalar*.
100AggScalar: TypeAlias = str | Callable[..., Any]
101# TODO: validate types on ScalarResult and move to _typing
102# Blocked from using by https://github.com/python/mypy/issues/1484
103# See note at _mangle_lambda_list
104ScalarResult = TypeVar("ScalarResult")
107@set_module("pandas")
108@dataclasses.dataclass
109class NamedAgg:
110 """
111 Helper for column specific aggregation with control over output column names.
113 A dataclass used with :meth:`DataFrame.groupby().agg()
114 <pandas.core.groupby.DataFrameGroupBy.aggregate>` to specify an
115 aggregation on a particular column and assign a custom name to the
116 resulting column. Additional positional and keyword arguments can be
117 forwarded to the aggregation function.
119 Parameters
120 ----------
121 column : Hashable
122 Column label in the DataFrame to apply aggfunc.
123 aggfunc : function or str
124 Function to apply to the provided column. If string, the name of a built-in
125 pandas function.
126 *args, **kwargs : Any
127 Optional positional and keyword arguments passed to ``aggfunc``.
129 See Also
130 --------
131 DataFrame.groupby : Group DataFrame using a mapper or by a Series of columns.
133 Examples
134 --------
135 >>> df = pd.DataFrame({"key": [1, 1, 2], "a": [-1, 0, 1], 1: [10, 11, 12]})
136 >>> agg_a = pd.NamedAgg(column="a", aggfunc="min")
137 >>> agg_1 = pd.NamedAgg(column=1, aggfunc=lambda x: np.mean(x))
138 >>> df.groupby("key").agg(result_a=agg_a, result_1=agg_1)
139 result_a result_1
140 key
141 1 -1 10.5
142 2 1 12.0
144 >>> def n_between(ser, low, high, **kwargs):
145 ... return ser.between(low, high, **kwargs).sum()
147 >>> agg_between = pd.NamedAgg("a", n_between, 0, 1)
148 >>> df.groupby("key").agg(count_between=agg_between)
149 count_between
150 key
151 1 1
152 2 1
154 >>> agg_between_kw = pd.NamedAgg("a", n_between, 0, 1, inclusive="both")
155 >>> df.groupby("key").agg(count_between_kw=agg_between_kw)
156 count_between_kw
157 key
158 1 1
159 2 1
160 """
162 column: Hashable
163 aggfunc: AggScalar
164 args: tuple[Any, ...] = ()
165 kwargs: dict[str, Any] = dataclasses.field(default_factory=dict)
167 def __init__(
168 self,
169 column: Hashable,
170 aggfunc: Callable[..., Any] | str,
171 *args: Any,
172 **kwargs: Any,
173 ) -> None:
174 self.column = column
175 self.aggfunc = aggfunc
176 self.args = args
177 self.kwargs = kwargs
179 def __getitem__(self, key: int) -> Any:
180 """Provide backward-compatible tuple-style access."""
181 if key == 0:
182 return self.column
183 elif key == 1:
184 return self.aggfunc
185 elif key == 2:
186 return self.args
187 elif key == 3:
188 return self.kwargs
189 raise IndexError("index out of range")
192@set_module("pandas.api.typing")
193class SeriesGroupBy(GroupBy[Series]):
194 def _wrap_agged_manager(self, mgr: Manager) -> Series:
195 out = self.obj._constructor_from_mgr(mgr, axes=mgr.axes)
196 out._name = self.obj.name
197 return out
199 def _get_data_to_aggregate(
200 self, *, numeric_only: bool = False, name: str | None = None
201 ) -> SingleBlockManager:
202 ser = self._obj_with_exclusions
203 single = ser._mgr
204 if numeric_only and not is_numeric_dtype(ser.dtype):
205 # GH#41291 match Series behavior
206 kwd_name = "numeric_only"
207 raise TypeError(
208 f"Cannot use {kwd_name}=True with "
209 f"{type(self).__name__}.{name} and non-numeric dtypes."
210 )
211 return single
213 def apply(self, func, *args, **kwargs) -> Series:
214 """
215 Apply function ``func`` group-wise and combine the results together.
217 The function passed to ``apply`` must take a series as its first
218 argument and return a DataFrame, Series or scalar. ``apply`` will
219 then take care of combining the results back together into a single
220 dataframe or series. ``apply`` is therefore a highly flexible
221 grouping method.
223 While ``apply`` is a very flexible method, its downside is that
224 using it can be quite a bit slower than using more specific methods
225 like ``agg`` or ``transform``. Pandas offers a wide range of method that will
226 be much faster than using ``apply`` for their specific purposes, so try to
227 use them before reaching for ``apply``.
229 Parameters
230 ----------
231 func : callable
232 A callable that takes a series as its first argument, and
233 returns a dataframe, a series or a scalar. In addition the
234 callable may take positional and keyword arguments.
236 *args : tuple
237 Optional positional arguments to pass to ``func``.
239 **kwargs : dict
240 Optional keyword arguments to pass to ``func``.
242 Returns
243 -------
244 Series or DataFrame
245 A pandas object with the result of applying ``func`` to each group.
247 See Also
248 --------
249 pipe : Apply function to the full GroupBy object instead of to each
250 group.
251 aggregate : Apply aggregate function to the GroupBy object.
252 transform : Apply function column-by-column to the GroupBy object.
253 Series.apply : Apply a function to a Series.
254 DataFrame.apply : Apply a function to each row or column of a DataFrame.
256 Notes
257 -----
258 See :ref:`groupby.apply` in the User Guide for more details and examples.
260 The resulting dtype will reflect the return value of the passed ``func``,
261 see the examples below.
263 Functions that mutate the passed object can produce unexpected
264 behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
265 for more details.
267 Each group passed to ``func`` has a ``name`` attribute set to the group
268 key (the value of the grouping for that group). This is useful for
269 identifying the current group, as in the examples below.
271 Examples
272 --------
273 >>> s = pd.Series([0, 1, 2], index="a a b".split())
274 >>> g1 = s.groupby(s.index, group_keys=False)
275 >>> g2 = s.groupby(s.index, group_keys=True)
277 From ``s`` above we can see that ``g`` has two groups, ``a`` and ``b``.
278 Notice that ``g1`` have ``g2`` have two groups, ``a`` and ``b``, and only
279 differ in their ``group_keys`` argument. Calling `apply` in various ways,
280 we can get different grouping results:
282 Example 1: The function passed to `apply` takes a Series as
283 its argument and returns a Series. `apply` combines the result for
284 each group together into a new Series.
286 The resulting dtype will reflect the return value of the passed ``func``.
288 >>> g1.apply(lambda x: x * 2 if x.name == "a" else x / 2)
289 a 0.0
290 a 2.0
291 b 1.0
292 dtype: float64
294 In the above, the groups are not part of the index. We can have them included
295 by using ``g2`` where ``group_keys=True``:
297 >>> g2.apply(lambda x: x * 2 if x.name == "a" else x / 2)
298 a a 0.0
299 a 2.0
300 b b 1.0
301 dtype: float64
303 Example 2: The function passed to `apply` takes a Series as
304 its argument and returns a scalar. `apply` combines the result for
305 each group together into a Series, including setting the index as
306 appropriate:
308 >>> g1.apply(lambda x: x.max() - x.min())
309 a 1
310 b 0
311 dtype: int64
313 The ``group_keys`` argument has no effect here because the result is not
314 like-indexed (i.e. :ref:`a transform <groupby.transform>`) when compared
315 to the input.
317 >>> g2.apply(lambda x: x.max() - x.min())
318 a 1
319 b 0
320 dtype: int64
321 """
322 return super().apply(func, *args, **kwargs)
324 def aggregate(
325 self, func=None, *args, engine=None, engine_kwargs=None, **kwargs
326 ) -> Series | DataFrame:
327 """
328 Aggregate using one or more operations.
330 The ``aggregate`` method enables flexible and efficient aggregation of grouped
331 data using a variety of functions, including built-in, user-defined, and
332 optimized JIT-compiled functions.
334 Parameters
335 ----------
336 func : function, str, list or None
337 Function to use for aggregating the data. If a function, must either
338 work when passed a Series or when passed to Series.apply.
340 Accepted combinations are:
342 - function
343 - string function name
344 - list of functions and/or function names, e.g. ``[np.sum, 'mean']``
345 - None, in which case ``**kwargs`` are used with Named Aggregation. Here
346 the output has one column for each element in ``**kwargs``. The name of
347 the column is keyword, whereas the value determines the aggregation
348 used to compute the values in the column.
350 Can also accept a Numba JIT function with
351 ``engine='numba'`` specified. Only passing a single function is supported
352 with this engine.
354 If the ``'numba'`` engine is chosen, the function must be
355 a user defined function with ``values`` and ``index`` as the
356 first and second arguments respectively in the function signature.
357 Each group's index will be passed to the user defined function
358 and optionally available for use.
360 *args
361 Positional arguments to pass to func.
362 engine : str, default None
363 * ``'cython'`` : Runs the function through C-extensions from cython.
364 * ``'numba'`` : Runs the function through JIT compiled code from numba.
365 * ``None`` : Defaults to ``'cython'`` or globally setting
366 ``compute.use_numba``
368 engine_kwargs : dict, default None
369 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
370 * For ``'numba'`` engine, the engine can accept ``nogil``
371 and ``parallel`` dictionary keys. The values must either be ``True`` or
372 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
373 ``{'nogil': False, 'parallel': False}`` and will be
374 applied to the function
376 **kwargs
377 * If ``func`` is None, ``**kwargs`` are used to define the output names and
378 aggregations via Named Aggregation. See ``func`` entry.
379 * Otherwise, keyword arguments to be passed into func.
381 Returns
382 -------
383 Series
384 Aggregated Series based on the grouping and the applied aggregation
385 functions.
387 See Also
388 --------
389 SeriesGroupBy.apply : Apply function func group-wise
390 and combine the results together.
391 SeriesGroupBy.transform : Transforms the Series on each group
392 based on the given function.
393 Series.aggregate : Aggregate using one or more operations.
395 Notes
396 -----
397 See :ref:`groupby.aggregate` in the User Guide for more details
398 and examples.
400 When using ``engine='numba'``, there will be no "fall back" behavior internally.
401 The group data and group index will be passed as numpy arrays to the JITed
402 user defined function, and no alternative execution attempts will be tried.
404 Functions that mutate the passed object can produce unexpected
405 behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
406 for more details.
408 The resulting dtype will reflect the return value of the passed ``func``,
409 see the examples below.
411 Examples
412 --------
413 >>> s = pd.Series([1, 2, 3, 4])
415 >>> s
416 0 1
417 1 2
418 2 3
419 3 4
420 dtype: int64
422 >>> s.groupby([1, 1, 2, 2]).min()
423 1 1
424 2 3
425 dtype: int64
427 >>> s.groupby([1, 1, 2, 2]).agg("min")
428 1 1
429 2 3
430 dtype: int64
432 >>> s.groupby([1, 1, 2, 2]).agg(["min", "max"])
433 min max
434 1 1 2
435 2 3 4
437 The output column names can be controlled by passing
438 the desired column names and aggregations as keyword arguments.
440 >>> s.groupby([1, 1, 2, 2]).agg(
441 ... minimum="min",
442 ... maximum="max",
443 ... )
444 minimum maximum
445 1 1 2
446 2 3 4
448 The resulting dtype will reflect the return value of the aggregating
449 function.
451 >>> s.groupby([1, 1, 2, 2]).agg(lambda x: x.astype(float).min())
452 1 1.0
453 2 3.0
454 dtype: float64
455 """
456 relabeling = func is None
457 columns = None
458 if relabeling:
459 columns, func = validate_func_kwargs(kwargs)
460 kwargs = {}
462 if isinstance(func, str):
463 if maybe_use_numba(engine) and engine is not None:
464 # Not all agg functions support numba, only propagate numba kwargs
465 # if user asks for numba, and engine is not None
466 # (if engine is None, the called function will handle the case where
467 # numba is requested via the global option)
468 kwargs["engine"] = engine
469 if engine_kwargs is not None:
470 kwargs["engine_kwargs"] = engine_kwargs
471 return getattr(self, func)(*args, **kwargs)
473 elif isinstance(func, abc.Iterable):
474 # Catch instances of lists / tuples
475 # but not the class list / tuple itself.
476 func = maybe_mangle_lambdas(func)
477 kwargs["engine"] = engine
478 kwargs["engine_kwargs"] = engine_kwargs
479 ret = self._aggregate_multiple_funcs(func, *args, **kwargs)
480 if relabeling:
481 # columns is not narrowed by mypy from relabeling flag
482 assert columns is not None # for mypy
483 ret.columns = columns
484 if not self.as_index:
485 ret = ret.reset_index()
486 return ret
488 else:
489 if maybe_use_numba(engine):
490 return self._aggregate_with_numba(
491 func, *args, engine_kwargs=engine_kwargs, **kwargs
492 )
494 if self.ngroups == 0:
495 # e.g. test_evaluate_with_empty_groups without any groups to
496 # iterate over, we have no output on which to do dtype
497 # inference. We default to using the existing dtype.
498 # xref GH#51445
499 obj = self._obj_with_exclusions
500 return self._wrap_aggregated_output(
501 self.obj._constructor(
502 [],
503 name=self.obj.name,
504 index=self._grouper.result_index,
505 dtype=obj.dtype,
506 )
507 )
508 return self._python_agg_general(func, *args, **kwargs)
510 agg = aggregate
512 def _python_agg_general(self, func, *args, **kwargs):
513 f = lambda x: func(x, *args, **kwargs)
515 obj = self._obj_with_exclusions
516 result = self._grouper.agg_series(obj, f)
517 res = obj._constructor(result, name=obj.name)
518 return self._wrap_aggregated_output(res)
520 def _aggregate_multiple_funcs(self, arg, *args, **kwargs) -> DataFrame:
521 if isinstance(arg, dict):
522 raise SpecificationError("nested renamer is not supported")
524 if any(isinstance(x, (tuple, list)) for x in arg):
525 arg = ((x, x) if not isinstance(x, (tuple, list)) else x for x in arg)
526 else:
527 # list of functions / function names
528 columns = (com.get_callable_name(f) or f for f in arg)
529 arg = zip(columns, arg, strict=True)
531 results: dict[base.OutputKey, DataFrame | Series] = {}
532 with com.temp_setattr(self, "as_index", True):
533 # Combine results using the index, need to adjust index after
534 # if as_index=False (GH#50724)
535 for idx, (name, func) in enumerate(arg):
536 key = base.OutputKey(label=name, position=idx)
537 results[key] = self.aggregate(func, *args, **kwargs)
539 if any(isinstance(x, DataFrame) for x in results.values()):
540 from pandas import concat
542 res_df = concat(
543 results.values(), axis=1, keys=[key.label for key in results]
544 )
545 return res_df
547 indexed_output = {key.position: val for key, val in results.items()}
548 # GH#39609 with no funcs, index by the groups
549 index = None if results else self._grouper.result_index
550 output = self.obj._constructor_expanddim(indexed_output, index=index)
551 output.columns = Index(key.label for key in results)
553 return output
555 def _wrap_applied_output(
556 self,
557 data: Series,
558 values: list[Any],
559 not_indexed_same: bool = False,
560 is_transform: bool = False,
561 ) -> DataFrame | Series:
562 """
563 Wrap the output of SeriesGroupBy.apply into the expected result.
565 Parameters
566 ----------
567 data : Series
568 Input data for groupby operation.
569 values : List[Any]
570 Applied output for each group.
571 not_indexed_same : bool, default False
572 Whether the applied outputs are not indexed the same as the group axes.
574 Returns
575 -------
576 DataFrame or Series
577 """
578 if len(values) == 0:
579 # GH #6265
580 if is_transform:
581 # GH#47787 see test_group_on_empty_multiindex
582 res_index = data.index
583 elif not self.group_keys:
584 res_index = None
585 else:
586 res_index = self._grouper.result_index
588 return self.obj._constructor(
589 [],
590 name=self.obj.name,
591 index=res_index,
592 dtype=data.dtype,
593 )
594 assert values is not None
596 if isinstance(values[0], dict):
597 # GH #823 #24880
598 index = self._grouper.result_index
599 res_df = self.obj._constructor_expanddim(values, index=index)
600 # if self.observed is False,
601 # keep all-NaN rows created while re-indexing
602 res_ser = res_df.stack()
603 res_ser.name = self.obj.name
604 return res_ser
605 elif isinstance(values[0], (Series, DataFrame)):
606 result = self._concat_objects(
607 values,
608 not_indexed_same=not_indexed_same,
609 is_transform=is_transform,
610 )
611 if isinstance(result, Series):
612 result.name = self.obj.name
613 return result.__finalize__(self.obj, method="groupby")
614 else:
615 # GH #6265 #24880
616 result = self.obj._constructor(
617 data=values, index=self._grouper.result_index, name=self.obj.name
618 )
619 if not self.as_index:
620 result = self._insert_inaxis_grouper(result)
621 result.index = default_index(len(result))
622 return result.__finalize__(self.obj, method="groupby")
624 def transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs):
625 """
626 Call function producing a same-indexed Series on each group.
628 Returns a Series having the same indexes as the original object
629 filled with the transformed values.
631 Parameters
632 ----------
633 func : function, str
634 Function to apply to each group.
635 See the Notes section below for requirements.
637 Accepted inputs are:
639 - String
640 - Python function
641 - Numba JIT function with ``engine='numba'`` specified.
643 Only passing a single function is supported with this engine.
644 If the ``'numba'`` engine is chosen, the function must be
645 a user defined function with ``values`` and ``index`` as the
646 first and second arguments respectively in the function signature.
647 Each group's index will be passed to the user defined function
648 and optionally available for use.
650 If a string is chosen, then it needs to be the name
651 of the groupby method you want to use.
652 *args
653 Positional arguments to pass to func.
654 engine : str, default None
655 * ``'cython'`` : Runs the function through C-extensions from cython.
656 * ``'numba'`` : Runs the function through JIT compiled code from numba.
657 * ``None`` : Defaults to ``'cython'``
658 or the global setting ``compute.use_numba``
660 engine_kwargs : dict, default None
661 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
662 * For ``'numba'`` engine, the engine can accept ``nogil``
663 and ``parallel`` dictionary keys. The values must either be ``True`` or
664 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
665 ``{'nogil': False, 'parallel': False}`` and will be
666 applied to the function
668 **kwargs
669 Keyword arguments to be passed into func.
671 Returns
672 -------
673 Series
674 Series with the same indexes as the original object filled
675 with transformed values.
677 See Also
678 --------
679 Series.groupby.apply : Apply function ``func`` group-wise and combine
680 the results together.
681 Series.groupby.aggregate : Aggregate using one or more operations.
682 Series.transform : Call ``func`` on self producing a Series with the
683 same axis shape as self.
685 Notes
686 -----
687 See :ref:`groupby.transform` in the User Guide for more details and examples.
689 Each group is endowed the attribute 'name' in case you need to know
690 which group you are working on.
692 ``func`` is passed each group as a whole :class:`Series`. The current
693 implementation imposes two requirements on ``func``:
695 * ``func`` must return a value that either has the same shape as the input
696 group or can be broadcast to the shape of the input group.
697 For example, if ``func`` returns a scalar it will be broadcast to have
698 the same shape as the input group.
699 * ``func`` must not mutate groups. Mutation is not supported and may
700 produce unexpected results. See :ref:`gotchas.udf-mutation` for more details.
702 When using ``engine='numba'``, there will be no "fall back" behavior internally.
703 The group data and group index will be passed as numpy arrays to the JITed
704 user defined function, and no alternative execution attempts will be tried.
706 The resulting dtype will reflect the return value of the passed ``func``,
707 see the examples below.
709 .. versionchanged:: 2.0.0
711 When using ``.transform`` on a grouped DataFrame and
712 the transformation function returns a DataFrame,
713 pandas now aligns the result's index with the input's index.
714 You can call ``.to_numpy()`` on the result of
715 the transformation function to avoid alignment.
717 Examples
718 --------
720 >>> ser = pd.Series(
721 ... [390.0, 350.0, 30.0, 20.0],
722 ... index=["Falcon", "Falcon", "Parrot", "Parrot"],
723 ... name="Max Speed",
724 ... )
725 >>> grouped = ser.groupby([1, 1, 2, 2])
726 >>> grouped.transform(lambda x: (x - x.mean()) / x.std())
727 Falcon 0.707107
728 Falcon -0.707107
729 Parrot 0.707107
730 Parrot -0.707107
731 Name: Max Speed, dtype: float64
733 Broadcast result of the transformation
735 >>> grouped.transform(lambda x: x.max() - x.min())
736 Falcon 40.0
737 Falcon 40.0
738 Parrot 10.0
739 Parrot 10.0
740 Name: Max Speed, dtype: float64
742 >>> grouped.transform("mean")
743 Falcon 370.0
744 Falcon 370.0
745 Parrot 25.0
746 Parrot 25.0
747 Name: Max Speed, dtype: float64
749 The resulting dtype will reflect the return value of the passed ``func``,
750 for example:
752 >>> grouped.transform(lambda x: x.astype(int).max())
753 Falcon 390
754 Falcon 390
755 Parrot 30
756 Parrot 30
757 Name: Max Speed, dtype: int64
758 """
760 if isinstance(func, list):
761 raise NotImplementedError(
762 "Passing a list to SeriesGroupBy.transform is not yet supported "
763 "and is intended to be implemented in a future release. "
764 "See https://github.com/pandas-dev/pandas/issues/58318."
765 )
767 return self._transform(
768 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
769 )
771 def _cython_transform(self, how: str, numeric_only: bool = False, **kwargs):
772 obj = self._obj_with_exclusions
774 try:
775 result = self._grouper._cython_operation(obj._values, how, 0, **kwargs)
776 except NotImplementedError as err:
777 # e.g. test_groupby_raises_string
778 raise TypeError(f"{how} is not supported for {obj.dtype} dtype") from err
780 return obj._constructor(result, index=self.obj.index, name=obj.name)
782 def _transform_general(
783 self, func: Callable, engine, engine_kwargs, *args, **kwargs
784 ) -> Series:
785 """
786 Transform with a callable `func`.
787 """
788 if maybe_use_numba(engine):
789 return self._transform_with_numba(
790 func, *args, engine_kwargs=engine_kwargs, **kwargs
791 )
792 assert callable(func)
793 klass = type(self.obj)
795 results = []
796 for name, group in self._grouper.get_iterator(
797 self._obj_with_exclusions,
798 ):
799 # this setattr is needed for test_transform_lambda_with_datetimetz
800 object.__setattr__(group, "name", name)
801 res = func(group, *args, **kwargs)
803 results.append(klass(res, index=group.index))
805 # check for empty "results" to avoid concat ValueError
806 if results:
807 from pandas.core.reshape.concat import concat
809 concatenated = concat(results, ignore_index=True)
810 result = self._set_result_index_ordered(concatenated)
811 else:
812 result = self.obj._constructor(dtype=np.float64)
814 result.name = self.obj.name
815 return result
817 def filter(self, func, dropna: bool = True, *args, **kwargs):
818 """
819 Filter elements from groups that don't satisfy a criterion.
821 Elements from groups are filtered if they do not satisfy the
822 boolean criterion specified by func.
824 Parameters
825 ----------
826 func : function
827 Criterion to apply to each group. Should return True or False.
828 dropna : bool, optional
829 Drop groups that do not pass the filter. True by default; if False,
830 groups that evaluate False are filled with NaNs.
831 *args : tuple
832 Optional positional arguments to pass to `func`.
833 **kwargs : dict
834 Optional keyword arguments to pass to `func`.
836 Returns
837 -------
838 Series
839 The filtered subset of the original Series.
841 See Also
842 --------
843 Series.filter: Filter elements of ungrouped Series.
844 DataFrameGroupBy.filter : Filter elements from groups based on criterion.
846 Notes
847 -----
848 Functions that mutate the passed object can produce unexpected
849 behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
850 for more details.
852 Examples
853 --------
854 >>> df = pd.DataFrame(
855 ... {
856 ... "A": ["foo", "bar", "foo", "bar", "foo", "bar"],
857 ... "B": [1, 2, 3, 4, 5, 6],
858 ... "C": [2.0, 5.0, 8.0, 1.0, 2.0, 9.0],
859 ... }
860 ... )
861 >>> grouped = df.groupby("A")
862 >>> df.groupby("A").B.filter(lambda x: x.mean() > 3.0)
863 1 2
864 3 4
865 5 6
866 Name: B, dtype: int64
867 """
868 if isinstance(func, str):
869 wrapper = lambda x: getattr(x, func)(*args, **kwargs)
870 else:
871 wrapper = lambda x: func(x, *args, **kwargs)
873 # Interpret np.nan as False.
874 def true_and_notna(x) -> bool:
875 b = wrapper(x)
876 return notna(b) and b
878 try:
879 indices = [
880 self._get_index(name)
881 for name, group in self._grouper.get_iterator(self._obj_with_exclusions)
882 if true_and_notna(group)
883 ]
884 except (ValueError, TypeError) as err:
885 raise TypeError("the filter must return a boolean result") from err
887 filtered = self._apply_filter(indices, dropna)
888 return filtered
890 def nunique(self, dropna: bool = True) -> Series | DataFrame:
891 """
892 Return number of unique elements in the group.
894 This method counts the number of distinct values within each group,
895 optionally excluding NaN values.
897 Parameters
898 ----------
899 dropna : bool, default True
900 Don't include NaN in the counts.
902 Returns
903 -------
904 Series
905 Number of unique values within each group.
907 See Also
908 --------
909 core.resample.Resampler.nunique : Method nunique for Resampler.
911 Examples
912 --------
913 >>> lst = ["a", "a", "b", "b"]
914 >>> ser = pd.Series([1, 2, 3, 3], index=lst)
915 >>> ser
916 a 1
917 a 2
918 b 3
919 b 3
920 dtype: int64
921 >>> ser.groupby(level=0).nunique()
922 a 2
923 b 1
924 dtype: int64
925 """
926 ids = self._grouper.ids
927 ngroups = self._grouper.ngroups
928 val = self.obj._values
929 codes, uniques = algorithms.factorize(val, use_na_sentinel=dropna, sort=False)
931 if self._grouper.has_dropped_na:
932 mask = ids >= 0
933 ids = ids[mask]
934 codes = codes[mask]
936 group_index = get_group_index(
937 labels=[ids, codes],
938 shape=(ngroups, len(uniques)),
939 sort=False,
940 xnull=dropna,
941 )
943 if dropna:
944 mask = group_index >= 0
945 if (~mask).any():
946 ids = ids[mask]
947 group_index = group_index[mask]
949 mask = duplicated(group_index, "first")
950 res = np.bincount(ids[~mask], minlength=ngroups)
951 res = ensure_int64(res)
953 ri = self._grouper.result_index
954 result: Series | DataFrame = self.obj._constructor(
955 res, index=ri, name=self.obj.name
956 )
957 if not self.as_index:
958 result = self._insert_inaxis_grouper(result)
959 result.index = default_index(len(result))
960 return result
962 def describe(self, percentiles=None, include=None, exclude=None) -> Series:
963 """
964 Generate descriptive statistics for each group.
966 Within each group, summarize the central tendency, dispersion,
967 and shape of the Series's distribution, excluding ``NaN`` values.
968 The per-group statistics depend on the Series's dtype; see Notes.
970 Parameters
971 ----------
972 percentiles : list-like of numbers, optional
973 The percentiles to include in the output. All should fall
974 between 0 and 1. The default, ``None``, returns the 25th,
975 50th, and 75th percentiles.
976 include : None
977 Has no effect on a Series groupby. Deprecated and will be
978 removed in a future version.
979 exclude : None
980 Has no effect on a Series groupby. Deprecated and will be
981 removed in a future version.
983 Returns
984 -------
985 DataFrame
986 One row per group; columns are the per-group statistics.
988 See Also
989 --------
990 Series.describe : Generate descriptive statistics of a Series.
991 DataFrameGroupBy.describe : Generate descriptive statistics for
992 each group of a DataFrame.
994 Notes
995 -----
996 For numeric Series, the per-group columns are ``count``, ``mean``,
997 ``std``, ``min``, ``max``, and the requested percentiles. By
998 default the lower percentile is ``25`` and the upper is ``75``;
999 the ``50`` percentile is the same as the median.
1001 For object Series (e.g. strings), the per-group columns are
1002 ``count``, ``unique``, ``top``, and ``freq``. The ``top`` is the
1003 most common value within the group and ``freq`` is its count.
1005 Examples
1006 --------
1007 >>> s = pd.Series([1, 2, 3, 4])
1008 >>> s.groupby([1, 1, 2, 2]).describe()
1009 count mean std min 25% 50% 75% max
1010 1 2.0 1.5 0.707107 1.0 1.25 1.5 1.75 2.0
1011 2 2.0 3.5 0.707107 3.0 3.25 3.5 3.75 4.0
1012 """
1013 return super().describe(
1014 percentiles=percentiles, include=include, exclude=exclude
1015 )
1017 def value_counts(
1018 self,
1019 normalize: bool = False,
1020 sort: bool = True,
1021 ascending: bool = False,
1022 bins=None,
1023 dropna: bool = True,
1024 ) -> Series | DataFrame:
1025 """
1026 Return a Series or DataFrame containing counts of unique rows.
1028 The resulting object will be in descending order by default so that
1029 the first element in each group is the most frequently-occurring
1030 value. NA values are excluded from the result by default.
1032 Parameters
1033 ----------
1034 normalize : bool, default False
1035 Return proportions rather than frequencies.
1036 sort : bool, default True
1037 Sort by frequencies.
1038 ascending : bool, default False
1039 Sort in ascending order.
1040 bins : int or list of ints, optional
1041 Rather than count values, group them into half-open bins,
1042 a convenience for pd.cut, only works with numeric data.
1043 dropna : bool, default True
1044 Don't include counts of rows that contain NA values.
1046 Returns
1047 -------
1048 Series or DataFrame
1049 Series if the groupby ``as_index`` is True, otherwise DataFrame.
1051 See Also
1052 --------
1053 Series.value_counts: Equivalent method on Series.
1054 DataFrame.value_counts: Equivalent method on DataFrame.
1055 DataFrameGroupBy.value_counts: Equivalent method on DataFrameGroupBy.
1057 Notes
1058 -----
1059 - If the groupby ``as_index`` is True then the returned Series will have a
1060 MultiIndex with one level per input column.
1061 - If the groupby ``as_index`` is False then the returned DataFrame will have an
1062 additional column with the value_counts. The column is labelled 'count' or
1063 'proportion', depending on the ``normalize`` parameter.
1065 By default, rows that contain any NA values are omitted from
1066 the result.
1068 By default, the result will be in descending order so that the
1069 first element of each group is the most frequently-occurring row.
1071 Examples
1072 --------
1073 >>> s = pd.Series(
1074 ... [1, 1, 2, 3, 2, 3, 3, 1, 1, 3, 3, 3],
1075 ... index=["A", "A", "A", "A", "A", "A", "B", "B", "B", "B", "B", "B"],
1076 ... )
1077 >>> s
1078 A 1
1079 A 1
1080 A 2
1081 A 3
1082 A 2
1083 A 3
1084 B 3
1085 B 1
1086 B 1
1087 B 3
1088 B 3
1089 B 3
1090 dtype: int64
1091 >>> g1 = s.groupby(s.index)
1092 >>> g1.value_counts(bins=2)
1093 A (0.997, 2.0] 4
1094 (2.0, 3.0] 2
1095 B (2.0, 3.0] 4
1096 (0.997, 2.0] 2
1097 Name: count, dtype: int64
1098 >>> g1.value_counts(normalize=True)
1099 A 1 0.333333
1100 2 0.333333
1101 3 0.333333
1102 B 3 0.666667
1103 1 0.333333
1104 Name: proportion, dtype: float64
1105 """
1106 name = "proportion" if normalize else "count"
1108 if bins is None:
1109 result = self._value_counts(
1110 normalize=normalize, sort=sort, ascending=ascending, dropna=dropna
1111 )
1112 result.name = name
1113 return result
1115 from pandas.core.reshape.merge import get_join_indexers
1116 from pandas.core.reshape.tile import cut
1118 ids = self._grouper.ids
1119 val = self.obj._values
1121 index_names = [*self._grouper.names, self.obj.name]
1123 if isinstance(val.dtype, CategoricalDtype) or not np.iterable(bins):
1124 # scalar bins cannot be done at top level
1125 # in a backward compatible way
1126 # GH38672 relates to categorical dtype
1127 ser = self.apply(
1128 Series.value_counts,
1129 normalize=normalize,
1130 sort=sort,
1131 ascending=ascending,
1132 bins=bins,
1133 )
1134 ser.name = name
1135 ser.index.names = index_names
1136 return ser
1138 # groupby removes null keys from groupings
1139 mask = ids != -1
1140 ids, val = ids[mask], val[mask]
1142 # lab is a Categorical with categories an IntervalIndex
1143 cat_ser = cut(Series(val, copy=False), bins, include_lowest=True)
1144 cat_obj = cast("Categorical", cat_ser._values)
1145 lev = cat_obj.categories
1146 lab = lev.take(
1147 cat_obj.codes,
1148 allow_fill=True,
1149 fill_value=lev._na_value,
1150 )
1151 llab = lambda lab, inc: lab[inc]._multiindex.codes[-1]
1153 if isinstance(lab.dtype, IntervalDtype):
1154 # TODO: should we do this inside II?
1155 lab_interval = cast("Interval", lab)
1157 sorter = np.lexsort((lab_interval.left, lab_interval.right, ids))
1158 else:
1159 sorter = np.lexsort((lab, ids))
1161 ids, lab = ids[sorter], lab[sorter]
1163 # group boundaries are where group ids change
1164 idchanges = 1 + np.nonzero(ids[1:] != ids[:-1])[0]
1165 idx = np.r_[0, idchanges]
1166 if not len(ids):
1167 idx = idchanges
1169 # new values are where sorted labels change
1170 lchanges = llab(lab, slice(1, None)) != llab(lab, slice(None, -1))
1171 inc = np.r_[True, lchanges]
1172 if not len(val):
1173 inc = lchanges
1174 inc[idx] = True # group boundaries are also new values
1175 out = np.diff(np.nonzero(np.r_[inc, True])[0]) # value counts
1177 # num. of times each group should be repeated
1178 rep = partial(np.repeat, repeats=np.add.reduceat(inc, idx))
1180 # multi-index components
1181 if isinstance(self._grouper.result_index, MultiIndex):
1182 codes = list(self._grouper.result_index.codes)
1183 else:
1184 codes = [
1185 algorithms.factorize(
1186 self._grouper.result_index,
1187 sort=self._grouper._sort,
1188 use_na_sentinel=self._grouper.dropna,
1189 )[0]
1190 ]
1191 codes = [rep(level_codes) for level_codes in codes] + [llab(lab, inc)]
1192 levels = [*self._grouper.levels, lev]
1194 if dropna:
1195 mask = codes[-1] != -1
1196 if mask.all():
1197 dropna = False
1198 else:
1199 out, codes = out[mask], [level_codes[mask] for level_codes in codes]
1201 if normalize:
1202 out = out.astype("float")
1203 d = np.diff(np.r_[idx, len(ids)])
1204 if dropna:
1205 m = ids[lab == -1]
1206 np.add.at(d, m, -1)
1207 acc = rep(d)[mask]
1208 else:
1209 acc = rep(d)
1210 out /= acc
1212 # for compat. with libgroupby.value_counts need to ensure every
1213 # bin is present at every index level, null filled with zeros
1214 diff = np.zeros(len(out), dtype="bool")
1215 for level_codes in codes[:-1]:
1216 diff |= np.r_[True, level_codes[1:] != level_codes[:-1]]
1218 ncat, nbin = diff.sum(), len(levels[-1])
1219 left = [np.repeat(np.arange(ncat), nbin), np.tile(np.arange(nbin), ncat)]
1220 right = [diff.cumsum() - 1, codes[-1]]
1222 # error: Argument 1 to "get_join_indexers" has incompatible type
1223 # "List[ndarray[Any, Any]]"; expected "List[Union[Union[ExtensionArray,
1224 # ndarray[Any, Any]], Index, Series]]
1225 _, idx = get_join_indexers(
1226 left, # type: ignore[arg-type]
1227 right,
1228 sort=False,
1229 how="left",
1230 )
1231 if idx is not None:
1232 out = np.where(idx != -1, out[idx], 0)
1234 if sort:
1235 sorter = np.lexsort((out if ascending else -out, left[0]))
1236 out, left[-1] = out[sorter], left[-1][sorter]
1238 # build the multi-index w/ full levels
1239 def build_codes(lev_codes: np.ndarray) -> np.ndarray:
1240 return np.repeat(lev_codes[diff], nbin)
1242 codes = [build_codes(lev_codes) for lev_codes in codes[:-1]]
1243 codes.append(left[-1])
1245 mi = MultiIndex(
1246 levels=levels, codes=codes, names=index_names, verify_integrity=False
1247 )
1249 if is_integer_dtype(out.dtype):
1250 out = ensure_int64(out)
1251 result = self.obj._constructor(out, index=mi, name=name)
1252 if not self.as_index:
1253 result = result.reset_index()
1254 return result
1256 def take(
1257 self,
1258 indices: TakeIndexer,
1259 **kwargs,
1260 ) -> Series:
1261 """
1262 Return the elements in the given *positional* indices in each group.
1264 This means that we are not indexing according to actual values in
1265 the index attribute of the object. We are indexing according to the
1266 actual position of the element in the object.
1268 If a requested index does not exist for some group, this method will raise.
1269 To get similar behavior that ignores indices that don't exist, see
1270 :meth:`.SeriesGroupBy.nth`.
1272 Parameters
1273 ----------
1274 indices : array-like
1275 An array of ints indicating which positions to take in each group.
1277 **kwargs
1278 For compatibility with :meth:`numpy.take`. Has no effect on the
1279 output.
1281 .. deprecated:: 3.1.0
1282 Passing ``**kwargs`` to SeriesGroupBy.take is deprecated
1283 and will be removed in a future version of pandas.
1285 Returns
1286 -------
1287 Series
1288 A Series containing the elements taken from each group.
1290 See Also
1291 --------
1292 Series.take : Take elements from a Series along an axis.
1293 Series.loc : Select a subset of a DataFrame by labels.
1294 Series.iloc : Select a subset of a DataFrame by positions.
1295 numpy.take : Take elements from an array along an axis.
1296 SeriesGroupBy.nth : Similar to take, won't raise if indices don't exist.
1298 Examples
1299 --------
1300 >>> df = pd.DataFrame(
1301 ... [
1302 ... ("falcon", "bird", 389.0),
1303 ... ("parrot", "bird", 24.0),
1304 ... ("lion", "mammal", 80.5),
1305 ... ("monkey", "mammal", np.nan),
1306 ... ("rabbit", "mammal", 15.0),
1307 ... ],
1308 ... columns=["name", "class", "max_speed"],
1309 ... index=[4, 3, 2, 1, 0],
1310 ... )
1311 >>> df
1312 name class max_speed
1313 4 falcon bird 389.0
1314 3 parrot bird 24.0
1315 2 lion mammal 80.5
1316 1 monkey mammal NaN
1317 0 rabbit mammal 15.0
1318 >>> gb = df["name"].groupby([1, 1, 2, 2, 2])
1320 Take elements at rows 0 and 1 in each group.
1322 >>> gb.take([0, 1])
1323 1 4 falcon
1324 3 parrot
1325 2 2 lion
1326 1 monkey
1327 Name: name, dtype: str
1329 We may take elements using negative integers for positive indices,
1330 starting from the end of the object, just like with Python lists.
1332 >>> gb.take([-1, -2])
1333 1 3 parrot
1334 4 falcon
1335 2 0 rabbit
1336 1 monkey
1337 Name: name, dtype: str
1338 """
1339 if kwargs:
1340 warnings.warn(
1341 "Passing additional arguments to SeriesGroupBy.take is "
1342 "deprecated and will be removed in a future version of pandas.",
1343 Pandas4Warning,
1344 stacklevel=find_stack_level(),
1345 )
1346 result = self._op_via_apply("take", indices=indices, **kwargs)
1347 return result
1349 def skew(
1350 self,
1351 skipna: bool = True,
1352 numeric_only: bool = False,
1353 **kwargs,
1354 ) -> Series:
1355 """
1356 Return unbiased skew within groups.
1358 Normalized by N-1.
1360 Parameters
1361 ----------
1362 skipna : bool, default True
1363 Exclude NA/null values when computing the result.
1365 numeric_only : bool, default False
1366 Include only float, int, boolean columns. Not implemented for Series.
1368 **kwargs
1369 Additional keyword arguments to be passed to the function.
1371 .. deprecated:: 3.1.0
1372 Passing ``**kwargs`` to SeriesGroupBy.skew is deprecated
1373 and will be removed in a future version of pandas.
1375 Returns
1376 -------
1377 Series
1378 Unbiased skew within groups.
1380 See Also
1381 --------
1382 Series.skew : Return unbiased skew over requested axis.
1384 Examples
1385 --------
1386 >>> ser = pd.Series(
1387 ... [390.0, 350.0, 357.0, np.nan, 22.0, 20.0, 30.0],
1388 ... index=[
1389 ... "Falcon",
1390 ... "Falcon",
1391 ... "Falcon",
1392 ... "Falcon",
1393 ... "Parrot",
1394 ... "Parrot",
1395 ... "Parrot",
1396 ... ],
1397 ... name="Max Speed",
1398 ... )
1399 >>> ser
1400 Falcon 390.0
1401 Falcon 350.0
1402 Falcon 357.0
1403 Falcon NaN
1404 Parrot 22.0
1405 Parrot 20.0
1406 Parrot 30.0
1407 Name: Max Speed, dtype: float64
1408 >>> ser.groupby(level=0).skew()
1409 Falcon 1.525174
1410 Parrot 1.457863
1411 Name: Max Speed, dtype: float64
1412 >>> ser.groupby(level=0).skew(skipna=False)
1413 Falcon NaN
1414 Parrot 1.457863
1415 Name: Max Speed, dtype: float64
1416 """
1417 if kwargs:
1418 warnings.warn(
1419 "Passing additional arguments to SeriesGroupBy.skew is "
1420 "deprecated and will be removed in a future version of pandas.",
1421 Pandas4Warning,
1422 stacklevel=find_stack_level(),
1423 )
1425 return self._cython_agg_general(
1426 "skew", alt=None, skipna=skipna, numeric_only=numeric_only, **kwargs
1427 )
1429 def kurt(
1430 self,
1431 skipna: bool = True,
1432 numeric_only: bool = False,
1433 **kwargs,
1434 ) -> Series:
1435 """
1436 Return unbiased kurtosis within groups.
1438 Kurtosis measures the tailedness of a distribution. This method
1439 computes Fisher's definition of kurtosis (normal distribution has
1440 a kurtosis of zero) for each group, using the unbiased estimator.
1442 Parameters
1443 ----------
1444 skipna : bool, default True
1445 Exclude NA/null values when computing the result.
1447 numeric_only : bool, default False
1448 Include only float, int, boolean columns. Not implemented for Series.
1450 **kwargs
1451 Additional keyword arguments to be passed to the function.
1453 Returns
1454 -------
1455 Series
1456 Unbiased kurtosis within groups.
1458 See Also
1459 --------
1460 Series.kurt : Return unbiased kurtosis over requested axis.
1462 Examples
1463 --------
1464 >>> ser = pd.Series(
1465 ... [390.0, 350.0, 357.0, 333.0, np.nan, 22.0, 20.0, 30.0, 40.0, 41.0],
1466 ... index=[
1467 ... "Falcon",
1468 ... "Falcon",
1469 ... "Falcon",
1470 ... "Falcon",
1471 ... "Falcon",
1472 ... "Parrot",
1473 ... "Parrot",
1474 ... "Parrot",
1475 ... "Parrot",
1476 ... "Parrot",
1477 ... ],
1478 ... name="Max Speed",
1479 ... )
1480 >>> ser
1481 Falcon 390.0
1482 Falcon 350.0
1483 Falcon 357.0
1484 Falcon 333.0
1485 Falcon NaN
1486 Parrot 22.0
1487 Parrot 20.0
1488 Parrot 30.0
1489 Parrot 40.0
1490 Parrot 41.0
1491 Name: Max Speed, dtype: float64
1492 >>> ser.groupby(level=0).kurt()
1493 Falcon 1.622109
1494 Parrot -2.878714
1495 Name: Max Speed, dtype: float64
1496 >>> ser.groupby(level=0).kurt(skipna=False)
1497 Falcon NaN
1498 Parrot -2.878714
1499 Name: Max Speed, dtype: float64
1500 """
1502 def alt(obj):
1503 # This should not be reached since the cython path should raise
1504 # TypeError and not NotImplementedError.
1505 raise TypeError(f"'kurt' is not supported for dtype={obj.dtype}")
1507 return self._cython_agg_general(
1508 "kurt", alt=alt, skipna=skipna, numeric_only=numeric_only, **kwargs
1509 )
1511 @property
1512 def plot(self) -> GroupByPlot:
1513 """
1514 Make plots of groups from a Series.
1516 Uses the backend specified by the option ``plotting.backend``.
1517 By default, matplotlib is used.
1519 Returns
1520 -------
1521 GroupByPlot
1522 A plotting object that can be used to create plots for each group.
1524 See Also
1525 --------
1526 Series.plot : Make plots of Series.
1528 Examples
1529 --------
1530 >>> ser = pd.Series([1, 2, 3, 4, 5], index=["a", "a", "b", "b", "c"])
1531 >>> g = ser.groupby(level=0)
1532 >>> g.plot() # doctest: +SKIP
1533 """
1534 result = GroupByPlot(self)
1535 return result
1537 def nlargest(
1538 self, n: int = 5, keep: Literal["first", "last", "all"] = "first"
1539 ) -> Series:
1540 """
1541 Return the largest `n` elements.
1543 Within each group, returns the `n` largest values sorted in
1544 descending order. The ``keep`` parameter controls how ties are
1545 handled when there are duplicate values at the boundary.
1547 Parameters
1548 ----------
1549 n : int, default 5
1550 Return this many descending sorted values.
1551 keep : {'first', 'last', 'all'}, default 'first'
1552 When there are duplicate values that cannot all fit in a
1553 Series of `n` elements:
1555 - ``first`` : return the first `n` occurrences in order
1556 of appearance.
1557 - ``last`` : return the last `n` occurrences in reverse
1558 order of appearance.
1559 - ``all`` : keep all occurrences. This can result in a Series of
1560 size larger than `n`.
1562 Returns
1563 -------
1564 Series
1565 The `n` largest values in the Series, sorted in decreasing order.
1567 See Also
1568 --------
1569 Series.nsmallest: Get the `n` smallest elements.
1570 Series.sort_values: Sort Series by values.
1571 Series.head: Return the first `n` rows.
1573 Notes
1574 -----
1575 Faster than ``.sort_values(ascending=False).head(n)`` for small `n`
1576 relative to the size of the ``Series`` object.
1578 Examples
1579 --------
1580 >>> s = pd.Series([1, 2, 3, 4, 5, 6])
1582 >>> s
1583 0 1
1584 1 2
1585 2 3
1586 3 4
1587 4 5
1588 5 6
1589 dtype: int64
1591 >>> s.groupby([1, 1, 1, 2, 2, 2]).nlargest(n=2)
1592 1 2 3
1593 1 2
1594 2 5 6
1595 4 5
1596 dtype: int64
1597 """
1598 f = partial(Series.nlargest, n=n, keep=keep)
1599 data = self._obj_with_exclusions
1600 # Don't change behavior if result index happens to be the same, i.e.
1601 # already ordered and n >= all group sizes.
1602 result = self._python_apply_general(f, data, not_indexed_same=True)
1603 return result
1605 def nsmallest(
1606 self, n: int = 5, keep: Literal["first", "last", "all"] = "first"
1607 ) -> Series:
1608 """
1609 Return the smallest `n` elements.
1611 Within each group, returns the `n` smallest values sorted in
1612 ascending order. The ``keep`` parameter controls how ties are
1613 handled when there are duplicate values at the boundary.
1615 Parameters
1616 ----------
1617 n : int, default 5
1618 Return this many ascending sorted values.
1619 keep : {'first', 'last', 'all'}, default 'first'
1620 When there are duplicate values that cannot all fit in a
1621 Series of `n` elements:
1623 - ``first`` : return the first `n` occurrences in order
1624 of appearance.
1625 - ``last`` : return the last `n` occurrences in reverse
1626 order of appearance.
1627 - ``all`` : keep all occurrences. This can result in a Series of
1628 size larger than `n`.
1630 Returns
1631 -------
1632 Series
1633 The `n` smallest values in the Series, sorted in increasing order.
1635 See Also
1636 --------
1637 Series.nlargest: Get the `n` largest elements.
1638 Series.sort_values: Sort Series by values.
1639 Series.head: Return the first `n` rows.
1641 Notes
1642 -----
1643 Faster than ``.sort_values().head(n)`` for small `n` relative to
1644 the size of the ``Series`` object.
1646 Examples
1647 --------
1648 >>> s = pd.Series([1, 2, 3, 4, 5, 6])
1650 >>> s
1651 0 1
1652 1 2
1653 2 3
1654 3 4
1655 4 5
1656 5 6
1657 dtype: int64
1659 >>> s.groupby([1, 1, 1, 2, 2, 2]).nsmallest(n=2)
1660 1 0 1
1661 1 2
1662 2 3 4
1663 4 5
1664 dtype: int64
1665 """
1666 f = partial(Series.nsmallest, n=n, keep=keep)
1667 data = self._obj_with_exclusions
1668 # Don't change behavior if result index happens to be the same, i.e.
1669 # already ordered and n >= all group sizes.
1670 result = self._python_apply_general(f, data, not_indexed_same=True)
1671 return result
1673 def idxmin(self, skipna: bool = True) -> Series:
1674 """
1675 Return the row label of the minimum value.
1677 If multiple values equal the minimum, the first row label with that
1678 value is returned.
1680 Parameters
1681 ----------
1682 skipna : bool, default True
1683 Exclude NA values.
1685 Returns
1686 -------
1687 Series
1688 Indexes of minima in each group.
1690 Raises
1691 ------
1692 ValueError
1693 When there are no valid values for a group. Then can happen if:
1695 * There is an unobserved group and ``observed=False``.
1696 * All values for a group are NA.
1697 * Some values for a group are NA and ``skipna=False``.
1699 .. versionchanged:: 3.0.0
1700 Previously if all values for a group are NA or some values for a group are
1701 NA and ``skipna=False``, this method would return NA. Now it raises instead.
1703 See Also
1704 --------
1705 numpy.argmin : Return indices of the minimum values
1706 along the given axis.
1707 DataFrame.idxmin : Return index of first occurrence of minimum
1708 over requested axis.
1709 Series.idxmax : Return index *label* of the first occurrence
1710 of maximum of values.
1712 Examples
1713 --------
1714 >>> ser = pd.Series(
1715 ... [1, 2, 3, 4],
1716 ... index=pd.DatetimeIndex(
1717 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
1718 ... ),
1719 ... )
1720 >>> ser
1721 2023-01-01 1
1722 2023-01-15 2
1723 2023-02-01 3
1724 2023-02-15 4
1725 dtype: int64
1727 >>> ser.groupby(["a", "a", "b", "b"]).idxmin()
1728 a 2023-01-01
1729 b 2023-02-01
1730 dtype: datetime64[us]
1731 """
1732 return self._idxmax_idxmin("idxmin", skipna=skipna)
1734 def idxmax(self, skipna: bool = True) -> Series:
1735 """
1736 Return the row label of the maximum value.
1738 If multiple values equal the maximum, the first row label with that
1739 value is returned.
1741 Parameters
1742 ----------
1743 skipna : bool, default True
1744 Exclude NA values.
1746 Returns
1747 -------
1748 Series
1749 Indexes of maxima in each group.
1751 Raises
1752 ------
1753 ValueError
1754 When there are no valid values for a group. Then can happen if:
1756 * There is an unobserved group and ``observed=False``.
1757 * All values for a group are NA.
1758 * Some values for a group are NA and ``skipna=False``.
1760 .. versionchanged:: 3.0.0
1761 Previously if all values for a group are NA or some values for a group are
1762 NA and ``skipna=False``, this method would return NA. Now it raises instead.
1764 See Also
1765 --------
1766 numpy.argmax : Return indices of the maximum values
1767 along the given axis.
1768 DataFrame.idxmax : Return index of first occurrence of maximum
1769 over requested axis.
1770 Series.idxmin : Return index *label* of the first occurrence
1771 of minimum of values.
1773 Examples
1774 --------
1775 >>> ser = pd.Series(
1776 ... [1, 2, 3, 4],
1777 ... index=pd.DatetimeIndex(
1778 ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
1779 ... ),
1780 ... )
1781 >>> ser
1782 2023-01-01 1
1783 2023-01-15 2
1784 2023-02-01 3
1785 2023-02-15 4
1786 dtype: int64
1788 >>> ser.groupby(["a", "a", "b", "b"]).idxmax()
1789 a 2023-01-15
1790 b 2023-02-15
1791 dtype: datetime64[us]
1792 """
1793 return self._idxmax_idxmin("idxmax", skipna=skipna)
1795 def corr(
1796 self,
1797 other: Series,
1798 method: CorrelationMethod = "pearson",
1799 min_periods: int | None = None,
1800 ) -> Series:
1801 """
1802 Compute correlation between each group and another Series.
1804 This method computes the pairwise correlation between each group's
1805 values and the corresponding values in ``other``, using the specified
1806 correlation method.
1808 Parameters
1809 ----------
1810 other : Series
1811 Series to compute correlation with.
1812 method : {'pearson', 'kendall', 'spearman'}, default 'pearson'
1813 Method of correlation to use.
1814 min_periods : int, optional
1815 Minimum number of observations required per pair of columns to
1816 have a valid result.
1818 Returns
1819 -------
1820 Series
1821 Correlation value for each group.
1823 See Also
1824 --------
1825 Series.corr : Equivalent method on ``Series``.
1827 Examples
1828 --------
1829 >>> s = pd.Series([1, 2, 3, 4], index=[0, 0, 1, 1])
1830 >>> g = s.groupby([0, 0, 1, 1])
1831 >>> g.corr() # doctest: +SKIP
1832 """
1833 result = self._op_via_apply(
1834 "corr", other=other, method=method, min_periods=min_periods
1835 )
1836 return result
1838 def cov(
1839 self, other: Series, min_periods: int | None = None, ddof: int | None = 1
1840 ) -> Series:
1841 """
1842 Compute covariance between each group and another Series.
1844 This method computes the sample covariance between each group's
1845 values and the corresponding values in ``other``.
1847 Parameters
1848 ----------
1849 other : Series
1850 Series to compute covariance with.
1851 min_periods : int, optional
1852 Minimum number of observations required per pair of columns to
1853 have a valid result.
1854 ddof : int, optional
1855 Delta degrees of freedom for variance calculation.
1857 Returns
1858 -------
1859 Series
1860 Covariance value for each group.
1862 See Also
1863 --------
1864 Series.cov : Equivalent method on ``Series``.
1866 Examples
1867 --------
1868 >>> s = pd.Series([1, 2, 3, 4], index=[0, 0, 1, 1])
1869 >>> g = s.groupby([0, 0, 1, 1])
1870 >>> g.cov() # doctest: +SKIP
1871 """
1872 result = self._op_via_apply(
1873 "cov", other=other, min_periods=min_periods, ddof=ddof
1874 )
1875 return result
1877 @property
1878 def is_monotonic_increasing(self) -> Series:
1879 """
1880 Return whether each group's values are monotonically increasing.
1882 A group is considered monotonically increasing if each successive
1883 element is greater than or equal to the previous one.
1885 Returns
1886 -------
1887 Series
1889 See Also
1890 --------
1891 SeriesGroupBy.is_monotonic_decreasing : Return whether each group's values
1892 are monotonically decreasing.
1894 Examples
1895 --------
1896 >>> s = pd.Series([2, 1, 3, 4], index=["Falcon", "Falcon", "Parrot", "Parrot"])
1897 >>> s.groupby(level=0).is_monotonic_increasing
1898 Falcon False
1899 Parrot True
1900 dtype: bool
1901 """
1902 return self.apply(lambda ser: ser.is_monotonic_increasing)
1904 @property
1905 def is_monotonic_decreasing(self) -> Series:
1906 """
1907 Return whether each group's values are monotonically decreasing.
1909 A group is considered monotonically decreasing if each successive
1910 element is less than or equal to the previous one.
1912 Returns
1913 -------
1914 Series
1916 See Also
1917 --------
1918 SeriesGroupBy.is_monotonic_increasing : Return whether each group's values
1919 are monotonically increasing.
1921 Examples
1922 --------
1923 >>> s = pd.Series([2, 1, 3, 4], index=["Falcon", "Falcon", "Parrot", "Parrot"])
1924 >>> s.groupby(level=0).is_monotonic_decreasing
1925 Falcon True
1926 Parrot False
1927 dtype: bool
1928 """
1929 return self.apply(lambda ser: ser.is_monotonic_decreasing)
1931 def hist(
1932 self,
1933 by=None,
1934 ax=None,
1935 grid: bool = True,
1936 xlabelsize: int | None = None,
1937 xrot: float | None = None,
1938 ylabelsize: int | None = None,
1939 yrot: float | None = None,
1940 figsize: tuple[float, float] | None = None,
1941 bins: int | Sequence[int] = 10,
1942 backend: str | None = None,
1943 legend: bool = False,
1944 **kwargs,
1945 ):
1946 """
1947 Draw histogram for each group's values using :meth:`Series.hist` API.
1949 A separate histogram subplot is generated for each group, making it
1950 easy to visually compare the distribution of values across groups.
1952 Parameters
1953 ----------
1954 by : object, optional
1955 Grouping key.
1956 ax : matplotlib.axes.Axes, optional
1957 Axis to draw the histogram on.
1958 grid : bool, default True
1959 Show axis grid lines.
1960 xlabelsize : int, default None
1961 X axis label size.
1962 xrot : float, default None
1963 Rotation for x ticks.
1964 ylabelsize : int, default None
1965 Y axis label size.
1966 yrot : float, default None
1967 Rotation for y ticks.
1968 figsize : tuple, optional
1969 Figure size in inches.
1970 bins : int or sequence, default 10
1971 Number of histogram bins or bin edges.
1972 backend : str or callable or None, optional
1973 Plotting backend to use (e.g. 'matplotlib'). If None, use the default
1974 plotting backend.
1975 legend : bool, default False
1976 Whether to draw the legend.
1977 **kwargs
1978 Additional keyword arguments passed to :meth:`Series.hist`.
1980 Returns
1981 -------
1982 matplotlib.axes.Axes or ndarray of Axes
1983 The returned matplotlib axes or array of axes depending on input.
1985 See Also
1986 --------
1987 Series.hist : Equivalent histogram plotting method on Series.
1989 Examples
1990 --------
1991 >>> df = pd.DataFrame({"val": [1, 2, 2, 3, 3, 3]}, index=[0, 0, 1, 1, 2, 2])
1992 >>> g = df["val"].groupby([0, 0, 1, 1, 2, 2])
1993 >>> g.hist() # doctest: +SKIP
1994 """
1995 result = self._op_via_apply(
1996 "hist",
1997 by=by,
1998 ax=ax,
1999 grid=grid,
2000 xlabelsize=xlabelsize,
2001 xrot=xrot,
2002 ylabelsize=ylabelsize,
2003 yrot=yrot,
2004 figsize=figsize,
2005 bins=bins,
2006 backend=backend,
2007 legend=legend,
2008 **kwargs,
2009 )
2010 return result
2012 @property
2013 def dtype(self) -> Series:
2014 """
2015 Return the dtype object of the underlying data for each group.
2017 Mirrors :meth:`Series.dtype` applied group-wise.
2019 Returns
2020 -------
2021 Series
2022 Dtype of each group's values.
2023 """
2024 return self.apply(lambda ser: ser.dtype)
2026 def unique(self) -> Series:
2027 """
2028 Return unique values for each group.
2030 It returns unique values for each of the grouped values. Returned in
2031 order of appearance. Hash table-based unique, therefore does NOT sort.
2033 Returns
2034 -------
2035 Series
2036 Unique values for each of the grouped values.
2038 See Also
2039 --------
2040 Series.unique : Return unique values of Series object.
2042 Examples
2043 --------
2044 >>> df = pd.DataFrame(
2045 ... [
2046 ... ("Chihuahua", "dog", 6.1),
2047 ... ("Beagle", "dog", 15.2),
2048 ... ("Chihuahua", "dog", 6.9),
2049 ... ("Persian", "cat", 9.2),
2050 ... ("Chihuahua", "dog", 7),
2051 ... ("Persian", "cat", 8.8),
2052 ... ],
2053 ... columns=["breed", "animal", "height_in"],
2054 ... )
2055 >>> df
2056 breed animal height_in
2057 0 Chihuahua dog 6.1
2058 1 Beagle dog 15.2
2059 2 Chihuahua dog 6.9
2060 3 Persian cat 9.2
2061 4 Chihuahua dog 7.0
2062 5 Persian cat 8.8
2063 >>> ser = df.groupby("animal")["breed"].unique()
2064 >>> ser
2065 animal
2066 cat [Persian]
2067 dog [Chihuahua, Beagle]
2068 Name: breed, dtype: object
2069 """
2070 result = self._op_via_apply("unique")
2071 return result
2074@set_module("pandas.api.typing")
2075class DataFrameGroupBy(GroupBy[DataFrame]):
2076 def aggregate(
2077 self, func=None, *args, engine=None, engine_kwargs=None, **kwargs
2078 ) -> DataFrame:
2079 """
2080 Aggregate using one or more operations.
2082 The ``aggregate`` function allows the application of one or more aggregation
2083 operations on groups of data within a DataFrameGroupBy object. It supports
2084 various aggregation methods, including user-defined functions and predefined
2085 functions such as 'sum', 'mean', etc.
2087 Parameters
2088 ----------
2089 func : function, str, list, dict or None
2090 Function to use for aggregating the data. If a function, must either
2091 work when passed a DataFrame or when passed to DataFrame.apply.
2093 Accepted combinations are:
2095 - function
2096 - string function name
2097 - list of functions and/or function names, e.g. ``[np.sum, 'mean']``
2098 - dict of index labels -> functions, function names or list of such.
2099 - None, in which case ``**kwargs`` are used with Named Aggregation. Here the
2100 output has one column for each element in ``**kwargs``. The name of the
2101 column is keyword, whereas the value determines the aggregation used to
2102 compute the values in the column.
2104 Can also accept a Numba JIT function with
2105 ``engine='numba'`` specified. Only passing a single function is supported
2106 with this engine.
2108 If the ``'numba'`` engine is chosen, the function must be
2109 a user defined function with ``values`` and ``index`` as the
2110 first and second arguments respectively in the function signature.
2111 Each group's index will be passed to the user defined function
2112 and optionally available for use.
2114 *args
2115 Positional arguments to pass to func.
2116 engine : str, default None
2117 * ``'cython'`` : Runs the function through C-extensions from cython.
2118 * ``'numba'`` : Runs the function through JIT compiled code from numba.
2119 * ``None`` : Defaults to ``'cython'`` or globally setting
2120 ``compute.use_numba``
2122 engine_kwargs : dict, default None
2123 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2124 * For ``'numba'`` engine, the engine can accept ``nogil``
2125 and ``parallel`` dictionary keys. The values must either be ``True`` or
2126 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2127 ``{'nogil': False, 'parallel': False}`` and will be
2128 applied to the function
2130 **kwargs
2131 * If ``func`` is None, ``**kwargs`` are used to define the output names and
2132 aggregations via Named Aggregation. See ``func`` entry.
2133 * Otherwise, keyword arguments to be passed into func.
2135 Returns
2136 -------
2137 DataFrame
2138 Aggregated DataFrame based on the grouping and the applied aggregation
2139 functions.
2141 See Also
2142 --------
2143 DataFrame.groupby.apply : Apply function func group-wise
2144 and combine the results together.
2145 DataFrame.groupby.transform : Transforms the Series on each group
2146 based on the given function.
2147 DataFrame.aggregate : Aggregate using one or more operations.
2149 Notes
2150 -----
2151 See :ref:`groupby.aggregate` in the User Guide for more details
2152 and examples.
2154 When using ``engine='numba'``, there will be no "fall back" behavior internally.
2155 The group data and group index will be passed as numpy arrays to the JITed
2156 user defined function, and no alternative execution attempts will be tried.
2158 Functions that mutate the passed object can produce unexpected
2159 behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
2160 for more details.
2162 The resulting dtype will reflect the return value of the passed ``func``,
2163 see the examples below.
2165 Examples
2166 --------
2167 >>> data = {
2168 ... "A": [1, 1, 2, 2],
2169 ... "B": [1, 2, 3, 4],
2170 ... "C": [0.362838, 0.227877, 1.267767, -0.562860],
2171 ... }
2172 >>> df = pd.DataFrame(data)
2173 >>> df
2174 A B C
2175 0 1 1 0.362838
2176 1 1 2 0.227877
2177 2 2 3 1.267767
2178 3 2 4 -0.562860
2180 The aggregation is for each column.
2182 >>> df.groupby("A").agg("min")
2183 B C
2184 A
2185 1 1 0.227877
2186 2 3 -0.562860
2188 Multiple aggregations
2190 >>> df.groupby("A").agg(["min", "max"])
2191 B C
2192 min max min max
2193 A
2194 1 1 2 0.227877 0.362838
2195 2 3 4 -0.562860 1.267767
2197 Select a column for aggregation
2199 >>> df.groupby("A").B.agg(["min", "max"])
2200 min max
2201 A
2202 1 1 2
2203 2 3 4
2205 User-defined function for aggregation
2207 >>> df.groupby("A").agg(lambda x: sum(x) + 2)
2208 B C
2209 A
2210 1 5 2.590715
2211 2 9 2.704907
2213 Different aggregations per column
2215 >>> df.groupby("A").agg({"B": ["min", "max"], "C": "sum"})
2216 B C
2217 min max sum
2218 A
2219 1 1 2 0.590715
2220 2 3 4 0.704907
2222 To control the output names with different aggregations per column,
2223 pandas supports "named aggregation"
2225 >>> df.groupby("A").agg(
2226 ... b_min=pd.NamedAgg(column="B", aggfunc="min"),
2227 ... c_sum=pd.NamedAgg(column="C", aggfunc="sum"),
2228 ... )
2229 b_min c_sum
2230 A
2231 1 1 0.590715
2232 2 3 0.704907
2234 - The keywords are the *output* column names
2235 - The values are tuples whose first element is the column to select
2236 and the second element is the aggregation to apply to that column.
2237 Pandas provides the ``pandas.NamedAgg`` namedtuple with the fields
2238 ``['column', 'aggfunc']`` to make it clearer what the arguments are.
2239 As usual, the aggregation can be a callable or a string alias.
2241 See :ref:`groupby.aggregate.named` for more.
2243 The resulting dtype will reflect the return value of the aggregating
2244 function.
2246 >>> df.groupby("A")[["B"]].agg(lambda x: x.astype(float).min())
2247 B
2248 A
2249 1 1.0
2250 2 3.0
2251 """
2252 # This method can consume the un-normalized {column: aggfunc} form, but only
2253 # when the columns are unique: dict aggregation fans a single key out to
2254 # every matching column, while named aggregation must produce exactly one
2255 # output per keyword. The columns must also be unnamed, since the dict
2256 # form keeps columns.name and the normalized form drops it.
2257 # `func is None` is a precondition for relabeling at all, and short-circuits
2258 # materializing _obj_with_exclusions on the far more common plain-agg path.
2259 allow_skip_normalization = func is None and (
2260 self._obj_with_exclusions.columns.is_unique
2261 and self._obj_with_exclusions.columns.name is None
2262 )
2263 relabeling, func, columns, order = reconstruct_func(
2264 func, allow_skip_normalization, **kwargs
2265 )
2266 func = maybe_mangle_lambdas(func)
2268 if maybe_use_numba(engine):
2269 # Not all agg functions support numba, only propagate numba kwargs
2270 # if user asks for numba
2271 kwargs["engine"] = engine
2272 kwargs["engine_kwargs"] = engine_kwargs
2274 op = GroupByApply(self, func, args=args, kwargs=kwargs)
2275 result = op.agg()
2276 if not is_dict_like(func) and result is not None:
2277 # GH #52849
2278 if not self.as_index and is_list_like(func):
2279 return result.reset_index()
2280 else:
2281 return cast("DataFrame", result)
2282 elif relabeling:
2283 # this should be the only (non-raising) case with relabeling
2284 # used reordered index of columns
2285 result = cast("DataFrame", result)
2286 result = result.iloc[:, order]
2287 result = cast("DataFrame", result)
2288 # error: Incompatible types in assignment (expression has type
2289 # "Optional[List[str]]", variable has type
2290 # "Union[Union[Union[ExtensionArray, ndarray[Any, Any]],
2291 # Index, Series], Sequence[Any]]")
2292 result.columns = columns # type: ignore[assignment]
2294 if result is None:
2295 # Remove the kwargs we inserted
2296 # (already stored in engine, engine_kwargs arguments)
2297 if "engine" in kwargs:
2298 del kwargs["engine"]
2299 del kwargs["engine_kwargs"]
2300 # at this point func is not a str, list-like, dict-like,
2301 # or a known callable(e.g. sum)
2302 if maybe_use_numba(engine):
2303 return self._aggregate_with_numba(
2304 func, *args, engine_kwargs=engine_kwargs, **kwargs
2305 )
2306 result = self._python_agg_general(func, *args, **kwargs)
2308 if not self.as_index:
2309 result = self._insert_inaxis_grouper(result)
2310 result.index = default_index(len(result))
2312 return cast("DataFrame", result)
2314 agg = aggregate
2316 def _python_agg_general(self, func, *args, **kwargs):
2317 f = lambda x: func(x, *args, **kwargs)
2319 obj = self._obj_with_exclusions
2321 if self.ngroups == 0 or len(obj.columns) == 0:
2322 res_index = self._grouper.result_index
2323 res = self.obj._constructor(index=res_index, columns=obj.columns).astype(
2324 obj.dtypes
2325 )
2326 else:
2327 output: dict[int, ArrayLike] = {
2328 idx: self._grouper.agg_series(ser, f)
2329 for idx, (name, ser) in enumerate(obj.items())
2330 }
2331 res = self.obj._constructor(output)
2332 res.columns = obj.columns.copy(deep=False)
2333 return self._wrap_aggregated_output(res)
2335 def _wrap_applied_output(
2336 self,
2337 data: DataFrame,
2338 values: list,
2339 not_indexed_same: bool = False,
2340 is_transform: bool = False,
2341 ):
2342 if len(values) == 0:
2343 if is_transform:
2344 # GH#47787 see test_group_on_empty_multiindex
2345 res_index = data.index
2346 elif not self.group_keys or not self.as_index:
2347 res_index = None
2348 else:
2349 res_index = self._grouper.result_index
2351 result = self.obj._constructor(index=res_index, columns=data.columns)
2352 result = result.astype(data.dtypes)
2353 return result
2355 # GH12824
2356 # using values[0] here breaks test_groupby_apply_none_first
2357 first_not_none = next(com.not_none(*values), None)
2359 if first_not_none is None:
2360 # GH9684 - All values are None, return an empty frame
2361 # GH57775 - Ensure that columns and dtypes from original frame are kept.
2362 result = self.obj._constructor(columns=data.columns)
2363 result = result.astype(data.dtypes)
2364 return result
2365 elif isinstance(first_not_none, DataFrame):
2366 return self._concat_objects(
2367 values,
2368 not_indexed_same=not_indexed_same,
2369 is_transform=is_transform,
2370 )
2372 key_index = self._grouper.result_index if self.as_index else None
2374 if isinstance(first_not_none, (np.ndarray, Index)):
2375 # GH#1738: values is list of arrays of unequal lengths
2376 # fall through to the outer else clause
2377 # TODO: sure this is right? we used to do this
2378 # after raising AttributeError above
2379 # GH 18930
2380 if not is_hashable(self._selection):
2381 # error: Need type annotation for "name"
2382 name = tuple(self._selection) # type: ignore[var-annotated, arg-type]
2383 else:
2384 # error: Incompatible types in assignment
2385 # (expression has type "Hashable", variable
2386 # has type "Tuple[Any, ...]")
2387 name = self._selection # type: ignore[assignment]
2388 return self.obj._constructor_sliced(values, index=key_index, name=name)
2389 elif not isinstance(first_not_none, Series):
2390 # values are not series or array-like but scalars
2391 # self._selection not passed through to Series as the
2392 # result should not take the name of original selection
2393 # of columns
2394 if self.as_index:
2395 return self.obj._constructor_sliced(values, index=key_index)
2396 else:
2397 result = self.obj._constructor(values, columns=[self._selection])
2398 result = self._insert_inaxis_grouper(result)
2399 return result
2400 else:
2401 # values are Series
2402 return self._wrap_applied_output_series(
2403 values,
2404 not_indexed_same,
2405 first_not_none,
2406 key_index,
2407 is_transform,
2408 )
2410 def _wrap_applied_output_series(
2411 self,
2412 values: list[Series],
2413 not_indexed_same: bool,
2414 first_not_none,
2415 key_index: Index | None,
2416 is_transform: bool,
2417 ) -> DataFrame | Series:
2418 kwargs = first_not_none._construct_axes_dict()
2419 backup = Series(**kwargs)
2420 values = [x if (x is not None) else backup for x in values]
2422 all_indexed_same = all_indexes_same(x.index for x in values)
2424 if not all_indexed_same:
2425 # GH 8467
2426 return self._concat_objects(
2427 values,
2428 not_indexed_same=True,
2429 is_transform=is_transform,
2430 )
2432 # Combine values
2433 # vstack+constructor is faster than concat and handles MI-columns
2434 stacked_values = np.vstack([np.asarray(v) for v in values])
2436 index = key_index
2437 columns = first_not_none.index.copy()
2438 if columns.name is None:
2439 # GH6124 - propagate name of Series when it's consistent
2440 names = {v.name for v in values}
2441 if len(names) == 1:
2442 columns.name = next(iter(names))
2444 if stacked_values.dtype == object:
2445 # We'll have the DataFrame constructor do inference
2446 stacked_values = stacked_values.tolist()
2447 result = self.obj._constructor(stacked_values, index=index, columns=columns)
2449 if not self.as_index:
2450 result = self._insert_inaxis_grouper(result)
2452 return result.__finalize__(self.obj, method="groupby")
2454 def _cython_transform(
2455 self,
2456 how: str,
2457 numeric_only: bool = False,
2458 **kwargs,
2459 ) -> DataFrame:
2460 # We have multi-block tests
2461 # e.g. test_rank_min_int, test_cython_transform_frame
2462 # test_transform_numeric_ret
2463 mgr: BlockManager = self._get_data_to_aggregate(
2464 numeric_only=numeric_only, name=how
2465 )
2467 def arr_func(bvalues: ArrayLike) -> ArrayLike:
2468 return self._grouper._cython_operation(bvalues, how, 1, **kwargs)
2470 res_mgr = mgr.apply(arr_func)
2472 res_df = self.obj._constructor_from_mgr(res_mgr, axes=res_mgr.axes)
2473 return res_df
2475 def _transform_general(self, func, engine, engine_kwargs, *args, **kwargs):
2476 if maybe_use_numba(engine):
2477 return self._transform_with_numba(
2478 func, *args, engine_kwargs=engine_kwargs, **kwargs
2479 )
2480 from pandas.core.reshape.concat import concat
2482 applied = []
2483 obj = self._obj_with_exclusions
2484 gen = self._grouper.get_iterator(obj)
2485 fast_path, slow_path = self._define_paths(func, *args, **kwargs)
2487 # Determine whether to use slow or fast path by evaluating on the first group.
2488 # Need to handle the case of an empty generator and process the result so that
2489 # it does not need to be computed again.
2490 try:
2491 name, group = next(gen)
2492 except StopIteration:
2493 pass
2494 else:
2495 # 2023-02-27 No tests broken by disabling this pinning
2496 object.__setattr__(group, "name", name)
2497 try:
2498 path, res = self._choose_path(fast_path, slow_path, group)
2499 except ValueError as err:
2500 # e.g. test_transform_with_non_scalar_group
2501 msg = "transform must return a scalar value for each group"
2502 raise ValueError(msg) from err
2503 if group.size > 0:
2504 res = _wrap_transform_general_frame(self.obj, group, res)
2505 applied.append(res)
2507 # Compute and process with the remaining groups
2508 for name, group in gen:
2509 if group.size == 0:
2510 continue
2511 # 2023-02-27 No tests broken by disabling this pinning
2512 object.__setattr__(group, "name", name)
2513 res = path(group)
2515 res = _wrap_transform_general_frame(self.obj, group, res)
2516 applied.append(res)
2518 concat_index = obj.columns
2519 concatenated = concat(
2520 applied, axis=0, verify_integrity=False, ignore_index=True
2521 )
2522 concatenated = concatenated.reindex(concat_index, axis=1)
2523 return self._set_result_index_ordered(concatenated)
2525 def transform(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs):
2526 """
2527 Call function producing a same-indexed DataFrame on each group.
2529 Returns a DataFrame having the same indexes as the original object
2530 filled with the transformed values.
2532 When ``func`` is a user-defined function, by default it operates on each
2533 column of each group separately (each column is passed as a
2534 :class:`Series`); see the Notes section below.
2536 Parameters
2537 ----------
2538 func : function, str, list, or dict
2539 Function to apply to each group.
2540 See the Notes section below for requirements.
2542 Accepted inputs are:
2544 - String
2545 - Python function
2546 - Numba JIT function with ``engine='numba'`` specified.
2547 - List of strings/functions: applied to every non-key column,
2548 returning a MultiIndex-column DataFrame ``(column, func)``.
2549 - Dict ``{column: func}`` or ``{name: NamedFunc(column, func)}``:
2550 applied per-column as specified.
2552 .. versionchanged:: 3.1.0
2553 Added support for list-like, dict, and :class:`NamedFunc` arguments.
2555 Only passing a single function is supported with the numba engine.
2556 If the ``'numba'`` engine is chosen, the function must be
2557 a user defined function with ``values`` and ``index`` as the
2558 first and second arguments respectively in the function signature.
2559 Each group's index will be passed to the user defined function
2560 and optionally available for use.
2562 If a string is chosen, then it needs to be the name
2563 of the groupby method you want to use.
2565 *args
2566 Positional arguments to pass to func.
2567 engine : str, default None
2568 * ``'cython'`` : Runs the function through C-extensions from cython.
2569 * ``'numba'`` : Runs the function through JIT compiled code from numba.
2570 * ``None`` : Defaults to ``'cython'``
2571 or the global setting ``compute.use_numba``
2573 engine_kwargs : dict, default None
2574 * For ``'cython'`` engine, there are no accepted ``engine_kwargs``
2575 * For ``'numba'`` engine, the engine can accept ``nogil``
2576 and ``parallel`` dictionary keys. The values must either be ``True`` or
2577 ``False``. The default ``engine_kwargs`` for the ``'numba'`` engine is
2578 ``{'nogil': False, 'parallel': False}`` and will be
2579 applied to the function
2581 **kwargs
2582 Keyword arguments to be passed into func.
2583 When ``func=None``, ``**kwargs`` should be pairs of
2584 ``output_name=NamedFunc(column, func)``.
2586 Returns
2587 -------
2588 DataFrame
2589 DataFrame with the same indexes as the original object filled
2590 with transformed values.
2592 See Also
2593 --------
2594 DataFrame.groupby.apply : Apply function ``func`` group-wise and combine
2595 the results together.
2596 DataFrame.groupby.aggregate : Aggregate using one or more operations.
2597 DataFrame.transform : Call ``func`` on self producing a DataFrame with the
2598 same axis shape as self.
2600 Notes
2601 -----
2602 See :ref:`groupby.transform` in the User Guide for more details and examples.
2604 Each group is endowed the attribute 'name' in case you need to know
2605 which group you are working on.
2607 The current implementation imposes three requirements on ``func``:
2609 * ``func`` must return a value that either has the same shape as the input
2610 subframe or can be broadcast to the shape of the input subframe.
2611 For example, if ``func`` returns a scalar it will be broadcast to have
2612 the same shape as the input subframe.
2613 * ``func`` is applied to each column of the group separately (each column
2614 is passed as a :class:`Series`). If ``func`` also supports application to
2615 the entire group :class:`DataFrame` and returns the same result, a faster
2616 path operating on the whole group is used starting from the second group.
2617 * ``func`` must not mutate groups. Mutation is not supported and may
2618 produce unexpected results. See :ref:`gotchas.udf-mutation` for more details.
2620 When using ``engine='numba'``, there will be no "fall back" behavior internally.
2621 The group data and group index will be passed as numpy arrays to the JITed
2622 user defined function, and no alternative execution attempts will be tried.
2624 The resulting dtype will reflect the return value of the passed ``func``,
2625 see the examples below.
2627 .. versionchanged:: 2.0.0
2629 When using ``.transform`` on a grouped DataFrame
2630 and the transformation function returns a DataFrame,
2631 pandas now aligns the result's index with the input's index.
2632 You can call ``.to_numpy()`` on the result of the
2633 transformation function to avoid alignment.
2635 Examples
2636 --------
2638 >>> df = pd.DataFrame(
2639 ... {
2640 ... "A": ["foo", "bar", "foo", "bar", "foo", "bar"],
2641 ... "B": ["one", "one", "two", "three", "two", "two"],
2642 ... "C": [1, 5, 5, 2, 5, 5],
2643 ... "D": [2.0, 5.0, 8.0, 1.0, 2.0, 9.0],
2644 ... }
2645 ... )
2646 >>> grouped = df.groupby("A")[["C", "D"]]
2647 >>> grouped.transform(lambda x: (x - x.mean()) / x.std())
2648 C D
2649 0 -1.154701 -0.577350
2650 1 0.577350 0.000000
2651 2 0.577350 1.154701
2652 3 -1.154701 -1.000000
2653 4 0.577350 -0.577350
2654 5 0.577350 1.000000
2656 Broadcast result of the transformation
2658 >>> grouped.transform(lambda x: x.max() - x.min())
2659 C D
2660 0 4.0 6.0
2661 1 3.0 8.0
2662 2 4.0 6.0
2663 3 3.0 8.0
2664 4 4.0 6.0
2665 5 3.0 8.0
2667 >>> grouped.transform("mean")
2668 C D
2669 0 3.666667 4.0
2670 1 4.000000 5.0
2671 2 3.666667 4.0
2672 3 4.000000 5.0
2673 4 3.666667 4.0
2674 5 4.000000 5.0
2676 The resulting dtype will reflect the return value of the passed ``func``,
2677 for example:
2679 >>> grouped.transform(lambda x: x.astype(int).max())
2680 C D
2681 0 5 8
2682 1 5 9
2683 2 5 8
2684 3 5 9
2685 4 5 8
2686 5 5 9
2688 List-like arguments
2690 >>> df2 = pd.DataFrame({"col": list("aab"), "val": range(3), "other": range(3)})
2691 >>> df2.groupby("col").transform(["sum", "min"])
2692 val other
2693 sum min sum min
2694 0 1 0 1 0
2695 1 1 0 1 0
2696 2 2 2 2 2
2698 Dictionary arguments
2700 >>> df2.groupby("col").transform({"val": "sum", "other": "min"})
2701 val other
2702 0 1 0
2703 1 1 0
2704 2 2 2
2706 Named aggregation
2708 >>> df2.groupby("col").transform(
2709 ... val_sum=pd.NamedAgg(column="val", aggfunc="sum"),
2710 ... other_min=pd.NamedAgg(column="other", aggfunc="min"),
2711 ... )
2712 val_sum other_min
2713 0 1 0
2714 1 1 0
2715 2 2 2
2716 """
2717 # GH#58318 - extended to accept list, dict, and NamedAgg kwargs.
2719 if func is None:
2720 # Named-aggregation style:
2721 # .transform(val_sum=NamedAgg(column="val", aggfunc="sum"), ...)
2722 return self._transform_multiple_funcs(
2723 kwargs, *args, engine=engine, engine_kwargs=engine_kwargs
2724 )
2725 elif is_dict_like(func):
2726 # e.g. .transform({"val": "sum"}) or {"name": NamedAgg(...)}
2727 # Dict-of-lists is not yet supported.
2728 for val in func.values():
2729 if is_list_like(val):
2730 raise NotImplementedError(
2731 "Passing a dict of lists to DataFrameGroupBy.transform is "
2732 "not yet supported and is intended to be implemented in a "
2733 "future release. "
2734 "See https://github.com/pandas-dev/pandas/issues/58318"
2735 )
2736 return self._transform_multiple_funcs(
2737 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
2738 )
2739 elif is_list_like(func):
2740 # e.g. .transform(["sum", "min"])
2741 func = maybe_mangle_lambdas(func)
2742 return self._transform_multiple_funcs(
2743 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
2744 )
2745 else:
2746 return self._transform(
2747 func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
2748 )
2750 def _transform_multiple_funcs(
2751 self,
2752 func: list | dict,
2753 *args,
2754 engine: str | None = None,
2755 engine_kwargs: dict | None = None,
2756 **kwargs,
2757 ) -> DataFrame:
2758 """
2759 Handle list-like and dict dispatch for DataFrameGroupBy.transform.
2761 Parameters
2762 ----------
2763 func : list or dict
2764 - list of str/callable: applied to every non-key column, producing
2765 a MultiIndex-column DataFrame (column, func_name).
2766 - dict mapping output_name -> str/callable or NamedAgg.
2767 """
2768 from pandas.core.reshape.concat import concat
2770 if is_dict_like(func):
2771 # Also includes NamedAgg / NamedFunc
2772 func = cast("dict", func)
2773 results: list[Series] = []
2774 for name, agg in func.items():
2775 if isinstance(agg, NamedAgg):
2776 column_name = agg.column
2777 agg_func = agg.aggfunc
2778 elif isinstance(agg, tuple) and len(agg) == 2:
2779 # plain tuple (column, func) — same semantics as NamedAgg
2780 column_name = agg[0]
2781 agg_func = agg[1]
2782 else:
2783 # plain {"col": "sum"} — column IS the key
2784 column_name = name
2785 agg_func = agg
2786 result = self._transform_single_column(
2787 column_name,
2788 agg_func,
2789 *args,
2790 engine=engine,
2791 engine_kwargs=engine_kwargs,
2792 **kwargs,
2793 )
2794 result.name = name
2795 results.append(result)
2796 return concat(results, axis=1)
2798 # list path
2799 # Apply every func to every non-key column.
2800 assert is_list_like(func)
2801 results_list: list[Series] = []
2802 col_order: list[tuple] = []
2803 for column in self._obj_with_exclusions.columns:
2804 for agg_func in func:
2805 col_result = self._transform_single_column(
2806 column,
2807 agg_func,
2808 *args,
2809 engine=engine,
2810 engine_kwargs=engine_kwargs,
2811 **kwargs,
2812 )
2813 col_result.name = (column, agg_func)
2814 results_list.append(col_result)
2815 col_order.append((column, agg_func))
2817 # Use ignore_index=True then assign MultiIndex — avoids redundant work.
2818 output = concat(results_list, ignore_index=True, axis=1)
2819 arrays = [list(x) for x in zip(*col_order, strict=False)]
2820 output.columns = MultiIndex.from_arrays(arrays)
2821 return output
2823 def _transform_single_column(
2824 self,
2825 column_name: Hashable,
2826 agg_func: Callable | str,
2827 *args,
2828 engine: str | None = None,
2829 engine_kwargs: dict | None = None,
2830 **kwargs,
2831 ) -> Series:
2832 """
2833 Apply a single transform function to one column via SeriesGroupBy.
2835 Parameters
2836 ----------
2837 column_name : Hashable
2838 Column label to select from the grouped DataFrame.
2839 agg_func : callable or str
2840 Transform function to apply to the selected column.
2841 *args
2842 Positional arguments to pass to ``agg_func``.
2843 engine : str, default None
2844 Passed through to ``SeriesGroupBy.transform``.
2845 engine_kwargs : dict, default None
2846 Passed through to ``SeriesGroupBy.transform``.
2847 **kwargs
2848 Keyword arguments to pass to ``agg_func``.
2850 Returns
2851 -------
2852 Series
2853 Transformed Series with the same index as the original DataFrame.
2854 """
2855 data = self._gotitem(column_name, ndim=1)
2856 return data.transform(
2857 agg_func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
2858 )
2860 def _define_paths(self, func, *args, **kwargs):
2861 if isinstance(func, str):
2862 fast_path = lambda group: getattr(group, func)(*args, **kwargs)
2863 slow_path = lambda group: group.apply(
2864 lambda x: getattr(x, func)(*args, **kwargs), axis=0
2865 )
2866 else:
2867 fast_path = lambda group: func(group, *args, **kwargs)
2868 slow_path = lambda group: group.apply(
2869 lambda x: func(x, *args, **kwargs), axis=0
2870 )
2871 return fast_path, slow_path
2873 def _choose_path(self, fast_path: Callable, slow_path: Callable, group: DataFrame):
2874 path = slow_path
2875 res = slow_path(group)
2877 if self.ngroups == 1:
2878 # no need to evaluate multiple paths when only
2879 # a single group exists
2880 return path, res
2882 # if we make it here, test if we can use the fast path
2883 try:
2884 res_fast = fast_path(group)
2885 except AssertionError:
2886 raise # pragma: no cover
2887 except Exception:
2888 # GH#29631 For user-defined function, we can't predict what may be
2889 # raised; see test_transform.test_transform_fastpath_raises
2890 return path, res
2892 # verify fast path returns either:
2893 # a DataFrame with columns equal to group.columns
2894 # OR a Series with index equal to group.columns
2895 if isinstance(res_fast, DataFrame):
2896 if not res_fast.columns.equals(group.columns):
2897 return path, res
2898 elif isinstance(res_fast, Series):
2899 if not res_fast.index.equals(group.columns):
2900 return path, res
2901 else:
2902 return path, res
2904 if res_fast.equals(res):
2905 path = fast_path
2907 return path, res
2909 def filter(self, func, dropna: bool = True, *args, **kwargs) -> DataFrame:
2910 """
2911 Filter elements from groups that don't satisfy a criterion.
2913 Elements from groups are filtered if they do not satisfy the
2914 boolean criterion specified by func.
2916 Parameters
2917 ----------
2918 func : function
2919 Criterion to apply to each group. Should return True or False.
2920 dropna : bool
2921 Drop groups that do not pass the filter. True by default; if False,
2922 groups that evaluate False are filled with NaNs.
2923 *args : tuple
2924 Additional positional arguments to pass to `func`.
2925 **kwargs : dict
2926 Additional keyword arguments to pass to `func`.
2928 Returns
2929 -------
2930 DataFrame
2931 The filtered subset of the original DataFrame.
2933 See Also
2934 --------
2935 DataFrame.filter: Filter elements of ungrouped DataFrame.
2936 SeriesGroupBy.filter : Filter elements from groups based on criterion.
2938 Notes
2939 -----
2940 Each subframe is endowed the attribute 'name' in case you need to know
2941 which group you are working on.
2943 Functions that mutate the passed object can produce unexpected
2944 behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
2945 for more details.
2947 Examples
2948 --------
2949 >>> df = pd.DataFrame(
2950 ... {
2951 ... "A": ["foo", "bar", "foo", "bar", "foo", "bar"],
2952 ... "B": [1, 2, 3, 4, 5, 6],
2953 ... "C": [2.0, 5.0, 8.0, 1.0, 2.0, 9.0],
2954 ... }
2955 ... )
2956 >>> grouped = df.groupby("A")
2957 >>> grouped.filter(lambda x: x["B"].mean() > 3.0)
2958 A B C
2959 1 bar 2 5.0
2960 3 bar 4 1.0
2961 5 bar 6 9.0
2962 """
2963 indices = []
2965 obj = self._selected_obj
2966 gen = self._grouper.get_iterator(obj)
2968 for name, group in gen:
2969 # 2023-02-27 no tests are broken this pinning, but it is documented in the
2970 # docstring above.
2971 object.__setattr__(group, "name", name)
2973 res = func(group, *args, **kwargs)
2975 try:
2976 res = res.squeeze()
2977 except AttributeError: # allow e.g., scalars and frames to pass
2978 pass
2980 # interpret the result of the filter
2981 if is_bool(res) or (is_scalar(res) and isna(res)):
2982 if notna(res) and res:
2983 indices.append(self._get_index(name))
2984 else:
2985 # non scalars aren't allowed
2986 raise TypeError(
2987 f"filter function returned a {type(res).__name__}, "
2988 "but expected a scalar bool"
2989 )
2991 return self._apply_filter(indices, dropna)
2993 def __getitem__(self, key) -> DataFrameGroupBy | SeriesGroupBy:
2994 # per GH 23566
2995 if isinstance(key, tuple) and len(key) > 1:
2996 # if len == 1, then it becomes a SeriesGroupBy and this is actually
2997 # valid syntax, so don't raise
2998 raise ValueError(
2999 "Cannot subset columns with a tuple with more than one element. "
3000 "Use a list instead."
3001 )
3002 return super().__getitem__(key)
3004 def _gotitem(self, key, ndim: int, subset=None):
3005 """
3006 sub-classes to define
3007 return a sliced object
3009 Parameters
3010 ----------
3011 key : string / list of selections
3012 ndim : {1, 2}
3013 requested ndim of result
3014 subset : object, default None
3015 subset to act on
3016 """
3017 if ndim == 2:
3018 if subset is None:
3019 subset = self.obj
3020 return DataFrameGroupBy(
3021 subset,
3022 self.keys,
3023 level=self.level,
3024 grouper=self._grouper,
3025 exclusions=self.exclusions,
3026 selection=key,
3027 as_index=self.as_index,
3028 sort=self.sort,
3029 group_keys=self.group_keys,
3030 observed=self.observed,
3031 dropna=self.dropna,
3032 )
3033 elif ndim == 1:
3034 if subset is None:
3035 subset = self.obj[key]
3036 return SeriesGroupBy(
3037 subset,
3038 self.keys,
3039 level=self.level,
3040 grouper=self._grouper,
3041 exclusions=self.exclusions,
3042 selection=key,
3043 as_index=self.as_index,
3044 sort=self.sort,
3045 group_keys=self.group_keys,
3046 observed=self.observed,
3047 dropna=self.dropna,
3048 )
3050 raise AssertionError("invalid ndim for _gotitem")
3052 def _get_data_to_aggregate(
3053 self, *, numeric_only: bool = False, name: str | None = None
3054 ) -> BlockManager:
3055 obj = self._obj_with_exclusions
3056 mgr = obj._mgr
3057 if numeric_only:
3058 mgr = mgr.get_numeric_data()
3059 return mgr
3061 def _wrap_agged_manager(self, mgr: BlockManager) -> DataFrame:
3062 return self.obj._constructor_from_mgr(mgr, axes=mgr.axes)
3064 def _apply_to_column_groupbys(self, func) -> DataFrame:
3065 from pandas.core.reshape.concat import concat
3067 obj = self._obj_with_exclusions
3068 columns = obj.columns
3069 sgbs = (
3070 SeriesGroupBy(
3071 obj.iloc[:, i],
3072 selection=colname,
3073 grouper=self._grouper,
3074 exclusions=self.exclusions,
3075 observed=self.observed,
3076 )
3077 for i, colname in enumerate(obj.columns)
3078 )
3079 results = [func(sgb) for sgb in sgbs]
3081 if not results:
3082 # concat would raise
3083 res_df = DataFrame([], columns=columns, index=self._grouper.result_index)
3084 else:
3085 res_df = concat(results, keys=columns, axis=1)
3087 if not self.as_index:
3088 res_df.index = default_index(len(res_df))
3089 res_df = self._insert_inaxis_grouper(res_df)
3090 return res_df
3092 def nunique(self, dropna: bool = True) -> DataFrame:
3093 """
3094 Return DataFrame with counts of unique elements in each position.
3096 This method counts the number of distinct values for each column
3097 within each group, optionally excluding NaN values.
3099 Parameters
3100 ----------
3101 dropna : bool, default True
3102 Don't include NaN in the counts.
3104 Returns
3105 -------
3106 nunique: DataFrame
3107 Counts of unique elements in each position.
3109 See Also
3110 --------
3111 DataFrame.nunique : Count number of distinct elements in specified axis.
3113 Examples
3114 --------
3115 >>> df = pd.DataFrame(
3116 ... {
3117 ... "id": ["spam", "egg", "egg", "spam", "ham", "ham"],
3118 ... "value1": [1, 5, 5, 2, 5, 5],
3119 ... "value2": list("abbaxy"),
3120 ... }
3121 ... )
3122 >>> df
3123 id value1 value2
3124 0 spam 1 a
3125 1 egg 5 b
3126 2 egg 5 b
3127 3 spam 2 a
3128 4 ham 5 x
3129 5 ham 5 y
3131 >>> df.groupby("id").nunique()
3132 value1 value2
3133 id
3134 egg 1 1
3135 ham 1 2
3136 spam 2 1
3138 Check for rows with the same id but conflicting values:
3140 >>> df.groupby("id").filter(lambda g: (g.nunique() > 1).any())
3141 id value1 value2
3142 0 spam 1 a
3143 3 spam 2 a
3144 4 ham 5 x
3145 5 ham 5 y
3146 """
3147 return self._apply_to_column_groupbys(lambda sgb: sgb.nunique(dropna))
3149 def idxmax(
3150 self,
3151 skipna: bool = True,
3152 numeric_only: bool = False,
3153 ) -> DataFrame:
3154 """
3155 Return index of first occurrence of maximum in each group.
3157 For each group and each column, identifies the row label where the
3158 maximum value first occurs. NA values are excluded by default.
3160 Parameters
3161 ----------
3162 skipna : bool, default True
3163 Exclude NA values.
3164 numeric_only : bool, default False
3165 Include only `float`, `int` or `boolean` data.
3167 Returns
3168 -------
3169 DataFrame
3170 Indexes of maxima in each column according to the group.
3172 Raises
3173 ------
3174 ValueError
3175 When there are no valid values for a group. Then can happen if:
3177 * There is an unobserved group and ``observed=False``.
3178 * All values for a group are NA.
3179 * Some values for a group are NA and ``skipna=False``.
3181 .. versionchanged:: 3.0.0
3182 Previously if all values for a group are NA or some values for a group are
3183 NA and ``skipna=False``, this method would return NA. Now it raises instead.
3185 See Also
3186 --------
3187 Series.idxmax : Return index of the maximum element.
3188 DataFrame.idxmax : Indexes of maxima along the specified axis.
3190 Notes
3191 -----
3192 This method is the DataFrame version of ``ndarray.argmax``.
3194 Examples
3195 --------
3196 Consider a dataset containing food consumption in Argentina.
3198 >>> df = pd.DataFrame(
3199 ... {
3200 ... "consumption": [10.51, 103.11, 55.48],
3201 ... "co2_emissions": [37.2, 19.66, 1712],
3202 ... "food_type": ["meat", "plant", "meat"],
3203 ... },
3204 ... index=["Pork", "Wheat Products", "Beef"],
3205 ... )
3207 >>> df
3208 consumption co2_emissions food_type
3209 Pork 10.51 37.20 meat
3210 Wheat Products 103.11 19.66 plant
3211 Beef 55.48 1712.00 meat
3213 By default, it returns the index for the maximum value in each column
3214 according to the group.
3216 >>> df.groupby("food_type").idxmax()
3217 consumption co2_emissions
3218 food_type
3219 meat Beef Beef
3220 plant Wheat Products Wheat Products
3221 """
3222 return self._idxmax_idxmin("idxmax", numeric_only=numeric_only, skipna=skipna)
3224 def idxmin(
3225 self,
3226 skipna: bool = True,
3227 numeric_only: bool = False,
3228 ) -> DataFrame:
3229 """
3230 Return index of first occurrence of minimum in each group.
3232 For each group and each column, identifies the row label where the
3233 minimum value first occurs. NA values are excluded by default.
3235 Parameters
3236 ----------
3237 skipna : bool, default True
3238 Exclude NA values.
3239 numeric_only : bool, default False
3240 Include only `float`, `int` or `boolean` data.
3242 Returns
3243 -------
3244 DataFrame
3245 Indexes of minima in each column according to the group.
3247 Raises
3248 ------
3249 ValueError
3250 When there are no valid values for a group. Then can happen if:
3252 * There is an unobserved group and ``observed=False``.
3253 * All values for a group are NA.
3254 * Some values for a group are NA and ``skipna=False``.
3256 .. versionchanged:: 3.0.0
3257 Previously if all values for a group are NA or some values for a group are
3258 NA and ``skipna=False``, this method would return NA. Now it raises instead.
3260 See Also
3261 --------
3262 Series.idxmin : Return index of the minimum element.
3263 DataFrame.idxmin : Indexes of minima along the specified axis.
3265 Notes
3266 -----
3267 This method is the DataFrame version of ``ndarray.argmin``.
3269 Examples
3270 --------
3271 Consider a dataset containing food consumption in Argentina.
3273 >>> df = pd.DataFrame(
3274 ... {
3275 ... "consumption": [10.51, 103.11, 55.48],
3276 ... "co2_emissions": [37.2, 19.66, 1712],
3277 ... "food_type": ["meat", "plant", "meat"],
3278 ... },
3279 ... index=["Pork", "Wheat Products", "Beef"],
3280 ... )
3282 >>> df
3283 consumption co2_emissions food_type
3284 Pork 10.51 37.20 meat
3285 Wheat Products 103.11 19.66 plant
3286 Beef 55.48 1712.00 meat
3288 By default, it returns the index for the minimum value in each column
3289 according to the group.
3291 >>> df.groupby("food_type").idxmin()
3292 consumption co2_emissions
3293 food_type
3294 meat Pork Pork
3295 plant Wheat Products Wheat Products
3296 """
3297 return self._idxmax_idxmin("idxmin", numeric_only=numeric_only, skipna=skipna)
3299 boxplot = boxplot_frame_groupby
3301 def describe(
3302 self,
3303 percentiles=None,
3304 include=None,
3305 exclude=None,
3306 ) -> DataFrame:
3307 """
3308 Generate descriptive statistics for each group.
3310 Within each group, summarize the central tendency, dispersion,
3311 and shape of each analyzed column's distribution, excluding
3312 ``NaN`` values. By default only numeric columns are analyzed;
3313 pass ``include`` to also analyze non-numeric columns (or
3314 ``exclude`` to omit columns by dtype).
3316 Parameters
3317 ----------
3318 percentiles : list-like of numbers, optional
3319 The percentiles to include in the output. All should fall
3320 between 0 and 1. The default, ``None``, returns the 25th,
3321 50th, and 75th percentiles.
3322 include : 'all', list-like of dtypes or None (default), optional
3323 Which column dtypes to include. Options:
3325 - ``'all'`` : Include all columns, including non-numeric ones.
3326 - list-like of dtypes : Limit the result to columns of the
3327 given dtypes, in the style of
3328 :meth:`DataFrame.select_dtypes` (e.g. ``include=[np.number]``
3329 or ``include=["category"]``).
3330 - ``None`` (default) : Include only numeric columns, falling
3331 back to object and categorical columns if there are no
3332 numeric columns.
3333 exclude : list-like of dtypes or None (default), optional
3334 Column dtypes to omit from the result, in the style of
3335 :meth:`DataFrame.select_dtypes`. ``None`` (default) excludes
3336 nothing.
3338 Returns
3339 -------
3340 DataFrame
3341 One row per group. The columns form a MultiIndex whose
3342 outer level is the analyzed column and whose inner level is
3343 the statistic name.
3345 See Also
3346 --------
3347 DataFrame.describe : Generate descriptive statistics of a DataFrame.
3348 SeriesGroupBy.describe : Generate descriptive statistics for each
3349 group of a Series.
3350 DataFrame.select_dtypes : Subset of a DataFrame including/excluding
3351 columns based on their dtype.
3353 Notes
3354 -----
3355 For numeric columns, the per-group statistics are ``count``,
3356 ``mean``, ``std``, ``min``, ``max``, and the requested
3357 percentiles. By default the lower percentile is ``25`` and the
3358 upper is ``75``; the ``50`` percentile is the same as the median.
3360 For object columns, the per-group statistics are ``count``,
3361 ``unique``, ``top``, and ``freq``. The ``top`` is the most common
3362 value within the group and ``freq`` is its count.
3364 Examples
3365 --------
3366 >>> df = pd.DataFrame({"group": ["a", "a", "b", "b"], "value": [1, 2, 3, 4]})
3367 >>> df.groupby("group").describe()
3368 value
3369 count mean std min 25% 50% 75% max
3370 group
3371 a 2.0 1.5 0.707107 1.0 1.25 1.5 1.75 2.0
3372 b 2.0 3.5 0.707107 3.0 3.25 3.5 3.75 4.0
3373 """
3374 return super().describe(
3375 percentiles=percentiles, include=include, exclude=exclude
3376 )
3378 def value_counts(
3379 self,
3380 subset: Sequence[Hashable] | None = None,
3381 normalize: bool = False,
3382 sort: bool = True,
3383 ascending: bool = False,
3384 dropna: bool = True,
3385 ) -> DataFrame | Series:
3386 """
3387 Return a Series or DataFrame containing counts of unique rows.
3389 The resulting object will be in descending order so that the
3390 first element in each group is the most frequently-occurring row.
3392 Parameters
3393 ----------
3394 subset : list-like, optional
3395 Columns to use when counting unique combinations.
3396 normalize : bool, default False
3397 Return proportions rather than frequencies.
3398 sort : bool, default True
3399 Stable sort by frequencies when True. When False, non-grouping
3400 columns will appear in the order they occur in within groups.
3402 .. versionchanged:: 3.0.0
3404 In prior versions, ``sort=False`` would sort the non-grouping columns
3405 by label.
3406 ascending : bool, default False
3407 Sort in ascending order.
3408 dropna : bool, default True
3409 Don't include counts of rows that contain NA values.
3411 Returns
3412 -------
3413 Series or DataFrame
3414 Series if the groupby ``as_index`` is True, otherwise DataFrame.
3416 See Also
3417 --------
3418 Series.value_counts: Equivalent method on Series.
3419 DataFrame.value_counts: Equivalent method on DataFrame.
3420 SeriesGroupBy.value_counts: Equivalent method on SeriesGroupBy.
3422 Notes
3423 -----
3424 - If the groupby ``as_index`` is True then the returned Series will have a
3425 MultiIndex with one level per input column.
3426 - If the groupby ``as_index`` is False then the returned DataFrame will have an
3427 additional column with the value_counts. The column is labelled 'count' or
3428 'proportion', depending on the ``normalize`` parameter.
3430 By default, rows that contain any NA values are omitted from
3431 the result.
3433 By default, the result will be in descending order so that the
3434 first element of each group is the most frequently-occurring row.
3436 Examples
3437 --------
3438 >>> df = pd.DataFrame(
3439 ... {
3440 ... "gender": ["male", "male", "female", "male", "female", "male"],
3441 ... "education": ["low", "medium", "high", "low", "high", "low"],
3442 ... "country": ["US", "FR", "US", "FR", "FR", "FR"],
3443 ... }
3444 ... )
3446 >>> df
3447 gender education country
3448 0 male low US
3449 1 male medium FR
3450 2 female high US
3451 3 male low FR
3452 4 female high FR
3453 5 male low FR
3455 >>> df.groupby("gender").value_counts()
3456 gender education country
3457 female high US 1
3458 FR 1
3459 male low FR 2
3460 US 1
3461 medium FR 1
3462 Name: count, dtype: int64
3464 >>> df.groupby("gender").value_counts(ascending=True)
3465 gender education country
3466 female high US 1
3467 FR 1
3468 male low US 1
3469 medium FR 1
3470 low FR 2
3471 Name: count, dtype: int64
3473 >>> df.groupby("gender").value_counts(normalize=True)
3474 gender education country
3475 female high US 0.50
3476 FR 0.50
3477 male low FR 0.50
3478 US 0.25
3479 medium FR 0.25
3480 Name: proportion, dtype: float64
3482 >>> df.groupby("gender", as_index=False).value_counts()
3483 gender education country count
3484 0 female high US 1
3485 1 female high FR 1
3486 2 male low FR 2
3487 3 male low US 1
3488 4 male medium FR 1
3490 >>> df.groupby("gender", as_index=False).value_counts(normalize=True)
3491 gender education country proportion
3492 0 female high US 0.50
3493 1 female high FR 0.50
3494 2 male low FR 0.50
3495 3 male low US 0.25
3496 4 male medium FR 0.25
3497 """
3498 return self._value_counts(subset, normalize, sort, ascending, dropna)
3500 def take(
3501 self,
3502 indices: TakeIndexer,
3503 **kwargs,
3504 ) -> DataFrame:
3505 """
3506 Return the elements in the given *positional* indices in each group.
3508 This means that we are not indexing according to actual values in
3509 the index attribute of the object. We are indexing according to the
3510 actual position of the element in the object.
3512 If a requested index does not exist for some group, this method will raise.
3513 To get similar behavior that ignores indices that don't exist, see
3514 :meth:`.DataFrameGroupBy.nth`.
3516 Parameters
3517 ----------
3518 indices : array-like
3519 An array of ints indicating which positions to take.
3521 **kwargs
3522 For compatibility with :meth:`numpy.take`. Has no effect on the
3523 output.
3525 .. deprecated:: 3.1.0
3526 Passing ``**kwargs`` to DataFrameGroupBy.take is deprecated
3527 and will be removed in a future version of pandas.
3529 Returns
3530 -------
3531 DataFrame
3532 A DataFrame containing the elements taken from each group.
3534 See Also
3535 --------
3536 DataFrame.take : Take elements from a Series along an axis.
3537 DataFrame.loc : Select a subset of a DataFrame by labels.
3538 DataFrame.iloc : Select a subset of a DataFrame by positions.
3539 numpy.take : Take elements from an array along an axis.
3541 Examples
3542 --------
3543 >>> df = pd.DataFrame(
3544 ... [
3545 ... ("falcon", "bird", 389.0),
3546 ... ("parrot", "bird", 24.0),
3547 ... ("lion", "mammal", 80.5),
3548 ... ("monkey", "mammal", np.nan),
3549 ... ("rabbit", "mammal", 15.0),
3550 ... ],
3551 ... columns=["name", "class", "max_speed"],
3552 ... index=[4, 3, 2, 1, 0],
3553 ... )
3554 >>> df
3555 name class max_speed
3556 4 falcon bird 389.0
3557 3 parrot bird 24.0
3558 2 lion mammal 80.5
3559 1 monkey mammal NaN
3560 0 rabbit mammal 15.0
3561 >>> gb = df.groupby([1, 1, 2, 2, 2])
3563 Take elements at rows 0 and 1.
3565 Note how the indices selected in the result do not correspond to
3566 our input indices 0 and 1. That's because we are selecting the 0th
3567 and 1st rows, not rows whose indices equal 0 and 1.
3569 >>> gb.take([0, 1])
3570 name class max_speed
3571 1 4 falcon bird 389.0
3572 3 parrot bird 24.0
3573 2 2 lion mammal 80.5
3574 1 monkey mammal NaN
3576 The order of the specified indices influences the order in the result.
3577 Here, the order is swapped from the previous example.
3579 >>> gb.take([1, 0])
3580 name class max_speed
3581 1 3 parrot bird 24.0
3582 4 falcon bird 389.0
3583 2 1 monkey mammal NaN
3584 2 lion mammal 80.5
3586 We may take elements using negative integers for positive indices,
3587 starting from the end of the object, just like with Python lists.
3589 >>> gb.take([-1, -2])
3590 name class max_speed
3591 1 3 parrot bird 24.0
3592 4 falcon bird 389.0
3593 2 0 rabbit mammal 15.0
3594 1 monkey mammal NaN
3595 """
3596 if kwargs:
3597 warnings.warn(
3598 "Passing additional arguments to DataFrameGroupBy.take is "
3599 "deprecated and will be removed in a future version of pandas.",
3600 Pandas4Warning,
3601 stacklevel=find_stack_level(),
3602 )
3603 result = self._op_via_apply("take", indices=indices, **kwargs)
3604 return result
3606 def skew(
3607 self,
3608 skipna: bool = True,
3609 numeric_only: bool = False,
3610 **kwargs,
3611 ) -> DataFrame:
3612 """
3613 Return unbiased skew within groups.
3615 Normalized by N-1.
3617 Parameters
3618 ----------
3619 skipna : bool, default True
3620 Exclude NA/null values when computing the result.
3622 numeric_only : bool, default False
3623 Include only float, int, boolean columns.
3625 **kwargs
3626 Additional keyword arguments to be passed to the function.
3628 .. deprecated:: 3.1.0
3629 Passing ``**kwargs`` to DataFrameGroupBy.skew is deprecated
3630 and will be removed in a future version of pandas.
3632 Returns
3633 -------
3634 DataFrame
3635 Unbiased skew within groups.
3637 See Also
3638 --------
3639 DataFrame.skew : Return unbiased skew over requested axis.
3641 Examples
3642 --------
3643 >>> arrays = [
3644 ... ["falcon", "parrot", "cockatoo", "kiwi", "lion", "monkey", "rabbit"],
3645 ... ["bird", "bird", "bird", "bird", "mammal", "mammal", "mammal"],
3646 ... ]
3647 >>> index = pd.MultiIndex.from_arrays(arrays, names=("name", "class"))
3648 >>> df = pd.DataFrame(
3649 ... {"max_speed": [389.0, 24.0, 70.0, np.nan, 80.5, 21.5, 15.0]},
3650 ... index=index,
3651 ... )
3652 >>> df
3653 max_speed
3654 name class
3655 falcon bird 389.0
3656 parrot bird 24.0
3657 cockatoo bird 70.0
3658 kiwi bird NaN
3659 lion mammal 80.5
3660 monkey mammal 21.5
3661 rabbit mammal 15.0
3662 >>> gb = df.groupby(["class"])
3663 >>> gb.skew()
3664 max_speed
3665 class
3666 bird 1.628296
3667 mammal 1.669046
3668 >>> gb.skew(skipna=False)
3669 max_speed
3670 class
3671 bird NaN
3672 mammal 1.669046
3673 """
3674 if kwargs:
3675 warnings.warn(
3676 "Passing additional arguments to DataFrameGroupBy.skew is "
3677 "deprecated and will be removed in a future version of pandas.",
3678 Pandas4Warning,
3679 stacklevel=find_stack_level(),
3680 )
3682 def alt(obj):
3683 # This should not be reached since the cython path should raise
3684 # TypeError and not NotImplementedError.
3685 raise TypeError(f"'skew' is not supported for dtype={obj.dtype}")
3687 return self._cython_agg_general(
3688 "skew", alt=alt, skipna=skipna, numeric_only=numeric_only, **kwargs
3689 )
3691 def kurt(
3692 self,
3693 skipna: bool = True,
3694 numeric_only: bool = False,
3695 **kwargs,
3696 ) -> DataFrame:
3697 """
3698 Return unbiased kurtosis within groups.
3700 Kurtosis obtained using Fisher's definition (kurtosis of normal == 0.0),
3701 normalized by N-1. Values are computed for each numeric column
3702 within each group.
3704 Parameters
3705 ----------
3706 skipna : bool, default True
3707 Exclude NA/null values when computing the result.
3709 numeric_only : bool, default False
3710 Include only float, int, boolean columns.
3712 **kwargs
3713 Additional keyword arguments to be passed to the function.
3715 Returns
3716 -------
3717 DataFrame
3718 Unbiased kurtosis within groups.
3720 See Also
3721 --------
3722 DataFrame.kurt : Return unbiased kurtosis over requested axis.
3724 Examples
3725 --------
3726 >>> arrays = [
3727 ... [
3728 ... "falcon",
3729 ... "parrot",
3730 ... "cockatoo",
3731 ... "kiwi",
3732 ... "eagle",
3733 ... "lion",
3734 ... "monkey",
3735 ... "rabbit",
3736 ... "dog",
3737 ... "wolf",
3738 ... ],
3739 ... [
3740 ... "bird",
3741 ... "bird",
3742 ... "bird",
3743 ... "bird",
3744 ... "bird",
3745 ... "mammal",
3746 ... "mammal",
3747 ... "mammal",
3748 ... "mammal",
3749 ... "mammal",
3750 ... ],
3751 ... ]
3752 >>> index = pd.MultiIndex.from_arrays(arrays, names=("name", "class"))
3753 >>> df = pd.DataFrame(
3754 ... {
3755 ... "max_speed": [
3756 ... 389.0,
3757 ... 24.0,
3758 ... 70.0,
3759 ... np.nan,
3760 ... 350.0,
3761 ... 80.5,
3762 ... 21.5,
3763 ... 15.0,
3764 ... 40.0,
3765 ... 50.0,
3766 ... ]
3767 ... },
3768 ... index=index,
3769 ... )
3770 >>> df
3771 max_speed
3772 name class
3773 falcon bird 389.0
3774 parrot bird 24.0
3775 cockatoo bird 70.0
3776 kiwi bird NaN
3777 eagle bird 350.0
3778 lion mammal 80.5
3779 monkey mammal 21.5
3780 rabbit mammal 15.0
3781 dog mammal 40.0
3782 wolf mammal 50.0
3783 >>> gb = df.groupby(["class"])
3784 >>> gb.kurt()
3785 max_speed
3786 class
3787 bird -5.493277
3788 mammal 0.204125
3789 >>> gb.kurt(skipna=False)
3790 max_speed
3791 class
3792 bird NaN
3793 mammal 0.204125
3794 """
3796 return self._cython_agg_general(
3797 "kurt", alt=None, skipna=skipna, numeric_only=numeric_only, **kwargs
3798 )
3800 @property
3801 def plot(self) -> GroupByPlot:
3802 """
3803 Make plots of groups from a DataFrame.
3805 Uses the backend specified by the option ``plotting.backend``.
3806 By default, matplotlib is used.
3808 Returns
3809 -------
3810 GroupByPlot
3811 A plotting object that can be used to create plots for each group.
3813 See Also
3814 --------
3815 DataFrame.plot : Make plots of DataFrame.
3817 Examples
3818 --------
3819 >>> df = pd.DataFrame(
3820 ... {"A": [1, 2, 3, 4], "B": [5, 6, 7, 8]}, index=["a", "a", "b", "b"]
3821 ... )
3822 >>> g = df.groupby(level=0)
3823 >>> g.plot() # doctest: +SKIP
3824 """
3825 result = GroupByPlot(self)
3826 return result
3828 def corr(
3829 self,
3830 method: str | Callable[[np.ndarray, np.ndarray], float] = "pearson",
3831 min_periods: int = 1,
3832 numeric_only: bool = False,
3833 ) -> DataFrame:
3834 """
3835 Compute pairwise correlation of columns, excluding NA/null values.
3837 Computes a correlation matrix for each group, measuring the linear
3838 or rank-based relationship between columns.
3840 Parameters
3841 ----------
3842 method : {'pearson', 'kendall', 'spearman'} or callable
3843 Method of correlation:
3845 * pearson : standard correlation coefficient
3846 * kendall : Kendall Tau correlation coefficient
3847 * spearman : Spearman rank correlation
3848 * callable: callable with input two 1d ndarrays
3849 and returning a float. Note that the returned matrix from corr
3850 will have 1 along the diagonals and will be symmetric
3851 regardless of the callable's behavior.
3852 min_periods : int, optional
3853 Minimum number of observations required per pair of columns
3854 to have a valid result. Currently only available for Pearson
3855 and Spearman correlation.
3856 numeric_only : bool, default False
3857 Include only `float`, `int` or `boolean` data.
3859 .. versionchanged:: 2.0.0
3860 The default value of ``numeric_only`` is now ``False``.
3862 Returns
3863 -------
3864 DataFrame
3865 Correlation matrix.
3867 See Also
3868 --------
3869 DataFrame.corrwith : Compute pairwise correlation with another
3870 DataFrame or Series.
3871 Series.corr : Compute the correlation between two Series.
3873 Notes
3874 -----
3875 Pearson, Kendall and Spearman correlation are currently computed using
3876 pairwise complete observations.
3878 * `Pearson correlation coefficient <https://en.wikipedia.org/wiki/Pearson_correlation_coefficient>`_
3879 * `Kendall rank correlation coefficient <https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient>`_
3880 * `Spearman's rank correlation coefficient <https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient>`_
3882 Examples
3883 --------
3884 >>> df = pd.DataFrame(
3885 ... {
3886 ... "age": [2, 3, 4, 6, 6, 1, 2, 1],
3887 ... "weight": [2.1, 3.2, 4.1, 6.5, 3.3, 2.1, 4.1, 1.9],
3888 ... "pet": ["dog", "cat", "dog", "cat", "dog", "cat", "dog", "cat"],
3889 ... }
3890 ... )
3891 >>> df
3892 age weight pet
3893 0 2 2.1 dog
3894 1 3 3.2 cat
3895 2 4 4.1 dog
3896 3 6 6.5 cat
3897 4 6 3.3 dog
3898 5 1 2.1 cat
3899 6 2 4.1 dog
3900 7 1 1.9 cat
3901 >>> df.groupby("pet").corr()
3902 age weight
3903 pet
3904 cat age 1.000000 0.989321
3905 weight 0.989321 1.000000
3906 dog age 1.000000 0.184177
3907 weight 0.184177 1.000000
3908 """
3909 result = self._op_via_apply(
3910 "corr", method=method, min_periods=min_periods, numeric_only=numeric_only
3911 )
3912 return result
3914 def cov(
3915 self,
3916 min_periods: int | None = None,
3917 ddof: int | None = 1,
3918 numeric_only: bool = False,
3919 ) -> DataFrame:
3920 """
3921 Compute pairwise covariance of columns, excluding NA/null values.
3923 Compute the pairwise covariance among the series of a DataFrame.
3924 The returned data frame is the `covariance matrix
3925 <https://en.wikipedia.org/wiki/Covariance_matrix>`__ of the columns
3926 of the DataFrame.
3928 Both NA and null values are automatically excluded from the
3929 calculation. (See the note below about bias from missing values.)
3930 A threshold can be set for the minimum number of
3931 observations for each value created. Comparisons with observations
3932 below this threshold will be returned as ``NaN``.
3934 This method is generally used for the analysis of time series data to
3935 understand the relationship between different measures
3936 across time.
3938 Parameters
3939 ----------
3940 min_periods : int, optional
3941 Minimum number of observations required per pair of columns
3942 to have a valid result.
3944 ddof : int, default 1
3945 Delta degrees of freedom. The divisor used in calculations
3946 is ``N - ddof``, where ``N`` represents the number of elements.
3947 This argument is applicable only when no ``nan`` is in the dataframe.
3949 numeric_only : bool, default False
3950 Include only `float`, `int` or `boolean` data.
3952 .. versionchanged:: 2.0.0
3953 The default value of ``numeric_only`` is now ``False``.
3955 Returns
3956 -------
3957 DataFrame
3958 The covariance matrix of the series of the DataFrame.
3960 See Also
3961 --------
3962 Series.cov : Compute covariance with another Series.
3963 core.window.ewm.ExponentialMovingWindow.cov : Exponential weighted sample
3964 covariance.
3965 core.window.expanding.Expanding.cov : Expanding sample covariance.
3966 core.window.rolling.Rolling.cov : Rolling sample covariance.
3968 Notes
3969 -----
3970 Returns the covariance matrix of the DataFrame's time series.
3971 The covariance is normalized by N-ddof.
3973 For DataFrames that have Series that are missing data (assuming that
3974 data is `missing at random
3975 <https://en.wikipedia.org/wiki/Missing_data#Missing_at_random>`__)
3976 the returned covariance matrix will be an unbiased estimate
3977 of the variance and covariance between the member Series.
3979 However, for many applications this estimate may not be acceptable
3980 because the estimate covariance matrix is not guaranteed to be positive
3981 semi-definite. This could lead to estimate correlations having
3982 absolute values which are greater than one, and/or a non-invertible
3983 covariance matrix. See `Estimation of covariance matrices
3984 <https://en.wikipedia.org/w/index.php?title=Estimation_of_covariance_
3985 matrices>`__ for more details.
3987 Examples
3988 --------
3989 >>> df = pd.DataFrame(
3990 ... {
3991 ... "age": [2, 3, 4, 6, 6, 1, 2, 1],
3992 ... "weight": [2.1, 3.2, 4.1, 6.5, 3.3, 2.1, 4.1, 1.9],
3993 ... "pet": ["dog", "cat", "dog", "cat", "dog", "cat", "dog", "cat"],
3994 ... }
3995 ... )
3996 >>> df
3997 age weight pet
3998 0 2 2.1 dog
3999 1 3 3.2 cat
4000 2 4 4.1 dog
4001 3 6 6.5 cat
4002 4 6 3.3 dog
4003 5 1 2.1 cat
4004 6 2 4.1 dog
4005 7 1 1.9 cat
4006 >>> df.groupby("pet").cov()
4007 age weight
4008 pet
4009 cat age 5.583333 4.975000
4010 weight 4.975000 4.529167
4011 dog age 3.666667 0.333333
4012 weight 0.333333 0.893333
4013 """
4014 result = self._op_via_apply(
4015 "cov", min_periods=min_periods, ddof=ddof, numeric_only=numeric_only
4016 )
4017 return result
4019 def hist(
4020 self,
4021 column: IndexLabel | None = None,
4022 by=None,
4023 grid: bool = True,
4024 xlabelsize: int | None = None,
4025 xrot: float | None = None,
4026 ylabelsize: int | None = None,
4027 yrot: float | None = None,
4028 ax=None,
4029 sharex: bool = False,
4030 sharey: bool = False,
4031 figsize: tuple[float, float] | None = None,
4032 layout: tuple[int, int] | None = None,
4033 bins: int | Sequence[int] = 10,
4034 backend: str | None = None,
4035 legend: bool = False,
4036 **kwargs,
4037 ):
4038 """
4039 Draw histogram of the DataFrame's columns for each group.
4041 A separate histogram subplot is generated for each group, making it
4042 easier to visually compare the distribution of each numeric column
4043 across groups. Internally this calls :meth:`DataFrame.hist` on every
4044 group's frame, so the same matplotlib options are accepted.
4046 Parameters
4047 ----------
4048 column : str or sequence, optional
4049 If passed, will be used to limit data to a subset of columns.
4050 by : object, optional
4051 If passed, then used to form histograms for separate groups.
4052 grid : bool, default True
4053 Whether to show axis grid lines.
4054 xlabelsize : int, default None
4055 If specified changes the x-axis label size.
4056 xrot : float, default None
4057 Rotation of x axis labels. For example, a value of 90 displays the
4058 x labels rotated 90 degrees clockwise.
4059 ylabelsize : int, default None
4060 If specified changes the y-axis label size.
4061 yrot : float, default None
4062 Rotation of y axis labels. For example, a value of 90 displays the
4063 y labels rotated 90 degrees clockwise.
4064 ax : Matplotlib axes object, default None
4065 The axes to plot the histogram on.
4066 sharex : bool, default True if ax is None else False
4067 In case subplots=True, share x axis and set some x axis labels to
4068 invisible; defaults to True if ax is None otherwise False if an ax
4069 is passed in.
4070 Note that passing in both an ax and sharex=True will alter all x axis
4071 labels for all subplots in a figure.
4072 sharey : bool, default False
4073 In case subplots=True, share y axis and set some y axis labels to
4074 invisible.
4075 figsize : tuple, optional
4076 The size in inches of the figure to create. Uses the value in
4077 `matplotlib.rcParams` by default.
4078 layout : tuple, optional
4079 Tuple of (rows, columns) for the layout of the histograms.
4080 bins : int or sequence, default 10
4081 Number of histogram bins to be used. If an integer is given, bins + 1
4082 bin edges are calculated and returned. If bins is a sequence, gives
4083 bin edges, including left edge of first bin and right edge of last
4084 bin. In this case, bins is returned unmodified.
4086 backend : str, default None
4087 Backend to use instead of the backend specified in the option
4088 ``plotting.backend``. For instance, 'matplotlib'. Alternatively, to
4089 specify the ``plotting.backend`` for the whole session, set
4090 ``pd.options.plotting.backend``.
4092 legend : bool, default False
4093 Whether to show the legend.
4095 **kwargs
4096 All other plotting keyword arguments to be passed to
4097 :meth:`matplotlib.pyplot.hist`.
4099 Returns
4100 -------
4101 matplotlib.Axes or numpy.ndarray
4102 A ``matplotlib.Axes`` object or an array of ``Axes`` objects, depending on
4103 the layout and grouping.
4105 See Also
4106 --------
4107 DataFrame.hist : Equivalent histogram plotting method on DataFrame.
4108 matplotlib.pyplot.hist : Plot a histogram using matplotlib.
4110 Examples
4111 --------
4112 For each animal category, draw a histogram of the numeric columns.
4113 The grouping column is used to split the data; one set of histogram
4114 subplots is produced per group.
4116 .. plot::
4117 :context: close-figs
4119 >>> df = pd.DataFrame(
4120 ... {
4121 ... "animal": ["cat", "cat", "dog", "dog", "dog"],
4122 ... "length": [1.0, 1.2, 1.5, 1.7, 2.0],
4123 ... "weight": [4.5, 5.0, 10.0, 12.5, 15.0],
4124 ... }
4125 ... )
4126 >>> hist = df.groupby("animal").hist(bins=3)
4127 """
4128 result = self._op_via_apply(
4129 "hist",
4130 column=column,
4131 by=by,
4132 grid=grid,
4133 xlabelsize=xlabelsize,
4134 xrot=xrot,
4135 ylabelsize=ylabelsize,
4136 yrot=yrot,
4137 ax=ax,
4138 sharex=sharex,
4139 sharey=sharey,
4140 figsize=figsize,
4141 layout=layout,
4142 bins=bins,
4143 backend=backend,
4144 legend=legend,
4145 **kwargs,
4146 )
4147 return result
4149 def corrwith(
4150 self,
4151 other: DataFrame | Series,
4152 drop: bool = False,
4153 method: CorrelationMethod = "pearson",
4154 numeric_only: bool = False,
4155 ) -> DataFrame:
4156 """
4157 Compute pairwise correlation.
4159 .. deprecated:: 3.0.0
4161 Pairwise correlation is computed between rows or columns of
4162 DataFrame with rows or columns of Series or DataFrame. DataFrames
4163 are first aligned along both axes before computing the
4164 correlations.
4166 Parameters
4167 ----------
4168 other : DataFrame, Series
4169 Object with which to compute correlations.
4170 drop : bool, default False
4171 Drop missing indices from result.
4172 method : {'pearson', 'kendall', 'spearman'} or callable
4173 Method of correlation:
4175 * pearson : standard correlation coefficient
4176 * kendall : Kendall Tau correlation coefficient
4177 * spearman : Spearman rank correlation
4178 * callable: callable with input two 1d ndarrays
4179 and returning a float.
4181 numeric_only : bool, default False
4182 Include only `float`, `int` or `boolean` data.
4184 .. versionchanged:: 2.0.0
4185 The default value of ``numeric_only`` is now ``False``.
4187 Returns
4188 -------
4189 Series
4190 Pairwise correlations.
4192 See Also
4193 --------
4194 DataFrame.corr : Compute pairwise correlation of columns.
4196 Examples
4197 --------
4198 >>> df1 = pd.DataFrame(
4199 ... {
4200 ... "Day": [1, 1, 1, 2, 2, 2, 3, 3, 3],
4201 ... "Data": [6, 6, 8, 5, 4, 2, 7, 3, 9],
4202 ... }
4203 ... )
4204 >>> df2 = pd.DataFrame(
4205 ... {
4206 ... "Day": [1, 1, 1, 2, 2, 2, 3, 3, 3],
4207 ... "Data": [5, 3, 8, 3, 1, 1, 2, 3, 6],
4208 ... }
4209 ... )
4211 >>> df1.groupby("Day").corrwith(df2)
4212 Data Day
4213 Day
4214 1 0.917663 NaN
4215 2 0.755929 NaN
4216 3 0.576557 NaN
4217 """
4218 warnings.warn(
4219 "DataFrameGroupBy.corrwith is deprecated",
4220 Pandas4Warning,
4221 stacklevel=find_stack_level(),
4222 )
4223 result = self._op_via_apply(
4224 "corrwith",
4225 other=other,
4226 drop=drop,
4227 method=method,
4228 numeric_only=numeric_only,
4229 )
4230 return result
4233def _wrap_transform_general_frame(
4234 obj: DataFrame, group: DataFrame, res: DataFrame | Series
4235) -> DataFrame:
4236 from pandas import concat
4238 if isinstance(res, Series):
4239 # we need to broadcast across the
4240 # other dimension; this will preserve dtypes
4241 # GH14457
4242 if res.index.is_(obj.index):
4243 res_frame = concat([res] * len(group.columns), axis=1, ignore_index=True)
4244 res_frame.columns = group.columns
4245 res_frame.index = group.index
4246 else:
4247 res_frame = obj._constructor(
4248 np.tile(res.values, (len(group.index), 1)),
4249 columns=group.columns,
4250 index=group.index,
4251 )
4252 assert isinstance(res_frame, DataFrame)
4253 return res_frame
4254 elif isinstance(res, DataFrame) and not res.index.is_(group.index):
4255 return res._align_frame(group)[0]
4256 else:
4257 return res