Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/apply.py: 17%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import abc
4from collections import defaultdict
5import functools
6from functools import partial
7import inspect
8from typing import (
9 TYPE_CHECKING,
10 Any,
11 Literal,
12 TypeAlias,
13 cast,
14)
16import numpy as np
18from pandas._libs.internals import BlockValuesRefs
19from pandas.compat._optional import import_optional_dependency
20from pandas.errors import SpecificationError
21from pandas.util._decorators import (
22 cache_readonly,
23 set_module,
24)
26from pandas.core.dtypes.cast import (
27 find_common_type,
28 is_nested_object,
29)
30from pandas.core.dtypes.common import (
31 is_dict_like,
32 is_extension_array_dtype,
33 is_list_like,
34 is_numeric_dtype,
35 is_sequence,
36)
37from pandas.core.dtypes.dtypes import (
38 ArrowDtype,
39 BaseMaskedDtype,
40 ExtensionDtype,
41)
42from pandas.core.dtypes.generic import (
43 ABCDataFrame,
44 ABCNDFrame,
45 ABCSeries,
46)
48from pandas.core._numba.executor import generate_apply_looper
49import pandas.core.common as com
50from pandas.core.construction import ensure_wrapped_if_datetimelike
51from pandas.core.reshape.concat import concat
52from pandas.core.util.numba_ import (
53 get_jit_arguments,
54 prepare_function_arguments,
55)
57if TYPE_CHECKING:
58 from collections.abc import (
59 Callable,
60 Generator,
61 Hashable,
62 Iterable,
63 MutableMapping,
64 Sequence,
65 )
67 from pandas._typing import (
68 AggFuncType,
69 AggFuncTypeBase,
70 AggFuncTypeDict,
71 AggObjType,
72 Axis,
73 AxisInt,
74 DtypeObj,
75 NDFrameT,
76 npt,
77 )
79 from pandas import (
80 DataFrame,
81 Index,
82 Series,
83 )
84 from pandas.core.arrays import (
85 ArrowExtensionArray,
86 BaseMaskedArray,
87 ExtensionArray,
88 )
89 from pandas.core.groupby import GroupBy
90 from pandas.core.groupby.groupby import BaseGroupBy
91 from pandas.core.resample import Resampler
92 from pandas.core.window.rolling import BaseWindow
94ResType: TypeAlias = dict[int, Any]
96# DataFrame methods that reduce column-wise to a Series indexed by the
97# columns, making them eligible for the fast path in
98# FrameApply._agg_list_like_frame_reductions.
99_frame_reduction_names = frozenset(
100 {
101 "all",
102 "any",
103 "count",
104 "idxmax",
105 "idxmin",
106 "kurt",
107 "kurtosis",
108 "max",
109 "mean",
110 "median",
111 "min",
112 "nunique",
113 "prod",
114 "product",
115 "quantile",
116 "sem",
117 "skew",
118 "std",
119 "sum",
120 "var",
121 }
122)
125def _stack_as_object(dtypes: list[DtypeObj]) -> bool:
126 """
127 Whether results of these dtypes must be stacked as object to keep their
128 values: bools mixed with other dtypes can be cast to numbers, and signed
129 with unsigned integers to a float, which rounds values above 2**53.
130 """
131 kinds = {dtype.kind for dtype in dtypes}
132 if "b" in kinds and len(kinds) > 1:
133 return True
134 return {"i", "u"} <= kinds and find_common_type(dtypes).kind == "f"
137@set_module("pandas.api.executors")
138class BaseExecutionEngine(abc.ABC):
139 """
140 Base class for execution engines for map and apply methods.
142 An execution engine receives all the parameters of a call to
143 ``apply`` or ``map``, such as the data container, the function,
144 etc. and takes care of running the execution.
146 Supporting different engines allows functions to be JIT compiled,
147 run in parallel, and others. Besides the default executor which
148 simply runs the code with the Python interpreter and pandas.
149 """
151 @staticmethod
152 @abc.abstractmethod
153 def map(
154 data: Series | DataFrame | np.ndarray,
155 func: AggFuncType,
156 args: tuple,
157 kwargs: dict[str, Any],
158 decorator: Callable | None,
159 skip_na: bool,
160 ):
161 """
162 Executor method to run functions elementwise.
164 In general, pandas uses ``map`` for running functions elementwise,
165 but ``Series.apply`` with the default ``by_row='compat'`` will also
166 call this executor function.
168 Parameters
169 ----------
170 data : Series, DataFrame or NumPy ndarray
171 The object to use for the data. Some methods implement a ``raw``
172 parameter which will convert the original pandas object to a
173 NumPy array, which will then be passed here to the executor.
174 func : function or NumPy ufunc
175 The function to execute.
176 args : tuple
177 Positional arguments to be passed to ``func``.
178 kwargs : dict
179 Keyword arguments to be passed to ``func``.
180 decorator : function, optional
181 For JIT compilers and other engines that need to decorate the
182 function ``func``, this is the decorator to use. While the
183 executor may already know which is the decorator to use, this
184 is useful as for a single executor the user can specify for
185 example ``numba.jit`` or ``numba.njit(nogil=True)``, and this
186 decorator parameter will contain the exact decorator from the
187 executor the user wants to use.
188 skip_na : bool
189 Whether the function should be called for missing values or not.
190 This is specified by the pandas user as ``map(na_action=None)``
191 or ``map(na_action='ignore')``.
192 """
194 @staticmethod
195 @abc.abstractmethod
196 def apply(
197 data: Series | DataFrame | np.ndarray,
198 func: AggFuncType,
199 args: tuple,
200 kwargs: dict[str, Any],
201 decorator: Callable,
202 axis: Axis,
203 ):
204 """
205 Executor method to run functions by an axis.
207 While we can see ``map`` as executing the function for each cell
208 in a ``DataFrame`` (or ``Series``), ``apply`` will execute the
209 function for each column (or row).
211 Parameters
212 ----------
213 data : Series, DataFrame or NumPy ndarray
214 The object to use for the data. Some methods implement a ``raw``
215 parameter which will convert the original pandas object to a
216 NumPy array, which will then be passed here to the executor.
217 func : function or NumPy ufunc
218 The function to execute.
219 args : tuple
220 Positional arguments to be passed to ``func``.
221 kwargs : dict
222 Keyword arguments to be passed to ``func``.
223 decorator : function, optional
224 For JIT compilers and other engines that need to decorate the
225 function ``func``, this is the decorator to use. While the
226 executor may already know which is the decorator to use, this
227 is useful as for a single executor the user can specify for
228 example ``numba.jit`` or ``numba.njit(nogil=True)``, and this
229 decorator parameter will contain the exact decorator from the
230 executor the user wants to use.
231 axis : {0 or 'index', 1 or 'columns'}
232 0 or 'index' should execute the function passing each column as
233 parameter. 1 or 'columns' should execute the function passing
234 each row as parameter. The default executor engine passes rows
235 as pandas ``Series``. Other executor engines should probably
236 expect functions to be implemented this way for compatibility.
237 But passing rows as other data structures is technically possible
238 as far as the function ``func`` is implemented accordingly.
239 """
242def frame_apply(
243 obj: DataFrame,
244 func: AggFuncType,
245 axis: Axis = 0,
246 raw: bool = False,
247 result_type: str | None = None,
248 by_row: Literal[False, "compat"] = "compat",
249 engine: str = "python",
250 engine_kwargs: dict[str, bool] | None = None,
251 args=None,
252 kwargs=None,
253) -> FrameApply:
254 """construct and return a row or column based frame apply object"""
255 _, func, columns, _ = reconstruct_func(func, **kwargs)
257 axis = obj._get_axis_number(axis)
258 klass: type[FrameApply]
259 if axis == 0:
260 klass = FrameRowApply
261 elif axis == 1:
262 if columns:
263 raise NotImplementedError(
264 f"Named aggregation is not supported when {axis=}."
265 )
266 klass = FrameColumnApply
268 return klass(
269 obj,
270 func,
271 raw=raw,
272 result_type=result_type,
273 by_row=by_row,
274 engine=engine,
275 engine_kwargs=engine_kwargs,
276 args=args,
277 kwargs=kwargs,
278 )
281class Apply(metaclass=abc.ABCMeta):
282 axis: AxisInt
284 def __init__(
285 self,
286 obj: AggObjType,
287 func: AggFuncType,
288 raw: bool,
289 result_type: str | None,
290 *,
291 by_row: Literal[False, "compat", "_compat"] = "compat",
292 engine: str = "python",
293 engine_kwargs: dict[str, bool] | None = None,
294 args,
295 kwargs,
296 ) -> None:
297 self.obj = obj
298 self.raw = raw
300 assert by_row is False or by_row in ["compat", "_compat"]
301 self.by_row = by_row
303 self.args = args or ()
304 self.kwargs = kwargs or {}
306 self.engine = engine
307 self.engine_kwargs = {} if engine_kwargs is None else engine_kwargs
309 if result_type not in [None, "reduce", "broadcast", "expand"]:
310 raise ValueError(
311 "invalid value for result_type, must be one "
312 "of {None, 'reduce', 'broadcast', 'expand'}"
313 )
315 self.result_type = result_type
317 self.func = func
319 @abc.abstractmethod
320 def apply(self) -> DataFrame | Series:
321 pass
323 @abc.abstractmethod
324 def agg_or_apply_list_like(
325 self, op_name: Literal["agg", "apply"]
326 ) -> DataFrame | Series:
327 pass
329 @abc.abstractmethod
330 def agg_or_apply_dict_like(
331 self, op_name: Literal["agg", "apply"]
332 ) -> DataFrame | Series:
333 pass
335 def agg(self) -> DataFrame | Series | None:
336 """
337 Provide an implementation for the aggregators.
339 Returns
340 -------
341 Result of aggregation, or None if agg cannot be performed by
342 this method.
343 """
344 func = self.func
346 if isinstance(func, str):
347 return self.apply_str()
349 if is_dict_like(func):
350 return self.agg_dict_like()
351 elif is_list_like(func):
352 # we require a list, but not a 'str'
353 return self.agg_list_like()
355 # caller can react
356 return None
358 def transform(self) -> DataFrame | Series:
359 """
360 Transform a DataFrame or Series.
362 Returns
363 -------
364 DataFrame or Series
365 Result of applying ``func`` along the given axis of the
366 Series or DataFrame.
368 Raises
369 ------
370 ValueError
371 If the transform function fails or does not transform.
372 """
373 obj = self.obj
374 func = self.func
375 axis = self.axis
376 args = self.args
377 kwargs = self.kwargs
379 is_series = obj.ndim == 1
381 if obj._get_axis_number(axis) == 1:
382 assert not is_series
383 return obj.T.transform(func, 0, *args, **kwargs).T
385 if is_list_like(func) and not is_dict_like(func):
386 func = cast("list[AggFuncTypeBase]", func)
387 # GH#54929 - raise if duplicate function names are passed
388 if len(func) > len(set(func)):
389 raise SpecificationError(
390 "Function names must be unique if there is no new column names "
391 "assigned"
392 )
393 # Convert func equivalent dict
394 if is_series:
395 func = {com.get_callable_name(v) or v: v for v in func}
396 else:
397 func = dict.fromkeys(obj, func)
399 if is_dict_like(func):
400 func = cast("AggFuncTypeDict", func)
401 return self.transform_dict_like(func)
403 # func is either str or callable
404 func = cast("AggFuncTypeBase", func)
405 try:
406 result = self.transform_str_or_callable(func)
407 except TypeError:
408 raise
409 except Exception as err:
410 raise ValueError("Transform function failed") from err
412 # Functions that transform may return empty Series/DataFrame
413 # when the dtype is not appropriate
414 if (
415 isinstance(result, (ABCSeries, ABCDataFrame))
416 and result.empty
417 and not obj.empty
418 ):
419 raise ValueError("Transform function failed")
420 if not isinstance(result, (ABCSeries, ABCDataFrame)) or not result.index.equals(
421 obj.index
422 ):
423 raise ValueError("Function did not transform")
425 return result
427 def transform_dict_like(self, func) -> DataFrame:
428 """
429 Compute transform in the case of a dict-like func
430 """
432 obj = self.obj
433 args = self.args
434 kwargs = self.kwargs
436 # transform is currently only for Series/DataFrame
437 assert isinstance(obj, ABCNDFrame)
439 if len(func) == 0:
440 raise ValueError("No transform functions were provided")
442 func = self.normalize_dictlike_arg("transform", obj, func)
444 results: dict[Hashable, DataFrame | Series] = {}
445 for name, how in func.items():
446 colg = obj._gotitem(name, ndim=1)
447 results[name] = colg.transform(how, 0, *args, **kwargs)
448 return concat(results, axis=1)
450 def transform_str_or_callable(self, func) -> DataFrame | Series:
451 """
452 Compute transform in the case of a string or callable func
453 """
454 obj = self.obj
455 args = self.args
456 kwargs = self.kwargs
458 if isinstance(func, str):
459 return self._apply_str(obj, func, *args, **kwargs)
461 # Two possible ways to use a UDF - apply or call directly
462 try:
463 return obj.apply(func, args=args, **kwargs)
464 except Exception:
465 return func(obj, *args, **kwargs)
467 def agg_list_like(self) -> DataFrame | Series:
468 """
469 Compute aggregation in the case of a list-like argument.
471 Returns
472 -------
473 Result of aggregation.
474 """
475 return self.agg_or_apply_list_like(op_name="agg")
477 def compute_list_like(
478 self,
479 op_name: Literal["agg", "apply"],
480 selected_obj: Series | DataFrame,
481 kwargs: dict[str, Any],
482 ) -> tuple[list[Hashable] | Index, list[Any]]:
483 """
484 Compute agg/apply results for like-like input.
486 Parameters
487 ----------
488 op_name : {"agg", "apply"}
489 Operation being performed.
490 selected_obj : Series or DataFrame
491 Data to perform operation on.
492 kwargs : dict
493 Keyword arguments to pass to the functions.
495 Returns
496 -------
497 keys : list[Hashable] or Index
498 Index labels for result.
499 results : list
500 Data for result. When aggregating with a Series, this can contain any
501 Python objects.
502 """
503 func = cast("list[AggFuncTypeBase]", self.func)
504 obj = self.obj
506 results = []
507 keys: list[Hashable] = []
509 # degenerate case
510 if selected_obj.ndim == 1:
511 for a in func:
512 colg = obj._gotitem(selected_obj.name, ndim=1, subset=selected_obj)
513 args = (
514 [self.axis, *self.args]
515 if include_axis(op_name, colg)
516 else self.args
517 )
518 new_res = getattr(colg, op_name)(a, *args, **kwargs)
519 results.append(new_res)
521 # make sure we find a good name
522 name = com.get_callable_name(a) or a
523 keys.append(name)
525 else:
526 indices = []
527 for index, col in enumerate(selected_obj):
528 colg = obj._gotitem(col, ndim=1, subset=selected_obj.iloc[:, index])
529 args = (
530 [self.axis, *self.args]
531 if include_axis(op_name, colg)
532 else self.args
533 )
534 new_res = getattr(colg, op_name)(func, *args, **kwargs)
535 results.append(new_res)
536 indices.append(index)
537 # error: Incompatible types in assignment (expression has type "Any |
538 # Index", variable has type "list[Any | Callable[..., Any] | str]")
539 keys = selected_obj.columns.take(indices) # type: ignore[assignment]
541 return keys, results
543 def wrap_results_list_like(
544 self, keys: Iterable[Hashable], results: list[Series | DataFrame]
545 ):
546 from pandas import Index
547 from pandas.core.groupby.groupby import BaseGroupBy
549 obj = self.obj
551 if not results:
552 # GH#39609
553 if isinstance(obj, ABCSeries):
554 return obj._constructor(index=Index([]), name=obj.name)
555 elif isinstance(obj, BaseGroupBy) and obj._selected_obj.ndim == 1:
556 return obj._selected_obj._constructor_expanddim(
557 index=_groupby_result_index(obj), columns=Index([])
558 )
560 try:
561 return concat(results, keys=keys, axis=1, sort=False)
562 except TypeError as err:
563 # we are concatting non-NDFrame objects,
564 # e.g. a list of scalars
565 from pandas import Series
567 result = Series(results, index=keys, name=obj.name)
568 if is_nested_object(result):
569 raise ValueError(
570 "cannot combine transform and aggregation operations"
571 ) from err
572 return result
574 def agg_dict_like(self) -> DataFrame | Series:
575 """
576 Compute aggregation in the case of a dict-like argument.
578 Returns
579 -------
580 Result of aggregation.
581 """
582 return self.agg_or_apply_dict_like(op_name="agg")
584 def compute_dict_like(
585 self,
586 op_name: Literal["agg", "apply"],
587 selected_obj: Series | DataFrame,
588 selection: Hashable | Sequence[Hashable],
589 kwargs: dict[str, Any],
590 ) -> tuple[list[Hashable], list[Any]]:
591 """
592 Compute agg/apply results for dict-like input.
594 Parameters
595 ----------
596 op_name : {"agg", "apply"}
597 Operation being performed.
598 selected_obj : Series or DataFrame
599 Data to perform operation on.
600 selection : hashable or sequence of hashables
601 Used by GroupBy, Window, and Resample if selection is applied to the object.
602 kwargs : dict
603 Keyword arguments to pass to the functions.
605 Returns
606 -------
607 keys : list[hashable]
608 Index labels for result.
609 results : list
610 Data for result. When aggregating with a Series, this can contain any
611 Python object.
612 """
613 from pandas.core.groupby.generic import (
614 DataFrameGroupBy,
615 SeriesGroupBy,
616 )
618 obj = self.obj
619 is_groupby = isinstance(obj, (DataFrameGroupBy, SeriesGroupBy))
620 func = cast("AggFuncTypeDict", self.func)
621 func = self.normalize_dictlike_arg(op_name, selected_obj, func)
623 is_non_unique_col = (
624 selected_obj.ndim == 2
625 and selected_obj.columns.nunique() < len(selected_obj.columns)
626 )
628 if selected_obj.ndim == 1:
629 # key only used for output
630 colg = obj._gotitem(selection, ndim=1)
631 results = [getattr(colg, op_name)(how, **kwargs) for _, how in func.items()]
632 keys = list(func.keys())
633 elif not is_groupby and is_non_unique_col:
634 # key used for column selection and output
635 # GH#51099
636 results = []
637 keys = []
638 for key, how in func.items():
639 indices = selected_obj.columns.get_indexer_for([key])
640 labels = selected_obj.columns.take(indices)
641 label_to_indices = defaultdict(list)
642 for index, label in zip(indices, labels, strict=True):
643 label_to_indices[label].append(index)
645 key_data = [
646 getattr(selected_obj._ixs(indice, axis=1), op_name)(how, **kwargs)
647 for label, indices in label_to_indices.items()
648 for indice in indices
649 ]
651 keys += [key] * len(key_data)
652 results += key_data
653 elif is_groupby:
654 # key used for column selection and output
656 df = selected_obj
657 results, keys = [], []
658 for key, how in func.items():
659 cols = df[key]
661 if cols.ndim == 1:
662 series = obj._gotitem(key, ndim=1, subset=cols)
663 results.append(getattr(series, op_name)(how, **kwargs))
664 keys.append(key)
665 else:
666 for _, col in cols.items():
667 series = obj._gotitem(key, ndim=1, subset=col)
668 results.append(getattr(series, op_name)(how, **kwargs))
669 keys.append(key)
670 else:
671 results = [
672 getattr(obj._gotitem(key, ndim=1), op_name)(how, **kwargs)
673 for key, how in func.items()
674 ]
675 keys = list(func.keys())
677 return keys, results
679 def wrap_results_dict_like(
680 self,
681 selected_obj: Series | DataFrame,
682 result_index: list[Hashable],
683 result_data: list,
684 ):
685 from pandas import Index
686 from pandas.core.groupby.groupby import BaseGroupBy
688 obj = self.obj
690 if not result_data:
691 # GH#39609
692 if isinstance(obj, ABCSeries):
693 return obj._constructor(index=Index([]), name=obj.name)
694 elif isinstance(obj, ABCDataFrame):
695 return obj._constructor(index=Index([]), columns=obj.columns[:0])
696 elif isinstance(obj, BaseGroupBy):
697 index = _groupby_result_index(obj)
698 if selected_obj.ndim == 2:
699 frame = cast("DataFrame", selected_obj)
700 return frame._constructor(index=index, columns=frame.columns[:0])
701 return selected_obj._constructor_expanddim(
702 index=index, columns=Index([])
703 )
705 # Avoid making two isinstance calls in all and any below
706 is_ndframe = [isinstance(r, ABCNDFrame) for r in result_data]
708 if all(is_ndframe):
709 results = [result for result in result_data if not result.empty]
710 keys_to_use: Iterable[Hashable]
711 keys_to_use = [
712 k for k, v in zip(result_index, result_data, strict=True) if not v.empty
713 ]
714 # Have to check, if at least one DataFrame is not empty.
715 if keys_to_use == []:
716 keys_to_use = result_index
717 results = result_data
719 if selected_obj.ndim == 2:
720 # keys are columns, so we can preserve names
721 ktu = Index(keys_to_use)
722 ktu._set_names(selected_obj.columns.names)
723 keys_to_use = ktu
725 axis: AxisInt = 0 if isinstance(obj, ABCSeries) else 1
726 result = concat(
727 results,
728 axis=axis,
729 keys=keys_to_use,
730 sort=False,
731 )
732 elif any(is_ndframe):
733 # There is a mix of NDFrames and scalars
734 raise ValueError(
735 "cannot perform both aggregation "
736 "and transformation operations "
737 "simultaneously"
738 )
739 else:
740 from pandas import Series
742 # we have a list of scalars
743 # GH 36212 use name only if obj is a series
744 if obj.ndim == 1:
745 obj = cast("Series", obj)
746 name = obj.name
747 else:
748 name = None
750 result = Series(result_data, index=result_index, name=name)
752 return result
754 def apply_str(self) -> DataFrame | Series:
755 """
756 Compute apply in case of a string.
758 Returns
759 -------
760 result: Series or DataFrame
761 """
762 # Caller is responsible for checking isinstance(self.f, str)
763 func = cast("str", self.func)
765 obj = self.obj
767 from pandas.core.groupby.generic import (
768 DataFrameGroupBy,
769 SeriesGroupBy,
770 )
772 # Support for `frame.transform('method')`
773 # Some methods (shift, etc.) require the axis argument, others
774 # don't, so inspect and insert if necessary.
775 method = getattr(obj, func, None)
776 if callable(method):
777 sig = inspect.getfullargspec(method)
778 arg_names = (*sig.args, *sig.kwonlyargs)
779 if self.axis != 0 and (
780 "axis" not in arg_names or func in ("corrwith", "skew")
781 ):
782 raise ValueError(f"Operation {func} does not support axis=1")
783 if "axis" in arg_names and not isinstance(
784 obj, (SeriesGroupBy, DataFrameGroupBy)
785 ):
786 self.kwargs["axis"] = self.axis
787 return self._apply_str(obj, func, *self.args, **self.kwargs)
789 def apply_list_or_dict_like(self) -> DataFrame | Series:
790 """
791 Compute apply in case of a list-like or dict-like.
793 Returns
794 -------
795 result: Series, DataFrame, or None
796 Result when self.func is a list-like or dict-like, None otherwise.
797 """
799 if self.engine == "numba":
800 raise NotImplementedError(
801 "The 'numba' engine doesn't support list-like/"
802 "dict likes of callables yet."
803 )
805 if self.axis == 1 and isinstance(self.obj, ABCDataFrame):
806 return self.obj.T.apply(self.func, 0, args=self.args, **self.kwargs).T
808 func = self.func
809 kwargs = self.kwargs
811 if is_dict_like(func):
812 result = self.agg_or_apply_dict_like(op_name="apply")
813 else:
814 result = self.agg_or_apply_list_like(op_name="apply")
816 result = reconstruct_and_relabel_result(result, func, **kwargs)
818 return result
820 def normalize_dictlike_arg(
821 self, how: str, obj: DataFrame | Series, func: AggFuncTypeDict
822 ) -> AggFuncTypeDict:
823 """
824 Handler for dict-like argument.
826 Ensures that necessary columns exist if obj is a DataFrame, and
827 that a nested renamer is not passed. Also normalizes to all lists
828 when values consists of a mix of list and non-lists.
829 """
830 assert how in ("apply", "agg", "transform")
832 # Can't use func.values(); wouldn't work for a Series
833 if (
834 how == "agg"
835 and isinstance(obj, ABCSeries)
836 and any(is_list_like(v) for _, v in func.items())
837 ) or (any(is_dict_like(v) for _, v in func.items())):
838 # GH 15931 - deprecation of renaming keys
839 raise SpecificationError("nested renamer is not supported")
841 if obj.ndim != 1:
842 # Check for missing columns on a frame
843 from pandas import Index
845 cols = Index(list(func.keys())).difference(obj.columns, sort=True)
846 if len(cols) > 0:
847 # GH 58474
848 raise KeyError(f"Label(s) {list(cols)} do not exist")
850 aggregator_types = (list, tuple, dict)
852 # if we have a dict of any non-scalars
853 # eg. {'A' : ['mean']}, normalize all to
854 # be list-likes
855 # Cannot use func.values() because arg may be a Series
856 if any(isinstance(x, aggregator_types) for _, x in func.items()):
857 new_func: AggFuncTypeDict = {}
858 for k, v in func.items():
859 if not isinstance(v, aggregator_types):
860 new_func[k] = [v]
861 else:
862 new_func[k] = v
863 func = new_func
864 return func
866 def _apply_str(self, obj, func: str, *args, **kwargs):
867 """
868 if arg is a string, then try to operate on it:
869 - try to find a function (or attribute) on obj
870 - try to find a numpy function
871 - raise
872 """
873 assert isinstance(func, str)
875 if hasattr(obj, func):
876 f = getattr(obj, func)
877 if callable(f):
878 return f(*args, **kwargs)
880 # people may aggregate on a non-callable attribute
881 # but don't let them think they can pass args to it
882 assert len(args) == 0
883 assert not any(kwarg == "axis" for kwarg in kwargs)
884 return f
885 elif hasattr(np, func) and hasattr(obj, "__array__"):
886 # in particular exclude Window
887 f = getattr(np, func)
888 return f(obj, *args, **kwargs)
889 else:
890 msg = f"'{func}' is not a valid function for '{type(obj).__name__}' object"
891 raise AttributeError(msg)
894class NDFrameApply(Apply):
895 """
896 Methods shared by FrameApply and SeriesApply but
897 not GroupByApply or ResamplerWindowApply
898 """
900 obj: DataFrame | Series
902 @property
903 def index(self) -> Index:
904 return self.obj.index
906 @property
907 def agg_axis(self) -> Index:
908 return self.obj._get_agg_axis(self.axis)
910 def agg_or_apply_list_like(
911 self, op_name: Literal["agg", "apply"]
912 ) -> DataFrame | Series:
913 obj = self.obj
914 kwargs = self.kwargs
916 if op_name == "apply":
917 if isinstance(self, FrameApply):
918 by_row = self.by_row
920 elif isinstance(self, SeriesApply):
921 by_row = "_compat" if self.by_row else False
922 else:
923 by_row = False
924 kwargs = {**kwargs, "by_row": by_row}
926 if getattr(obj, "axis", 0) == 1:
927 raise NotImplementedError("axis other than 0 is not supported")
929 if op_name == "agg" and isinstance(self, FrameApply):
930 result = self._agg_list_like_frame_reductions()
931 if result is not None:
932 return result
934 keys, results = self.compute_list_like(op_name, obj, kwargs)
935 result = self.wrap_results_list_like(keys, results)
936 return result
938 def agg_or_apply_dict_like(
939 self, op_name: Literal["agg", "apply"]
940 ) -> DataFrame | Series:
941 assert op_name in ["agg", "apply"]
942 obj = self.obj
944 kwargs = {}
945 if op_name == "apply":
946 by_row = "_compat" if self.by_row else False
947 kwargs.update({"by_row": by_row})
949 if getattr(obj, "axis", 0) == 1:
950 raise NotImplementedError("axis other than 0 is not supported")
952 selection = None
953 result_index, result_data = self.compute_dict_like(
954 op_name, obj, selection, kwargs
955 )
956 result = self.wrap_results_dict_like(obj, result_index, result_data)
957 return result
960class FrameApply(NDFrameApply):
961 obj: DataFrame
963 def __init__(
964 self,
965 obj: AggObjType,
966 func: AggFuncType,
967 raw: bool,
968 result_type: str | None,
969 *,
970 by_row: Literal[False, "compat"] = False,
971 engine: str = "python",
972 engine_kwargs: dict[str, bool] | None = None,
973 args,
974 kwargs,
975 ) -> None:
976 if by_row is not False and by_row != "compat":
977 raise ValueError(f"by_row={by_row} not allowed")
978 super().__init__(
979 obj,
980 func,
981 raw,
982 result_type,
983 by_row=by_row,
984 engine=engine,
985 engine_kwargs=engine_kwargs,
986 args=args,
987 kwargs=kwargs,
988 )
990 # ---------------------------------------------------------------
991 # Abstract Methods
993 @property
994 @abc.abstractmethod
995 def result_index(self) -> Index:
996 pass
998 @property
999 @abc.abstractmethod
1000 def result_columns(self) -> Index:
1001 pass
1003 @property
1004 @abc.abstractmethod
1005 def series_generator(self) -> Generator[Series]:
1006 pass
1008 @staticmethod
1009 @functools.cache
1010 @abc.abstractmethod
1011 def generate_numba_apply_func(
1012 func, nogil: bool = True, parallel: bool = False
1013 ) -> Callable[[npt.NDArray, Index, Index], dict[int, Any]]:
1014 pass
1016 @abc.abstractmethod
1017 def apply_with_numba(self):
1018 pass
1020 def validate_values_for_numba(self) -> None:
1021 # Validate column dtypes all OK
1022 for colname, dtype in self.obj.dtypes.items():
1023 if not is_numeric_dtype(dtype):
1024 raise ValueError(
1025 f"Column {colname} must have a numeric dtype. "
1026 f"Found '{dtype}' instead"
1027 )
1028 if is_extension_array_dtype(dtype):
1029 raise ValueError(
1030 f"Column {colname} is backed by an extension array, "
1031 f"which is not supported by the numba engine."
1032 )
1034 @abc.abstractmethod
1035 def wrap_results_for_axis(
1036 self, results: ResType, res_index: Index
1037 ) -> DataFrame | Series:
1038 pass
1040 # ---------------------------------------------------------------
1042 @property
1043 def res_columns(self) -> Index:
1044 return self.result_columns
1046 @property
1047 def columns(self) -> Index:
1048 return self.obj.columns
1050 @cache_readonly
1051 def values(self):
1052 return self.obj.values
1054 def apply(self) -> DataFrame | Series:
1055 """compute the results"""
1057 # dispatch to handle list-like or dict-like
1058 if is_list_like(self.func):
1059 if self.engine == "numba":
1060 raise NotImplementedError(
1061 "the 'numba' engine doesn't support lists of callables yet"
1062 )
1063 return self.apply_list_or_dict_like()
1065 # all empty
1066 if len(self.columns) == 0 and len(self.index) == 0:
1067 return self.apply_empty_result()
1069 # string dispatch
1070 if isinstance(self.func, str):
1071 if self.engine == "numba":
1072 raise NotImplementedError(
1073 "the 'numba' engine doesn't support using "
1074 "a string as the callable function"
1075 )
1076 return self.apply_str()
1078 # ufunc
1079 elif isinstance(self.func, np.ufunc):
1080 if self.engine == "numba":
1081 raise NotImplementedError(
1082 "the 'numba' engine doesn't support "
1083 "using a numpy ufunc as the callable function"
1084 )
1085 with np.errstate(all="ignore"):
1086 results = self.obj._mgr.apply("apply", func=self.func)
1087 # _constructor will retain self.index and self.columns
1088 return self.obj._constructor_from_mgr(results, axes=results.axes)
1090 # broadcasting
1091 if self.result_type == "broadcast":
1092 if self.engine == "numba":
1093 raise NotImplementedError(
1094 "the 'numba' engine doesn't support result_type='broadcast'"
1095 )
1096 return self.apply_broadcast(self.obj)
1098 # one axis empty
1099 elif not all(self.obj.shape):
1100 return self.apply_empty_result()
1102 # raw
1103 elif self.raw:
1104 return self.apply_raw(engine=self.engine, engine_kwargs=self.engine_kwargs)
1106 return self.apply_standard()
1108 def agg(self):
1109 obj = self.obj
1110 axis = self.axis
1112 # TODO: Avoid having to change state
1113 self.obj = self.obj if self.axis == 0 else self.obj.T
1114 self.axis = 0
1116 result = None
1117 try:
1118 result = super().agg()
1119 finally:
1120 self.obj = obj
1121 self.axis = axis
1123 if axis == 1:
1124 result = result.T if result is not None else result
1126 if result is None:
1127 result = self.obj.apply(self.func, axis, args=self.args, **self.kwargs)
1129 return result
1131 def _agg_list_like_frame_reductions(self) -> DataFrame | None:
1132 """
1133 Aggregate a list of named functions using DataFrame-level reductions.
1135 Instead of extracting each column as a Series and calling
1136 Series.agg per column, call DataFrame-level reductions directly.
1137 Operates per dtype group to preserve per-column dtypes.
1139 Returns None if the fast path cannot be used (e.g. non-string
1140 functions or functions that aren't known column-wise reductions).
1141 """
1142 func = cast("list[AggFuncTypeBase]", self.func)
1144 if not all(isinstance(f, str) for f in func):
1145 return None
1147 obj = self.obj
1148 func_names = cast("list[str]", func)
1150 # Cannot reindex with duplicate column names
1151 if not obj.columns.is_unique:
1152 return None
1154 for func_name in func_names:
1155 if func_name not in _frame_reduction_names:
1156 # GH#65031 anything that isn't a known column-wise reduction
1157 # (e.g. value_counts, duplicated) must use the per-column
1158 # fallback; a Series-returning method could otherwise slip
1159 # through the shape checks below and give wrong results.
1160 return None
1162 if self.kwargs.get("numeric_only"):
1163 obj = obj._get_numeric_data()
1164 elif self.kwargs.get("bool_only"):
1165 obj = obj._get_bool_data()
1167 if obj.columns.empty or len(func_names) == 0:
1168 return obj._constructor(index=func_names, columns=obj.columns)
1170 # Compute reductions per dtype group to preserve per-column dtypes.
1171 groups = obj.columns.groupby(obj.dtypes)
1172 pieces: list[DataFrame] = []
1173 for dtype in groups:
1174 cols = groups[dtype]
1175 sub = obj[cols]
1176 rows: list[Series] = []
1177 for func_name in func_names:
1178 try:
1179 row = getattr(sub, func_name)(*self.args, **self.kwargs)
1180 except TypeError:
1181 return None
1182 if not isinstance(row, ABCSeries):
1183 # Not a reduction result for these args (e.g. quantile
1184 # with a list q returns a DataFrame); fall back
1185 return None
1186 if not row.index.equals(sub.columns):
1187 # Backstop: a genuine column-wise reduction is indexed
1188 # by the columns; anything else would silently misalign
1189 # in the concat below.
1190 return None
1191 rows.append(row)
1193 # to_frame().T avoids the slow DataFrame(list-of-Series) path
1194 frames = [
1195 row.to_frame(name).T for name, row in zip(func_names, rows, strict=True)
1196 ]
1197 if _stack_as_object([row.dtype for row in rows]):
1198 # GH#65031 see test_agg_list_like_unsigned_and_signed_is_object
1199 # and test_agg_list_like_bool_with_numeric_is_object
1200 frames = [frame.astype(object) for frame in frames]
1201 pieces.append(concat(frames))
1203 result = concat(pieces, axis=1)
1204 result = result.reindex(columns=obj.columns)
1205 return result
1207 def apply_empty_result(self):
1208 """
1209 we have an empty result; at least 1 axis is 0
1211 we will try to apply the function to an empty
1212 series in order to see if this is a reduction function
1213 """
1214 assert callable(self.func)
1216 # we are not asked to reduce or infer reduction
1217 # so just return a copy of the existing object
1218 if self.result_type not in ["reduce", None]:
1219 return self.obj.copy()
1221 # we may need to infer
1222 should_reduce = self.result_type == "reduce"
1224 from pandas import Series
1226 if not should_reduce:
1227 try:
1228 if self.axis == 0:
1229 r = self.func(
1230 Series([], dtype=np.float64), *self.args, **self.kwargs
1231 )
1232 else:
1233 r = self.func(
1234 Series(index=self.columns, dtype=np.float64),
1235 *self.args,
1236 **self.kwargs,
1237 )
1238 except Exception:
1239 pass
1240 else:
1241 should_reduce = not isinstance(r, Series)
1243 if should_reduce:
1244 if len(self.agg_axis):
1245 r = self.func(Series([], dtype=np.float64), *self.args, **self.kwargs)
1246 else:
1247 r = np.nan
1249 return self.obj._constructor_sliced(r, index=self.agg_axis)
1250 else:
1251 return self.obj.copy()
1253 def apply_raw(self, engine="python", engine_kwargs=None):
1254 """apply to the values as a numpy array"""
1256 def wrap_function(func):
1257 """
1258 Wrap user supplied function to work around numpy issue.
1260 see https://github.com/numpy/numpy/issues/8352
1261 """
1263 def wrapper(*args, **kwargs):
1264 result = func(*args, **kwargs)
1265 if isinstance(result, str):
1266 result = np.array(result, dtype=object)
1267 return result
1269 return wrapper
1271 if engine == "numba":
1272 args, kwargs = prepare_function_arguments(
1273 self.func, # type: ignore[arg-type]
1274 self.args,
1275 self.kwargs,
1276 num_required_args=1,
1277 )
1278 # error: Argument 1 to "__call__" of "_lru_cache_wrapper" has
1279 # incompatible type "Callable[..., Any] | str | list[Callable
1280 # [..., Any] | str] | dict[Hashable,Callable[..., Any] | str |
1281 # list[Callable[..., Any] | str]]"; expected "Hashable"
1282 nb_looper = generate_apply_looper(
1283 self.func, # type: ignore[arg-type]
1284 **get_jit_arguments(engine_kwargs),
1285 )
1286 result = nb_looper(self.values, self.axis, *args)
1287 # If we made the result 2-D, squeeze it back to 1-D
1288 result = np.squeeze(result)
1289 else:
1290 result = np.apply_along_axis(
1291 wrap_function(self.func),
1292 self.axis,
1293 self.values,
1294 *self.args,
1295 **self.kwargs,
1296 )
1298 # TODO: mixed type case
1299 if result.ndim == 2:
1300 return self.obj._constructor(result, index=self.index, columns=self.columns)
1301 else:
1302 return self.obj._constructor_sliced(result, index=self.agg_axis)
1304 def apply_broadcast(self, target: DataFrame) -> DataFrame:
1305 assert callable(self.func)
1307 result_values = np.empty_like(target.values)
1309 # axis which we want to compare compliance
1310 result_compare = target.shape[0]
1312 for i, col in enumerate(target.columns):
1313 res = self.func(target[col], *self.args, **self.kwargs)
1314 ares = np.asarray(res).ndim
1316 # must be a scalar or 1d
1317 if ares > 1:
1318 raise ValueError("too many dims to broadcast")
1319 if ares == 1:
1320 # must match return dim
1321 if result_compare != len(res):
1322 raise ValueError("cannot broadcast result")
1324 result_values[:, i] = res
1326 # we *always* preserve the original index / columns
1327 result = self.obj._constructor(
1328 result_values, index=target.index, columns=target.columns
1329 )
1330 return result
1332 def apply_standard(self):
1333 if self.engine == "python":
1334 results, res_index = self.apply_series_generator()
1335 else:
1336 results, res_index = self.apply_series_numba()
1338 # wrap results
1339 return self.wrap_results(results, res_index)
1341 def apply_series_generator(self) -> tuple[ResType, Index]:
1342 assert callable(self.func)
1344 series_gen = self.series_generator
1345 res_index = self.result_index
1347 results = {}
1349 for i, v in enumerate(series_gen):
1350 results[i] = self.func(v, *self.args, **self.kwargs)
1351 if isinstance(results[i], ABCSeries):
1352 # If we have a view on v, we need to make a copy because
1353 # series_generator will swap out the underlying data
1354 results[i] = results[i].copy(deep=False)
1356 return results, res_index
1358 def apply_series_numba(self):
1359 if self.engine_kwargs.get("parallel", False):
1360 raise NotImplementedError(
1361 "Parallel apply is not supported when raw=False and engine='numba'"
1362 )
1363 if not self.obj.index.is_unique or not self.columns.is_unique:
1364 raise NotImplementedError(
1365 "The index/columns must be unique when raw=False and engine='numba'"
1366 )
1367 self.validate_values_for_numba()
1368 results = self.apply_with_numba()
1369 return results, self.result_index
1371 def wrap_results(self, results: ResType, res_index: Index) -> DataFrame | Series:
1372 from pandas import Series
1374 # see if we can infer the results
1375 if len(results) > 0 and 0 in results and is_sequence(results[0]):
1376 return self.wrap_results_for_axis(results, res_index)
1378 # dict of scalars
1380 # the default dtype of an empty Series is `object`, but this
1381 # code can be hit by df.mean() where the result should have dtype
1382 # float64 even if it's an empty Series.
1383 constructor_sliced = self.obj._constructor_sliced
1384 if len(results) == 0 and constructor_sliced is Series:
1385 result = constructor_sliced(results, dtype=np.float64)
1386 else:
1387 result = constructor_sliced(results)
1388 result.index = res_index
1390 return result
1392 def apply_str(self) -> DataFrame | Series:
1393 # Caller is responsible for checking isinstance(self.func, str)
1394 # TODO: GH#39993 - Avoid special-casing by replacing with lambda
1395 if self.func == "size":
1396 # Special-cased because DataFrame.size returns a single scalar
1397 obj = self.obj
1398 value = obj.shape[self.axis]
1399 return obj._constructor_sliced(value, index=self.agg_axis)
1400 return super().apply_str()
1403class FrameRowApply(FrameApply):
1404 axis: AxisInt = 0
1406 @property
1407 def series_generator(self) -> Generator[Series]:
1408 return (self.obj._ixs(i, axis=1) for i in range(len(self.columns)))
1410 @staticmethod
1411 @functools.cache
1412 def generate_numba_apply_func(
1413 func, nogil: bool = True, parallel: bool = False
1414 ) -> Callable[[npt.NDArray, Index, Index], dict[int, Any]]:
1415 numba = import_optional_dependency("numba")
1416 from pandas import Series
1418 # Import helper from extensions to cast string object -> np strings
1419 # Note: This also has the side effect of loading our numba extensions
1420 from pandas.core._numba.extensions import maybe_cast_str
1422 jitted_udf = numba.extending.register_jitable(func)
1424 # Currently the parallel argument doesn't get passed through here
1425 # (it's disabled) since the dicts in numba aren't thread-safe.
1426 @numba.jit(nogil=nogil, parallel=parallel)
1427 def numba_func(values, col_names, df_index, *args):
1428 results = {}
1429 for j in range(values.shape[1]):
1430 # Create the series
1431 ser = Series(
1432 values[:, j], index=df_index, name=maybe_cast_str(col_names[j])
1433 )
1434 results[j] = jitted_udf(ser, *args)
1435 return results
1437 return numba_func
1439 def apply_with_numba(self) -> dict[int, Any]:
1440 func = cast("Callable", self.func)
1441 args, kwargs = prepare_function_arguments(
1442 func, self.args, self.kwargs, num_required_args=1
1443 )
1444 nb_func = self.generate_numba_apply_func(
1445 func, **get_jit_arguments(self.engine_kwargs)
1446 )
1447 from pandas.core._numba.extensions import set_numba_data
1449 index = self.obj.index
1450 columns = self.obj.columns
1452 # Convert from numba dict to regular dict
1453 # Our isinstance checks in the df constructor don't pass for numbas typed dict
1454 with set_numba_data(index) as index, set_numba_data(columns) as columns:
1455 res = dict(nb_func(self.values, columns, index, *args))
1456 return res
1458 @property
1459 def result_index(self) -> Index:
1460 return self.columns
1462 @property
1463 def result_columns(self) -> Index:
1464 return self.index
1466 def wrap_results_for_axis(
1467 self, results: ResType, res_index: Index
1468 ) -> DataFrame | Series:
1469 """return the results for the rows"""
1471 if self.result_type == "reduce":
1472 # e.g. test_apply_dict GH#8735
1473 res = self.obj._constructor_sliced(results)
1474 res.index = res_index
1475 return res
1477 elif self.result_type is None and all(
1478 isinstance(x, dict) for x in results.values()
1479 ):
1480 # Our operation was a to_dict op e.g.
1481 # test_apply_dict GH#8735, test_apply_reduce_to_dict GH#25196 #37544
1482 res = self.obj._constructor_sliced(results)
1483 res.index = res_index
1484 return res
1486 try:
1487 result = self.obj._constructor(data=results)
1488 except ValueError as err:
1489 if "All arrays must be of the same length" in str(err):
1490 # e.g. result = [[2, 3], [1.5], ['foo', 'bar']]
1491 # see test_agg_listlike_result GH#29587
1492 res = self.obj._constructor_sliced(results)
1493 res.index = res_index
1494 return res
1495 else:
1496 raise
1498 if not isinstance(results[0], ABCSeries):
1499 if len(result.index) == len(self.res_columns):
1500 result.index = self.res_columns
1502 if len(result.columns) == len(res_index):
1503 result.columns = res_index
1505 return result
1508class FrameColumnApply(FrameApply):
1509 axis: AxisInt = 1
1511 def apply_broadcast(self, target: DataFrame) -> DataFrame:
1512 result = super().apply_broadcast(target.T)
1513 return result.T
1515 @property
1516 def series_generator(self) -> Generator[Series]:
1517 assert len(self.obj) > 0
1519 # We create one Series object, and will swap out the data inside
1520 # of it. Kids: don't do this at home.
1521 ser = self.obj._ixs(0, axis=0)
1522 mgr = ser._mgr
1524 is_view = mgr.blocks[0].refs.has_reference()
1526 if isinstance(ser.dtype, ExtensionDtype):
1527 # GH#61747 - Pre-extract column arrays and reuse a template
1528 # Series to avoid the per-row block traversal and
1529 # Series/manager construction overhead in fast_xs.
1530 obj = self.obj
1531 col_arrays = [obj._get_column_array(i) for i in range(len(obj.columns))]
1532 dtype = ser.dtype
1533 cls = dtype.construct_array_type()
1534 mgr = ser._mgr
1536 build_row = self._make_ea_row_builder(col_arrays, dtype, cls)
1538 blk = mgr.blocks[0]
1539 for row_idx in range(len(obj)):
1540 ser._mgr = mgr
1541 mgr.set_values(build_row(row_idx))
1542 # EABackedBlock.array_values is @cache_readonly;
1543 # clear it so Series.array sees the new values.
1544 blk._cache.pop("array_values", None)
1545 object.__setattr__(ser, "_name", obj.index[row_idx])
1546 blk.refs = BlockValuesRefs(blk)
1547 yield ser
1549 else:
1550 values = self.values
1551 values = ensure_wrapped_if_datetimelike(values)
1553 for arr, name in zip(values, self.index, strict=True):
1554 # GH#35462 re-pin mgr in case setitem changed it
1555 ser._mgr = mgr
1556 mgr.set_values(arr)
1557 object.__setattr__(ser, "_name", name)
1558 if not is_view:
1559 # In apply_series_generator we store a shallow copy of the
1560 # result, which potentially increases the ref count of this reused
1561 # `ser` object (depending on the result of the applied function)
1562 # -> if that happened and `ser` is already a copy, then we reset
1563 # the refs here to avoid triggering an unnecessary CoW inside the
1564 # applied function (https://github.com/pandas-dev/pandas/pull/56212)
1565 mgr.blocks[0].refs = BlockValuesRefs(mgr.blocks[0])
1566 yield ser
1568 @staticmethod
1569 def _make_ea_row_builder(
1570 col_arrays: list, dtype: ExtensionDtype, cls: type[ExtensionArray]
1571 ) -> Callable[[int], ExtensionArray]:
1572 """Build a callable that constructs an EA row for a given row index.
1574 Special-cases BaseMaskedArray and ArrowExtensionArray to avoid the
1575 overhead of _from_sequence per row. The fast paths require all columns
1576 to share ``dtype``; mixed-dtype frames fall through to the generic
1577 ``_from_sequence`` path (GH#65097).
1578 """
1579 homogeneous = all(col_arr.dtype == dtype for col_arr in col_arrays)
1581 if homogeneous and isinstance(dtype, BaseMaskedDtype):
1582 col_data = [col_arr._data for col_arr in col_arrays]
1583 col_masks = [col_arr._mask for col_arr in col_arrays]
1584 np_dtype = dtype.numpy_dtype
1585 masked_cls = cast("type[BaseMaskedArray]", cls)
1587 def build_row(row_idx: int) -> ExtensionArray:
1588 data = np.array([cd[row_idx] for cd in col_data], dtype=np_dtype)
1589 mask = np.array([cm[row_idx] for cm in col_masks], dtype=np.bool_)
1590 return masked_cls._simple_new(data, mask)
1592 elif homogeneous and isinstance(dtype, ArrowDtype):
1593 import pyarrow as pa
1595 pa_arrays = [col_arr._pa_array for col_arr in col_arrays]
1596 pa_type = dtype.pyarrow_dtype
1597 arrow_cls = cast("type[ArrowExtensionArray]", cls)
1599 def build_row(row_idx: int) -> ExtensionArray:
1600 pa_scalars = [pa_col[row_idx] for pa_col in pa_arrays]
1601 return arrow_cls(pa.array(pa_scalars, type=pa_type))
1603 else:
1605 def build_row(row_idx: int) -> ExtensionArray:
1606 row_data = np.array(
1607 [col_arr[row_idx] for col_arr in col_arrays], dtype=object
1608 )
1609 return cls._from_sequence(row_data, dtype=dtype)
1611 return build_row
1613 @staticmethod
1614 @functools.cache
1615 def generate_numba_apply_func(
1616 func, nogil: bool = True, parallel: bool = False
1617 ) -> Callable[[npt.NDArray, Index, Index], dict[int, Any]]:
1618 numba = import_optional_dependency("numba")
1619 from pandas import Series
1620 from pandas.core._numba.extensions import maybe_cast_str
1622 jitted_udf = numba.extending.register_jitable(func)
1624 @numba.jit(nogil=nogil, parallel=parallel)
1625 def numba_func(values, col_names_index, index, *args):
1626 results = {}
1627 # Currently the parallel argument doesn't get passed through here
1628 # (it's disabled) since the dicts in numba aren't thread-safe.
1629 for i in range(values.shape[0]):
1630 # Create the series
1631 # TODO: values corrupted without the copy
1632 ser = Series(
1633 values[i].copy(),
1634 index=col_names_index,
1635 name=maybe_cast_str(index[i]),
1636 )
1637 results[i] = jitted_udf(ser, *args)
1639 return results
1641 return numba_func
1643 def apply_with_numba(self) -> dict[int, Any]:
1644 func = cast("Callable", self.func)
1645 args, kwargs = prepare_function_arguments(
1646 func, self.args, self.kwargs, num_required_args=1
1647 )
1648 nb_func = self.generate_numba_apply_func(
1649 func, **get_jit_arguments(self.engine_kwargs)
1650 )
1652 from pandas.core._numba.extensions import set_numba_data
1654 # Convert from numba dict to regular dict
1655 # Our isinstance checks in the df constructor don't pass for numbas typed dict
1656 with (
1657 set_numba_data(self.obj.index) as index,
1658 set_numba_data(self.columns) as columns,
1659 ):
1660 res = dict(nb_func(self.values, columns, index, *args))
1662 return res
1664 @property
1665 def result_index(self) -> Index:
1666 return self.index
1668 @property
1669 def result_columns(self) -> Index:
1670 return self.columns
1672 def wrap_results_for_axis(
1673 self, results: ResType, res_index: Index
1674 ) -> DataFrame | Series:
1675 """return the results for the columns"""
1676 result: DataFrame | Series
1678 # we have requested to expand
1679 if self.result_type == "expand":
1680 result = self.infer_to_same_shape(results, res_index)
1682 # we have a non-series and don't want inference
1683 elif not isinstance(results[0], ABCSeries):
1684 result = self.obj._constructor_sliced(results)
1685 result.index = res_index
1687 # we may want to infer results
1688 else:
1689 result = self.infer_to_same_shape(results, res_index)
1691 return result
1693 def infer_to_same_shape(self, results: ResType, res_index: Index) -> DataFrame:
1694 """infer the results to the same shape as the input object"""
1695 result = self.obj._constructor(data=results)
1696 result = result.T
1698 # set the index
1699 result.index = res_index
1701 # infer dtypes
1702 result = result.infer_objects()
1704 return result
1707class SeriesApply(NDFrameApply):
1708 obj: Series
1709 axis: AxisInt = 0
1710 by_row: Literal[False, "compat", "_compat"] # only relevant for apply()
1712 def __init__(
1713 self,
1714 obj: Series,
1715 func: AggFuncType,
1716 *,
1717 by_row: Literal[False, "compat", "_compat"] = "compat",
1718 args,
1719 kwargs,
1720 ) -> None:
1721 super().__init__(
1722 obj,
1723 func,
1724 raw=False,
1725 result_type=None,
1726 by_row=by_row,
1727 args=args,
1728 kwargs=kwargs,
1729 )
1731 def apply(self) -> DataFrame | Series:
1732 obj = self.obj
1734 if len(obj) == 0:
1735 return self.apply_empty_result()
1737 # dispatch to handle list-like or dict-like
1738 if is_list_like(self.func):
1739 return self.apply_list_or_dict_like()
1741 if isinstance(self.func, str):
1742 # if we are a string, try to dispatch
1743 return self.apply_str()
1745 if self.by_row == "_compat":
1746 return self.apply_compat()
1748 # self.func is Callable
1749 return self.apply_standard()
1751 def agg(self):
1752 result = super().agg()
1753 if result is None:
1754 obj = self.obj
1755 func = self.func
1756 # string, list-like, and dict-like are entirely handled in super
1757 assert callable(func)
1758 result = func(obj, *self.args, **self.kwargs)
1759 return result
1761 def apply_empty_result(self) -> Series:
1762 obj = self.obj
1763 return obj._constructor(dtype=obj.dtype, index=obj.index).__finalize__(
1764 obj, method="apply"
1765 )
1767 def apply_compat(self):
1768 """compat apply method for funcs in listlikes and dictlikes.
1770 Used for each callable when giving listlikes and dictlikes of callables to
1771 apply. Needed for compatibility with Pandas < v2.1.
1773 .. versionadded:: 2.1.0
1774 """
1775 obj = self.obj
1776 func = self.func
1778 if callable(func):
1779 f = com.get_cython_func(func)
1780 if f and not self.args and not self.kwargs:
1781 return obj.apply(func, by_row=False)
1783 try:
1784 result = obj.apply(func, by_row="compat")
1785 except (ValueError, AttributeError, TypeError):
1786 result = obj.apply(func, by_row=False)
1787 return result
1789 def apply_standard(self) -> DataFrame | Series:
1790 # caller is responsible for ensuring that f is Callable
1791 func = cast("Callable", self.func)
1792 obj = self.obj
1794 if isinstance(func, np.ufunc):
1795 with np.errstate(all="ignore"):
1796 return func(obj, *self.args, **self.kwargs)
1797 elif not self.by_row:
1798 return func(obj, *self.args, **self.kwargs)
1800 if self.args or self.kwargs:
1801 # _map_values does not support args/kwargs
1802 def curried(x):
1803 return func(x, *self.args, **self.kwargs)
1805 else:
1806 curried = func
1807 mapped = obj._map_values(mapper=curried)
1809 if len(mapped) and isinstance(mapped[0], ABCSeries):
1810 # GH#43986 Need to do list(mapped) in order to get treated as nested
1811 # See also GH#25959 regarding EA support
1812 return obj._constructor_expanddim(list(mapped), index=obj.index)
1813 else:
1814 return obj._constructor(mapped, index=obj.index).__finalize__(
1815 obj, method="apply"
1816 )
1819class GroupByApply(Apply):
1820 obj: GroupBy | Resampler | BaseWindow
1822 def __init__(
1823 self,
1824 obj: GroupBy[NDFrameT],
1825 func: AggFuncType,
1826 *,
1827 args,
1828 kwargs,
1829 ) -> None:
1830 kwargs = kwargs.copy()
1831 self.axis = obj.obj._get_axis_number(kwargs.get("axis", 0))
1832 super().__init__(
1833 obj,
1834 func,
1835 raw=False,
1836 result_type=None,
1837 args=args,
1838 kwargs=kwargs,
1839 )
1841 def apply(self):
1842 raise NotImplementedError
1844 def transform(self):
1845 raise NotImplementedError
1847 def agg_or_apply_list_like(
1848 self, op_name: Literal["agg", "apply"]
1849 ) -> DataFrame | Series:
1850 obj = self.obj
1851 kwargs = self.kwargs
1852 if op_name == "apply":
1853 kwargs = {**kwargs, "by_row": False}
1855 if getattr(obj, "axis", 0) == 1:
1856 raise NotImplementedError("axis other than 0 is not supported")
1858 if obj._selected_obj.ndim == 1:
1859 # For SeriesGroupBy this matches _obj_with_exclusions
1860 selected_obj = obj._selected_obj
1861 else:
1862 selected_obj = obj._obj_with_exclusions
1864 # Only set as_index=True on groupby objects, not Window or Resample
1865 # that inherit from this class.
1866 with com.temp_setattr(
1867 obj, "as_index", True, condition=hasattr(obj, "as_index")
1868 ):
1869 keys, results = self.compute_list_like(op_name, selected_obj, kwargs)
1870 result = self.wrap_results_list_like(keys, results)
1871 return result
1873 def agg_or_apply_dict_like(
1874 self, op_name: Literal["agg", "apply"]
1875 ) -> DataFrame | Series:
1876 from pandas.core.groupby.generic import (
1877 DataFrameGroupBy,
1878 SeriesGroupBy,
1879 )
1881 assert op_name in ["agg", "apply"]
1883 obj = self.obj
1884 kwargs: dict[str, Any] = {}
1885 if op_name == "apply":
1886 by_row = "_compat" if self.by_row else False
1887 kwargs.update({"by_row": by_row})
1889 if getattr(obj, "axis", 0) == 1:
1890 raise NotImplementedError("axis other than 0 is not supported")
1892 selected_obj = obj._selected_obj
1893 selection = obj._selection
1895 is_groupby = isinstance(obj, (DataFrameGroupBy, SeriesGroupBy))
1897 # Numba Groupby engine/engine-kwargs passthrough
1898 if is_groupby:
1899 engine = self.kwargs.get("engine", None)
1900 engine_kwargs = self.kwargs.get("engine_kwargs", None)
1901 kwargs.update({"engine": engine, "engine_kwargs": engine_kwargs})
1903 with com.temp_setattr(
1904 obj, "as_index", True, condition=hasattr(obj, "as_index")
1905 ):
1906 result_index, result_data = self.compute_dict_like(
1907 op_name, selected_obj, selection, kwargs
1908 )
1909 result = self.wrap_results_dict_like(selected_obj, result_index, result_data)
1910 return result
1913class ResamplerWindowApply(GroupByApply):
1914 axis: AxisInt = 0
1915 obj: Resampler | BaseWindow
1917 def __init__(
1918 self,
1919 obj: Resampler | BaseWindow,
1920 func: AggFuncType,
1921 *,
1922 args,
1923 kwargs,
1924 ) -> None:
1925 super(GroupByApply, self).__init__(
1926 obj,
1927 func,
1928 raw=False,
1929 result_type=None,
1930 args=args,
1931 kwargs=kwargs,
1932 )
1934 def apply(self):
1935 raise NotImplementedError
1937 def transform(self):
1938 raise NotImplementedError
1941def _groupby_result_index(obj: BaseGroupBy) -> Index:
1942 """
1943 Index of a groupby aggregation result, used when there is nothing to aggregate.
1944 """
1945 from pandas.core.resample import Resampler
1947 if isinstance(obj, Resampler):
1948 # a groupby-resample has no single grouper to take the index from
1949 return obj.size().index
1950 return obj._grouper.result_index
1953def reconstruct_func(
1954 func: AggFuncType | None, allow_skip_normalization: bool = False, /, **kwargs
1955) -> tuple[bool, AggFuncType, tuple[str, ...] | None, npt.NDArray[np.intp] | None]:
1956 """
1957 This is the internal function to reconstruct func given if there is relabeling
1958 or not and also normalize the keyword to get new order of columns.
1960 If named aggregation is applied, `func` will be None, and kwargs contains the
1961 column and aggregation function information to be parsed;
1962 If named aggregation is not applied, `func` is either string (e.g. 'min') or
1963 Callable, or list of them (e.g. ['min', np.max]), or the dictionary of column name
1964 and str/Callable/list of them (e.g. {'A': 'min'}, or {'A': [np.min, lambda x: x]})
1966 If relabeling is True, will return relabeling, reconstructed func, column
1967 names, and the reconstructed order of columns.
1968 If relabeling is False, the columns and order will be None.
1970 Named aggregation is the one exception: when ``allow_skip_normalization`` is
1971 True, every output name equals its source column name, and every aggfunc
1972 reduces to a scalar, relabeling is reported as False (and columns/order as
1973 None) even though named aggregation was used, because the caller can consume
1974 the un-normalized func directly.
1976 Parameters
1977 ----------
1978 func: agg function (e.g. 'min' or Callable) or list of agg functions
1979 (e.g. ['min', np.max]) or dictionary (e.g. {'A': ['min', np.max]}).
1980 allow_skip_normalization: bool, default False
1981 Whether the caller can handle the un-normalized ``{column: aggfunc}`` form
1982 that named aggregation reduces to when every output name equals its source
1983 column name and every aggfunc is a scalar reduction. Callers that rely on
1984 ``columns``/``order`` being returned whenever named aggregation was used
1985 must leave this False.
1986 **kwargs: dict, kwargs used in is_multi_agg_with_relabel and
1987 normalize_keyword_aggregation function for relabelling
1989 Returns
1990 -------
1991 relabelling: bool, if there is relabelling or not
1992 func: normalized and mangled func
1993 columns: tuple of column names
1994 order: array of columns indices
1996 Examples
1997 --------
1998 >>> reconstruct_func(None, **{"foo": ("col", "min")})
1999 (True, defaultdict(<class 'list'>, {'col': ['min']}), ('foo',), array([0]))
2001 >>> reconstruct_func("min")
2002 (False, 'min', None, None)
2003 """
2004 # deferred: pandas.core.groupby.generic imports from this module at import
2005 # time, so a top-level import here would be circular
2006 from pandas.core.groupby.base import reduction_kernels
2007 from pandas.core.groupby.generic import NamedAgg
2009 relabeling = func is None and (
2010 is_multi_agg_with_relabel(**kwargs)
2011 or any(isinstance(v, NamedAgg) for v in kwargs.values())
2012 )
2014 columns: tuple[str, ...] | None = None
2015 order: npt.NDArray[np.intp] | None = None
2017 if not relabeling:
2018 if isinstance(func, list) and len(func) > len(set(func)):
2019 # GH 28426 will raise error if duplicated function names are used and
2020 # there is no reassigned name
2021 raise SpecificationError(
2022 "Function names must be unique if there is no new column names assigned"
2023 )
2024 if func is None:
2025 # nicer error message
2026 raise TypeError("Must provide 'func' or tuples of '(column, aggfunc).")
2028 if relabeling:
2029 normalization_needed = not allow_skip_normalization
2030 # error: Incompatible types in assignment (expression has type
2031 # "MutableMapping[Hashable, list[Callable[..., Any] | str]]", variable has type
2032 # "Callable[..., Any] | str | list[Callable[..., Any] | str] |
2033 # MutableMapping[Hashable, Callable[..., Any] | str | list[Callable[..., Any] |
2034 # str]] | None")
2035 converted_kwargs = {}
2036 for key, val in kwargs.items():
2037 if isinstance(val, NamedAgg):
2038 column = val.column
2039 aggfunc = val.aggfunc
2040 if val.args or val.kwargs:
2041 aggfunc = lambda x, func=aggfunc, a=val.args, kw=val.kwargs: func(
2042 x, *a, **kw
2043 )
2044 else:
2045 column, aggfunc = val
2047 # The un-normalized {column: aggfunc} form only matches the normalized
2048 # one when the aggfunc reduces each group to a single scalar. A
2049 # list-like aggfunc, or a string naming a non-reduction groupby method
2050 # (e.g. "describe"/"ohlc", which give a frame per group), would widen
2051 # the result past the one output column per keyword named aggregation
2052 # promises.
2053 if (
2054 column != key
2055 or is_list_like(aggfunc)
2056 or (isinstance(aggfunc, str) and aggfunc not in reduction_kernels)
2057 ):
2058 normalization_needed = True
2059 converted_kwargs[key] = column, aggfunc
2061 if normalization_needed:
2062 func, columns, order = normalize_keyword_aggregation( # type: ignore[assignment]
2063 converted_kwargs
2064 )
2065 else:
2066 # When names match, convert to dict format {column: aggfunc}
2067 # and disable relabeling since no reordering is needed
2068 func = {
2069 column: aggfunc for key, (column, aggfunc) in converted_kwargs.items()
2070 }
2071 # Normalization is skipped if all kwargs keys are the same as their
2072 # corresponding column name.
2073 relabeling = False
2075 assert func is not None
2077 return relabeling, func, columns, order
2080def is_multi_agg_with_relabel(**kwargs) -> bool:
2081 """
2082 Check whether kwargs passed to .agg look like multi-agg with relabeling.
2084 Parameters
2085 ----------
2086 **kwargs : dict
2088 Returns
2089 -------
2090 bool
2092 Examples
2093 --------
2094 >>> is_multi_agg_with_relabel(a="max")
2095 False
2096 >>> is_multi_agg_with_relabel(a_max=("a", "max"), a_min=("a", "min"))
2097 True
2098 >>> is_multi_agg_with_relabel()
2099 False
2100 """
2101 return all(isinstance(v, tuple) and len(v) == 2 for v in kwargs.values()) and (
2102 len(kwargs) > 0
2103 )
2106def normalize_keyword_aggregation(
2107 kwargs: dict,
2108) -> tuple[
2109 MutableMapping[Hashable, list[AggFuncTypeBase]],
2110 tuple[str, ...],
2111 npt.NDArray[np.intp],
2112]:
2113 """
2114 Normalize user-provided "named aggregation" kwargs.
2115 Transforms from the new ``Mapping[str, NamedAgg]`` style kwargs
2116 to the old Dict[str, List[scalar]]].
2118 Parameters
2119 ----------
2120 kwargs : dict
2122 Returns
2123 -------
2124 aggspec : dict
2125 The transformed kwargs.
2126 columns : tuple[str, ...]
2127 The user-provided keys.
2128 col_idx_order : List[int]
2129 List of columns indices.
2131 Examples
2132 --------
2133 >>> normalize_keyword_aggregation({"output": ("input", "sum")})
2134 (defaultdict(<class 'list'>, {'input': ['sum']}), ('output',), array([0]))
2135 """
2136 from pandas.core.indexes.base import Index
2138 # Normalize the aggregation functions as Mapping[column, List[func]],
2139 # process normally, then fixup the names.
2140 # TODO: aggspec type: typing.Dict[str, List[AggScalar]]
2141 aggspec = defaultdict(list)
2142 order = []
2143 columns = tuple(kwargs.keys())
2145 for column, aggfunc in kwargs.values():
2146 aggspec[column].append(aggfunc)
2147 order.append((column, com.get_callable_name(aggfunc) or aggfunc))
2149 # uniquify aggfunc name if duplicated in order list
2150 uniquified_order = _make_unique_kwarg_list(order)
2152 # GH 25719, due to aggspec will change the order of assigned columns in aggregation
2153 # uniquified_aggspec will store uniquified order list and will compare it with order
2154 # based on index
2155 aggspec_order = [
2156 (column, com.get_callable_name(aggfunc) or aggfunc)
2157 for column, aggfuncs in aggspec.items()
2158 for aggfunc in aggfuncs
2159 ]
2160 uniquified_aggspec = _make_unique_kwarg_list(aggspec_order)
2162 # get the new index of columns by comparison
2163 col_idx_order = Index(uniquified_aggspec).get_indexer(uniquified_order) # type: ignore[arg-type]
2164 return aggspec, columns, col_idx_order
2167def _make_unique_kwarg_list(
2168 seq: Sequence[tuple[Any, Any]],
2169) -> Sequence[tuple[Any, Any]]:
2170 """
2171 Uniquify aggfunc name of the pairs in the order list
2173 Examples:
2174 --------
2175 >>> kwarg_list = [("a", "<lambda>"), ("a", "<lambda>"), ("b", "<lambda>")]
2176 >>> _make_unique_kwarg_list(kwarg_list)
2177 [('a', '<lambda>_0'), ('a', '<lambda>_1'), ('b', '<lambda>')]
2178 """
2179 return [
2180 (pair[0], f"{pair[1]}_{seq[:i].count(pair)}") if seq.count(pair) > 1 else pair
2181 for i, pair in enumerate(seq)
2182 ]
2185def relabel_result(
2186 result: DataFrame | Series,
2187 func: dict[str, list[Callable | str]],
2188 columns: Iterable[Hashable],
2189 order: Iterable[int],
2190) -> dict[Hashable, Series]:
2191 """
2192 Internal function to reorder result if relabelling is True for
2193 dataframe.agg, and return the reordered result in dict.
2195 Parameters:
2196 ----------
2197 result: Result from aggregation
2198 func: Dict of (column name, funcs)
2199 columns: New columns name for relabelling
2200 order: New order for relabelling
2202 Examples
2203 --------
2204 >>> from pandas.core.apply import relabel_result
2205 >>> result = pd.DataFrame(
2206 ... {"A": [np.nan, 2, np.nan], "C": [6, np.nan, np.nan], "B": [np.nan, 4, 2.5]},
2207 ... index=["max", "mean", "min"],
2208 ... )
2209 >>> funcs = {"A": ["max"], "C": ["max"], "B": ["mean", "min"]}
2210 >>> columns = ("foo", "aab", "bar", "dat")
2211 >>> order = [0, 1, 2, 3]
2212 >>> result_in_dict = relabel_result(result, funcs, columns, order)
2213 >>> pd.DataFrame(result_in_dict, index=columns)
2214 A C B
2215 foo 2.0 NaN NaN
2216 aab NaN 6.0 NaN
2217 bar NaN NaN 4.0
2218 dat NaN NaN 2.5
2219 """
2220 from pandas.core.indexes.base import Index
2222 reordered_indexes = [
2223 pair[0] for pair in sorted(zip(columns, order, strict=True), key=lambda t: t[1])
2224 ]
2225 reordered_result_in_dict: dict[Hashable, Series] = {}
2226 idx = 0
2228 reorder_mask = not isinstance(result, ABCSeries) and len(result.columns) > 1
2229 for col, fun in func.items():
2230 s = result[col].dropna()
2232 # In the `_aggregate`, the callable names are obtained and used in `result`, and
2233 # these names are ordered alphabetically. e.g.
2234 # C2 C1
2235 # <lambda> 1 NaN
2236 # amax NaN 4.0
2237 # max NaN 4.0
2238 # sum 18.0 6.0
2239 # Therefore, the order of functions for each column could be shuffled
2240 # accordingly so need to get the callable name if it is not parsed names, and
2241 # reorder the aggregated result for each column.
2242 # e.g. if df.agg(c1=("C2", sum), c2=("C2", lambda x: min(x))), correct order is
2243 # [sum, <lambda>], but in `result`, it will be [<lambda>, sum], and we need to
2244 # reorder so that aggregated values map to their functions regarding the order.
2246 # However there is only one column being used for aggregation, not need to
2247 # reorder since the index is not sorted, and keep as is in `funcs`, e.g.
2248 # A
2249 # min 1.0
2250 # mean 1.5
2251 # mean 1.5
2252 if reorder_mask:
2253 fun = [
2254 com.get_callable_name(f) if not isinstance(f, str) else f # type: ignore[misc]
2255 for f in fun
2256 ]
2257 col_idx_order = Index(s.index, copy=False).get_indexer(fun)
2258 valid_idx = col_idx_order != -1
2259 if valid_idx.any():
2260 s = s.iloc[col_idx_order[valid_idx]]
2261 # assign the new user-provided "named aggregation" as index names, and reindex
2262 # it based on the whole user-provided names.
2263 if not s.empty:
2264 s.index = reordered_indexes[idx : idx + len(fun)]
2265 reordered_result_in_dict[col] = s.reindex(columns)
2266 idx = idx + len(fun)
2267 return reordered_result_in_dict
2270def reconstruct_and_relabel_result(result, func, **kwargs) -> DataFrame | Series:
2271 from pandas import DataFrame
2273 relabeling, func, columns, order = reconstruct_func(func, **kwargs)
2275 if relabeling:
2276 # This is to keep the order to columns occurrence unchanged, and also
2277 # keep the order of new columns occurrence unchanged
2279 # For the return values of reconstruct_func, if relabeling is
2280 # False, columns and order will be None.
2281 assert columns is not None
2282 assert order is not None
2284 result_in_dict = relabel_result(result, func, columns, order)
2285 result = DataFrame(result_in_dict, index=columns)
2287 return result
2290# TODO: Can't use, because mypy doesn't like us setting __name__
2291# error: "partial[Any]" has no attribute "__name__"
2292# the type is:
2293# typing.Sequence[Callable[..., ScalarResult]]
2294# -> typing.Sequence[Callable[..., ScalarResult]]:
2297def _managle_lambda_list(aggfuncs: Sequence[Any]) -> Sequence[Any]:
2298 """
2299 Possibly mangle a list of aggfuncs.
2301 Parameters
2302 ----------
2303 aggfuncs : Sequence
2305 Returns
2306 -------
2307 mangled: list-like
2308 A new AggSpec sequence, where lambdas have been converted
2309 to have unique names.
2311 Notes
2312 -----
2313 If just one aggfunc is passed, the name will not be mangled.
2314 """
2315 if len(aggfuncs) <= 1:
2316 # don't mangle for .agg([lambda x: .])
2317 return aggfuncs
2318 i = 0
2319 mangled_aggfuncs = []
2320 for aggfunc in aggfuncs:
2321 if com.get_callable_name(aggfunc) == "<lambda>":
2322 aggfunc = partial(aggfunc)
2323 # error: "partial[Any]" has no attribute "__name__"; maybe "__new__"?
2324 aggfunc.__name__ = f"<lambda_{i}>" # type: ignore[attr-defined]
2325 i += 1
2326 mangled_aggfuncs.append(aggfunc)
2328 return mangled_aggfuncs
2331def maybe_mangle_lambdas(agg_spec: Any) -> Any:
2332 """
2333 Make new lambdas with unique names.
2335 Parameters
2336 ----------
2337 agg_spec : Any
2338 An argument to GroupBy.agg.
2339 Non-dict-like `agg_spec` are pass through as is.
2340 For dict-like `agg_spec` a new spec is returned
2341 with name-mangled lambdas.
2343 Returns
2344 -------
2345 mangled : Any
2346 Same type as the input.
2348 Examples
2349 --------
2350 >>> maybe_mangle_lambdas("sum")
2351 'sum'
2352 >>> maybe_mangle_lambdas([lambda: 1, lambda: 2]) # doctest: +SKIP
2353 [<function __main__.<lambda_0>,
2354 <function pandas...._make_lambda.<locals>.f(*args, **kwargs)>]
2355 """
2356 is_dict = is_dict_like(agg_spec)
2357 if not (is_dict or is_list_like(agg_spec)):
2358 return agg_spec
2359 mangled_aggspec = type(agg_spec)() # dict or OrderedDict
2361 if is_dict:
2362 for key, aggfuncs in agg_spec.items():
2363 if is_list_like(aggfuncs) and not is_dict_like(aggfuncs):
2364 mangled_aggfuncs = _managle_lambda_list(aggfuncs)
2365 else:
2366 mangled_aggfuncs = aggfuncs
2368 mangled_aggspec[key] = mangled_aggfuncs
2369 else:
2370 mangled_aggspec = _managle_lambda_list(agg_spec)
2372 return mangled_aggspec
2375def validate_func_kwargs(
2376 kwargs: dict,
2377) -> tuple[list[str], list[str | Callable[..., Any]]]:
2378 """
2379 Validates types of user-provided "named aggregation" kwargs.
2380 `TypeError` is raised if aggfunc is not `str` or callable.
2382 Parameters
2383 ----------
2384 kwargs : dict
2386 Returns
2387 -------
2388 columns : List[str]
2389 List of user-provided keys.
2390 func : List[Union[str, callable[...,Any]]]
2391 List of user-provided aggfuncs
2393 Examples
2394 --------
2395 >>> validate_func_kwargs({"one": "min", "two": "max"})
2396 (['one', 'two'], ['min', 'max'])
2397 """
2398 tuple_given_message = "func is expected but received {} in **kwargs."
2399 columns = list(kwargs)
2400 func = []
2401 for col_func in kwargs.values():
2402 if not (isinstance(col_func, str) or callable(col_func)):
2403 raise TypeError(tuple_given_message.format(type(col_func).__name__))
2404 func.append(col_func)
2405 if not columns:
2406 no_arg_message = "Must provide 'func' or named aggregation **kwargs."
2407 raise TypeError(no_arg_message)
2408 return columns, func
2411def include_axis(op_name: Literal["agg", "apply"], colg: Series | DataFrame) -> bool:
2412 return isinstance(colg, ABCDataFrame) or (
2413 isinstance(colg, ABCSeries) and op_name == "agg"
2414 )