Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/nanops.py: 17%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import functools
4import itertools
5from typing import (
6 TYPE_CHECKING,
7 Any,
8 Literal,
9 cast,
10)
11import warnings
13import numpy as np
15from pandas._config.config import _global_config as config
17from pandas._libs import (
18 NaT,
19 NaTType,
20 iNaT,
21 lib,
22)
23import pandas._libs.algos as libalgos
24from pandas._libs.tslibs import OutOfBoundsTimedelta
25from pandas.compat._optional import import_optional_dependency
27from pandas.core.dtypes.common import (
28 ensure_float64,
29 is_float,
30 is_float_dtype,
31 is_integer,
32 is_numeric_dtype,
33 is_object_dtype,
34 needs_i8_conversion,
35 pandas_dtype,
36)
37from pandas.core.dtypes.missing import (
38 isna,
39 na_value_for_dtype,
40 notna,
41)
43if TYPE_CHECKING:
44 from collections.abc import Callable
46 from pandas._typing import (
47 ArrayLike,
48 AxisInt,
49 CorrelationMethod,
50 Dtype,
51 DtypeObj,
52 F,
53 Scalar,
54 Shape,
55 npt,
56 )
58bn = import_optional_dependency("bottleneck", errors="warn")
59_BOTTLENECK_INSTALLED = bn is not None
60_USE_BOTTLENECK = False
63def set_use_bottleneck(v: bool = True) -> None:
64 # set/unset to use bottleneck
65 global _USE_BOTTLENECK
66 if _BOTTLENECK_INSTALLED:
67 _USE_BOTTLENECK = v
70set_use_bottleneck(config["compute"]["use_bottleneck"])
73class disallow:
74 def __init__(self, *dtypes: Dtype) -> None:
75 super().__init__()
76 self.dtypes = tuple(pandas_dtype(dtype).type for dtype in dtypes)
78 def check(self, obj) -> bool:
79 return hasattr(obj, "dtype") and issubclass(obj.dtype.type, self.dtypes)
81 def __call__(self, f: F) -> F:
82 @functools.wraps(f)
83 def _f(*args, **kwargs):
84 obj_iter = itertools.chain(args, kwargs.values())
85 if any(self.check(obj) for obj in obj_iter):
86 f_name = f.__name__.replace("nan", "")
87 raise TypeError(
88 f"reduction operation '{f_name}' not allowed for this dtype"
89 )
90 try:
91 return f(*args, **kwargs)
92 except ValueError as e:
93 # we want to transform an object array
94 # ValueError message to the more typical TypeError
95 # e.g. this is normally a disallowed function on
96 # object arrays that contain strings
97 if is_object_dtype(args[0]):
98 raise TypeError(e) from e
99 raise
101 return cast("F", _f)
104class bottleneck_switch:
105 def __init__(self, name=None, **kwargs) -> None:
106 self.name = name
107 self.kwargs = kwargs
109 def __call__(self, alt: F) -> F:
110 bn_name = self.name or alt.__name__
112 try:
113 bn_func = getattr(bn, bn_name)
114 except (AttributeError, NameError): # pragma: no cover
115 bn_func = None
117 @functools.wraps(alt)
118 def f(
119 values: np.ndarray,
120 *,
121 axis: AxisInt | None = None,
122 skipna: bool = True,
123 **kwds,
124 ):
125 if len(self.kwargs) > 0:
126 for k, v in self.kwargs.items():
127 if k not in kwds:
128 kwds[k] = v
130 # GH#18976 bottleneck's nanmin/nanmax raise on empty input; skip it
131 # and let each nanops function return the NA for its own dtype.
132 if (
133 _USE_BOTTLENECK
134 and skipna
135 and values.size > 0
136 and _bn_ok_dtype(values.dtype, bn_name)
137 ):
138 if kwds.get("mask", None) is None:
139 # `mask` is not recognised by bottleneck, would raise
140 # TypeError if called
141 kwds.pop("mask", None)
142 result = bn_func(values, axis=axis, **kwds) # pyright: ignore[reportOptionalCall]
144 # prefer to treat inf/-inf as NA, but must compute the func
145 # twice :(
146 if _has_infs(result):
147 result = alt(values, axis=axis, skipna=skipna, **kwds)
148 else:
149 result = alt(values, axis=axis, skipna=skipna, **kwds)
150 else:
151 result = alt(values, axis=axis, skipna=skipna, **kwds)
153 return result
155 return cast("F", f)
158def _bn_ok_dtype(dtype: DtypeObj, name: str) -> bool:
159 # Bottleneck chokes on datetime64, PeriodDtype (or an EA)
160 if dtype != object and not needs_i8_conversion(dtype):
161 # GH 42878
162 # Bottleneck uses naive summation leading to O(n) loss of precision
163 # unlike numpy which implements pairwise summation, which has O(log(n)) loss
164 # crossref: https://github.com/pydata/bottleneck/issues/379
166 # GH 15507
167 # bottleneck does not properly upcast during the sum
168 # so can overflow
170 # GH 9422
171 # further we also want to preserve NaN when all elements
172 # are NaN, unlike bottleneck/numpy which consider this
173 # to be 0
174 return name not in ["nansum", "nanprod", "nanmean"]
175 return False
178def _has_infs(result) -> bool:
179 if isinstance(result, np.ndarray):
180 if result.dtype in ("f8", "f4"):
181 # Note: outside of a nanops-specific test, we always have
182 # result.ndim == 1, so there is no risk of this ravel making a copy.
183 return lib.has_infs(result.ravel("K"))
184 try:
185 return np.isinf(result).any()
186 except (TypeError, NotImplementedError):
187 # if it doesn't support infs, then it can't have infs
188 return False
191def _get_fill_value(
192 dtype: DtypeObj, fill_value: Scalar | None = None, fill_value_typ=None
193):
194 """return the correct fill value for the dtype of the values"""
195 if fill_value is not None:
196 return fill_value
197 if _na_ok_dtype(dtype):
198 if fill_value_typ is None:
199 return np.nan
200 elif fill_value_typ == "+inf":
201 return np.inf
202 else:
203 return -np.inf
204 elif dtype.kind == "u":
205 # Unsigned ints: use a uint64 sentinel so np.where keeps the values
206 # unsigned. An int64 fill would force a uint64/int64 mix to promote to
207 # float64, losing precision for values above 2**53 and giving the wrong
208 # idxmin/idxmax/min/max (GH#64478). u8max/0 also bracket the full
209 # uint64 range, unlike i8max which real values can exceed.
210 if fill_value_typ == "+inf":
211 return np.uint64(lib.u8max)
212 return np.uint64(0)
213 elif fill_value_typ == "+inf":
214 # need the max int here
215 # Return as np.int64 so that np.where promotes the dtype
216 # instead of raising OverflowError (numpy 2.5+) when the
217 # value doesn't fit in the array's dtype (e.g. int8).
218 return np.int64(lib.i8max)
219 else:
220 return np.int64(iNaT)
223def _maybe_get_mask(
224 values: np.ndarray, skipna: bool, mask: npt.NDArray[np.bool_] | None
225) -> npt.NDArray[np.bool_] | None:
226 """
227 Compute a mask if and only if necessary.
229 An explicit `mask` is returned unchanged; the values it marks need not be
230 NaN (a masked array stores fill values there). Otherwise one is computed
231 with isna(), but only where it is needed: never for boolean or integer
232 values, which cannot store NaN, and under skipna=False only for
233 datetime64/timedelta64, whose NaT stops being detectable once `_get_values`
234 views it as i8 (GH#37392).
236 A caller that already holds that i8 view must pass its own mask.
238 Parameters
239 ----------
240 values : ndarray
241 input array to potentially compute mask for
242 skipna : bool
243 boolean for whether NaNs should be skipped
244 mask : Optional[ndarray]
245 nan-mask if known
247 Returns
248 -------
249 Optional[np.ndarray[bool]]
250 """
251 if mask is None:
252 if values.dtype.kind in "biu":
253 # Boolean data cannot contain nulls, so signal via mask being None
254 return None
256 if skipna or values.dtype.kind in "mM":
257 mask = isna(values)
259 return mask
262def _get_values(
263 values: np.ndarray,
264 skipna: bool,
265 fill_value: Any = None,
266 fill_value_typ: str | None = None,
267 mask: npt.NDArray[np.bool_] | None = None,
268) -> tuple[np.ndarray, npt.NDArray[np.bool_] | None]:
269 """
270 Utility to get the values view, mask, dtype, dtype_max, and fill_value.
272 If both mask and fill_value/fill_value_typ are not None and skipna is True,
273 the values array will be copied.
275 For input arrays of boolean or integer dtypes, copies will only occur if a
276 precomputed mask, a fill_value/fill_value_typ, and skipna=True are
277 provided.
279 Parameters
280 ----------
281 values : ndarray
282 input array to potentially compute mask for
283 skipna : bool
284 boolean for whether NaNs should be skipped
285 fill_value : Any
286 value to fill NaNs with
287 fill_value_typ : str
288 Set to '+inf' or '-inf' to handle dtype-specific infinities
289 mask : Optional[np.ndarray[bool]]
290 nan-mask if known
292 Returns
293 -------
294 values : ndarray
295 Potential copy of input value array
296 mask : Optional[ndarray[bool]]
297 Mask for values, if deemed necessary to compute
298 """
299 # In _get_values is only called from within nanops, and in all cases
300 # with scalar fill_value. This guarantee is important for the
301 # np.where call below
303 mask = _maybe_get_mask(values, skipna, mask)
305 dtype = values.dtype
307 datetimelike = False
308 if values.dtype.kind in "mM":
309 # changing timedelta64/datetime64 to int64 needs to happen after
310 # finding `mask` above
311 values = np.asarray(values.view("i8"))
312 datetimelike = True
314 if skipna and (mask is not None):
315 # get our fill value (in case we need to provide an alternative
316 # dtype for it)
317 fill_value = _get_fill_value(
318 dtype, fill_value=fill_value, fill_value_typ=fill_value_typ
319 )
321 if fill_value is not None:
322 if mask.any():
323 if datetimelike or _na_ok_dtype(dtype):
324 values = values.copy()
325 np.putmask(values, mask, fill_value)
326 else:
327 # np.where will promote if needed
328 values = np.where(~mask, values, fill_value)
330 return values, mask
333def _get_dtype_max(dtype: np.dtype) -> np.dtype:
334 # return a platform independent precision dtype
335 dtype_max = dtype
336 if dtype.kind in "bi":
337 dtype_max = np.dtype(np.int64)
338 elif dtype.kind == "u":
339 dtype_max = np.dtype(np.uint64)
340 elif dtype.kind == "f":
341 dtype_max = np.dtype(np.float64)
342 return dtype_max
345def _na_ok_dtype(dtype: DtypeObj) -> bool:
346 if needs_i8_conversion(dtype):
347 return False
348 return not issubclass(dtype.type, np.integer)
351def _wrap_results(result, dtype: np.dtype, fill_value=None, result_mask=None):
352 """wrap our results if needed"""
353 if result is NaT:
354 pass
356 elif dtype.kind == "M":
357 if fill_value is None:
358 # GH#24293
359 fill_value = iNaT
360 if not isinstance(result, np.ndarray):
361 assert not isna(fill_value), "Expected non-null fill_value"
362 if result == fill_value:
363 result = np.nan
365 if isna(result):
366 result = np.datetime64("NaT", "ns").astype(dtype)
367 else:
368 result = np.int64(result).view(dtype)
369 # retain original unit
370 result = result.astype(dtype, copy=False)
371 else:
372 # If we have float dtype, taking a view will give the wrong result
373 result = result.astype(dtype)
374 elif dtype.kind == "m":
375 if not isinstance(result, np.ndarray):
376 if result_mask or result == fill_value or np.isnan(result):
377 unit = np.datetime_data(dtype)[0]
378 result = np.timedelta64("NaT", unit) # type: ignore[call-overload]
380 elif np.fabs(result) > lib.i8max:
381 # GH#43178: raise if the result is too large for the dtype's unit
382 raise OutOfBoundsTimedelta("overflow in timedelta operation")
383 else:
384 # return a timedelta64 with the original unit
385 result = np.int64(result).astype(dtype, copy=False)
387 else:
388 overflow = np.abs(result) > lib.i8max
389 if result_mask is not None:
390 # GH#43178: positions that will be set to NaT (skipna=False) are
391 # exempt from the overflow check and must not trip the cast below
392 overflow &= ~result_mask
393 result = np.where(result_mask, 0, result)
394 if overflow.any():
395 # GH#43178: raise if any result is too large for the dtype's unit
396 raise OutOfBoundsTimedelta("overflow in timedelta operation")
397 result = result.astype("m8[ns]").view(dtype)
399 return result
402def _datetimelike_compat(func: F) -> F:
403 """
404 If we have datetime64 or timedelta64 values, ensure we have a correct
405 mask before calling the wrapped function, then cast back afterwards.
406 """
408 @functools.wraps(func)
409 def new_func(
410 values: np.ndarray,
411 *,
412 axis: AxisInt | None = None,
413 skipna: bool = True,
414 mask: npt.NDArray[np.bool_] | None = None,
415 **kwargs,
416 ):
417 orig_values = values
419 datetimelike = values.dtype.kind in "mM"
420 if datetimelike and mask is None:
421 mask = isna(values)
423 result = func(values, axis=axis, skipna=skipna, mask=mask, **kwargs)
425 if datetimelike:
426 if getattr(result, "dtype", None) == orig_values.dtype:
427 # GH#18976 empty input short-circuits to NA of the original
428 # dtype; _wrap_results expects an i8 result and would compare
429 # this against iNaT.
430 return result
432 result_mask = None
433 if not skipna:
434 assert mask is not None # checked above
435 # positions with any NA reduce to NaT, so exempt them from the
436 # timedelta overflow guard in _wrap_results
437 if isinstance(result, np.ndarray):
438 result_mask = mask.any(axis=axis)
439 else:
440 result_mask = mask.any()
441 result = _wrap_results(
442 result, orig_values.dtype, fill_value=iNaT, result_mask=result_mask
443 )
444 if not skipna:
445 assert mask is not None # checked above
446 result = _mask_datetimelike_result(result, axis, mask, orig_values)
448 return result
450 return cast("F", new_func)
453def _na_for_min_count(values: np.ndarray, axis: AxisInt | None) -> Scalar | np.ndarray:
454 """
455 Return the missing value for `values`.
457 Parameters
458 ----------
459 values : ndarray
460 axis : int or None
461 axis for the reduction, required if values.ndim > 1.
463 Returns
464 -------
465 result : scalar or ndarray
466 For 1-D values, returns a scalar of the correct missing type.
467 For 2-D values, returns a 1-D array where each element is missing.
468 """
469 # we either return np.nan or pd.NaT
470 if values.dtype.kind in "iufcb":
471 values = values.astype("float64")
472 fill_value = na_value_for_dtype(values.dtype)
474 if values.ndim == 1:
475 return fill_value
476 elif axis is None:
477 return fill_value
478 else:
479 result_shape = values.shape[:axis] + values.shape[axis + 1 :]
481 return np.full(result_shape, fill_value, dtype=values.dtype)
484def maybe_operate_rowwise(func: F) -> F:
485 """
486 NumPy operations on C-contiguous ndarrays with axis=1 can be
487 very slow if axis 1 >> axis 0.
488 Operate row-by-row and concatenate the results.
489 """
491 @functools.wraps(func)
492 def newfunc(values: np.ndarray, *, axis: AxisInt | None = None, **kwargs):
493 if (
494 axis == 1
495 and values.ndim == 2
496 and values.flags["C_CONTIGUOUS"]
497 # only takes this path for wide arrays (long dataframes), for threshold see
498 # https://github.com/pandas-dev/pandas/pull/43311#issuecomment-974891737
499 and (values.shape[1] / 1000) > values.shape[0]
500 and values.dtype not in (object, bool)
501 ):
502 arrs = list(values)
503 if kwargs.get("mask") is not None:
504 mask = kwargs.pop("mask")
505 results = [
506 func(arrs[i], mask=mask[i], **kwargs) for i in range(len(arrs))
507 ]
508 else:
509 results = [func(x, **kwargs) for x in arrs]
510 return np.array(results)
512 return func(values, axis=axis, **kwargs)
514 return cast("F", newfunc)
517def _ensure_numeric(values: np.ndarray) -> np.ndarray:
518 """
519 Convert an object-dtype ndarray to the numeric dtype it represents.
521 Object-dtype input is normalized once, up front, so that every statistic
522 below can treat it exactly as it treats the equivalent float64/complex128
523 array. Whatever a reduction does for ``np.float64`` input it now also does
524 for an object array of floats, and any object array that does not hold
525 numbers raises ``TypeError`` from here instead of from whichever ufunc
526 happened to touch it first.
528 Parameters
529 ----------
530 values : np.ndarray
532 Returns
533 -------
534 np.ndarray
535 `values` unchanged if it is not object-dtype, else the equivalent
536 float64 or complex128 array.
538 Raises
539 ------
540 TypeError
541 If the values are not numbers.
542 """
543 if values.dtype.kind in "SUT":
544 raise TypeError(f"Could not convert {values.dtype} values to numeric")
545 if values.dtype != object:
546 return values
548 # GH#44008, GH#36703 strings (and datetime64s) convert to a number without
549 # complaint, so reject them before numpy gets the chance
550 non_numeric, has_nat = lib.first_non_numeric(values)
551 if non_numeric is not None:
552 raise TypeError(f"Could not convert {non_numeric!r} to numeric")
554 if has_nat:
555 # A datetime64/timedelta64 NaT is an NA, but numpy casts it to the
556 # int64 sentinel rather than to NaN. Swap in None, which every cast
557 # below maps to the same NaN it maps pd.NaT to. Any *non*-NaT
558 # datetime64 was already rejected above, so every one left is an NA.
559 nat_mask = np.array(
560 [isinstance(val, (np.datetime64, np.timedelta64)) for val in values.ravel()]
561 )
562 values = values.copy()
563 values[nat_mask.reshape(values.shape)] = None
565 try:
566 with warnings.catch_warnings():
567 # numpy *scalars* have __float__ and so cast to float64 while
568 # discarding the imaginary part; only Python complex refuses
569 # outright. Treat the warning as a failed cast either way, so a
570 # complex payload falls through to complex128 below (GH#34671).
571 warnings.simplefilter("error", np.exceptions.ComplexWarning)
572 return values.astype(np.float64)
573 except (TypeError, ValueError, np.exceptions.ComplexWarning):
574 pass
576 mask = isna(values)
577 filled = values
578 if mask.any():
579 # None/NaT/pd.NA have no float(); retry with the NaN sentinel
580 filled = values.copy()
581 filled[mask] = np.nan
582 try:
583 with warnings.catch_warnings():
584 warnings.simplefilter("error", np.exceptions.ComplexWarning)
585 return filled.astype(np.float64)
586 except (TypeError, ValueError, np.exceptions.ComplexWarning):
587 pass
589 if mask.any():
590 # e.g. complex, which has no __float__. Convert each NA on its own so
591 # it lands where an object->complex128 cast would have put it: None
592 # and the NAs numpy refuses outright on nan+nanj, np.nan on nan+0j.
593 filled[mask] = [_to_complex(val) for val in values[mask]]
595 try:
596 return filled.astype(np.complex128)
597 except (TypeError, ValueError) as err:
598 # GH#29941 e.g. Timestamps, or elements that are themselves list-like
599 raise TypeError(
600 f"Could not convert {_first_unconvertible(filled)!r} to numeric"
601 ) from err
604def _to_complex(val: Any) -> complex:
605 """
606 Convert `val` to a complex, mapping an NA that has no complex() to nan+nanj.
607 """
608 try:
609 return complex(val)
610 except (TypeError, ValueError):
611 return complex(np.nan, np.nan)
614def _complex_castable(val: Any) -> bool:
615 """
616 Whether numpy's object->complex128 cast can handle `val` on its own.
617 """
618 try:
619 complex(val)
620 except (TypeError, ValueError):
621 return False
622 return True
625def _first_unconvertible(values: np.ndarray) -> object:
626 """
627 Return the element responsible for a failed `_ensure_numeric` conversion.
628 """
629 for val in values.ravel():
630 if not _complex_castable(val):
631 return val
632 # numpy balked at something complex() accepts; report the values as a whole
633 return values
636def _ensure_numeric_input(func: F) -> F:
637 """
638 Normalize object-dtype values before the wrapped reduction sees them.
639 """
641 @functools.wraps(func)
642 def new_func(values: np.ndarray, **kwargs):
643 return func(_ensure_numeric(values), **kwargs)
645 return cast("F", new_func)
648def dt64_any_all_msg(how: str) -> str:
649 """
650 Message for the GH#34479 removal of any/all on datetime64 data.
652 Shared so that the reduction, groupby and sparse paths cannot drift apart.
653 """
654 return (
655 f"'{how}' with datetime64 dtypes is not supported. "
656 f"Use (obj != pd.Timestamp(0)).{how}() instead."
657 )
660def nanany(
661 values: np.ndarray,
662 *,
663 axis: AxisInt | None = None,
664 skipna: bool = True,
665 mask: npt.NDArray[np.bool_] | None = None,
666) -> bool:
667 """
668 Check if any elements along an axis evaluate to True.
670 Parameters
671 ----------
672 values : ndarray
673 axis : int, optional
674 skipna : bool, default True
675 mask : ndarray[bool], optional
676 nan-mask if known
678 Returns
679 -------
680 result : bool
682 Examples
683 --------
684 >>> from pandas.core import nanops
685 >>> s = pd.Series([1, 2])
686 >>> nanops.nanany(s.values)
687 np.True_
689 >>> from pandas.core import nanops
690 >>> s = pd.Series([np.nan])
691 >>> nanops.nanany(s.values)
692 np.False_
693 """
694 if values.dtype.kind in "iub" and mask is None:
695 # GH#26032 fastpath
696 # error: Incompatible return value type (got "Union[bool_, ndarray]",
697 # expected "bool")
698 return values.any(axis) # type: ignore[return-value]
700 if values.dtype.kind == "M":
701 # GH#34479
702 raise TypeError(dt64_any_all_msg("any"))
704 values, _ = _get_values(values, skipna, fill_value=False, mask=mask)
706 # For object type, any won't necessarily return
707 # boolean values (numpy/numpy#4352)
708 if values.dtype == object:
709 values = values.astype(bool)
711 # error: Incompatible return value type (got "Union[bool_, ndarray]", expected
712 # "bool")
713 return values.any(axis) # type: ignore[return-value]
716def nanall(
717 values: np.ndarray,
718 *,
719 axis: AxisInt | None = None,
720 skipna: bool = True,
721 mask: npt.NDArray[np.bool_] | None = None,
722) -> bool:
723 """
724 Check if all elements along an axis evaluate to True.
726 Parameters
727 ----------
728 values : ndarray
729 axis : int, optional
730 skipna : bool, default True
731 mask : ndarray[bool], optional
732 nan-mask if known
734 Returns
735 -------
736 result : bool
738 Examples
739 --------
740 >>> from pandas.core import nanops
741 >>> s = pd.Series([1, 2, np.nan])
742 >>> nanops.nanall(s.values)
743 np.True_
745 >>> from pandas.core import nanops
746 >>> s = pd.Series([1, 0])
747 >>> nanops.nanall(s.values)
748 np.False_
749 """
750 if values.dtype.kind in "iub" and mask is None:
751 # GH#26032 fastpath
752 # error: Incompatible return value type (got "Union[bool_, ndarray]",
753 # expected "bool")
754 return values.all(axis) # type: ignore[return-value]
756 if values.dtype.kind == "M":
757 # GH#34479
758 raise TypeError(dt64_any_all_msg("all"))
760 values, _ = _get_values(values, skipna, fill_value=True, mask=mask)
762 # For object type, all won't necessarily return
763 # boolean values (numpy/numpy#4352)
764 if values.dtype == object:
765 values = values.astype(bool)
767 # error: Incompatible return value type (got "Union[bool_, ndarray]", expected
768 # "bool")
769 return values.all(axis) # type: ignore[return-value]
772@disallow("M8")
773@_datetimelike_compat
774@maybe_operate_rowwise
775def nansum(
776 values: np.ndarray,
777 *,
778 axis: AxisInt | None = None,
779 skipna: bool = True,
780 min_count: int = 0,
781 mask: npt.NDArray[np.bool_] | None = None,
782) -> np.ndarray | np.int64 | float | NaTType:
783 """
784 Sum the elements along an axis ignoring NaNs
786 Parameters
787 ----------
788 values : ndarray[dtype]
789 axis : int, optional
790 skipna : bool, default True
791 min_count: int, default 0
792 mask : ndarray[bool], optional
793 nan-mask if known
795 Returns
796 -------
797 result : dtype
799 Examples
800 --------
801 >>> from pandas.core import nanops
802 >>> s = pd.Series([1, 2, np.nan])
803 >>> nanops.nansum(s.values)
804 np.float64(3.0)
805 """
806 dtype = values.dtype
807 if dtype == object and not skipna:
808 if mask is None:
809 mask = isna(values)
810 if mask.any():
811 # GH#4147: an object NA that supports addition propagates on its
812 # own and carries more type information than a bare nan would
813 # (pd.NA, Decimal("NaN"), NaT), so prefer it. Step in only when it
814 # does not, e.g. None, or values that cannot be added at all.
815 try:
816 the_sum = values.sum(axis, dtype=_get_dtype_max(dtype))
817 except TypeError:
818 pass
819 else:
820 return _maybe_null_out(
821 the_sum, axis, mask, values.shape, min_count=min_count
822 )
823 if values.ndim == 1 or axis is None:
824 # error: Incompatible return value type (got "Union[Scalar,
825 # ndarray]", expected "Union[ndarray, float, NaTType]")
826 return _na_for_min_count(values, axis) # type: ignore[return-value]
827 values = _blank_object_na_slices(values, mask, axis, 0)
828 min_count = _na_min_count(values, axis, min_count)
829 else:
830 values, mask = _get_values(values, skipna, fill_value=0, mask=mask)
831 dtype_sum = _get_dtype_max(dtype)
832 if dtype.kind == "f":
833 # GH#43929 float16 sum overflows easily; upcast like numpy does
834 dtype_sum = np.dtype(np.float64) if dtype == np.float16 else dtype
835 elif dtype.kind == "m":
836 # GH#66551 float64 has a 53-bit mantissa, so accumulating i8 values there
837 # silently rounds the result; sum in int64 instead.
838 the_sum = _sum_timedelta_i8(values, axis, mask, skipna, min_count)
839 return _maybe_null_out(
840 the_sum, axis, mask, values.shape, min_count=min_count, datetimelike=True
841 )
843 the_sum = values.sum(axis, dtype=dtype_sum)
844 the_sum = _maybe_null_out(the_sum, axis, mask, values.shape, min_count=min_count)
846 return the_sum
849def _sum_timedelta_i8(
850 values: np.ndarray,
851 axis: AxisInt | None,
852 mask: npt.NDArray[np.bool_] | None,
853 skipna: bool,
854 min_count: int,
855) -> np.ndarray | np.int64:
856 """
857 Sum i8-viewed timedelta64 values exactly, raising on a result that does not
858 fit in int64.
860 int64 addition is modular, so an intermediate that leaves the range does not
861 corrupt the total; only the total itself has to be checked. Values are
862 summed in blocks short enough that each block sum provably stays in range,
863 and the block sums are then combined in Python ints.
865 Parameters
866 ----------
867 values : ndarray[int64]
868 axis : int, optional
869 mask : ndarray[bool], optional
870 skipna : bool
871 min_count : int
873 Returns
874 -------
875 int64 scalar or ndarray[int64]
877 Raises
878 ------
879 OutOfBoundsTimedelta
880 If the exact sum is not representable as a timedelta64, i.e. it lands
881 outside (iNaT, i8max].
882 """
883 # Results that get discarded downstream are exempt from the bounds check:
884 # positions that reduce to NaT under skipna=False, as in _wrap_results, and
885 # positions that min_count nulls out in _maybe_null_out.
886 exempt: np.ndarray | np.bool_ = np.False_
887 if not skipna and mask is not None:
888 exempt = mask.any(axis=axis)
889 if min_count > 0:
890 if axis is not None and values.ndim > 1:
891 counts = values.shape[axis] - (0 if mask is None else mask.sum(axis))
892 exempt = exempt | (counts < min_count)
893 else:
894 exempt = exempt | check_below_min_count(values.shape, mask, min_count)
896 if not skipna and mask is not None and mask.any():
897 # These positions reduce to NaT regardless of what they sum to, so keep
898 # their raw iNaT out of the accumulator.
899 values = np.where(mask, 0, values)
901 if values.size == 0:
902 return values.sum(axis, dtype=np.int64)
904 reduce_axis: AxisInt = 0
905 if axis is None:
906 values = values.ravel()
907 else:
908 reduce_axis = axis
910 length = values.shape[reduce_axis]
911 largest = max(abs(int(values.min())), abs(int(values.max())))
912 block = lib.i8max // largest if largest else length
914 if block >= length:
915 # Every partial sum is bounded by length * largest <= i8max, so the
916 # int64 sum is exact and in range.
917 return values.sum(reduce_axis, dtype=np.int64)
919 starts = np.arange(0, length, block)
920 block_sums = np.add.reduceat(values, starts, axis=reduce_axis)
921 total = block_sums.sum(reduce_axis, dtype=object)
923 out_of_range = (total <= iNaT) | (total > lib.i8max)
924 if np.any(out_of_range & ~exempt):
925 raise OutOfBoundsTimedelta("overflow in timedelta operation")
927 if isinstance(total, np.ndarray):
928 # anything still out of range here is exempt, i.e. is discarded anyway
929 return np.where(out_of_range, 0, total).astype(np.int64)
930 return np.int64(0 if out_of_range else total)
933def _mask_datetimelike_result(
934 result: np.ndarray | np.datetime64 | np.timedelta64,
935 axis: AxisInt | None,
936 mask: npt.NDArray[np.bool_],
937 orig_values: np.ndarray,
938) -> np.ndarray | np.datetime64 | np.timedelta64 | NaTType:
939 if isinstance(result, np.ndarray):
940 # we need to apply the mask
941 result = result.astype("i8").view(orig_values.dtype)
942 axis_mask = mask.any(axis=axis)
943 result[axis_mask] = iNaT
944 elif mask.any():
945 return np.int64(iNaT).view(orig_values.dtype)
946 return result
949@_ensure_numeric_input
950@bottleneck_switch()
951@_datetimelike_compat
952def nanmean(
953 values: np.ndarray,
954 *,
955 axis: AxisInt | None = None,
956 skipna: bool = True,
957 mask: npt.NDArray[np.bool_] | None = None,
958) -> float:
959 """
960 Compute the mean of the element along an axis ignoring NaNs
962 Parameters
963 ----------
964 values : ndarray
965 axis : int, optional
966 skipna : bool, default True
967 mask : ndarray[bool], optional
968 nan-mask if known
970 Returns
971 -------
972 float
973 Unless input is a float array, in which case use the same
974 precision as the input array.
976 Examples
977 --------
978 >>> from pandas.core import nanops
979 >>> s = pd.Series([1, 2, np.nan])
980 >>> nanops.nanmean(s.values)
981 np.float64(1.5)
982 """
983 if values.size == 0:
984 # GH#18976
985 return cast("float", _na_for_min_count(values, axis))
987 dtype = values.dtype
988 values, mask = _get_values(values, skipna, fill_value=0, mask=mask)
989 dtype_sum = _get_dtype_max(dtype)
990 dtype_count = np.dtype(np.float64)
992 # not using needs_i8_conversion because that includes period
993 if dtype.kind in "mM":
994 dtype_sum = np.dtype(np.float64)
995 elif dtype.kind in "iu":
996 dtype_sum = np.dtype(np.float64)
997 elif dtype.kind == "f":
998 # GH#43929 upcast float16 sum and count, which lose precision or overflow
999 dtype_sum = np.dtype(np.float64) if dtype == np.float16 else dtype
1000 dtype_count = dtype_sum
1002 count = _get_counts(values.shape, mask, axis, dtype=dtype_count)
1003 the_sum = values.sum(axis, dtype=dtype_sum)
1005 if axis is not None and getattr(the_sum, "ndim", False):
1006 count = cast("np.ndarray", count)
1007 with np.errstate(all="ignore"):
1008 # suppress division by zero warnings
1009 the_mean = the_sum / count
1010 ct_mask = count == 0
1011 if ct_mask.any():
1012 the_mean[ct_mask] = np.nan
1013 else:
1014 the_mean = the_sum / count if count > 0 else np.nan
1016 return the_mean
1019@_ensure_numeric_input
1020@bottleneck_switch()
1021def nanmedian(
1022 values: np.ndarray, *, axis: AxisInt | None = None, skipna: bool = True, mask=None
1023) -> float | np.ndarray:
1024 """
1025 Parameters
1026 ----------
1027 values : ndarray
1028 axis : int, optional
1029 skipna : bool, default True
1030 mask : ndarray[bool], optional
1031 nan-mask if known
1033 Returns
1034 -------
1035 result : float | ndarray
1036 Unless input is a float or complex array, in which case use the same
1037 precision as the input array.
1039 Examples
1040 --------
1041 >>> from pandas.core import nanops
1042 >>> s = pd.Series([1, np.nan, 2, 2])
1043 >>> nanops.nanmedian(s.values)
1044 2.0
1046 >>> s = pd.Series([np.nan, np.nan, np.nan])
1047 >>> nanops.nanmedian(s.values)
1048 nan
1049 """
1050 # for floats without mask, the data already uses NaN as missing value
1051 # indicator, and `mask` will be calculated from that below -> in those
1052 # cases we never need to set NaN to the masked values
1053 using_nan_sentinel = values.dtype.kind == "f" and mask is None
1055 def get_median(x: np.ndarray, _mask=None):
1056 if _mask is None:
1057 _mask = notna(x)
1058 else:
1059 _mask = ~_mask
1060 if not skipna and not _mask.all():
1061 # x's own NaN: np.apply_along_axis below takes the result dtype
1062 # from the first slice, so a bare float would cast a later
1063 # complex slice to real
1064 return x.dtype.type(np.nan)
1065 with warnings.catch_warnings():
1066 # Suppress RuntimeWarning about All-NaN slice
1067 warnings.filterwarnings(
1068 "ignore", "All-NaN slice encountered", RuntimeWarning
1069 )
1070 warnings.filterwarnings("ignore", "Mean of empty slice", RuntimeWarning)
1071 res = np.nanmedian(x[_mask])
1072 return res
1074 dtype = values.dtype
1075 values, mask = _get_values(values, skipna, mask=mask, fill_value=None)
1076 if values.dtype.kind not in "fc":
1077 # complex is left alone; np.nanmedian handles it, and casting it to f8
1078 # would silently drop the imaginary part
1079 values = values.astype("f8")
1080 if not using_nan_sentinel and mask is not None:
1081 if not values.flags.writeable:
1082 values = values.copy()
1083 values[mask] = np.nan
1085 notempty = values.size
1087 res: float | np.ndarray
1089 # an array from a frame
1090 if values.ndim > 1 and axis is not None:
1091 # there's a non-empty array to apply over otherwise numpy raises
1092 if notempty:
1093 if not skipna:
1094 res = np.apply_along_axis(get_median, axis, values)
1096 else:
1097 # fastpath for the skipna case
1098 with warnings.catch_warnings():
1099 # Suppress RuntimeWarning about All-NaN slice
1100 warnings.filterwarnings(
1101 "ignore", "All-NaN slice encountered", RuntimeWarning
1102 )
1103 if (values.shape[1] == 1 and axis == 0) or (
1104 values.shape[0] == 1 and axis == 1
1105 ):
1106 # GH52788: fastpath when squeezable, nanmedian for 2D array slow
1107 # atleast_1d: see test_nanmedian_2d_matches_numpy (GH#68191)
1108 res = np.nanmedian(
1109 np.atleast_1d(np.squeeze(values)), keepdims=True
1110 )
1111 else:
1112 res = np.nanmedian(values, axis=axis)
1114 else:
1115 # must return the correct shape, but median is not defined for the
1116 # empty set so return nans of shape "everything but the passed axis"
1117 # since "axis" is where the reduction would occur if we had a nonempty
1118 # array
1119 res = _get_empty_reduction_result(values.shape, axis)
1121 else:
1122 # otherwise return a scalar value
1123 res = get_median(values, mask) if notempty else np.nan
1124 return _wrap_results(res, dtype)
1127def _get_empty_reduction_result(
1128 shape: Shape,
1129 axis: AxisInt,
1130) -> np.ndarray:
1131 """
1132 The result from a reduction on an empty ndarray.
1134 Parameters
1135 ----------
1136 shape : Tuple[int, ...]
1137 axis : int
1139 Returns
1140 -------
1141 np.ndarray
1142 """
1143 shp = np.array(shape)
1144 dims = np.arange(len(shape))
1145 ret = np.empty(shp[dims != axis], dtype=np.float64)
1146 ret.fill(np.nan)
1147 return ret
1150def _get_counts_nanvar(
1151 values_shape: Shape,
1152 mask: npt.NDArray[np.bool_] | None,
1153 axis: AxisInt | None,
1154 ddof: int,
1155 dtype: np.dtype = np.dtype(np.float64),
1156) -> tuple[float | np.ndarray, float | np.ndarray]:
1157 """
1158 Get the count of non-null values along an axis, accounting
1159 for degrees of freedom.
1161 Parameters
1162 ----------
1163 values_shape : Tuple[int, ...]
1164 shape tuple from values ndarray, used if mask is None
1165 mask : Optional[ndarray[bool]]
1166 locations in values that should be considered missing
1167 axis : Optional[int]
1168 axis to count along
1169 ddof : int
1170 degrees of freedom
1171 dtype : type, optional
1172 type to use for count
1174 Returns
1175 -------
1176 count : np.floating or np.ndarray
1177 d : np.floating or np.ndarray
1178 """
1179 count = _get_counts(values_shape, mask, axis, dtype=dtype)
1180 d = count - dtype.type(ddof)
1182 # always return NaN, never inf
1183 if is_float(count):
1184 if count <= ddof:
1185 # dtype's own NaN, not a bare float: nansem divides by sqrt(count),
1186 # which would widen a float32 result to float64
1187 count = d = dtype.type(np.nan)
1188 else:
1189 # count is not narrowed by is_float check
1190 count = cast("np.ndarray", count)
1191 mask = count <= ddof
1192 if mask.any():
1193 np.putmask(d, mask, np.nan)
1194 np.putmask(count, mask, np.nan)
1195 return count, d
1198@_ensure_numeric_input
1199@bottleneck_switch(ddof=1)
1200def nanstd(
1201 values,
1202 *,
1203 axis: AxisInt | None = None,
1204 skipna: bool = True,
1205 ddof: int = 1,
1206 mask=None,
1207):
1208 """
1209 Compute the standard deviation along given axis while ignoring NaNs
1211 Parameters
1212 ----------
1213 values : ndarray
1214 axis : int, optional
1215 skipna : bool, default True
1216 ddof : int, default 1
1217 Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
1218 where N represents the number of elements.
1219 mask : ndarray[bool], optional
1220 NA-mask if known
1222 Returns
1223 -------
1224 result : float
1225 Unless input is a float array, in which case use the same
1226 precision as the input array.
1228 Examples
1229 --------
1230 >>> from pandas.core import nanops
1231 >>> s = pd.Series([1, np.nan, 2, 3])
1232 >>> nanops.nanstd(s.values)
1233 1.0
1234 """
1235 if values.dtype.kind == "M":
1236 unit = np.datetime_data(values.dtype)[0]
1237 values = values.view(f"m8[{unit}]")
1239 if values.size == 0:
1240 # GH#18976
1241 return cast("float", _na_for_min_count(values, axis))
1243 orig_dtype = values.dtype
1244 values, mask = _get_values(values, skipna, mask=mask)
1246 result = np.sqrt(nanvar(values, axis=axis, skipna=skipna, ddof=ddof, mask=mask))
1247 return _wrap_results(result, orig_dtype)
1250@_ensure_numeric_input
1251@disallow("M8", "m8")
1252@bottleneck_switch(ddof=1)
1253def nanvar(
1254 values: np.ndarray,
1255 *,
1256 axis: AxisInt | None = None,
1257 skipna: bool = True,
1258 ddof: int = 1,
1259 mask=None,
1260):
1261 """
1262 Compute the variance along given axis while ignoring NaNs
1264 Parameters
1265 ----------
1266 values : ndarray
1267 axis : int, optional
1268 skipna : bool, default True
1269 ddof : int, default 1
1270 Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
1271 where N represents the number of elements.
1272 mask : ndarray[bool], optional
1273 NA-mask if known
1275 Returns
1276 -------
1277 result : float
1278 Unless input is a float array, in which case use the same
1279 precision as the input array.
1281 Examples
1282 --------
1283 >>> from pandas.core import nanops
1284 >>> s = pd.Series([1, np.nan, 2, 3])
1285 >>> nanops.nanvar(s.values)
1286 1.0
1287 """
1288 if values.size == 0:
1289 # GH#18976
1290 return cast("float", _na_for_min_count(values, axis))
1291 dtype = values.dtype
1292 mask = _maybe_get_mask(values, skipna, mask)
1293 if dtype.kind in "biu":
1294 # bool: the np.nan putmask below would write True into a bool array
1295 values = values.astype("f8")
1296 elif dtype.kind == "c":
1297 # https://en.wikipedia.org/wiki/Complex_random_variable#Variance_and_pseudo-variance
1298 # The variance is equal to the sum of
1299 # the variances of the real and imaginary part of the complex random variable.
1300 return nanvar(
1301 values.real, axis=axis, skipna=skipna, ddof=ddof, mask=mask
1302 ) + nanvar(values.imag, axis=axis, skipna=skipna, ddof=ddof, mask=mask)
1304 if values.dtype.kind == "f":
1305 count, d = _get_counts_nanvar(values.shape, mask, axis, ddof, values.dtype)
1306 else:
1307 count, d = _get_counts_nanvar(values.shape, mask, axis, ddof)
1309 if mask is not None:
1310 values = values.copy()
1311 # GH#65373 an explicit mask marks NA, so skipna=False propagates
1312 np.putmask(values, mask, 0 if skipna else np.nan)
1314 # xref GH10242
1315 # Compute variance via two-pass algorithm, which is stable against
1316 # cancellation errors and relatively accurate for small numbers of
1317 # observations.
1318 #
1319 # See https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
1320 avg = values.sum(axis=axis, dtype=np.float64) / count
1321 if axis is not None:
1322 avg = np.expand_dims(avg, axis)
1324 sqr = (avg - values) ** 2
1325 if mask is not None:
1326 np.putmask(sqr, mask, 0)
1327 # numpy's stubs allow the division to produce a plain float, which it never
1328 # does here; d is an ndarray or an np.float64 and so is the sum
1329 result: np.ndarray | np.float64 = cast(
1330 "np.ndarray | np.float64", sqr.sum(axis=axis, dtype=np.float64) / d
1331 )
1333 # Return variance as np.float64 (the datatype used in the accumulator),
1334 # unless we were dealing with a float array, in which case use the same
1335 # precision as the original values array.
1336 if dtype.kind == "f":
1337 result = result.astype(dtype, copy=False)
1338 return result
1341@_ensure_numeric_input
1342@disallow("M8", "m8")
1343def nansem(
1344 values: np.ndarray,
1345 *,
1346 axis: AxisInt | None = None,
1347 skipna: bool = True,
1348 ddof: int = 1,
1349 mask: npt.NDArray[np.bool_] | None = None,
1350) -> float:
1351 """
1352 Compute the standard error in the mean along given axis while ignoring NaNs
1354 Parameters
1355 ----------
1356 values : ndarray
1357 axis : int, optional
1358 skipna : bool, default True
1359 ddof : int, default 1
1360 Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
1361 where N represents the number of elements.
1362 mask : ndarray[bool], optional
1363 NA-mask if known
1365 Returns
1366 -------
1367 result : float64
1368 Unless input is a float array, in which case use the same
1369 precision as the input array.
1371 Examples
1372 --------
1373 >>> from pandas.core import nanops
1374 >>> s = pd.Series([1, np.nan, 2, 3])
1375 >>> nanops.nansem(s.values)
1376 np.float64(0.5773502691896258)
1377 """
1378 # This checks if non-numeric-like data is passed with numeric_only=False
1379 # and raises a TypeError otherwise
1380 nanvar(values, axis=axis, skipna=skipna, ddof=ddof, mask=mask)
1382 mask = _maybe_get_mask(values, skipna, mask)
1383 # Convert to bottleneck return a float
1384 if values.dtype.kind not in "fc":
1385 values = values.astype("f8")
1387 dtype_count = np.dtype(np.float64)
1388 if values.dtype.kind == "f":
1389 dtype_count = values.dtype
1390 count, _ = _get_counts_nanvar(values.shape, mask, axis, ddof, dtype_count)
1391 var = nanvar(values, axis=axis, skipna=skipna, ddof=ddof, mask=mask)
1393 return np.sqrt(var) / np.sqrt(count)
1396def _blank_object_na_slices(
1397 values: np.ndarray,
1398 mask: npt.NDArray[np.bool_],
1399 axis: AxisInt,
1400 fill_value: Any,
1401) -> np.ndarray:
1402 """
1403 Replace every reduction slice of an object-dtype array that holds an NA
1404 with ``fill_value``.
1406 For ``skipna=False`` those slices are nulled out afterwards, so their values
1407 never reach the result and need not even support the operation being applied
1408 (GH#4147). A float NaN propagates through arithmetic and comparison on its
1409 own; an object NA does not, and raises instead.
1411 Only for reductions over an axis of a 2-D array. When the reduction covers
1412 the whole array the result is simply NA, and callers short-circuit instead.
1413 """
1414 any_na = np.expand_dims(mask.any(axis=axis), axis)
1415 return np.where(any_na, fill_value, values)
1418def _na_min_count(values: np.ndarray, axis: AxisInt, min_count: int) -> int:
1419 """
1420 ``min_count`` that nulls out any slice holding an NA, without weakening an
1421 explicit larger ``min_count`` from the caller (GH#4147).
1422 """
1423 return max(min_count, values.shape[axis])
1426def _fill_object_na(
1427 values: np.ndarray,
1428 mask: npt.NDArray[np.bool_],
1429 axis: AxisInt | None,
1430 fill_value: Any,
1431) -> np.ndarray:
1432 """
1433 Replace NA entries in an object-dtype array with a valid value taken from
1434 the same reduction slice.
1436 GH#65500: the +/-inf fill used for other dtypes is not comparable with
1437 arbitrary objects (Timestamps, strings, tuples) and raises. Substituting a
1438 value already present in the slice cannot change a min/max, and for
1439 argmin/argmax a result landing on a filled position is corrected by
1440 ``_maybe_fix_arg_at_na``. Entirely-NA slices get ``fill_value``, since
1441 their result is discarded by the caller either way.
1442 """
1443 if mask.all():
1444 return np.full(values.shape, fill_value, dtype=object)
1445 if values.ndim == 1 or axis is None:
1446 fill = np.empty(1, dtype=object)
1447 # wrap in an array so that e.g. tuple fill values
1448 # are not broadcast by np.where
1449 fill[0] = values[~mask][0]
1450 return np.where(mask, fill, values)
1451 indexer = np.expand_dims((~mask).argmax(axis=axis), axis)
1452 fill = np.take_along_axis(values, indexer, axis=axis)
1453 # argmax selects an NA entry for entirely-NA slices; fill
1454 # those with +/-inf instead so the reduction does not warn
1455 all_na = np.expand_dims(mask.all(axis=axis), axis)
1456 fill = np.where(all_na, fill_value, fill)
1457 return np.where(mask, fill, values)
1460def _get_arg_values(
1461 values: np.ndarray,
1462 *,
1463 fill_value_typ: str,
1464 mask: npt.NDArray[np.bool_] | None,
1465 skipna: bool,
1466 axis: AxisInt | None = None,
1467) -> tuple[np.ndarray, npt.NDArray[np.bool_] | None]:
1468 """
1469 Prepare values for nanargmin/nanargmax by filling NA entries so that the
1470 comparison does not see them, and raise for ``skipna=False``.
1472 Mirrors ``_get_values(values, True, ...)`` except for object dtype, where
1473 the +/-inf fill is not comparable with arbitrary objects (GH#4147). The
1474 object fill is a same-slice value, which *can* win the comparison; a result
1475 landing on it is corrected by ``_maybe_fix_arg_at_na``.
1476 """
1477 if values.dtype == object:
1478 if mask is None:
1479 mask = isna(values)
1480 if not skipna and mask.any():
1481 # _maybe_arg_null_out would raise this once the argmin/argmax is
1482 # known, but the comparison must not run first: values in a slice
1483 # holding an NA need not be comparable with each other (GH#4147)
1484 raise ValueError("Encountered an NA value with skipna=False")
1485 if mask.any():
1486 fill_value = _get_fill_value(values.dtype, fill_value_typ=fill_value_typ)
1487 values = _fill_object_na(values, mask, axis, fill_value)
1488 return values, mask
1489 return _get_values(values, True, fill_value_typ=fill_value_typ, mask=mask)
1492def _nanminmax(meth, fill_value_typ):
1493 @bottleneck_switch(name=f"nan{meth}")
1494 @_datetimelike_compat
1495 def reduction(
1496 values: np.ndarray,
1497 *,
1498 axis: AxisInt | None = None,
1499 skipna: bool = True,
1500 mask: npt.NDArray[np.bool_] | None = None,
1501 ):
1502 if values.size == 0:
1503 return _na_for_min_count(values, axis)
1505 dtype = values.dtype
1506 min_count = 1
1507 if dtype == object:
1508 # GH#65500: _get_values' +/-inf fill isn't comparable with arbitrary
1509 # objects (Timestamps, strings) and raises; see _fill_object_na.
1510 if mask is None:
1511 mask = isna(values)
1512 if mask.any():
1513 fill_value = _get_fill_value(dtype, fill_value_typ=fill_value_typ)
1514 if skipna:
1515 values = _fill_object_na(values, mask, axis, fill_value)
1516 elif values.ndim == 1 or axis is None:
1517 # GH#4147: an object NA does not propagate through the
1518 # comparison the way a float NaN does, so null out every
1519 # slice that holds one, matching float64/str/dt64.
1520 return _na_for_min_count(values, axis)
1521 else:
1522 values = _blank_object_na_slices(values, mask, axis, fill_value)
1523 min_count = _na_min_count(values, axis, min_count)
1524 else:
1525 values, mask = _get_values(
1526 values, skipna, fill_value_typ=fill_value_typ, mask=mask
1527 )
1528 result = getattr(values, meth)(axis)
1529 result = _maybe_null_out(
1530 result,
1531 axis,
1532 mask,
1533 values.shape,
1534 min_count=min_count,
1535 datetimelike=dtype.kind in "mM",
1536 )
1537 return result
1539 return reduction
1542nanmin = _nanminmax("min", fill_value_typ="+inf")
1543nanmax = _nanminmax("max", fill_value_typ="-inf")
1546def nanargmax(
1547 values: np.ndarray,
1548 *,
1549 axis: AxisInt | None = None,
1550 skipna: bool = True,
1551 mask: npt.NDArray[np.bool_] | None = None,
1552) -> int | np.ndarray:
1553 """
1554 Parameters
1555 ----------
1556 values : ndarray
1557 axis : int, optional
1558 skipna : bool, default True
1559 mask : ndarray[bool], optional
1560 nan-mask if known
1562 Returns
1563 -------
1564 result : int or ndarray[int]
1565 The index/indices of max value in specified axis or -1 in the NA case
1567 Examples
1568 --------
1569 >>> from pandas.core import nanops
1570 >>> arr = np.array([1, 2, 3, np.nan, 4])
1571 >>> nanops.nanargmax(arr)
1572 np.int64(4)
1574 >>> arr = np.array(range(12), dtype=np.float64).reshape(4, 3)
1575 >>> arr[2:, 2] = np.nan
1576 >>> arr
1577 array([[ 0., 1., 2.],
1578 [ 3., 4., 5.],
1579 [ 6., 7., nan],
1580 [ 9., 10., nan]])
1581 >>> nanops.nanargmax(arr, axis=1)
1582 array([2, 2, 1, 1])
1583 """
1584 values, mask = _get_arg_values(
1585 values, fill_value_typ="-inf", mask=mask, skipna=skipna, axis=axis
1586 )
1587 result = values.argmax(axis)
1588 # error: Argument 1 to "_maybe_fix_arg_at_na" has incompatible type "Any |
1589 # signedinteger[Any]"; expected "ndarray[Any, Any]"
1590 result = _maybe_fix_arg_at_na(result, mask, axis) # type: ignore[arg-type]
1591 result = _maybe_arg_null_out(result, axis, mask, skipna) # type: ignore[assignment]
1592 return result
1595def nanargmin(
1596 values: np.ndarray,
1597 *,
1598 axis: AxisInt | None = None,
1599 skipna: bool = True,
1600 mask: npt.NDArray[np.bool_] | None = None,
1601) -> int | np.ndarray:
1602 """
1603 Parameters
1604 ----------
1605 values : ndarray
1606 axis : int, optional
1607 skipna : bool, default True
1608 mask : ndarray[bool], optional
1609 nan-mask if known
1611 Returns
1612 -------
1613 result : int or ndarray[int]
1614 The index/indices of min value in specified axis or -1 in the NA case
1616 Examples
1617 --------
1618 >>> from pandas.core import nanops
1619 >>> arr = np.array([1, 2, 3, np.nan, 4])
1620 >>> nanops.nanargmin(arr)
1621 np.int64(0)
1623 >>> arr = np.array(range(12), dtype=np.float64).reshape(4, 3)
1624 >>> arr[2:, 0] = np.nan
1625 >>> arr
1626 array([[ 0., 1., 2.],
1627 [ 3., 4., 5.],
1628 [nan, 7., 8.],
1629 [nan, 10., 11.]])
1630 >>> nanops.nanargmin(arr, axis=1)
1631 array([0, 0, 1, 1])
1632 """
1633 values, mask = _get_arg_values(
1634 values, fill_value_typ="+inf", mask=mask, skipna=skipna, axis=axis
1635 )
1636 result = values.argmin(axis)
1637 # error: Argument 1 to "_maybe_fix_arg_at_na" has incompatible type "Any |
1638 # signedinteger[Any]"; expected "ndarray[Any, Any]"
1639 result = _maybe_fix_arg_at_na(result, mask, axis) # type: ignore[arg-type]
1640 result = _maybe_arg_null_out(result, axis, mask, skipna) # type: ignore[assignment]
1641 return result
1644@_ensure_numeric_input
1645@disallow("M8", "m8")
1646@maybe_operate_rowwise
1647def nanskew(
1648 values: np.ndarray,
1649 *,
1650 axis: AxisInt | None = None,
1651 skipna: bool = True,
1652 mask: npt.NDArray[np.bool_] | None = None,
1653) -> np.ndarray | float:
1654 """
1655 Compute the sample skewness.
1657 The statistic computed here is the adjusted Fisher-Pearson standardized
1658 moment coefficient G1. The algorithm computes this coefficient directly
1659 from the second and third central moment.
1661 Parameters
1662 ----------
1663 values : ndarray
1664 axis : int, optional
1665 skipna : bool, default True
1666 mask : ndarray[bool], optional
1667 nan-mask if known
1669 Returns
1670 -------
1671 result : float64
1672 Unless input is a float array, in which case use the same
1673 precision as the input array.
1675 Examples
1676 --------
1677 >>> from pandas.core import nanops
1678 >>> s = pd.Series([1, np.nan, 1, 2])
1679 >>> round(nanops.nanskew(s.values), 6)
1680 np.float64(1.732051)
1681 """
1682 dtype = values.dtype
1683 values = ensure_float64(values)
1685 result: npt.NDArray[np.floating] | np.floating
1686 if axis is None or (values.ndim == 1 and axis == 0):
1687 order: Literal["F", "C"] = "F" if values.flags.f_contiguous else "C"
1688 result_float = libalgos.scalar_skew(
1689 values.ravel(order), skipna, mask.ravel(order) if mask is not None else None
1690 )
1691 result = np.float64(result_float)
1692 elif axis in {0, 1}:
1693 result = libalgos.axis_skew(values, axis, skipna, mask)
1694 else:
1695 raise ValueError("axis must be 0, 1 or None")
1697 if dtype.kind == "f":
1698 result = result.astype(dtype, copy=False)
1700 return result
1703@_ensure_numeric_input
1704@disallow("M8", "m8")
1705@maybe_operate_rowwise
1706def nankurt(
1707 values: np.ndarray,
1708 *,
1709 axis: AxisInt | None = None,
1710 skipna: bool = True,
1711 mask: npt.NDArray[np.bool_] | None = None,
1712) -> np.ndarray | float:
1713 """
1714 Compute the sample excess kurtosis
1716 The statistic computed here is the adjusted Fisher-Pearson standardized
1717 moment coefficient G2, computed directly from the second and fourth
1718 central moment.
1720 Parameters
1721 ----------
1722 values : ndarray
1723 axis : int, optional
1724 skipna : bool, default True
1725 mask : ndarray[bool], optional
1726 nan-mask if known
1728 Returns
1729 -------
1730 result : float64
1731 Unless input is a float array, in which case use the same
1732 precision as the input array.
1734 Examples
1735 --------
1736 >>> from pandas.core import nanops
1737 >>> s = pd.Series([1, np.nan, 1, 3, 2])
1738 >>> round(nanops.nankurt(s.values), 6)
1739 np.float64(-1.289256)
1740 """
1741 dtype = values.dtype
1742 values = ensure_float64(values)
1744 result: npt.NDArray[np.floating] | np.floating
1745 if axis is None or (values.ndim == 1 and axis == 0):
1746 order: Literal["F", "C"] = "F" if values.flags.f_contiguous else "C"
1747 result_float = libalgos.scalar_kurt(
1748 values.ravel(order), skipna, mask.ravel(order) if mask is not None else None
1749 )
1750 result = np.float64(result_float)
1751 elif axis in {0, 1}:
1752 result = libalgos.axis_kurt(values, axis, skipna, mask)
1753 else:
1754 raise ValueError("axis must be 0, 1 or None")
1756 if dtype.kind == "f":
1757 result = result.astype(dtype, copy=False)
1759 return result
1762@disallow("M8", "m8")
1763@maybe_operate_rowwise
1764def nanprod(
1765 values: np.ndarray,
1766 *,
1767 axis: AxisInt | None = None,
1768 skipna: bool = True,
1769 min_count: int = 0,
1770 mask: npt.NDArray[np.bool_] | None = None,
1771) -> float:
1772 """
1773 Parameters
1774 ----------
1775 values : ndarray[dtype]
1776 axis : int, optional
1777 skipna : bool, default True
1778 min_count: int, default 0
1779 mask : ndarray[bool], optional
1780 nan-mask if known
1782 Returns
1783 -------
1784 Dtype
1785 The product of all elements on a given axis. ( NaNs are treated as 1)
1787 Examples
1788 --------
1789 >>> from pandas.core import nanops
1790 >>> s = pd.Series([1, 2, 3, np.nan])
1791 >>> nanops.nanprod(s.values)
1792 np.float64(6.0)
1793 """
1794 mask = _maybe_get_mask(values, skipna, mask)
1796 if values.dtype == object and not skipna:
1797 # GH#4147: see nansum
1798 if mask is None:
1799 mask = isna(values)
1800 if mask.any():
1801 try:
1802 result = values.prod(axis)
1803 except TypeError:
1804 pass
1805 else:
1806 return _maybe_null_out( # type: ignore[return-value]
1807 result, axis, mask, values.shape, min_count=min_count
1808 )
1809 if values.ndim == 1 or axis is None:
1810 # error: Incompatible return value type (got "Union[Scalar,
1811 # ndarray]", expected "float")
1812 return _na_for_min_count(values, axis) # type: ignore[return-value]
1813 values = _blank_object_na_slices(values, mask, axis, 1)
1814 min_count = _na_min_count(values, axis, min_count)
1815 elif skipna and mask is not None:
1816 values = values.copy()
1817 values[mask] = 1
1818 result = values.prod(axis)
1819 # error: Incompatible return value type (got "Union[ndarray, float]", expected
1820 # "float")
1821 return _maybe_null_out( # type: ignore[return-value]
1822 result, axis, mask, values.shape, min_count=min_count
1823 )
1826def _maybe_fix_arg_at_na(
1827 result: np.ndarray,
1828 mask: npt.NDArray[np.bool_] | None,
1829 axis: AxisInt | None,
1830) -> np.ndarray:
1831 # helper function for nanargmin/nanargmax. Masked positions carry either
1832 # the +/-inf sentinel or, for object dtype, the value at the first unmasked
1833 # position (GH#4147). Either way an argmin/argmax landing on a masked
1834 # position means the fill is the extremum, so the first unmasked position
1835 # holds it and is the correct result (GH#64478)
1836 if mask is None or not mask.any():
1837 return result
1838 if axis is None or mask.ndim == 1:
1839 if mask.ravel()[result] and not mask.all():
1840 # error: Incompatible return value type (got "signedinteger[_32Bit
1841 # | _64Bit]", expected "ndarray[tuple[Any, ...], dtype[Any]]")
1842 return (~mask).ravel().argmax() # type: ignore[return-value]
1843 else:
1844 indexer = np.expand_dims(result, axis)
1845 arg_is_na = np.take_along_axis(mask, indexer, axis).squeeze(axis)
1846 if arg_is_na.any():
1847 result = np.where(arg_is_na, (~mask).argmax(axis), result)
1848 return result
1851def _maybe_arg_null_out(
1852 result: np.ndarray,
1853 axis: AxisInt | None,
1854 mask: npt.NDArray[np.bool_] | None,
1855 skipna: bool,
1856) -> np.ndarray | int:
1857 # helper function for nanargmin/nanargmax
1858 if mask is None:
1859 return result
1861 if axis is None or not getattr(result, "ndim", False):
1862 if skipna and mask.all():
1863 raise ValueError("Encountered all NA values")
1864 elif not skipna and mask.any():
1865 raise ValueError("Encountered an NA value with skipna=False")
1866 elif skipna and mask.all(axis).any():
1867 raise ValueError("Encountered all NA values")
1868 elif not skipna and mask.any(axis).any():
1869 raise ValueError("Encountered an NA value with skipna=False")
1870 return result
1873def _get_counts(
1874 values_shape: Shape,
1875 mask: npt.NDArray[np.bool_] | None,
1876 axis: AxisInt | None,
1877 dtype: np.dtype[np.floating] = np.dtype(np.float64),
1878) -> np.floating | npt.NDArray[np.floating]:
1879 """
1880 Get the count of non-null values along an axis
1882 Parameters
1883 ----------
1884 values_shape : tuple of int
1885 shape tuple from values ndarray, used if mask is None
1886 mask : Optional[ndarray[bool]]
1887 locations in values that should be considered missing
1888 axis : Optional[int]
1889 axis to count along
1890 dtype : type, optional
1891 type to use for count
1893 Returns
1894 -------
1895 count : scalar or array
1896 """
1897 if axis is None:
1898 if mask is not None:
1899 n = mask.size - mask.sum()
1900 else:
1901 n = np.prod(values_shape)
1902 return dtype.type(n)
1904 if mask is not None:
1905 count = mask.shape[axis] - mask.sum(axis)
1906 else:
1907 count = values_shape[axis]
1909 if is_integer(count):
1910 return dtype.type(count)
1911 return count.astype(dtype, copy=False)
1914def _maybe_null_out(
1915 result: np.ndarray | np.int64 | float | NaTType,
1916 axis: AxisInt | None,
1917 mask: npt.NDArray[np.bool_] | None,
1918 shape: tuple[int, ...],
1919 min_count: int = 1,
1920 datetimelike: bool = False,
1921) -> np.ndarray | np.int64 | float | NaTType:
1922 """
1923 Returns
1924 -------
1925 Dtype
1926 The product of all elements on a given axis. ( NaNs are treated as 1)
1927 """
1928 if min_count <= 0:
1929 # min_count <= 0 never nulls out; short-circuit
1930 return result
1932 if axis is not None and isinstance(result, np.ndarray):
1933 if mask is not None:
1934 null_mask = (mask.shape[axis] - mask.sum(axis) - min_count) < 0
1935 else:
1936 # we have no nulls, kept mask=None in _maybe_get_mask
1937 below_count = shape[axis] - min_count < 0
1938 new_shape = shape[:axis] + shape[axis + 1 :]
1939 null_mask = np.broadcast_to(below_count, new_shape)
1941 if np.any(null_mask):
1942 if datetimelike:
1943 # GH#60646 For datetimelike, no need to cast to float
1944 result[null_mask] = iNaT
1945 elif is_numeric_dtype(result):
1946 if np.iscomplexobj(result):
1947 result = result.astype("c16")
1948 elif not is_float_dtype(result):
1949 result = result.astype("f8", copy=False)
1950 result[null_mask] = np.nan
1951 else:
1952 # GH12941, use None to auto cast null
1953 result[null_mask] = None
1954 elif result is not NaT:
1955 if check_below_min_count(shape, mask, min_count):
1956 result_dtype = getattr(result, "dtype", None)
1957 if is_float_dtype(result_dtype):
1958 # error: Item "None" of "Optional[Any]" has no attribute "type"
1959 result = result_dtype.type("nan") # type: ignore[union-attr]
1960 else:
1961 result = np.nan
1963 return result
1966def check_below_min_count(
1967 shape: tuple[int, ...], mask: npt.NDArray[np.bool_] | None, min_count: int
1968) -> bool:
1969 """
1970 Check for the `min_count` keyword. Returns True if below `min_count` (when
1971 missing value should be returned from the reduction).
1973 Parameters
1974 ----------
1975 shape : tuple
1976 The shape of the values (`values.shape`).
1977 mask : ndarray[bool] or None
1978 Boolean numpy array (typically of same shape as `shape`) or None.
1979 min_count : int
1980 Keyword passed through from sum/prod call.
1982 Returns
1983 -------
1984 bool
1985 """
1986 if min_count > 0:
1987 if mask is None:
1988 # no missing values, only check size
1989 non_nulls = np.prod(shape)
1990 else:
1991 non_nulls = mask.size - mask.sum()
1992 if non_nulls < min_count:
1993 return True
1994 return False
1997@disallow("M8", "m8")
1998def nancorr(
1999 a: np.ndarray,
2000 b: np.ndarray,
2001 *,
2002 method: CorrelationMethod = "pearson",
2003 min_periods: int | None = None,
2004) -> float:
2005 """
2006 a, b: ndarrays
2007 """
2008 if len(a) != len(b):
2009 raise AssertionError("Operands to nancorr must have same size")
2011 if min_periods is None:
2012 min_periods = 1
2014 valid = notna(a) & notna(b)
2015 if not valid.all():
2016 a = a[valid]
2017 b = b[valid]
2019 if len(a) < min_periods:
2020 return np.nan
2022 a = _ensure_numeric(a)
2023 b = _ensure_numeric(b)
2025 f = get_corr_func(method)
2026 return f(a, b)
2029def get_corr_func(
2030 method: CorrelationMethod,
2031) -> Callable[[np.ndarray, np.ndarray], float]:
2032 if method == "kendall":
2033 from scipy.stats import kendalltau
2035 def func(a, b): # pyright: ignore[reportRedeclaration]
2036 return kendalltau(a, b)[0]
2038 return func
2039 elif method == "spearman":
2040 from scipy.stats import spearmanr
2042 def func(a, b): # pyright: ignore[reportRedeclaration]
2043 return spearmanr(a, b)[0]
2045 return func
2046 elif method == "pearson":
2048 def func(a, b):
2049 return _pearson_corr(a, b)
2051 return func
2052 elif callable(method):
2053 return method
2055 raise ValueError(
2056 f"Unknown method '{method}', expected one of "
2057 "'kendall', 'spearman', 'pearson', or callable"
2058 )
2061def _pearson_corr(a: np.ndarray, b: np.ndarray) -> float:
2062 if a.ndim != 1 or b.ndim != 1 or np.iscomplexobj(a) or np.iscomplexobj(b):
2063 return np.corrcoef(a, b)[0, 1]
2065 if len(a) < 2:
2066 return np.nan
2068 a = a.astype(np.float64, copy=False)
2069 b = b.astype(np.float64, copy=False)
2071 a = a - a.mean()
2072 b = b - b.mean()
2073 a_scale = np.max(np.abs(a))
2074 b_scale = np.max(np.abs(b))
2076 if a_scale == 0 or b_scale == 0:
2077 return np.nan
2079 a = a / a_scale
2080 b = b / b_scale
2081 fact = len(a) - 1
2082 divisor = np.sqrt(np.dot(a, a) / fact)
2084 if divisor == 0:
2085 return np.nan
2087 result = np.dot(a, b) / fact / divisor
2088 divisor = np.sqrt(np.dot(b, b) / fact)
2090 if divisor == 0:
2091 return np.nan
2093 return np.clip(result / divisor, -1.0, 1.0)
2096@disallow("M8", "m8")
2097def nancov(
2098 a: np.ndarray,
2099 b: np.ndarray,
2100 *,
2101 min_periods: int | None = None,
2102 ddof: int | None = 1,
2103) -> float:
2104 if len(a) != len(b):
2105 raise AssertionError("Operands to nancov must have same size")
2107 if min_periods is None:
2108 min_periods = 1
2110 valid = notna(a) & notna(b)
2111 if not valid.all():
2112 a = a[valid]
2113 b = b[valid]
2115 if len(a) < min_periods:
2116 return np.nan
2118 a = _ensure_numeric(a)
2119 b = _ensure_numeric(b)
2121 return np.cov(a, b, ddof=ddof)[0, 1]
2124def na_accum_func(values: ArrayLike, accum_func, *, skipna: bool) -> ArrayLike:
2125 """
2126 Cumulative function with skipna support.
2128 Parameters
2129 ----------
2130 values : np.ndarray or ExtensionArray
2131 accum_func : {np.cumprod, np.maximum.accumulate, np.cumsum, np.minimum.accumulate}
2132 skipna : bool
2134 Returns
2135 -------
2136 np.ndarray or ExtensionArray
2137 """
2138 mask_a, mask_b = {
2139 np.cumprod: (1.0, np.nan),
2140 np.maximum.accumulate: (-np.inf, np.nan),
2141 np.cumsum: (0.0, np.nan),
2142 np.minimum.accumulate: (np.inf, np.nan),
2143 }[accum_func]
2145 # This should go through ea interface
2146 assert values.dtype.kind not in "mM"
2148 # We will be applying this function to block values
2149 if skipna and not issubclass(values.dtype.type, (np.integer, np.bool_)):
2150 vals = values.copy()
2151 mask = isna(vals)
2152 vals[mask] = mask_a
2153 result = accum_func(vals, axis=0)
2154 result[mask] = mask_b
2155 else:
2156 result = accum_func(values, axis=0)
2158 return result