Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/nanops.py: 17%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import functools
4import itertools
5from typing import (
6 TYPE_CHECKING,
7 Any,
8 cast,
9)
10import warnings
12import numpy as np
14from pandas._config import get_option
16from pandas._libs import (
17 NaT,
18 NaTType,
19 iNaT,
20 lib,
21)
22from pandas._typing import (
23 ArrayLike,
24 AxisInt,
25 CorrelationMethod,
26 Dtype,
27 DtypeObj,
28 F,
29 Scalar,
30 Shape,
31 npt,
32)
33from pandas.compat._optional import import_optional_dependency
35from pandas.core.dtypes.common import (
36 is_complex,
37 is_float,
38 is_float_dtype,
39 is_integer,
40 is_numeric_dtype,
41 is_object_dtype,
42 needs_i8_conversion,
43 pandas_dtype,
44)
45from pandas.core.dtypes.missing import (
46 isna,
47 na_value_for_dtype,
48 notna,
49)
51if TYPE_CHECKING:
52 from collections.abc import Callable
54bn = import_optional_dependency("bottleneck", errors="warn")
55_BOTTLENECK_INSTALLED = bn is not None
56_USE_BOTTLENECK = False
59def set_use_bottleneck(v: bool = True) -> None:
60 # set/unset to use bottleneck
61 global _USE_BOTTLENECK
62 if _BOTTLENECK_INSTALLED:
63 _USE_BOTTLENECK = v
66set_use_bottleneck(get_option("compute.use_bottleneck"))
69class disallow:
70 def __init__(self, *dtypes: Dtype) -> None:
71 super().__init__()
72 self.dtypes = tuple(pandas_dtype(dtype).type for dtype in dtypes)
74 def check(self, obj) -> bool:
75 return hasattr(obj, "dtype") and issubclass(obj.dtype.type, self.dtypes)
77 def __call__(self, f: F) -> F:
78 @functools.wraps(f)
79 def _f(*args, **kwargs):
80 obj_iter = itertools.chain(args, kwargs.values())
81 if any(self.check(obj) for obj in obj_iter):
82 f_name = f.__name__.replace("nan", "")
83 raise TypeError(
84 f"reduction operation '{f_name}' not allowed for this dtype"
85 )
86 try:
87 return f(*args, **kwargs)
88 except ValueError as e:
89 # we want to transform an object array
90 # ValueError message to the more typical TypeError
91 # e.g. this is normally a disallowed function on
92 # object arrays that contain strings
93 if is_object_dtype(args[0]):
94 raise TypeError(e) from e
95 raise
97 return cast(F, _f)
100class bottleneck_switch:
101 def __init__(self, name=None, **kwargs) -> None:
102 self.name = name
103 self.kwargs = kwargs
105 def __call__(self, alt: F) -> F:
106 bn_name = self.name or alt.__name__
108 try:
109 bn_func = getattr(bn, bn_name)
110 except (AttributeError, NameError): # pragma: no cover
111 bn_func = None
113 @functools.wraps(alt)
114 def f(
115 values: np.ndarray,
116 *,
117 axis: AxisInt | None = None,
118 skipna: bool = True,
119 **kwds,
120 ):
121 if len(self.kwargs) > 0:
122 for k, v in self.kwargs.items():
123 if k not in kwds:
124 kwds[k] = v
126 if values.size == 0 and kwds.get("min_count") is None:
127 # We are empty, returning NA for our type
128 # Only applies for the default `min_count` of None
129 # since that affects how empty arrays are handled.
130 # TODO(GH-18976) update all the nanops methods to
131 # correctly handle empty inputs and remove this check.
132 # It *may* just be `var`
133 return _na_for_min_count(values, axis)
135 if _USE_BOTTLENECK and skipna and _bn_ok_dtype(values.dtype, bn_name):
136 if kwds.get("mask", None) is None:
137 # `mask` is not recognised by bottleneck, would raise
138 # TypeError if called
139 kwds.pop("mask", None)
140 result = bn_func(values, axis=axis, **kwds)
142 # prefer to treat inf/-inf as NA, but must compute the func
143 # twice :(
144 if _has_infs(result):
145 result = alt(values, axis=axis, skipna=skipna, **kwds)
146 else:
147 result = alt(values, axis=axis, skipna=skipna, **kwds)
148 else:
149 result = alt(values, axis=axis, skipna=skipna, **kwds)
151 return result
153 return cast(F, f)
156def _bn_ok_dtype(dtype: DtypeObj, name: str) -> bool:
157 # Bottleneck chokes on datetime64, PeriodDtype (or and EA)
158 if dtype != object and not needs_i8_conversion(dtype):
159 # GH 42878
160 # Bottleneck uses naive summation leading to O(n) loss of precision
161 # unlike numpy which implements pairwise summation, which has O(log(n)) loss
162 # crossref: https://github.com/pydata/bottleneck/issues/379
164 # GH 15507
165 # bottleneck does not properly upcast during the sum
166 # so can overflow
168 # GH 9422
169 # further we also want to preserve NaN when all elements
170 # are NaN, unlike bottleneck/numpy which consider this
171 # to be 0
172 return name not in ["nansum", "nanprod", "nanmean"]
173 return False
176def _has_infs(result) -> bool:
177 if isinstance(result, np.ndarray):
178 if result.dtype in ("f8", "f4"):
179 # Note: outside of a nanops-specific test, we always have
180 # result.ndim == 1, so there is no risk of this ravel making a copy.
181 return lib.has_infs(result.ravel("K"))
182 try:
183 return np.isinf(result).any()
184 except (TypeError, NotImplementedError):
185 # if it doesn't support infs, then it can't have infs
186 return False
189def _get_fill_value(
190 dtype: DtypeObj, fill_value: Scalar | None = None, fill_value_typ=None
191):
192 """return the correct fill value for the dtype of the values"""
193 if fill_value is not None:
194 return fill_value
195 if _na_ok_dtype(dtype):
196 if fill_value_typ is None:
197 return np.nan
198 elif fill_value_typ == "+inf":
199 return np.inf
200 else:
201 return -np.inf
202 elif fill_value_typ == "+inf":
203 # need the max int here
204 # Return as np.int64 so that np.where promotes the dtype
205 # instead of raising OverflowError (numpy 2.5+) when the
206 # value doesn't fit in the array's dtype (e.g. int8).
207 return np.int64(lib.i8max)
208 else:
209 return np.int64(iNaT)
212def _maybe_get_mask(
213 values: np.ndarray, skipna: bool, mask: npt.NDArray[np.bool_] | None
214) -> npt.NDArray[np.bool_] | None:
215 """
216 Compute a mask if and only if necessary.
218 This function will compute a mask iff it is necessary. Otherwise,
219 return the provided mask (potentially None) when a mask does not need to be
220 computed.
222 A mask is never necessary if the values array is of boolean or integer
223 dtypes, as these are incapable of storing NaNs. If passing a NaN-capable
224 dtype that is interpretable as either boolean or integer data (eg,
225 timedelta64), a mask must be provided.
227 If the skipna parameter is False, a new mask will not be computed.
229 The mask is computed using isna() by default. Setting invert=True selects
230 notna() as the masking function.
232 Parameters
233 ----------
234 values : ndarray
235 input array to potentially compute mask for
236 skipna : bool
237 boolean for whether NaNs should be skipped
238 mask : Optional[ndarray]
239 nan-mask if known
241 Returns
242 -------
243 Optional[np.ndarray[bool]]
244 """
245 if mask is None:
246 if values.dtype.kind in "biu":
247 # Boolean data cannot contain nulls, so signal via mask being None
248 return None
250 if skipna or values.dtype.kind in "mM":
251 mask = isna(values)
253 return mask
256def _get_values(
257 values: np.ndarray,
258 skipna: bool,
259 fill_value: Any = None,
260 fill_value_typ: str | None = None,
261 mask: npt.NDArray[np.bool_] | None = None,
262) -> tuple[np.ndarray, npt.NDArray[np.bool_] | None]:
263 """
264 Utility to get the values view, mask, dtype, dtype_max, and fill_value.
266 If both mask and fill_value/fill_value_typ are not None and skipna is True,
267 the values array will be copied.
269 For input arrays of boolean or integer dtypes, copies will only occur if a
270 precomputed mask, a fill_value/fill_value_typ, and skipna=True are
271 provided.
273 Parameters
274 ----------
275 values : ndarray
276 input array to potentially compute mask for
277 skipna : bool
278 boolean for whether NaNs should be skipped
279 fill_value : Any
280 value to fill NaNs with
281 fill_value_typ : str
282 Set to '+inf' or '-inf' to handle dtype-specific infinities
283 mask : Optional[np.ndarray[bool]]
284 nan-mask if known
286 Returns
287 -------
288 values : ndarray
289 Potential copy of input value array
290 mask : Optional[ndarray[bool]]
291 Mask for values, if deemed necessary to compute
292 """
293 # In _get_values is only called from within nanops, and in all cases
294 # with scalar fill_value. This guarantee is important for the
295 # np.where call below
297 mask = _maybe_get_mask(values, skipna, mask)
299 dtype = values.dtype
301 datetimelike = False
302 if values.dtype.kind in "mM":
303 # changing timedelta64/datetime64 to int64 needs to happen after
304 # finding `mask` above
305 values = np.asarray(values.view("i8"))
306 datetimelike = True
308 if skipna and (mask is not None):
309 # get our fill value (in case we need to provide an alternative
310 # dtype for it)
311 fill_value = _get_fill_value(
312 dtype, fill_value=fill_value, fill_value_typ=fill_value_typ
313 )
315 if fill_value is not None:
316 if mask.any():
317 if datetimelike or _na_ok_dtype(dtype):
318 values = values.copy()
319 np.putmask(values, mask, fill_value)
320 else:
321 # np.where will promote if needed
322 values = np.where(~mask, values, fill_value)
324 return values, mask
327def _get_dtype_max(dtype: np.dtype) -> np.dtype:
328 # return a platform independent precision dtype
329 dtype_max = dtype
330 if dtype.kind in "bi":
331 dtype_max = np.dtype(np.int64)
332 elif dtype.kind == "u":
333 dtype_max = np.dtype(np.uint64)
334 elif dtype.kind == "f":
335 dtype_max = np.dtype(np.float64)
336 return dtype_max
339def _na_ok_dtype(dtype: DtypeObj) -> bool:
340 if needs_i8_conversion(dtype):
341 return False
342 return not issubclass(dtype.type, np.integer)
345def _wrap_results(result, dtype: np.dtype, fill_value=None):
346 """wrap our results if needed"""
347 if result is NaT:
348 pass
350 elif dtype.kind == "M":
351 if fill_value is None:
352 # GH#24293
353 fill_value = iNaT
354 if not isinstance(result, np.ndarray):
355 assert not isna(fill_value), "Expected non-null fill_value"
356 if result == fill_value:
357 result = np.nan
359 if isna(result):
360 result = np.datetime64("NaT", "ns").astype(dtype)
361 else:
362 result = np.int64(result).view(dtype)
363 # retain original unit
364 result = result.astype(dtype, copy=False)
365 else:
366 # If we have float dtype, taking a view will give the wrong result
367 result = result.astype(dtype)
368 elif dtype.kind == "m":
369 if not isinstance(result, np.ndarray):
370 if result == fill_value or np.isnan(result):
371 unit = np.datetime_data(dtype)[0]
372 result = np.timedelta64("NaT", unit) # type: ignore[call-overload]
374 elif np.fabs(result) > lib.i8max:
375 # raise if we have a timedelta64[ns] which is too large
376 raise ValueError("overflow in timedelta operation")
377 else:
378 # return a timedelta64 with the original unit
379 result = np.int64(result).astype(dtype, copy=False)
381 else:
382 result = result.astype("m8[ns]").view(dtype)
384 return result
387def _datetimelike_compat(func: F) -> F:
388 """
389 If we have datetime64 or timedelta64 values, ensure we have a correct
390 mask before calling the wrapped function, then cast back afterwards.
391 """
393 @functools.wraps(func)
394 def new_func(
395 values: np.ndarray,
396 *,
397 axis: AxisInt | None = None,
398 skipna: bool = True,
399 mask: npt.NDArray[np.bool_] | None = None,
400 **kwargs,
401 ):
402 orig_values = values
404 datetimelike = values.dtype.kind in "mM"
405 if datetimelike and mask is None:
406 mask = isna(values)
408 result = func(values, axis=axis, skipna=skipna, mask=mask, **kwargs)
410 if datetimelike:
411 result = _wrap_results(result, orig_values.dtype, fill_value=iNaT)
412 if not skipna:
413 assert mask is not None # checked above
414 result = _mask_datetimelike_result(result, axis, mask, orig_values)
416 return result
418 return cast(F, new_func)
421def _na_for_min_count(values: np.ndarray, axis: AxisInt | None) -> Scalar | np.ndarray:
422 """
423 Return the missing value for `values`.
425 Parameters
426 ----------
427 values : ndarray
428 axis : int or None
429 axis for the reduction, required if values.ndim > 1.
431 Returns
432 -------
433 result : scalar or ndarray
434 For 1-D values, returns a scalar of the correct missing type.
435 For 2-D values, returns a 1-D array where each element is missing.
436 """
437 # we either return np.nan or pd.NaT
438 if values.dtype.kind in "iufcb":
439 values = values.astype("float64")
440 fill_value = na_value_for_dtype(values.dtype)
442 if values.ndim == 1:
443 return fill_value
444 elif axis is None:
445 return fill_value
446 else:
447 result_shape = values.shape[:axis] + values.shape[axis + 1 :]
449 return np.full(result_shape, fill_value, dtype=values.dtype)
452def maybe_operate_rowwise(func: F) -> F:
453 """
454 NumPy operations on C-contiguous ndarrays with axis=1 can be
455 very slow if axis 1 >> axis 0.
456 Operate row-by-row and concatenate the results.
457 """
459 @functools.wraps(func)
460 def newfunc(values: np.ndarray, *, axis: AxisInt | None = None, **kwargs):
461 if (
462 axis == 1
463 and values.ndim == 2
464 and values.flags["C_CONTIGUOUS"]
465 # only takes this path for wide arrays (long dataframes), for threshold see
466 # https://github.com/pandas-dev/pandas/pull/43311#issuecomment-974891737
467 and (values.shape[1] / 1000) > values.shape[0]
468 and values.dtype not in (object, bool)
469 ):
470 arrs = list(values)
471 if kwargs.get("mask") is not None:
472 mask = kwargs.pop("mask")
473 results = [
474 func(arrs[i], mask=mask[i], **kwargs) for i in range(len(arrs))
475 ]
476 else:
477 results = [func(x, **kwargs) for x in arrs]
478 return np.array(results)
480 return func(values, axis=axis, **kwargs)
482 return cast(F, newfunc)
485def nanany(
486 values: np.ndarray,
487 *,
488 axis: AxisInt | None = None,
489 skipna: bool = True,
490 mask: npt.NDArray[np.bool_] | None = None,
491) -> bool:
492 """
493 Check if any elements along an axis evaluate to True.
495 Parameters
496 ----------
497 values : ndarray
498 axis : int, optional
499 skipna : bool, default True
500 mask : ndarray[bool], optional
501 nan-mask if known
503 Returns
504 -------
505 result : bool
507 Examples
508 --------
509 >>> from pandas.core import nanops
510 >>> s = pd.Series([1, 2])
511 >>> nanops.nanany(s.values)
512 np.True_
514 >>> from pandas.core import nanops
515 >>> s = pd.Series([np.nan])
516 >>> nanops.nanany(s.values)
517 np.False_
518 """
519 if values.dtype.kind in "iub" and mask is None:
520 # GH#26032 fastpath
521 # error: Incompatible return value type (got "Union[bool_, ndarray]",
522 # expected "bool")
523 return values.any(axis) # type: ignore[return-value]
525 if values.dtype.kind == "M":
526 # GH#34479
527 raise TypeError("datetime64 type does not support operation 'any'")
529 values, _ = _get_values(values, skipna, fill_value=False, mask=mask)
531 # For object type, any won't necessarily return
532 # boolean values (numpy/numpy#4352)
533 if values.dtype == object:
534 values = values.astype(bool)
536 # error: Incompatible return value type (got "Union[bool_, ndarray]", expected
537 # "bool")
538 return values.any(axis) # type: ignore[return-value]
541def nanall(
542 values: np.ndarray,
543 *,
544 axis: AxisInt | None = None,
545 skipna: bool = True,
546 mask: npt.NDArray[np.bool_] | None = None,
547) -> bool:
548 """
549 Check if all elements along an axis evaluate to True.
551 Parameters
552 ----------
553 values : ndarray
554 axis : int, optional
555 skipna : bool, default True
556 mask : ndarray[bool], optional
557 nan-mask if known
559 Returns
560 -------
561 result : bool
563 Examples
564 --------
565 >>> from pandas.core import nanops
566 >>> s = pd.Series([1, 2, np.nan])
567 >>> nanops.nanall(s.values)
568 np.True_
570 >>> from pandas.core import nanops
571 >>> s = pd.Series([1, 0])
572 >>> nanops.nanall(s.values)
573 np.False_
574 """
575 if values.dtype.kind in "iub" and mask is None:
576 # GH#26032 fastpath
577 # error: Incompatible return value type (got "Union[bool_, ndarray]",
578 # expected "bool")
579 return values.all(axis) # type: ignore[return-value]
581 if values.dtype.kind == "M":
582 # GH#34479
583 raise TypeError("datetime64 type does not support operation 'all'")
585 values, _ = _get_values(values, skipna, fill_value=True, mask=mask)
587 # For object type, all won't necessarily return
588 # boolean values (numpy/numpy#4352)
589 if values.dtype == object:
590 values = values.astype(bool)
592 # error: Incompatible return value type (got "Union[bool_, ndarray]", expected
593 # "bool")
594 return values.all(axis) # type: ignore[return-value]
597@disallow("M8")
598@_datetimelike_compat
599@maybe_operate_rowwise
600def nansum(
601 values: np.ndarray,
602 *,
603 axis: AxisInt | None = None,
604 skipna: bool = True,
605 min_count: int = 0,
606 mask: npt.NDArray[np.bool_] | None = None,
607) -> npt.NDArray[np.floating] | float | NaTType:
608 """
609 Sum the elements along an axis ignoring NaNs
611 Parameters
612 ----------
613 values : ndarray[dtype]
614 axis : int, optional
615 skipna : bool, default True
616 min_count: int, default 0
617 mask : ndarray[bool], optional
618 nan-mask if known
620 Returns
621 -------
622 result : dtype
624 Examples
625 --------
626 >>> from pandas.core import nanops
627 >>> s = pd.Series([1, 2, np.nan])
628 >>> nanops.nansum(s.values)
629 np.float64(3.0)
630 """
631 dtype = values.dtype
632 values, mask = _get_values(values, skipna, fill_value=0, mask=mask)
633 dtype_sum = _get_dtype_max(dtype)
634 if dtype.kind == "f":
635 dtype_sum = dtype
636 elif dtype.kind == "m":
637 dtype_sum = np.dtype(np.float64)
639 the_sum = values.sum(axis, dtype=dtype_sum)
640 the_sum = _maybe_null_out(the_sum, axis, mask, values.shape, min_count=min_count)
642 return the_sum
645def _mask_datetimelike_result(
646 result: np.ndarray | np.datetime64 | np.timedelta64,
647 axis: AxisInt | None,
648 mask: npt.NDArray[np.bool_],
649 orig_values: np.ndarray,
650) -> np.ndarray | np.datetime64 | np.timedelta64 | NaTType:
651 if isinstance(result, np.ndarray):
652 # we need to apply the mask
653 result = result.astype("i8").view(orig_values.dtype)
654 axis_mask = mask.any(axis=axis)
655 result[axis_mask] = iNaT
656 elif mask.any():
657 return np.int64(iNaT).view(orig_values.dtype)
658 return result
661@bottleneck_switch()
662@_datetimelike_compat
663def nanmean(
664 values: np.ndarray,
665 *,
666 axis: AxisInt | None = None,
667 skipna: bool = True,
668 mask: npt.NDArray[np.bool_] | None = None,
669) -> float:
670 """
671 Compute the mean of the element along an axis ignoring NaNs
673 Parameters
674 ----------
675 values : ndarray
676 axis : int, optional
677 skipna : bool, default True
678 mask : ndarray[bool], optional
679 nan-mask if known
681 Returns
682 -------
683 float
684 Unless input is a float array, in which case use the same
685 precision as the input array.
687 Examples
688 --------
689 >>> from pandas.core import nanops
690 >>> s = pd.Series([1, 2, np.nan])
691 >>> nanops.nanmean(s.values)
692 np.float64(1.5)
693 """
694 if values.dtype == object and len(values) > 1_000 and mask is None:
695 # GH#54754 if we are going to fail, try to fail-fast
696 nanmean(values[:1000], axis=axis, skipna=skipna)
698 dtype = values.dtype
699 values, mask = _get_values(values, skipna, fill_value=0, mask=mask)
700 dtype_sum = _get_dtype_max(dtype)
701 dtype_count = np.dtype(np.float64)
703 # not using needs_i8_conversion because that includes period
704 if dtype.kind in "mM":
705 dtype_sum = np.dtype(np.float64)
706 elif dtype.kind in "iu":
707 dtype_sum = np.dtype(np.float64)
708 elif dtype.kind == "f":
709 dtype_sum = dtype
710 dtype_count = dtype
712 count = _get_counts(values.shape, mask, axis, dtype=dtype_count)
713 the_sum = values.sum(axis, dtype=dtype_sum)
714 the_sum = _ensure_numeric(the_sum)
716 if axis is not None and getattr(the_sum, "ndim", False):
717 count = cast(np.ndarray, count)
718 with np.errstate(all="ignore"):
719 # suppress division by zero warnings
720 the_mean = the_sum / count
721 ct_mask = count == 0
722 if ct_mask.any():
723 the_mean[ct_mask] = np.nan
724 else:
725 the_mean = the_sum / count if count > 0 else np.nan
727 return the_mean
730@bottleneck_switch()
731def nanmedian(
732 values: np.ndarray, *, axis: AxisInt | None = None, skipna: bool = True, mask=None
733) -> float | np.ndarray:
734 """
735 Parameters
736 ----------
737 values : ndarray
738 axis : int, optional
739 skipna : bool, default True
740 mask : ndarray[bool], optional
741 nan-mask if known
743 Returns
744 -------
745 result : float | ndarray
746 Unless input is a float array, in which case use the same
747 precision as the input array.
749 Examples
750 --------
751 >>> from pandas.core import nanops
752 >>> s = pd.Series([1, np.nan, 2, 2])
753 >>> nanops.nanmedian(s.values)
754 2.0
756 >>> s = pd.Series([np.nan, np.nan, np.nan])
757 >>> nanops.nanmedian(s.values)
758 nan
759 """
760 # for floats without mask, the data already uses NaN as missing value
761 # indicator, and `mask` will be calculated from that below -> in those
762 # cases we never need to set NaN to the masked values
763 using_nan_sentinel = values.dtype.kind == "f" and mask is None
765 def get_median(x: np.ndarray, _mask=None):
766 if _mask is None:
767 _mask = notna(x)
768 else:
769 _mask = ~_mask
770 if not skipna and not _mask.all():
771 return np.nan
772 with warnings.catch_warnings():
773 # Suppress RuntimeWarning about All-NaN slice
774 warnings.filterwarnings(
775 "ignore", "All-NaN slice encountered", RuntimeWarning
776 )
777 warnings.filterwarnings("ignore", "Mean of empty slice", RuntimeWarning)
778 res = np.nanmedian(x[_mask])
779 return res
781 dtype = values.dtype
782 values, mask = _get_values(values, skipna, mask=mask, fill_value=None)
783 if values.dtype.kind != "f":
784 if values.dtype == object:
785 # GH#34671 avoid casting strings to numeric
786 inferred = lib.infer_dtype(values)
787 if inferred in ["string", "mixed"]:
788 raise TypeError(f"Cannot convert {values} to numeric")
789 try:
790 values = values.astype("f8")
791 except ValueError as err:
792 # e.g. "could not convert string to float: 'a'"
793 raise TypeError(str(err)) from err
794 if not using_nan_sentinel and mask is not None:
795 if not values.flags.writeable:
796 values = values.copy()
797 values[mask] = np.nan
799 notempty = values.size
801 res: float | np.ndarray
803 # an array from a frame
804 if values.ndim > 1 and axis is not None:
805 # there's a non-empty array to apply over otherwise numpy raises
806 if notempty:
807 if not skipna:
808 res = np.apply_along_axis(get_median, axis, values)
810 else:
811 # fastpath for the skipna case
812 with warnings.catch_warnings():
813 # Suppress RuntimeWarning about All-NaN slice
814 warnings.filterwarnings(
815 "ignore", "All-NaN slice encountered", RuntimeWarning
816 )
817 if (values.shape[1] == 1 and axis == 0) or (
818 values.shape[0] == 1 and axis == 1
819 ):
820 # GH52788: fastpath when squeezable, nanmedian for 2D array slow
821 res = np.nanmedian(np.squeeze(values), keepdims=True)
822 else:
823 res = np.nanmedian(values, axis=axis)
825 else:
826 # must return the correct shape, but median is not defined for the
827 # empty set so return nans of shape "everything but the passed axis"
828 # since "axis" is where the reduction would occur if we had a nonempty
829 # array
830 res = _get_empty_reduction_result(values.shape, axis)
832 else:
833 # otherwise return a scalar value
834 res = get_median(values, mask) if notempty else np.nan
835 return _wrap_results(res, dtype)
838def _get_empty_reduction_result(
839 shape: Shape,
840 axis: AxisInt,
841) -> np.ndarray:
842 """
843 The result from a reduction on an empty ndarray.
845 Parameters
846 ----------
847 shape : Tuple[int, ...]
848 axis : int
850 Returns
851 -------
852 np.ndarray
853 """
854 shp = np.array(shape)
855 dims = np.arange(len(shape))
856 ret = np.empty(shp[dims != axis], dtype=np.float64)
857 ret.fill(np.nan)
858 return ret
861def _get_counts_nanvar(
862 values_shape: Shape,
863 mask: npt.NDArray[np.bool_] | None,
864 axis: AxisInt | None,
865 ddof: int,
866 dtype: np.dtype = np.dtype(np.float64),
867) -> tuple[float | np.ndarray, float | np.ndarray]:
868 """
869 Get the count of non-null values along an axis, accounting
870 for degrees of freedom.
872 Parameters
873 ----------
874 values_shape : Tuple[int, ...]
875 shape tuple from values ndarray, used if mask is None
876 mask : Optional[ndarray[bool]]
877 locations in values that should be considered missing
878 axis : Optional[int]
879 axis to count along
880 ddof : int
881 degrees of freedom
882 dtype : type, optional
883 type to use for count
885 Returns
886 -------
887 count : int, np.nan or np.ndarray
888 d : int, np.nan or np.ndarray
889 """
890 count = _get_counts(values_shape, mask, axis, dtype=dtype)
891 d = count - dtype.type(ddof)
893 # always return NaN, never inf
894 if is_float(count):
895 if count <= ddof:
896 # error: Incompatible types in assignment (expression has type
897 # "float", variable has type "Union[floating[Any], ndarray[Any,
898 # dtype[floating[Any]]]]")
899 count = np.nan # type: ignore[assignment]
900 d = np.nan
901 else:
902 # count is not narrowed by is_float check
903 count = cast(np.ndarray, count)
904 mask = count <= ddof
905 if mask.any():
906 np.putmask(d, mask, np.nan)
907 np.putmask(count, mask, np.nan)
908 return count, d
911@bottleneck_switch(ddof=1)
912def nanstd(
913 values,
914 *,
915 axis: AxisInt | None = None,
916 skipna: bool = True,
917 ddof: int = 1,
918 mask=None,
919):
920 """
921 Compute the standard deviation along given axis while ignoring NaNs
923 Parameters
924 ----------
925 values : ndarray
926 axis : int, optional
927 skipna : bool, default True
928 ddof : int, default 1
929 Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
930 where N represents the number of elements.
931 mask : ndarray[bool], optional
932 nan-mask if known
934 Returns
935 -------
936 result : float
937 Unless input is a float array, in which case use the same
938 precision as the input array.
940 Examples
941 --------
942 >>> from pandas.core import nanops
943 >>> s = pd.Series([1, np.nan, 2, 3])
944 >>> nanops.nanstd(s.values)
945 1.0
946 """
947 if values.dtype.kind == "M":
948 unit = np.datetime_data(values.dtype)[0]
949 values = values.view(f"m8[{unit}]")
951 orig_dtype = values.dtype
952 values, mask = _get_values(values, skipna, mask=mask)
954 result = np.sqrt(nanvar(values, axis=axis, skipna=skipna, ddof=ddof, mask=mask))
955 return _wrap_results(result, orig_dtype)
958@disallow("M8", "m8")
959@bottleneck_switch(ddof=1)
960def nanvar(
961 values: np.ndarray,
962 *,
963 axis: AxisInt | None = None,
964 skipna: bool = True,
965 ddof: int = 1,
966 mask=None,
967):
968 """
969 Compute the variance along given axis while ignoring NaNs
971 Parameters
972 ----------
973 values : ndarray
974 axis : int, optional
975 skipna : bool, default True
976 ddof : int, default 1
977 Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
978 where N represents the number of elements.
979 mask : ndarray[bool], optional
980 nan-mask if known
982 Returns
983 -------
984 result : float
985 Unless input is a float array, in which case use the same
986 precision as the input array.
988 Examples
989 --------
990 >>> from pandas.core import nanops
991 >>> s = pd.Series([1, np.nan, 2, 3])
992 >>> nanops.nanvar(s.values)
993 1.0
994 """
995 dtype = values.dtype
996 mask = _maybe_get_mask(values, skipna, mask)
997 if dtype.kind in "iu":
998 values = values.astype("f8")
999 if mask is not None:
1000 values[mask] = np.nan
1001 elif dtype.kind == "c":
1002 # https://en.wikipedia.org/wiki/Complex_random_variable#Variance_and_pseudo-variance
1003 # The variance is equal to the sum of
1004 # the variances of the real and imaginary part of the complex random variable.
1005 return nanvar(
1006 values.real, axis=axis, skipna=skipna, ddof=ddof, mask=mask
1007 ) + nanvar(values.imag, axis=axis, skipna=skipna, ddof=ddof, mask=mask)
1009 if values.dtype.kind == "f":
1010 count, d = _get_counts_nanvar(values.shape, mask, axis, ddof, values.dtype)
1011 else:
1012 count, d = _get_counts_nanvar(values.shape, mask, axis, ddof)
1014 if skipna and mask is not None:
1015 values = values.copy()
1016 np.putmask(values, mask, 0)
1018 # xref GH10242
1019 # Compute variance via two-pass algorithm, which is stable against
1020 # cancellation errors and relatively accurate for small numbers of
1021 # observations.
1022 #
1023 # See https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
1024 avg = _ensure_numeric(values.sum(axis=axis, dtype=np.float64)) / count
1025 if axis is not None:
1026 avg = np.expand_dims(avg, axis)
1028 sqr = _ensure_numeric((avg - values) ** 2)
1029 if mask is not None:
1030 np.putmask(sqr, mask, 0)
1031 result = sqr.sum(axis=axis, dtype=np.float64) / d
1033 # Return variance as np.float64 (the datatype used in the accumulator),
1034 # unless we were dealing with a float array, in which case use the same
1035 # precision as the original values array.
1036 if dtype.kind == "f":
1037 result = result.astype(dtype, copy=False)
1038 return result
1041@disallow("M8", "m8")
1042def nansem(
1043 values: np.ndarray,
1044 *,
1045 axis: AxisInt | None = None,
1046 skipna: bool = True,
1047 ddof: int = 1,
1048 mask: npt.NDArray[np.bool_] | None = None,
1049) -> float:
1050 """
1051 Compute the standard error in the mean along given axis while ignoring NaNs
1053 Parameters
1054 ----------
1055 values : ndarray
1056 axis : int, optional
1057 skipna : bool, default True
1058 ddof : int, default 1
1059 Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
1060 where N represents the number of elements.
1061 mask : ndarray[bool], optional
1062 nan-mask if known
1064 Returns
1065 -------
1066 result : float64
1067 Unless input is a float array, in which case use the same
1068 precision as the input array.
1070 Examples
1071 --------
1072 >>> from pandas.core import nanops
1073 >>> s = pd.Series([1, np.nan, 2, 3])
1074 >>> nanops.nansem(s.values)
1075 np.float64(0.5773502691896258)
1076 """
1077 # This checks if non-numeric-like data is passed with numeric_only=False
1078 # and raises a TypeError otherwise
1079 nanvar(values, axis=axis, skipna=skipna, ddof=ddof, mask=mask)
1081 mask = _maybe_get_mask(values, skipna, mask)
1082 # Convert to bottleneck return a float
1083 if values.dtype.kind not in "fc":
1084 values = values.astype("f8")
1086 if not skipna and mask is not None and mask.any():
1087 return np.nan
1089 dtype_count = np.dtype(np.float64)
1090 if values.dtype.kind == "f":
1091 dtype_count = values.dtype
1092 count, _ = _get_counts_nanvar(values.shape, mask, axis, ddof, dtype_count)
1093 var = nanvar(values, axis=axis, skipna=skipna, ddof=ddof, mask=mask)
1095 return np.sqrt(var) / np.sqrt(count)
1098def _nanminmax(meth, fill_value_typ):
1099 @bottleneck_switch(name=f"nan{meth}")
1100 @_datetimelike_compat
1101 def reduction(
1102 values: np.ndarray,
1103 *,
1104 axis: AxisInt | None = None,
1105 skipna: bool = True,
1106 mask: npt.NDArray[np.bool_] | None = None,
1107 ):
1108 if values.size == 0:
1109 return _na_for_min_count(values, axis)
1111 dtype = values.dtype
1112 values, mask = _get_values(
1113 values, skipna, fill_value_typ=fill_value_typ, mask=mask
1114 )
1115 result = getattr(values, meth)(axis)
1116 result = _maybe_null_out(
1117 result, axis, mask, values.shape, datetimelike=dtype.kind in "mM"
1118 )
1119 return result
1121 return reduction
1124nanmin = _nanminmax("min", fill_value_typ="+inf")
1125nanmax = _nanminmax("max", fill_value_typ="-inf")
1128def nanargmax(
1129 values: np.ndarray,
1130 *,
1131 axis: AxisInt | None = None,
1132 skipna: bool = True,
1133 mask: npt.NDArray[np.bool_] | None = None,
1134) -> int | np.ndarray:
1135 """
1136 Parameters
1137 ----------
1138 values : ndarray
1139 axis : int, optional
1140 skipna : bool, default True
1141 mask : ndarray[bool], optional
1142 nan-mask if known
1144 Returns
1145 -------
1146 result : int or ndarray[int]
1147 The index/indices of max value in specified axis or -1 in the NA case
1149 Examples
1150 --------
1151 >>> from pandas.core import nanops
1152 >>> arr = np.array([1, 2, 3, np.nan, 4])
1153 >>> nanops.nanargmax(arr)
1154 np.int64(4)
1156 >>> arr = np.array(range(12), dtype=np.float64).reshape(4, 3)
1157 >>> arr[2:, 2] = np.nan
1158 >>> arr
1159 array([[ 0., 1., 2.],
1160 [ 3., 4., 5.],
1161 [ 6., 7., nan],
1162 [ 9., 10., nan]])
1163 >>> nanops.nanargmax(arr, axis=1)
1164 array([2, 2, 1, 1])
1165 """
1166 values, mask = _get_values(values, True, fill_value_typ="-inf", mask=mask)
1167 result = values.argmax(axis)
1168 # error: Argument 1 to "_maybe_arg_null_out" has incompatible type "Any |
1169 # signedinteger[Any]"; expected "ndarray[Any, Any]"
1170 result = _maybe_arg_null_out(result, axis, mask, skipna) # type: ignore[arg-type]
1171 return result
1174def nanargmin(
1175 values: np.ndarray,
1176 *,
1177 axis: AxisInt | None = None,
1178 skipna: bool = True,
1179 mask: npt.NDArray[np.bool_] | None = None,
1180) -> int | np.ndarray:
1181 """
1182 Parameters
1183 ----------
1184 values : ndarray
1185 axis : int, optional
1186 skipna : bool, default True
1187 mask : ndarray[bool], optional
1188 nan-mask if known
1190 Returns
1191 -------
1192 result : int or ndarray[int]
1193 The index/indices of min value in specified axis or -1 in the NA case
1195 Examples
1196 --------
1197 >>> from pandas.core import nanops
1198 >>> arr = np.array([1, 2, 3, np.nan, 4])
1199 >>> nanops.nanargmin(arr)
1200 np.int64(0)
1202 >>> arr = np.array(range(12), dtype=np.float64).reshape(4, 3)
1203 >>> arr[2:, 0] = np.nan
1204 >>> arr
1205 array([[ 0., 1., 2.],
1206 [ 3., 4., 5.],
1207 [nan, 7., 8.],
1208 [nan, 10., 11.]])
1209 >>> nanops.nanargmin(arr, axis=1)
1210 array([0, 0, 1, 1])
1211 """
1212 values, mask = _get_values(values, True, fill_value_typ="+inf", mask=mask)
1213 result = values.argmin(axis)
1214 # error: Argument 1 to "_maybe_arg_null_out" has incompatible type "Any |
1215 # signedinteger[Any]"; expected "ndarray[Any, Any]"
1216 result = _maybe_arg_null_out(result, axis, mask, skipna) # type: ignore[arg-type]
1217 return result
1220@disallow("M8", "m8")
1221@maybe_operate_rowwise
1222def nanskew(
1223 values: np.ndarray,
1224 *,
1225 axis: AxisInt | None = None,
1226 skipna: bool = True,
1227 mask: npt.NDArray[np.bool_] | None = None,
1228) -> float:
1229 """
1230 Compute the sample skewness.
1232 The statistic computed here is the adjusted Fisher-Pearson standardized
1233 moment coefficient G1. The algorithm computes this coefficient directly
1234 from the second and third central moment.
1236 Parameters
1237 ----------
1238 values : ndarray
1239 axis : int, optional
1240 skipna : bool, default True
1241 mask : ndarray[bool], optional
1242 nan-mask if known
1244 Returns
1245 -------
1246 result : float64
1247 Unless input is a float array, in which case use the same
1248 precision as the input array.
1250 Examples
1251 --------
1252 >>> from pandas.core import nanops
1253 >>> s = pd.Series([1, np.nan, 1, 2])
1254 >>> nanops.nanskew(s.values)
1255 np.float64(1.7320508075688787)
1256 """
1257 mask = _maybe_get_mask(values, skipna, mask)
1258 if values.dtype.kind != "f":
1259 values = values.astype("f8")
1260 count = _get_counts(values.shape, mask, axis)
1261 else:
1262 count = _get_counts(values.shape, mask, axis, dtype=values.dtype)
1264 if skipna and mask is not None:
1265 values = values.copy()
1266 np.putmask(values, mask, 0)
1267 elif not skipna and mask is not None and mask.any():
1268 return np.nan
1270 with np.errstate(invalid="ignore", divide="ignore"):
1271 mean = values.sum(axis, dtype=np.float64) / count
1272 if axis is not None:
1273 mean = np.expand_dims(mean, axis)
1275 adjusted = values - mean
1276 if skipna and mask is not None:
1277 np.putmask(adjusted, mask, 0)
1278 adjusted2 = adjusted**2
1279 adjusted3 = adjusted2 * adjusted
1280 m2 = adjusted2.sum(axis, dtype=np.float64)
1281 m3 = adjusted3.sum(axis, dtype=np.float64)
1283 # floating point error. See comment in [nankurt]
1284 max_abs = np.abs(values).max(axis, initial=0.0)
1285 eps = np.finfo(m2.dtype).eps
1286 constant_tolerance2 = ((eps * max_abs) ** 2) * count
1287 constant_tolerance3 = ((eps * max_abs) ** 3) * count
1288 m2 = _zero_out_fperr(m2, constant_tolerance2)
1289 m3 = _zero_out_fperr(m3, constant_tolerance3)
1291 with np.errstate(invalid="ignore", divide="ignore"):
1292 result = (count * (count - 1) ** 0.5 / (count - 2)) * (m3 / m2**1.5)
1294 dtype = values.dtype
1295 if dtype.kind == "f":
1296 result = result.astype(dtype, copy=False)
1298 if isinstance(result, np.ndarray):
1299 result = np.where(m2 == 0, 0, result)
1300 result[count < 3] = np.nan
1301 else:
1302 result = dtype.type(0) if m2 == 0 else result
1303 if count < 3:
1304 return np.nan
1306 return result
1309@disallow("M8", "m8")
1310@maybe_operate_rowwise
1311def nankurt(
1312 values: np.ndarray,
1313 *,
1314 axis: AxisInt | None = None,
1315 skipna: bool = True,
1316 mask: npt.NDArray[np.bool_] | None = None,
1317) -> float:
1318 """
1319 Compute the sample excess kurtosis
1321 The statistic computed here is the adjusted Fisher-Pearson standardized
1322 moment coefficient G2, computed directly from the second and fourth
1323 central moment.
1325 Parameters
1326 ----------
1327 values : ndarray
1328 axis : int, optional
1329 skipna : bool, default True
1330 mask : ndarray[bool], optional
1331 nan-mask if known
1333 Returns
1334 -------
1335 result : float64
1336 Unless input is a float array, in which case use the same
1337 precision as the input array.
1339 Examples
1340 --------
1341 >>> from pandas.core import nanops
1342 >>> s = pd.Series([1, np.nan, 1, 3, 2])
1343 >>> nanops.nankurt(s.values)
1344 np.float64(-1.2892561983471076)
1345 """
1346 mask = _maybe_get_mask(values, skipna, mask)
1347 if values.dtype.kind != "f":
1348 values = values.astype("f8")
1349 count = _get_counts(values.shape, mask, axis)
1350 else:
1351 count = _get_counts(values.shape, mask, axis, dtype=values.dtype)
1353 if skipna and mask is not None:
1354 values = values.copy()
1355 np.putmask(values, mask, 0)
1356 elif not skipna and mask is not None and mask.any():
1357 return np.nan
1359 with np.errstate(invalid="ignore", divide="ignore"):
1360 mean = values.sum(axis, dtype=np.float64) / count
1361 if axis is not None:
1362 mean = np.expand_dims(mean, axis)
1364 adjusted = values - mean
1365 if skipna and mask is not None:
1366 np.putmask(adjusted, mask, 0)
1367 adjusted2 = adjusted**2
1368 adjusted4 = adjusted2**2
1369 m2 = adjusted2.sum(axis, dtype=np.float64)
1370 m4 = adjusted4.sum(axis, dtype=np.float64)
1372 # Several floating point errors may occur during the summation due to rounding.
1373 # This computation is similar to the one in Scipy
1374 # https://github.com/scipy/scipy/blob/04d6d9c460b1fed83f2919ecec3d743cfa2e8317/scipy/stats/_stats_py.py#L1429
1375 # With a few modifications, like using the maximum value instead of the averages
1376 # and some adaptations because they use the average and we use the sum for `m2`.
1377 # We need to estimate an upper bound to the error to consider the data constant.
1378 # Let's call:
1379 # x: true value in data
1380 # y: floating point representation
1381 # e: relative approximation error
1382 # n: number of observations in array
1383 #
1384 # We have that:
1385 # |x - y|/|x| <= e (See https://en.wikipedia.org/wiki/Machine_epsilon)
1386 # (|x - y|/|x|)² <= e²
1387 # Σ (|x - y|/|x|)² <= ne²
1388 #
1389 # Let's say that the fperr upper bound for m2 is constrained by the summation.
1390 # |m2 - y|/|m2| <= ne²
1391 # |m2 - y| <= n|m2|e²
1392 #
1393 # We will use max (x²) to estimate |m2|
1394 max_abs = np.abs(values).max(axis, initial=0.0)
1395 eps = np.finfo(m2.dtype).eps
1396 constant_tolerance2 = ((eps * max_abs) ** 2) * count
1397 constant_tolerance4 = ((eps * max_abs) ** 4) * count
1398 m2 = _zero_out_fperr(m2, constant_tolerance2)
1399 m4 = _zero_out_fperr(m4, constant_tolerance4)
1401 with np.errstate(invalid="ignore", divide="ignore"):
1402 adj = 3 * (count - 1) ** 2 / ((count - 2) * (count - 3))
1403 numerator = count * (count + 1) * (count - 1) * m4
1404 denominator = (count - 2) * (count - 3) * m2**2
1406 if not isinstance(denominator, np.ndarray):
1407 # if ``denom`` is a scalar, check these corner cases first before
1408 # doing division
1409 if count < 4:
1410 return np.nan
1411 if denominator == 0:
1412 return values.dtype.type(0)
1414 with np.errstate(invalid="ignore", divide="ignore"):
1415 result = numerator / denominator - adj
1417 dtype = values.dtype
1418 if dtype.kind == "f":
1419 result = result.astype(dtype, copy=False)
1421 if isinstance(result, np.ndarray):
1422 result = np.where(denominator == 0, 0, result)
1423 result[count < 4] = np.nan
1425 return result
1428@disallow("M8", "m8")
1429@maybe_operate_rowwise
1430def nanprod(
1431 values: np.ndarray,
1432 *,
1433 axis: AxisInt | None = None,
1434 skipna: bool = True,
1435 min_count: int = 0,
1436 mask: npt.NDArray[np.bool_] | None = None,
1437) -> float:
1438 """
1439 Parameters
1440 ----------
1441 values : ndarray[dtype]
1442 axis : int, optional
1443 skipna : bool, default True
1444 min_count: int, default 0
1445 mask : ndarray[bool], optional
1446 nan-mask if known
1448 Returns
1449 -------
1450 Dtype
1451 The product of all elements on a given axis. ( NaNs are treated as 1)
1453 Examples
1454 --------
1455 >>> from pandas.core import nanops
1456 >>> s = pd.Series([1, 2, 3, np.nan])
1457 >>> nanops.nanprod(s.values)
1458 np.float64(6.0)
1459 """
1460 mask = _maybe_get_mask(values, skipna, mask)
1462 if skipna and mask is not None:
1463 values = values.copy()
1464 values[mask] = 1
1465 result = values.prod(axis)
1466 # error: Incompatible return value type (got "Union[ndarray, float]", expected
1467 # "float")
1468 return _maybe_null_out( # type: ignore[return-value]
1469 result, axis, mask, values.shape, min_count=min_count
1470 )
1473def _maybe_arg_null_out(
1474 result: np.ndarray,
1475 axis: AxisInt | None,
1476 mask: npt.NDArray[np.bool_] | None,
1477 skipna: bool,
1478) -> np.ndarray | int:
1479 # helper function for nanargmin/nanargmax
1480 if mask is None:
1481 return result
1483 if axis is None or not getattr(result, "ndim", False):
1484 if skipna and mask.all():
1485 raise ValueError("Encountered all NA values")
1486 elif not skipna and mask.any():
1487 raise ValueError("Encountered an NA value with skipna=False")
1488 elif skipna and mask.all(axis).any():
1489 raise ValueError("Encountered all NA values")
1490 elif not skipna and mask.any(axis).any():
1491 raise ValueError("Encountered an NA value with skipna=False")
1492 return result
1495def _get_counts(
1496 values_shape: Shape,
1497 mask: npt.NDArray[np.bool_] | None,
1498 axis: AxisInt | None,
1499 dtype: np.dtype[np.floating] = np.dtype(np.float64),
1500) -> np.floating | npt.NDArray[np.floating]:
1501 """
1502 Get the count of non-null values along an axis
1504 Parameters
1505 ----------
1506 values_shape : tuple of int
1507 shape tuple from values ndarray, used if mask is None
1508 mask : Optional[ndarray[bool]]
1509 locations in values that should be considered missing
1510 axis : Optional[int]
1511 axis to count along
1512 dtype : type, optional
1513 type to use for count
1515 Returns
1516 -------
1517 count : scalar or array
1518 """
1519 if axis is None:
1520 if mask is not None:
1521 n = mask.size - mask.sum()
1522 else:
1523 n = np.prod(values_shape)
1524 return dtype.type(n)
1526 if mask is not None:
1527 count = mask.shape[axis] - mask.sum(axis)
1528 else:
1529 count = values_shape[axis]
1531 if is_integer(count):
1532 return dtype.type(count)
1533 return count.astype(dtype, copy=False)
1536def _maybe_null_out(
1537 result: np.ndarray | float | NaTType,
1538 axis: AxisInt | None,
1539 mask: npt.NDArray[np.bool_] | None,
1540 shape: tuple[int, ...],
1541 min_count: int = 1,
1542 datetimelike: bool = False,
1543) -> np.ndarray | float | NaTType:
1544 """
1545 Returns
1546 -------
1547 Dtype
1548 The product of all elements on a given axis. ( NaNs are treated as 1)
1549 """
1550 if mask is None and min_count == 0:
1551 # nothing to check; short-circuit
1552 return result
1554 if axis is not None and isinstance(result, np.ndarray):
1555 if mask is not None:
1556 null_mask = (mask.shape[axis] - mask.sum(axis) - min_count) < 0
1557 else:
1558 # we have no nulls, kept mask=None in _maybe_get_mask
1559 below_count = shape[axis] - min_count < 0
1560 new_shape = shape[:axis] + shape[axis + 1 :]
1561 null_mask = np.broadcast_to(below_count, new_shape)
1563 if np.any(null_mask):
1564 if datetimelike:
1565 # GH#60646 For datetimelike, no need to cast to float
1566 result[null_mask] = iNaT
1567 elif is_numeric_dtype(result):
1568 if np.iscomplexobj(result):
1569 result = result.astype("c16")
1570 elif not is_float_dtype(result):
1571 result = result.astype("f8", copy=False)
1572 result[null_mask] = np.nan
1573 else:
1574 # GH12941, use None to auto cast null
1575 result[null_mask] = None
1576 elif result is not NaT:
1577 if check_below_min_count(shape, mask, min_count):
1578 result_dtype = getattr(result, "dtype", None)
1579 if is_float_dtype(result_dtype):
1580 # error: Item "None" of "Optional[Any]" has no attribute "type"
1581 result = result_dtype.type("nan") # type: ignore[union-attr]
1582 else:
1583 result = np.nan
1585 return result
1588def check_below_min_count(
1589 shape: tuple[int, ...], mask: npt.NDArray[np.bool_] | None, min_count: int
1590) -> bool:
1591 """
1592 Check for the `min_count` keyword. Returns True if below `min_count` (when
1593 missing value should be returned from the reduction).
1595 Parameters
1596 ----------
1597 shape : tuple
1598 The shape of the values (`values.shape`).
1599 mask : ndarray[bool] or None
1600 Boolean numpy array (typically of same shape as `shape`) or None.
1601 min_count : int
1602 Keyword passed through from sum/prod call.
1604 Returns
1605 -------
1606 bool
1607 """
1608 if min_count > 0:
1609 if mask is None:
1610 # no missing values, only check size
1611 non_nulls = np.prod(shape)
1612 else:
1613 non_nulls = mask.size - mask.sum()
1614 if non_nulls < min_count:
1615 return True
1616 return False
1619def _zero_out_fperr(arg, tol: float | np.ndarray):
1620 # #18044 reference this behavior to fix rolling skew/kurt issue
1621 if isinstance(arg, np.ndarray):
1622 return np.where(np.abs(arg) < tol, 0, arg)
1623 else:
1624 return arg.dtype.type(0) if np.abs(arg) < tol else arg
1627@disallow("M8", "m8")
1628def nancorr(
1629 a: np.ndarray,
1630 b: np.ndarray,
1631 *,
1632 method: CorrelationMethod = "pearson",
1633 min_periods: int | None = None,
1634) -> float:
1635 """
1636 a, b: ndarrays
1637 """
1638 if len(a) != len(b):
1639 raise AssertionError("Operands to nancorr must have same size")
1641 if min_periods is None:
1642 min_periods = 1
1644 valid = notna(a) & notna(b)
1645 if not valid.all():
1646 a = a[valid]
1647 b = b[valid]
1649 if len(a) < min_periods:
1650 return np.nan
1652 a = _ensure_numeric(a)
1653 b = _ensure_numeric(b)
1655 f = get_corr_func(method)
1656 return f(a, b)
1659def get_corr_func(
1660 method: CorrelationMethod,
1661) -> Callable[[np.ndarray, np.ndarray], float]:
1662 if method == "kendall":
1663 from scipy.stats import kendalltau
1665 def func(a, b):
1666 return kendalltau(a, b)[0]
1668 return func
1669 elif method == "spearman":
1670 from scipy.stats import spearmanr
1672 def func(a, b):
1673 return spearmanr(a, b)[0]
1675 return func
1676 elif method == "pearson":
1678 def func(a, b):
1679 return np.corrcoef(a, b)[0, 1]
1681 return func
1682 elif callable(method):
1683 return method
1685 raise ValueError(
1686 f"Unknown method '{method}', expected one of "
1687 "'kendall', 'spearman', 'pearson', or callable"
1688 )
1691@disallow("M8", "m8")
1692def nancov(
1693 a: np.ndarray,
1694 b: np.ndarray,
1695 *,
1696 min_periods: int | None = None,
1697 ddof: int | None = 1,
1698) -> float:
1699 if len(a) != len(b):
1700 raise AssertionError("Operands to nancov must have same size")
1702 if min_periods is None:
1703 min_periods = 1
1705 valid = notna(a) & notna(b)
1706 if not valid.all():
1707 a = a[valid]
1708 b = b[valid]
1710 if len(a) < min_periods:
1711 return np.nan
1713 a = _ensure_numeric(a)
1714 b = _ensure_numeric(b)
1716 return np.cov(a, b, ddof=ddof)[0, 1]
1719def _ensure_numeric(x):
1720 if isinstance(x, np.ndarray):
1721 if x.dtype.kind in "biu":
1722 x = x.astype(np.float64)
1723 elif x.dtype == object:
1724 inferred = lib.infer_dtype(x)
1725 if inferred in ["string", "mixed"]:
1726 # GH#44008, GH#36703 avoid casting e.g. strings to numeric
1727 raise TypeError(f"Could not convert {x} to numeric")
1728 try:
1729 x = x.astype(np.complex128)
1730 except (TypeError, ValueError):
1731 try:
1732 x = x.astype(np.float64)
1733 except ValueError as err:
1734 # GH#29941 we get here with object arrays containing strs
1735 raise TypeError(f"Could not convert {x} to numeric") from err
1736 else:
1737 if not np.any(np.imag(x)):
1738 x = x.real
1739 elif not (is_float(x) or is_integer(x) or is_complex(x)):
1740 if isinstance(x, str):
1741 # GH#44008, GH#36703 avoid casting e.g. strings to numeric
1742 raise TypeError(f"Could not convert string '{x}' to numeric")
1743 try:
1744 x = float(x)
1745 except (TypeError, ValueError):
1746 # e.g. "1+1j" or "foo"
1747 try:
1748 x = complex(x)
1749 except ValueError as err:
1750 # e.g. "foo"
1751 raise TypeError(f"Could not convert {x} to numeric") from err
1752 return x
1755def na_accum_func(values: ArrayLike, accum_func, *, skipna: bool) -> ArrayLike:
1756 """
1757 Cumulative function with skipna support.
1759 Parameters
1760 ----------
1761 values : np.ndarray or ExtensionArray
1762 accum_func : {np.cumprod, np.maximum.accumulate, np.cumsum, np.minimum.accumulate}
1763 skipna : bool
1765 Returns
1766 -------
1767 np.ndarray or ExtensionArray
1768 """
1769 mask_a, mask_b = {
1770 np.cumprod: (1.0, np.nan),
1771 np.maximum.accumulate: (-np.inf, np.nan),
1772 np.cumsum: (0.0, np.nan),
1773 np.minimum.accumulate: (np.inf, np.nan),
1774 }[accum_func]
1776 # This should go through ea interface
1777 assert values.dtype.kind not in "mM"
1779 # We will be applying this function to block values
1780 if skipna and not issubclass(values.dtype.type, (np.integer, np.bool_)):
1781 vals = values.copy()
1782 mask = isna(vals)
1783 vals[mask] = mask_a
1784 result = accum_func(vals, axis=0)
1785 result[mask] = mask_b
1786 else:
1787 result = accum_func(values, axis=0)
1789 return result