Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2missing types & inference
3"""
5from __future__ import annotations
7from decimal import Decimal
8from typing import (
9 TYPE_CHECKING,
10 TypeVar,
11 cast,
12 overload,
13)
14import warnings
16import numpy as np
18from pandas._libs import lib
19import pandas._libs.missing as libmissing
20from pandas._libs.tslibs import (
21 NaT,
22 iNaT,
23)
24from pandas.util._decorators import set_module
26from pandas.core.dtypes.common import (
27 ensure_object,
28 is_scalar,
29 is_string_or_object_np_dtype,
30)
31from pandas.core.dtypes.dtypes import (
32 CategoricalDtype,
33 DatetimeTZDtype,
34 ExtensionDtype,
35 IntervalDtype,
36 PeriodDtype,
37)
38from pandas.core.dtypes.generic import (
39 ABCDataFrame,
40 ABCExtensionArray,
41 ABCIndex,
42 ABCMultiIndex,
43 ABCSeries,
44)
46if TYPE_CHECKING:
47 from re import Pattern
49 from pandas._libs.missing import NAType
50 from pandas._libs.tslibs import NaTType
51 from pandas._typing import (
52 ArrayLike,
53 DtypeObj,
54 NDFrame,
55 NDFrameT,
56 Scalar,
57 npt,
58 )
60 from pandas import Series
61 from pandas.core.indexes.base import Index
64isposinf_scalar = libmissing.isposinf_scalar
65isneginf_scalar = libmissing.isneginf_scalar
67_ArrLikeT = TypeVar("_ArrLikeT", "Series", "Index", np.ndarray)
69_dtype_object = np.dtype("object")
70_dtype_str = np.dtype(str)
73@overload
74def isna(obj: Scalar | Pattern | NAType | NaTType) -> bool: ...
77@overload
78def isna(
79 obj: ArrayLike | Index | list,
80) -> npt.NDArray[np.bool_]: ...
83@overload
84def isna(obj: NDFrameT) -> NDFrameT: ...
87# handle unions
88@overload
89def isna(
90 obj: NDFrameT | ArrayLike | Index | list,
91) -> NDFrameT | npt.NDArray[np.bool_]: ...
94@overload
95def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: ...
98@set_module("pandas")
99def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
100 """
101 Detect missing values for an array-like object.
103 This function takes a scalar or array-like object and indicates
104 whether values are missing (``NaN`` in numeric arrays, ``None`` or ``NaN``
105 in object arrays, ``NaT`` in datetimelike).
107 Parameters
108 ----------
109 obj : scalar or array-like
110 Object to check for null or missing values.
112 Returns
113 -------
114 bool or array-like of bool
115 For scalar input, returns a scalar boolean.
116 For array input, returns an array of boolean indicating whether each
117 corresponding element is missing.
119 See Also
120 --------
121 notna : Boolean inverse of pandas.isna.
122 Series.isna : Detect missing values in a Series.
123 DataFrame.isna : Detect missing values in a DataFrame.
124 Index.isna : Detect missing values in an Index.
126 Examples
127 --------
128 Scalar arguments (including strings) result in a scalar boolean.
130 >>> pd.isna("dog")
131 False
133 >>> pd.isna(pd.NA)
134 True
136 >>> pd.isna(np.nan)
137 True
139 ndarrays result in an ndarray of booleans.
141 >>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]])
142 >>> array
143 array([[ 1., nan, 3.],
144 [ 4., 5., nan]])
145 >>> pd.isna(array)
146 array([[False, True, False],
147 [False, False, True]])
149 For indexes, an ndarray of booleans is returned.
151 >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, "2017-07-08"])
152 >>> index
153 DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
154 dtype='datetime64[us]', freq=None)
155 >>> pd.isna(index)
156 array([False, False, True, False])
158 For Series and DataFrame, the same type is returned, containing booleans.
160 >>> df = pd.DataFrame([["ant", "bee", "cat"], ["dog", None, "fly"]])
161 >>> df
162 0 1 2
163 0 ant bee cat
164 1 dog NaN fly
165 >>> pd.isna(df)
166 0 1 2
167 0 False False False
168 1 False True False
170 >>> pd.isna(df[1])
171 0 False
172 1 True
173 Name: 1, dtype: bool
174 """
175 return _isna(obj)
178isnull = isna
181def _isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
182 """
183 Detect missing values, treating None, NaN or NA as null.
185 Parameters
186 ----------
187 obj: ndarray or object value
188 Input array or scalar value.
190 Returns
191 -------
192 boolean ndarray or boolean
193 """
194 if is_scalar(obj):
195 return libmissing.checknull(obj)
196 elif isinstance(obj, ABCMultiIndex):
197 raise NotImplementedError("isna is not defined for MultiIndex")
198 elif isinstance(obj, type):
199 return False
200 elif isinstance(obj, (np.ndarray, ABCExtensionArray)):
201 return _isna_array(obj)
202 elif isinstance(obj, ABCIndex):
203 # Try to use cached isna, which also short-circuits for integer dtypes
204 # and avoids materializing RangeIndex._values
205 if not obj._can_hold_na:
206 return obj.isna()
207 return _isna_array(obj._values)
209 elif isinstance(obj, ABCSeries):
210 result = _isna_array(obj._values)
211 # box
212 result = obj._constructor(result, index=obj.index, name=obj.name, copy=False)
213 return result
214 elif isinstance(obj, ABCDataFrame):
215 return obj.isna()
216 elif isinstance(obj, list):
217 return _isna_array(np.asarray(obj, dtype=object))
218 elif hasattr(obj, "__array__"):
219 return _isna_array(np.asarray(obj))
220 else:
221 return False
224def _isna_array(values: ArrayLike) -> npt.NDArray[np.bool_] | NDFrame:
225 """
226 Return an array indicating which values of the input array are NaN / NA.
228 Parameters
229 ----------
230 values : ndarray or ExtensionArray
231 The input array whose elements are to be checked.
233 Returns
234 -------
235 array-like
236 Array of boolean values denoting the NA status of each element.
237 """
238 dtype = values.dtype
239 result: npt.NDArray[np.bool_] | NDFrame
241 if not isinstance(values, np.ndarray):
242 # i.e. ExtensionArray
243 # error: Incompatible types in assignment (expression has type
244 # "Union[ndarray[Any, Any], ExtensionArrayNaResult]", variable has
245 # type "ndarray[Any, dtype[bool_]]")
246 result = values.isna() # type: ignore[assignment]
247 elif dtype.names is not None:
248 # GH#48526 (np.rec.recarray), GH#55011 (structured np.ndarray)
249 result = _isna_recarray_dtype(values)
250 elif is_string_or_object_np_dtype(values.dtype):
251 result = _isna_string_dtype(values)
252 elif dtype.kind in "mM":
253 # this is the NaT pattern
254 result = values.view("i8") == iNaT
255 else:
256 result = np.isnan(values)
258 return result
261def _isna_string_dtype(values: np.ndarray) -> npt.NDArray[np.bool_]:
262 # Working around NumPy ticket 1542
263 dtype = values.dtype
265 if dtype.kind in ("S", "U"):
266 result = np.zeros(values.shape, dtype=bool)
267 elif values.ndim in {1, 2}:
268 result = libmissing.isnaobj(values)
269 else:
270 # 0-D, reached via e.g. mask_missing
271 result = libmissing.isnaobj(values.ravel())
272 result = result.reshape(values.shape)
274 return result
277def _isna_recarray_dtype(values: np.ndarray) -> npt.NDArray[np.bool_]:
278 result = np.zeros(values.shape, dtype=bool)
279 for i, record in enumerate(values):
280 record_as_array = np.array(record.tolist())
281 does_record_contain_nan = isna_all(record_as_array)
282 result[i] = np.any(does_record_contain_nan)
284 return result
287@overload
288def notna(obj: Scalar | Pattern | NAType | NaTType) -> bool: ...
291@overload
292def notna(
293 obj: ArrayLike | Index | list,
294) -> npt.NDArray[np.bool_]: ...
297@overload
298def notna(obj: NDFrameT) -> NDFrameT: ...
301# handle unions
302@overload
303def notna(
304 obj: NDFrameT | ArrayLike | Index | list,
305) -> NDFrameT | npt.NDArray[np.bool_]: ...
308@overload
309def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: ...
312@set_module("pandas")
313def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
314 """
315 Detect non-missing values for an array-like object.
317 This function takes a scalar or array-like object and indicates
318 whether values are valid (not missing, which is ``NaN`` in numeric
319 arrays, ``None`` or ``NaN`` in object arrays, ``NaT`` in datetimelike).
321 Parameters
322 ----------
323 obj : array-like or object value
324 Object to check for *not* null or *non*-missing values.
326 Returns
327 -------
328 bool or array-like of bool
329 For scalar input, returns a scalar boolean.
330 For array input, returns an array of boolean indicating whether each
331 corresponding element is valid.
333 See Also
334 --------
335 isna : Boolean inverse of pandas.notna.
336 Series.notna : Detect valid values in a Series.
337 DataFrame.notna : Detect valid values in a DataFrame.
338 Index.notna : Detect valid values in an Index.
340 Examples
341 --------
342 Scalar arguments (including strings) result in a scalar boolean.
344 >>> pd.notna("dog")
345 True
347 >>> pd.notna(pd.NA)
348 False
350 >>> pd.notna(np.nan)
351 False
353 ndarrays result in an ndarray of booleans.
355 >>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]])
356 >>> array
357 array([[ 1., nan, 3.],
358 [ 4., 5., nan]])
359 >>> pd.notna(array)
360 array([[ True, False, True],
361 [ True, True, False]])
363 For indexes, an ndarray of booleans is returned.
365 >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, "2017-07-08"])
366 >>> index
367 DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
368 dtype='datetime64[us]', freq=None)
369 >>> pd.notna(index)
370 array([ True, True, False, True])
372 For Series and DataFrame, the same type is returned, containing booleans.
374 >>> df = pd.DataFrame([["ant", "bee", "cat"], ["dog", None, "fly"]])
375 >>> df
376 0 1 2
377 0 ant bee cat
378 1 dog NaN fly
379 >>> pd.notna(df)
380 0 1 2
381 0 True True True
382 1 True False True
384 >>> pd.notna(df[1])
385 0 True
386 1 False
387 Name: 1, dtype: bool
388 """
389 res = isna(obj)
390 if isinstance(res, bool):
391 return not res
392 return ~res
395notnull = notna
398def array_equivalent(
399 left: ArrayLike,
400 right: ArrayLike,
401 strict_nan: bool = False,
402 dtype_equal: bool = False,
403) -> bool:
404 """
405 True if two arrays, left and right, have equal non-NaN elements, and NaNs
406 in corresponding locations. False otherwise. It is assumed that left and
407 right are NumPy arrays of the same dtype. The behavior of this function
408 (particularly with respect to NaNs) is not defined if the dtypes are
409 different.
411 Parameters
412 ----------
413 left, right : ndarrays
414 strict_nan : bool, default False
415 If True, consider NaN and None to be different.
416 dtype_equal : bool, default False
417 Whether `left` and `right` are known to have the same dtype
418 according to `is_dtype_equal`. Some methods like `BlockManager.equals`.
419 require that the dtypes match. Setting this to ``True`` can improve
420 performance, but will give different results for arrays that are
421 equal but different dtypes.
423 Returns
424 -------
425 b : bool
426 Returns True if the arrays are equivalent.
428 Examples
429 --------
430 >>> array_equivalent(np.array([1, 2, np.nan]), np.array([1, 2, np.nan]))
431 np.True_
432 >>> array_equivalent(np.array([1, np.nan, 2]), np.array([1, 2, np.nan]))
433 np.False_
434 """
435 left, right = np.asarray(left), np.asarray(right)
437 # shape compat
438 if left.shape != right.shape:
439 return False
441 if dtype_equal:
442 # fastpath when we require that the dtypes match (Block.equals)
443 if left.dtype.kind in "fc":
444 return _array_equivalent_float(left, right)
445 elif left.dtype.kind in "mM":
446 return _array_equivalent_datetimelike(left, right)
447 elif is_string_or_object_np_dtype(left.dtype):
448 # TODO: fastpath for pandas' StringDtype
449 return _array_equivalent_object(left, right, strict_nan)
450 else:
451 return lib.array_equivalent_bytes(left, right)
453 # Slow path when we allow comparing different dtypes.
454 # Object arrays can contain None, NaN and NaT.
455 # string dtypes must come to this path for NumPy 1.7.1 compat
456 if left.dtype.kind in "OSU" or right.dtype.kind in "OSU":
457 # Note: `in "OSU"` is non-trivially faster than `in ["O", "S", "U"]`
458 # or `in ("O", "S", "U")`
459 return _array_equivalent_object(left, right, strict_nan)
461 # NaNs can occur in float and complex arrays.
462 if left.dtype.kind in "fc":
463 if not (left.size and right.size):
464 return True
465 return ((left == right) | (isna(left) & isna(right))).all()
467 elif left.dtype.kind in "mM" or right.dtype.kind in "mM":
468 # datetime64, timedelta64, Period
469 if left.dtype != right.dtype:
470 return False
472 left = left.view("i8")
473 right = right.view("i8")
475 # if we have structured dtypes, compare first
476 if (
477 left.dtype.type is np.void or right.dtype.type is np.void
478 ) and left.dtype != right.dtype:
479 return False
481 if left.dtype == right.dtype and left.dtype.kind != "V":
482 return lib.array_equivalent_bytes(left, right)
483 return np.array_equal(left, right)
486def _array_equivalent_float(left: np.ndarray, right: np.ndarray) -> bool:
487 # lib.array_equivalent_float only accepts native float32/float64. Complex is
488 # compared with numpy: viewing it to float pairs would test the real and
489 # imaginary parts independently, breaking whole-element NA semantics -- a
490 # complex value is NA if either part is NaN (GH#66160). float16, longdouble,
491 # and byte-swapped floats also fall back to numpy (GH#65192).
492 if left.dtype.kind == "c" or not (
493 left.dtype.isnative and left.dtype.itemsize in (4, 8)
494 ):
495 return bool(((left == right) | (np.isnan(left) & np.isnan(right))).all())
496 if left.ndim > 1:
497 if left.flags.f_contiguous and right.flags.f_contiguous:
498 # .T is a C-contiguous view of an F-contiguous array
499 left = left.T
500 right = right.T
501 if not (left.flags.c_contiguous and right.flags.c_contiguous):
502 return bool(((left == right) | (np.isnan(left) & np.isnan(right))).all())
503 left = left.ravel()
504 right = right.ravel()
505 return lib.array_equivalent_float(left, right)
508def _array_equivalent_datetimelike(left: np.ndarray, right: np.ndarray) -> bool:
509 return lib.array_equivalent_bytes(left.view("i8"), right.view("i8"))
512def _array_equivalent_object(
513 left: np.ndarray, right: np.ndarray, strict_nan: bool
514) -> bool:
515 left = ensure_object(left)
516 right = ensure_object(right)
518 mask: npt.NDArray[np.bool_] | None = None
519 if strict_nan:
520 mask = isna(left) & isna(right)
521 if not mask.any():
522 mask = None
524 try:
525 if mask is None:
526 return lib.array_equivalent_object(left, right)
527 if not lib.array_equivalent_object(left[~mask], right[~mask]):
528 return False
529 left_remaining = left[mask]
530 right_remaining = right[mask]
531 except ValueError:
532 # can raise a ValueError if left and right cannot be
533 # compared (e.g. nested arrays)
534 left_remaining = left
535 right_remaining = right
537 for left_value, right_value in zip(left_remaining, right_remaining, strict=True):
538 if left_value is NaT and right_value is not NaT:
539 return False
541 elif left_value is libmissing.NA and right_value is not libmissing.NA:
542 return False
544 elif isinstance(left_value, float) and np.isnan(left_value):
545 if not isinstance(right_value, float) or not np.isnan(right_value):
546 return False
547 else:
548 with warnings.catch_warnings():
549 # suppress numpy's "elementwise comparison failed"
550 warnings.simplefilter("ignore", DeprecationWarning)
551 try:
552 if np.any(np.asarray(left_value != right_value)):
553 return False
554 except TypeError as err:
555 if "boolean value of NA is ambiguous" in str(err):
556 return False
557 raise
558 except ValueError:
559 # numpy can raise a ValueError if left and right cannot be
560 # compared (e.g. nested arrays)
561 return False
562 return True
565def array_equals(left: ArrayLike, right: ArrayLike) -> bool:
566 """
567 ExtensionArray-compatible implementation of array_equivalent.
568 """
569 if left.dtype != right.dtype:
570 return False
571 elif isinstance(left, ABCExtensionArray):
572 return left.equals(right)
573 else:
574 return array_equivalent(left, right, dtype_equal=True)
577def construct_1d_array_from_inferred_fill_value(
578 value: object, length: int
579) -> ArrayLike:
580 # Find our empty_value dtype by constructing an array
581 # from our value and doing a .take on it
582 from pandas.core.algorithms import take_nd
583 from pandas.core.construction import sanitize_array
584 from pandas.core.indexes.base import Index
586 arr = sanitize_array(value, Index(range(1)), copy=False)
587 taker = -1 * np.ones(length, dtype=np.intp)
588 return take_nd(arr, taker)
591def maybe_fill(arr: np.ndarray) -> np.ndarray:
592 """
593 Fill numpy.ndarray with NaN, unless we have an integer or boolean dtype.
594 """
595 if arr.dtype.kind not in "iub":
596 arr.fill(np.nan)
597 return arr
600def na_value_for_dtype(dtype: DtypeObj, compat: bool = True) -> Scalar:
601 """
602 Return a dtype compat na value
604 Parameters
605 ----------
606 dtype : string / dtype
607 compat : bool, default True
609 Returns
610 -------
611 np.dtype or a pandas dtype
613 Examples
614 --------
615 >>> na_value_for_dtype(np.dtype("int64"))
616 0
617 >>> na_value_for_dtype(np.dtype("int64"), compat=False)
618 nan
619 >>> na_value_for_dtype(np.dtype("float64"))
620 nan
621 >>> na_value_for_dtype(np.dtype("complex128"))
622 nan
623 >>> na_value_for_dtype(np.dtype("bool"))
624 False
625 >>> na_value_for_dtype(np.dtype("datetime64[ns]"))
626 np.datetime64('NaT','ns')
627 """
629 if isinstance(dtype, ExtensionDtype):
630 return cast("Scalar", dtype.na_value)
631 elif dtype.kind in "mM":
632 unit = np.datetime_data(dtype)[0]
633 return dtype.type("NaT", unit)
634 elif dtype.kind in "fc":
635 return np.nan
636 elif dtype.kind in "iu":
637 if compat:
638 return 0
639 return np.nan
640 elif dtype.kind == "b":
641 if compat:
642 return False
643 return np.nan
644 return np.nan
647def remove_na_arraylike(arr: _ArrLikeT) -> _ArrLikeT:
648 """
649 Return array-like containing only true/non-NaN values, possibly empty.
650 """
651 if isinstance(arr.dtype, ExtensionDtype):
652 return arr[notna(arr)] # pyright: ignore[reportReturnType]
653 else:
654 return arr[notna(np.asarray(arr))] # pyright: ignore[reportReturnType]
657def is_valid_na_for_dtype(obj: object, dtype: DtypeObj) -> bool:
658 """
659 isna check that excludes incompatible dtypes
661 Parameters
662 ----------
663 obj : object
664 dtype : np.datetime64, np.timedelta64, DatetimeTZDtype, or PeriodDtype
666 Returns
667 -------
668 bool
669 """
670 if not lib.is_scalar(obj) or not isna(obj): # pyright: ignore[reportGeneralTypeIssues]
671 return False
672 elif dtype.kind == "M":
673 if isinstance(dtype, np.dtype):
674 # i.e. not tzaware
675 return not isinstance(obj, (np.timedelta64, Decimal))
676 # we have to rule out tznaive dt64("NaT")
677 return not isinstance(obj, (np.timedelta64, np.datetime64, Decimal))
678 elif dtype.kind == "m":
679 return not isinstance(obj, (np.datetime64, Decimal))
680 elif dtype.kind in "iufc":
681 # Numeric
682 return obj is not NaT and not isinstance(obj, (np.datetime64, np.timedelta64))
683 elif dtype.kind == "b":
684 # We allow pd.NA, None, np.nan in BooleanArray (same as IntervalDtype)
685 return lib.is_float(obj) or obj is None or obj is libmissing.NA
687 elif dtype == _dtype_str:
688 # numpy string dtypes to avoid float np.nan
689 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal, float))
691 elif dtype == _dtype_object:
692 # This is needed for Categorical, but is kind of weird
693 return True
695 elif isinstance(dtype, PeriodDtype):
696 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))
698 elif isinstance(dtype, IntervalDtype):
699 return lib.is_float(obj) or obj is None or obj is libmissing.NA
701 elif isinstance(dtype, CategoricalDtype):
702 return is_valid_na_for_dtype(obj, dtype.categories.dtype)
704 # fallback, default to allowing NaN, None, NA, NaT
705 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))
708def isna_all(arr: ArrayLike) -> bool:
709 """
710 Optimized equivalent to isna(arr).all()
711 """
712 total_len = len(arr)
714 # Usually it's enough to check but a small fraction of values to see if
715 # a block is NOT null, chunks should help in such cases.
716 # parameters 1000 and 40 were chosen arbitrarily
717 chunk_len = max(total_len // 40, 1000)
719 dtype = arr.dtype
720 if lib.is_np_dtype(dtype, "f"):
721 checker = np.isnan
723 elif (lib.is_np_dtype(dtype, "mM")) or isinstance(
724 dtype, (DatetimeTZDtype, PeriodDtype)
725 ):
726 # error: Incompatible types in assignment (expression has type
727 # "Callable[[Any], Any]", variable has type "ufunc")
728 checker = lambda x: np.asarray(x.view("i8")) == iNaT # type: ignore[assignment]
730 else:
731 # error: Incompatible types in assignment (expression has type "Callable[[Any],
732 # Any]", variable has type "ufunc")
733 checker = _isna_array # type: ignore[assignment]
735 return all(
736 checker(arr[i : i + chunk_len]).all() for i in range(0, total_len, chunk_len)
737 )