Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2missing types & inference
3"""
5from __future__ import annotations
7from decimal import Decimal
8from typing import (
9 TYPE_CHECKING,
10 overload,
11)
12import warnings
14import numpy as np
16from pandas._libs import lib
17import pandas._libs.missing as libmissing
18from pandas._libs.tslibs import (
19 NaT,
20 iNaT,
21)
22from pandas.util._decorators import set_module
24from pandas.core.dtypes.common import (
25 DT64NS_DTYPE,
26 TD64NS_DTYPE,
27 ensure_object,
28 is_scalar,
29 is_string_or_object_np_dtype,
30)
31from pandas.core.dtypes.dtypes import (
32 CategoricalDtype,
33 DatetimeTZDtype,
34 ExtensionDtype,
35 IntervalDtype,
36 PeriodDtype,
37)
38from pandas.core.dtypes.generic import (
39 ABCDataFrame,
40 ABCExtensionArray,
41 ABCIndex,
42 ABCMultiIndex,
43 ABCSeries,
44)
45from pandas.core.dtypes.inference import is_list_like
47if TYPE_CHECKING:
48 from re import Pattern
50 from pandas._libs.missing import NAType
51 from pandas._libs.tslibs import NaTType
52 from pandas._typing import (
53 ArrayLike,
54 DtypeObj,
55 NDFrame,
56 NDFrameT,
57 Scalar,
58 npt,
59 )
61 from pandas import Series
62 from pandas.core.indexes.base import Index
65isposinf_scalar = libmissing.isposinf_scalar
66isneginf_scalar = libmissing.isneginf_scalar
68_dtype_object = np.dtype("object")
69_dtype_str = np.dtype(str)
72@overload
73def isna(obj: Scalar | Pattern | NAType | NaTType) -> bool: ...
76@overload
77def isna(
78 obj: ArrayLike | Index | list,
79) -> npt.NDArray[np.bool_]: ...
82@overload
83def isna(obj: NDFrameT) -> NDFrameT: ...
86# handle unions
87@overload
88def isna(
89 obj: NDFrameT | ArrayLike | Index | list,
90) -> NDFrameT | npt.NDArray[np.bool_]: ...
93@overload
94def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: ...
97@set_module("pandas")
98def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
99 """
100 Detect missing values for an array-like object.
102 This function takes a scalar or array-like object and indicates
103 whether values are missing (``NaN`` in numeric arrays, ``None`` or ``NaN``
104 in object arrays, ``NaT`` in datetimelike).
106 Parameters
107 ----------
108 obj : scalar or array-like
109 Object to check for null or missing values.
111 Returns
112 -------
113 bool or array-like of bool
114 For scalar input, returns a scalar boolean.
115 For array input, returns an array of boolean indicating whether each
116 corresponding element is missing.
118 See Also
119 --------
120 notna : Boolean inverse of pandas.isna.
121 Series.isna : Detect missing values in a Series.
122 DataFrame.isna : Detect missing values in a DataFrame.
123 Index.isna : Detect missing values in an Index.
125 Examples
126 --------
127 Scalar arguments (including strings) result in a scalar boolean.
129 >>> pd.isna("dog")
130 False
132 >>> pd.isna(pd.NA)
133 True
135 >>> pd.isna(np.nan)
136 True
138 ndarrays result in an ndarray of booleans.
140 >>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]])
141 >>> array
142 array([[ 1., nan, 3.],
143 [ 4., 5., nan]])
144 >>> pd.isna(array)
145 array([[False, True, False],
146 [False, False, True]])
148 For indexes, an ndarray of booleans is returned.
150 >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, "2017-07-08"])
151 >>> index
152 DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
153 dtype='datetime64[us]', freq=None)
154 >>> pd.isna(index)
155 array([False, False, True, False])
157 For Series and DataFrame, the same type is returned, containing booleans.
159 >>> df = pd.DataFrame([["ant", "bee", "cat"], ["dog", None, "fly"]])
160 >>> df
161 0 1 2
162 0 ant bee cat
163 1 dog NaN fly
164 >>> pd.isna(df)
165 0 1 2
166 0 False False False
167 1 False True False
169 >>> pd.isna(df[1])
170 0 False
171 1 True
172 Name: 1, dtype: bool
173 """
174 return _isna(obj)
177isnull = isna
180def _isna(obj):
181 """
182 Detect missing values, treating None, NaN or NA as null.
184 Parameters
185 ----------
186 obj: ndarray or object value
187 Input array or scalar value.
189 Returns
190 -------
191 boolean ndarray or boolean
192 """
193 if is_scalar(obj):
194 return libmissing.checknull(obj)
195 elif isinstance(obj, ABCMultiIndex):
196 raise NotImplementedError("isna is not defined for MultiIndex")
197 elif isinstance(obj, type):
198 return False
199 elif isinstance(obj, (np.ndarray, ABCExtensionArray)):
200 return _isna_array(obj)
201 elif isinstance(obj, ABCIndex):
202 # Try to use cached isna, which also short-circuits for integer dtypes
203 # and avoids materializing RangeIndex._values
204 if not obj._can_hold_na:
205 return obj.isna()
206 return _isna_array(obj._values)
208 elif isinstance(obj, ABCSeries):
209 result = _isna_array(obj._values)
210 # box
211 result = obj._constructor(result, index=obj.index, name=obj.name, copy=False)
212 return result
213 elif isinstance(obj, ABCDataFrame):
214 return obj.isna()
215 elif isinstance(obj, list):
216 return _isna_array(np.asarray(obj, dtype=object))
217 elif hasattr(obj, "__array__"):
218 return _isna_array(np.asarray(obj))
219 else:
220 return False
223def _isna_array(values: ArrayLike) -> npt.NDArray[np.bool_] | NDFrame:
224 """
225 Return an array indicating which values of the input array are NaN / NA.
227 Parameters
228 ----------
229 obj: ndarray or ExtensionArray
230 The input array whose elements are to be checked.
232 Returns
233 -------
234 array-like
235 Array of boolean values denoting the NA status of each element.
236 """
237 dtype = values.dtype
238 result: npt.NDArray[np.bool_] | NDFrame
240 if not isinstance(values, np.ndarray):
241 # i.e. ExtensionArray
242 # error: Incompatible types in assignment (expression has type
243 # "Union[ndarray[Any, Any], ExtensionArraySupportsAnyAll]", variable has
244 # type "ndarray[Any, dtype[bool_]]")
245 result = values.isna() # type: ignore[assignment]
246 elif isinstance(values, np.rec.recarray):
247 # GH 48526
248 result = _isna_recarray_dtype(values)
249 elif is_string_or_object_np_dtype(values.dtype):
250 result = _isna_string_dtype(values)
251 elif dtype.kind in "mM":
252 # this is the NaT pattern
253 result = values.view("i8") == iNaT
254 else:
255 result = np.isnan(values)
257 return result
260def _isna_string_dtype(values: np.ndarray) -> npt.NDArray[np.bool_]:
261 # Working around NumPy ticket 1542
262 dtype = values.dtype
264 if dtype.kind in ("S", "U"):
265 result = np.zeros(values.shape, dtype=bool)
266 elif values.ndim in {1, 2}:
267 result = libmissing.isnaobj(values)
268 else:
269 # 0-D, reached via e.g. mask_missing
270 result = libmissing.isnaobj(values.ravel())
271 result = result.reshape(values.shape)
273 return result
276def _isna_recarray_dtype(values: np.rec.recarray) -> npt.NDArray[np.bool_]:
277 result = np.zeros(values.shape, dtype=bool)
278 for i, record in enumerate(values):
279 record_as_array = np.array(record.tolist())
280 does_record_contain_nan = isna_all(record_as_array)
281 result[i] = np.any(does_record_contain_nan)
283 return result
286@overload
287def notna(obj: Scalar | Pattern | NAType | NaTType) -> bool: ...
290@overload
291def notna(
292 obj: ArrayLike | Index | list,
293) -> npt.NDArray[np.bool_]: ...
296@overload
297def notna(obj: NDFrameT) -> NDFrameT: ...
300# handle unions
301@overload
302def notna(
303 obj: NDFrameT | ArrayLike | Index | list,
304) -> NDFrameT | npt.NDArray[np.bool_]: ...
307@overload
308def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: ...
311@set_module("pandas")
312def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
313 """
314 Detect non-missing values for an array-like object.
316 This function takes a scalar or array-like object and indicates
317 whether values are valid (not missing, which is ``NaN`` in numeric
318 arrays, ``None`` or ``NaN`` in object arrays, ``NaT`` in datetimelike).
320 Parameters
321 ----------
322 obj : array-like or object value
323 Object to check for *not* null or *non*-missing values.
325 Returns
326 -------
327 bool or array-like of bool
328 For scalar input, returns a scalar boolean.
329 For array input, returns an array of boolean indicating whether each
330 corresponding element is valid.
332 See Also
333 --------
334 isna : Boolean inverse of pandas.notna.
335 Series.notna : Detect valid values in a Series.
336 DataFrame.notna : Detect valid values in a DataFrame.
337 Index.notna : Detect valid values in an Index.
339 Examples
340 --------
341 Scalar arguments (including strings) result in a scalar boolean.
343 >>> pd.notna("dog")
344 True
346 >>> pd.notna(pd.NA)
347 False
349 >>> pd.notna(np.nan)
350 False
352 ndarrays result in an ndarray of booleans.
354 >>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]])
355 >>> array
356 array([[ 1., nan, 3.],
357 [ 4., 5., nan]])
358 >>> pd.notna(array)
359 array([[ True, False, True],
360 [ True, True, False]])
362 For indexes, an ndarray of booleans is returned.
364 >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, "2017-07-08"])
365 >>> index
366 DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
367 dtype='datetime64[us]', freq=None)
368 >>> pd.notna(index)
369 array([ True, True, False, True])
371 For Series and DataFrame, the same type is returned, containing booleans.
373 >>> df = pd.DataFrame([["ant", "bee", "cat"], ["dog", None, "fly"]])
374 >>> df
375 0 1 2
376 0 ant bee cat
377 1 dog NaN fly
378 >>> pd.notna(df)
379 0 1 2
380 0 True True True
381 1 True False True
383 >>> pd.notna(df[1])
384 0 True
385 1 False
386 Name: 1, dtype: bool
387 """
388 res = isna(obj)
389 if isinstance(res, bool):
390 return not res
391 return ~res
394notnull = notna
397def array_equivalent(
398 left,
399 right,
400 strict_nan: bool = False,
401 dtype_equal: bool = False,
402) -> bool:
403 """
404 True if two arrays, left and right, have equal non-NaN elements, and NaNs
405 in corresponding locations. False otherwise. It is assumed that left and
406 right are NumPy arrays of the same dtype. The behavior of this function
407 (particularly with respect to NaNs) is not defined if the dtypes are
408 different.
410 Parameters
411 ----------
412 left, right : ndarrays
413 strict_nan : bool, default False
414 If True, consider NaN and None to be different.
415 dtype_equal : bool, default False
416 Whether `left` and `right` are known to have the same dtype
417 according to `is_dtype_equal`. Some methods like `BlockManager.equals`.
418 require that the dtypes match. Setting this to ``True`` can improve
419 performance, but will give different results for arrays that are
420 equal but different dtypes.
422 Returns
423 -------
424 b : bool
425 Returns True if the arrays are equivalent.
427 Examples
428 --------
429 >>> array_equivalent(np.array([1, 2, np.nan]), np.array([1, 2, np.nan]))
430 np.True_
431 >>> array_equivalent(np.array([1, np.nan, 2]), np.array([1, 2, np.nan]))
432 np.False_
433 """
434 left, right = np.asarray(left), np.asarray(right)
436 # shape compat
437 if left.shape != right.shape:
438 return False
440 if dtype_equal:
441 # fastpath when we require that the dtypes match (Block.equals)
442 if left.dtype.kind in "fc":
443 return _array_equivalent_float(left, right)
444 elif left.dtype.kind in "mM":
445 return _array_equivalent_datetimelike(left, right)
446 elif is_string_or_object_np_dtype(left.dtype):
447 # TODO: fastpath for pandas' StringDtype
448 return _array_equivalent_object(left, right, strict_nan)
449 else:
450 return np.array_equal(left, right)
452 # Slow path when we allow comparing different dtypes.
453 # Object arrays can contain None, NaN and NaT.
454 # string dtypes must be come to this path for NumPy 1.7.1 compat
455 if left.dtype.kind in "OSU" or right.dtype.kind in "OSU":
456 # Note: `in "OSU"` is non-trivially faster than `in ["O", "S", "U"]`
457 # or `in ("O", "S", "U")`
458 return _array_equivalent_object(left, right, strict_nan)
460 # NaNs can occur in float and complex arrays.
461 if left.dtype.kind in "fc":
462 if not (left.size and right.size):
463 return True
464 return ((left == right) | (isna(left) & isna(right))).all()
466 elif left.dtype.kind in "mM" or right.dtype.kind in "mM":
467 # datetime64, timedelta64, Period
468 if left.dtype != right.dtype:
469 return False
471 left = left.view("i8")
472 right = right.view("i8")
474 # if we have structured dtypes, compare first
475 if (
476 left.dtype.type is np.void or right.dtype.type is np.void
477 ) and left.dtype != right.dtype:
478 return False
480 return np.array_equal(left, right)
483def _array_equivalent_float(left: np.ndarray, right: np.ndarray) -> bool:
484 return bool(((left == right) | (np.isnan(left) & np.isnan(right))).all())
487def _array_equivalent_datetimelike(left: np.ndarray, right: np.ndarray) -> bool:
488 return np.array_equal(left.view("i8"), right.view("i8"))
491def _array_equivalent_object(
492 left: np.ndarray, right: np.ndarray, strict_nan: bool
493) -> bool:
494 left = ensure_object(left)
495 right = ensure_object(right)
497 mask: npt.NDArray[np.bool_] | None = None
498 if strict_nan:
499 mask = isna(left) & isna(right)
500 if not mask.any():
501 mask = None
503 try:
504 if mask is None:
505 return lib.array_equivalent_object(left, right)
506 if not lib.array_equivalent_object(left[~mask], right[~mask]):
507 return False
508 left_remaining = left[mask]
509 right_remaining = right[mask]
510 except ValueError:
511 # can raise a ValueError if left and right cannot be
512 # compared (e.g. nested arrays)
513 left_remaining = left
514 right_remaining = right
516 for left_value, right_value in zip(left_remaining, right_remaining, strict=True):
517 if left_value is NaT and right_value is not NaT:
518 return False
520 elif left_value is libmissing.NA and right_value is not libmissing.NA:
521 return False
523 elif isinstance(left_value, float) and np.isnan(left_value):
524 if not isinstance(right_value, float) or not np.isnan(right_value):
525 return False
526 else:
527 with warnings.catch_warnings():
528 # suppress numpy's "elementwise comparison failed"
529 warnings.simplefilter("ignore", DeprecationWarning)
530 try:
531 if np.any(np.asarray(left_value != right_value)):
532 return False
533 except TypeError as err:
534 if "boolean value of NA is ambiguous" in str(err):
535 return False
536 raise
537 except ValueError:
538 # numpy can raise a ValueError if left and right cannot be
539 # compared (e.g. nested arrays)
540 return False
541 return True
544def array_equals(left: ArrayLike, right: ArrayLike) -> bool:
545 """
546 ExtensionArray-compatible implementation of array_equivalent.
547 """
548 if left.dtype != right.dtype:
549 return False
550 elif isinstance(left, ABCExtensionArray):
551 return left.equals(right)
552 else:
553 return array_equivalent(left, right, dtype_equal=True)
556def infer_fill_value(val):
557 """
558 infer the fill value for the nan/NaT from the provided
559 scalar/ndarray/list-like if we are a NaT, return the correct dtyped
560 element to provide proper block construction
561 """
562 if not is_list_like(val):
563 val = [val]
564 val = np.asarray(val)
565 if val.dtype.kind in "mM":
566 return np.array("NaT", dtype=val.dtype)
567 elif val.dtype == object:
568 dtype = lib.infer_dtype(ensure_object(val), skipna=False)
569 if dtype in ["datetime", "datetime64"]:
570 return np.array("NaT", dtype=DT64NS_DTYPE)
571 elif dtype in ["timedelta", "timedelta64"]:
572 return np.array("NaT", dtype=TD64NS_DTYPE)
573 return np.array(np.nan, dtype=object)
574 elif val.dtype.kind == "U":
575 return np.array(np.nan, dtype=val.dtype)
576 return np.nan
579def construct_1d_array_from_inferred_fill_value(
580 value: object, length: int
581) -> ArrayLike:
582 # Find our empty_value dtype by constructing an array
583 # from our value and doing a .take on it
584 from pandas.core.algorithms import take_nd
585 from pandas.core.construction import sanitize_array
586 from pandas.core.indexes.base import Index
588 arr = sanitize_array(value, Index(range(1)), copy=False)
589 taker = -1 * np.ones(length, dtype=np.intp)
590 return take_nd(arr, taker)
593def maybe_fill(arr: np.ndarray) -> np.ndarray:
594 """
595 Fill numpy.ndarray with NaN, unless we have an integer or boolean dtype.
596 """
597 if arr.dtype.kind not in "iub":
598 arr.fill(np.nan)
599 return arr
602def na_value_for_dtype(dtype: DtypeObj, compat: bool = True):
603 """
604 Return a dtype compat na value
606 Parameters
607 ----------
608 dtype : string / dtype
609 compat : bool, default True
611 Returns
612 -------
613 np.dtype or a pandas dtype
615 Examples
616 --------
617 >>> na_value_for_dtype(np.dtype("int64"))
618 0
619 >>> na_value_for_dtype(np.dtype("int64"), compat=False)
620 nan
621 >>> na_value_for_dtype(np.dtype("float64"))
622 nan
623 >>> na_value_for_dtype(np.dtype("complex128"))
624 nan
625 >>> na_value_for_dtype(np.dtype("bool"))
626 False
627 >>> na_value_for_dtype(np.dtype("datetime64[ns]"))
628 np.datetime64('NaT','ns')
629 """
631 if isinstance(dtype, ExtensionDtype):
632 return dtype.na_value
633 elif dtype.kind in "mM":
634 unit = np.datetime_data(dtype)[0]
635 return dtype.type("NaT", unit)
636 elif dtype.kind in "fc":
637 return np.nan
638 elif dtype.kind in "iu":
639 if compat:
640 return 0
641 return np.nan
642 elif dtype.kind == "b":
643 if compat:
644 return False
645 return np.nan
646 return np.nan
649def remove_na_arraylike(arr: Series | Index | np.ndarray):
650 """
651 Return array-like containing only true/non-NaN values, possibly empty.
652 """
653 if isinstance(arr.dtype, ExtensionDtype):
654 return arr[notna(arr)]
655 else:
656 return arr[notna(np.asarray(arr))]
659def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool:
660 """
661 isna check that excludes incompatible dtypes
663 Parameters
664 ----------
665 obj : object
666 dtype : np.datetime64, np.timedelta64, DatetimeTZDtype, or PeriodDtype
668 Returns
669 -------
670 bool
671 """
672 if not lib.is_scalar(obj) or not isna(obj):
673 return False
674 elif dtype.kind == "M":
675 if isinstance(dtype, np.dtype):
676 # i.e. not tzaware
677 return not isinstance(obj, (np.timedelta64, Decimal))
678 # we have to rule out tznaive dt64("NaT")
679 return not isinstance(obj, (np.timedelta64, np.datetime64, Decimal))
680 elif dtype.kind == "m":
681 return not isinstance(obj, (np.datetime64, Decimal))
682 elif dtype.kind in "iufc":
683 # Numeric
684 return obj is not NaT and not isinstance(obj, (np.datetime64, np.timedelta64))
685 elif dtype.kind == "b":
686 # We allow pd.NA, None, np.nan in BooleanArray (same as IntervalDtype)
687 return lib.is_float(obj) or obj is None or obj is libmissing.NA
689 elif dtype == _dtype_str:
690 # numpy string dtypes to avoid float np.nan
691 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal, float))
693 elif dtype == _dtype_object:
694 # This is needed for Categorical, but is kind of weird
695 return True
697 elif isinstance(dtype, PeriodDtype):
698 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))
700 elif isinstance(dtype, IntervalDtype):
701 return lib.is_float(obj) or obj is None or obj is libmissing.NA
703 elif isinstance(dtype, CategoricalDtype):
704 return is_valid_na_for_dtype(obj, dtype.categories.dtype)
706 # fallback, default to allowing NaN, None, NA, NaT
707 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))
710def isna_all(arr: ArrayLike) -> bool:
711 """
712 Optimized equivalent to isna(arr).all()
713 """
714 total_len = len(arr)
716 # Usually it's enough to check but a small fraction of values to see if
717 # a block is NOT null, chunks should help in such cases.
718 # parameters 1000 and 40 were chosen arbitrarily
719 chunk_len = max(total_len // 40, 1000)
721 dtype = arr.dtype
722 if lib.is_np_dtype(dtype, "f"):
723 checker = np.isnan
725 elif (lib.is_np_dtype(dtype, "mM")) or isinstance(
726 dtype, (DatetimeTZDtype, PeriodDtype)
727 ):
728 # error: Incompatible types in assignment (expression has type
729 # "Callable[[Any], Any]", variable has type "ufunc")
730 checker = lambda x: np.asarray(x.view("i8")) == iNaT # type: ignore[assignment]
732 else:
733 # error: Incompatible types in assignment (expression has type "Callable[[Any],
734 # Any]", variable has type "ufunc")
735 checker = _isna_array # type: ignore[assignment]
737 return all(
738 checker(arr[i : i + chunk_len]).all() for i in range(0, total_len, chunk_len)
739 )