Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.9/dist-packages/pandas/core/dtypes/missing.py: 46%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

297 statements  

1""" 

2missing types & inference 

3""" 

4from __future__ import annotations 

5 

6from decimal import Decimal 

7from functools import partial 

8from typing import ( 

9 TYPE_CHECKING, 

10 overload, 

11) 

12import warnings 

13 

14import numpy as np 

15 

16from pandas._config import get_option 

17 

18from pandas._libs import lib 

19import pandas._libs.missing as libmissing 

20from pandas._libs.tslibs import ( 

21 NaT, 

22 iNaT, 

23) 

24 

25from pandas.core.dtypes.common import ( 

26 DT64NS_DTYPE, 

27 TD64NS_DTYPE, 

28 ensure_object, 

29 is_scalar, 

30 is_string_or_object_np_dtype, 

31) 

32from pandas.core.dtypes.dtypes import ( 

33 CategoricalDtype, 

34 DatetimeTZDtype, 

35 ExtensionDtype, 

36 IntervalDtype, 

37 PeriodDtype, 

38) 

39from pandas.core.dtypes.generic import ( 

40 ABCDataFrame, 

41 ABCExtensionArray, 

42 ABCIndex, 

43 ABCMultiIndex, 

44 ABCSeries, 

45) 

46from pandas.core.dtypes.inference import is_list_like 

47 

48if TYPE_CHECKING: 

49 from re import Pattern 

50 

51 from pandas._typing import ( 

52 ArrayLike, 

53 DtypeObj, 

54 NDFrame, 

55 NDFrameT, 

56 Scalar, 

57 npt, 

58 ) 

59 

60 from pandas import Series 

61 from pandas.core.indexes.base import Index 

62 

63 

64isposinf_scalar = libmissing.isposinf_scalar 

65isneginf_scalar = libmissing.isneginf_scalar 

66 

67nan_checker = np.isnan 

68INF_AS_NA = False 

69_dtype_object = np.dtype("object") 

70_dtype_str = np.dtype(str) 

71 

72 

73@overload 

74def isna(obj: Scalar | Pattern) -> bool: 

75 ... 

76 

77 

78@overload 

79def isna( 

80 obj: ArrayLike | Index | list, 

81) -> npt.NDArray[np.bool_]: 

82 ... 

83 

84 

85@overload 

86def isna(obj: NDFrameT) -> NDFrameT: 

87 ... 

88 

89 

90# handle unions 

91@overload 

92def isna(obj: NDFrameT | ArrayLike | Index | list) -> NDFrameT | npt.NDArray[np.bool_]: 

93 ... 

94 

95 

96@overload 

97def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: 

98 ... 

99 

100 

101def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: 

102 """ 

103 Detect missing values for an array-like object. 

104 

105 This function takes a scalar or array-like object and indicates 

106 whether values are missing (``NaN`` in numeric arrays, ``None`` or ``NaN`` 

107 in object arrays, ``NaT`` in datetimelike). 

108 

109 Parameters 

110 ---------- 

111 obj : scalar or array-like 

112 Object to check for null or missing values. 

113 

114 Returns 

115 ------- 

116 bool or array-like of bool 

117 For scalar input, returns a scalar boolean. 

118 For array input, returns an array of boolean indicating whether each 

119 corresponding element is missing. 

120 

121 See Also 

122 -------- 

123 notna : Boolean inverse of pandas.isna. 

124 Series.isna : Detect missing values in a Series. 

125 DataFrame.isna : Detect missing values in a DataFrame. 

126 Index.isna : Detect missing values in an Index. 

127 

128 Examples 

129 -------- 

130 Scalar arguments (including strings) result in a scalar boolean. 

131 

132 >>> pd.isna('dog') 

133 False 

134 

135 >>> pd.isna(pd.NA) 

136 True 

137 

138 >>> pd.isna(np.nan) 

139 True 

140 

141 ndarrays result in an ndarray of booleans. 

142 

143 >>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]]) 

144 >>> array 

145 array([[ 1., nan, 3.], 

146 [ 4., 5., nan]]) 

147 >>> pd.isna(array) 

148 array([[False, True, False], 

149 [False, False, True]]) 

150 

151 For indexes, an ndarray of booleans is returned. 

152 

153 >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, 

154 ... "2017-07-08"]) 

155 >>> index 

156 DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'], 

157 dtype='datetime64[ns]', freq=None) 

158 >>> pd.isna(index) 

159 array([False, False, True, False]) 

160 

161 For Series and DataFrame, the same type is returned, containing booleans. 

162 

163 >>> df = pd.DataFrame([['ant', 'bee', 'cat'], ['dog', None, 'fly']]) 

164 >>> df 

165 0 1 2 

166 0 ant bee cat 

167 1 dog None fly 

168 >>> pd.isna(df) 

169 0 1 2 

170 0 False False False 

171 1 False True False 

172 

173 >>> pd.isna(df[1]) 

174 0 False 

175 1 True 

176 Name: 1, dtype: bool 

177 """ 

178 return _isna(obj) 

179 

180 

181isnull = isna 

182 

183 

184def _isna(obj, inf_as_na: bool = False): 

185 """ 

186 Detect missing values, treating None, NaN or NA as null. Infinite 

187 values will also be treated as null if inf_as_na is True. 

188 

189 Parameters 

190 ---------- 

191 obj: ndarray or object value 

192 Input array or scalar value. 

193 inf_as_na: bool 

194 Whether to treat infinity as null. 

195 

196 Returns 

197 ------- 

198 boolean ndarray or boolean 

199 """ 

200 if is_scalar(obj): 

201 return libmissing.checknull(obj, inf_as_na=inf_as_na) 

202 elif isinstance(obj, ABCMultiIndex): 

203 raise NotImplementedError("isna is not defined for MultiIndex") 

204 elif isinstance(obj, type): 

205 return False 

206 elif isinstance(obj, (np.ndarray, ABCExtensionArray)): 

207 return _isna_array(obj, inf_as_na=inf_as_na) 

208 elif isinstance(obj, ABCIndex): 

209 # Try to use cached isna, which also short-circuits for integer dtypes 

210 # and avoids materializing RangeIndex._values 

211 if not obj._can_hold_na: 

212 return obj.isna() 

213 return _isna_array(obj._values, inf_as_na=inf_as_na) 

214 

215 elif isinstance(obj, ABCSeries): 

216 result = _isna_array(obj._values, inf_as_na=inf_as_na) 

217 # box 

218 result = obj._constructor(result, index=obj.index, name=obj.name, copy=False) 

219 return result 

220 elif isinstance(obj, ABCDataFrame): 

221 return obj.isna() 

222 elif isinstance(obj, list): 

223 return _isna_array(np.asarray(obj, dtype=object), inf_as_na=inf_as_na) 

224 elif hasattr(obj, "__array__"): 

225 return _isna_array(np.asarray(obj), inf_as_na=inf_as_na) 

226 else: 

227 return False 

228 

229 

230def _use_inf_as_na(key) -> None: 

231 """ 

232 Option change callback for na/inf behaviour. 

233 

234 Choose which replacement for numpy.isnan / -numpy.isfinite is used. 

235 

236 Parameters 

237 ---------- 

238 flag: bool 

239 True means treat None, NaN, INF, -INF as null (old way), 

240 False means None and NaN are null, but INF, -INF are not null 

241 (new way). 

242 

243 Notes 

244 ----- 

245 This approach to setting global module values is discussed and 

246 approved here: 

247 

248 * https://stackoverflow.com/questions/4859217/ 

249 programmatically-creating-variables-in-python/4859312#4859312 

250 """ 

251 inf_as_na = get_option(key) 

252 globals()["_isna"] = partial(_isna, inf_as_na=inf_as_na) 

253 if inf_as_na: 

254 globals()["nan_checker"] = lambda x: ~np.isfinite(x) 

255 globals()["INF_AS_NA"] = True 

256 else: 

257 globals()["nan_checker"] = np.isnan 

258 globals()["INF_AS_NA"] = False 

259 

260 

261def _isna_array(values: ArrayLike, inf_as_na: bool = False): 

262 """ 

263 Return an array indicating which values of the input array are NaN / NA. 

264 

265 Parameters 

266 ---------- 

267 obj: ndarray or ExtensionArray 

268 The input array whose elements are to be checked. 

269 inf_as_na: bool 

270 Whether or not to treat infinite values as NA. 

271 

272 Returns 

273 ------- 

274 array-like 

275 Array of boolean values denoting the NA status of each element. 

276 """ 

277 dtype = values.dtype 

278 

279 if not isinstance(values, np.ndarray): 

280 # i.e. ExtensionArray 

281 if inf_as_na and isinstance(dtype, CategoricalDtype): 

282 result = libmissing.isnaobj(values.to_numpy(), inf_as_na=inf_as_na) 

283 else: 

284 # error: Incompatible types in assignment (expression has type 

285 # "Union[ndarray[Any, Any], ExtensionArraySupportsAnyAll]", variable has 

286 # type "ndarray[Any, dtype[bool_]]") 

287 result = values.isna() # type: ignore[assignment] 

288 elif isinstance(values, np.rec.recarray): 

289 # GH 48526 

290 result = _isna_recarray_dtype(values, inf_as_na=inf_as_na) 

291 elif is_string_or_object_np_dtype(values.dtype): 

292 result = _isna_string_dtype(values, inf_as_na=inf_as_na) 

293 elif dtype.kind in "mM": 

294 # this is the NaT pattern 

295 result = values.view("i8") == iNaT 

296 else: 

297 if inf_as_na: 

298 result = ~np.isfinite(values) 

299 else: 

300 result = np.isnan(values) 

301 

302 return result 

303 

304 

305def _isna_string_dtype(values: np.ndarray, inf_as_na: bool) -> npt.NDArray[np.bool_]: 

306 # Working around NumPy ticket 1542 

307 dtype = values.dtype 

308 

309 if dtype.kind in ("S", "U"): 

310 result = np.zeros(values.shape, dtype=bool) 

311 else: 

312 if values.ndim in {1, 2}: 

313 result = libmissing.isnaobj(values, inf_as_na=inf_as_na) 

314 else: 

315 # 0-D, reached via e.g. mask_missing 

316 result = libmissing.isnaobj(values.ravel(), inf_as_na=inf_as_na) 

317 result = result.reshape(values.shape) 

318 

319 return result 

320 

321 

322def _has_record_inf_value(record_as_array: np.ndarray) -> np.bool_: 

323 is_inf_in_record = np.zeros(len(record_as_array), dtype=bool) 

324 for i, value in enumerate(record_as_array): 

325 is_element_inf = False 

326 try: 

327 is_element_inf = np.isinf(value) 

328 except TypeError: 

329 is_element_inf = False 

330 is_inf_in_record[i] = is_element_inf 

331 

332 return np.any(is_inf_in_record) 

333 

334 

335def _isna_recarray_dtype( 

336 values: np.rec.recarray, inf_as_na: bool 

337) -> npt.NDArray[np.bool_]: 

338 result = np.zeros(values.shape, dtype=bool) 

339 for i, record in enumerate(values): 

340 record_as_array = np.array(record.tolist()) 

341 does_record_contain_nan = isna_all(record_as_array) 

342 does_record_contain_inf = False 

343 if inf_as_na: 

344 does_record_contain_inf = bool(_has_record_inf_value(record_as_array)) 

345 result[i] = np.any( 

346 np.logical_or(does_record_contain_nan, does_record_contain_inf) 

347 ) 

348 

349 return result 

350 

351 

352@overload 

353def notna(obj: Scalar) -> bool: 

354 ... 

355 

356 

357@overload 

358def notna( 

359 obj: ArrayLike | Index | list, 

360) -> npt.NDArray[np.bool_]: 

361 ... 

362 

363 

364@overload 

365def notna(obj: NDFrameT) -> NDFrameT: 

366 ... 

367 

368 

369# handle unions 

370@overload 

371def notna(obj: NDFrameT | ArrayLike | Index | list) -> NDFrameT | npt.NDArray[np.bool_]: 

372 ... 

373 

374 

375@overload 

376def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: 

377 ... 

378 

379 

380def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: 

381 """ 

382 Detect non-missing values for an array-like object. 

383 

384 This function takes a scalar or array-like object and indicates 

385 whether values are valid (not missing, which is ``NaN`` in numeric 

386 arrays, ``None`` or ``NaN`` in object arrays, ``NaT`` in datetimelike). 

387 

388 Parameters 

389 ---------- 

390 obj : array-like or object value 

391 Object to check for *not* null or *non*-missing values. 

392 

393 Returns 

394 ------- 

395 bool or array-like of bool 

396 For scalar input, returns a scalar boolean. 

397 For array input, returns an array of boolean indicating whether each 

398 corresponding element is valid. 

399 

400 See Also 

401 -------- 

402 isna : Boolean inverse of pandas.notna. 

403 Series.notna : Detect valid values in a Series. 

404 DataFrame.notna : Detect valid values in a DataFrame. 

405 Index.notna : Detect valid values in an Index. 

406 

407 Examples 

408 -------- 

409 Scalar arguments (including strings) result in a scalar boolean. 

410 

411 >>> pd.notna('dog') 

412 True 

413 

414 >>> pd.notna(pd.NA) 

415 False 

416 

417 >>> pd.notna(np.nan) 

418 False 

419 

420 ndarrays result in an ndarray of booleans. 

421 

422 >>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]]) 

423 >>> array 

424 array([[ 1., nan, 3.], 

425 [ 4., 5., nan]]) 

426 >>> pd.notna(array) 

427 array([[ True, False, True], 

428 [ True, True, False]]) 

429 

430 For indexes, an ndarray of booleans is returned. 

431 

432 >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, 

433 ... "2017-07-08"]) 

434 >>> index 

435 DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'], 

436 dtype='datetime64[ns]', freq=None) 

437 >>> pd.notna(index) 

438 array([ True, True, False, True]) 

439 

440 For Series and DataFrame, the same type is returned, containing booleans. 

441 

442 >>> df = pd.DataFrame([['ant', 'bee', 'cat'], ['dog', None, 'fly']]) 

443 >>> df 

444 0 1 2 

445 0 ant bee cat 

446 1 dog None fly 

447 >>> pd.notna(df) 

448 0 1 2 

449 0 True True True 

450 1 True False True 

451 

452 >>> pd.notna(df[1]) 

453 0 True 

454 1 False 

455 Name: 1, dtype: bool 

456 """ 

457 res = isna(obj) 

458 if isinstance(res, bool): 

459 return not res 

460 return ~res 

461 

462 

463notnull = notna 

464 

465 

466def array_equivalent( 

467 left, 

468 right, 

469 strict_nan: bool = False, 

470 dtype_equal: bool = False, 

471) -> bool: 

472 """ 

473 True if two arrays, left and right, have equal non-NaN elements, and NaNs 

474 in corresponding locations. False otherwise. It is assumed that left and 

475 right are NumPy arrays of the same dtype. The behavior of this function 

476 (particularly with respect to NaNs) is not defined if the dtypes are 

477 different. 

478 

479 Parameters 

480 ---------- 

481 left, right : ndarrays 

482 strict_nan : bool, default False 

483 If True, consider NaN and None to be different. 

484 dtype_equal : bool, default False 

485 Whether `left` and `right` are known to have the same dtype 

486 according to `is_dtype_equal`. Some methods like `BlockManager.equals`. 

487 require that the dtypes match. Setting this to ``True`` can improve 

488 performance, but will give different results for arrays that are 

489 equal but different dtypes. 

490 

491 Returns 

492 ------- 

493 b : bool 

494 Returns True if the arrays are equivalent. 

495 

496 Examples 

497 -------- 

498 >>> array_equivalent( 

499 ... np.array([1, 2, np.nan]), 

500 ... np.array([1, 2, np.nan])) 

501 True 

502 >>> array_equivalent( 

503 ... np.array([1, np.nan, 2]), 

504 ... np.array([1, 2, np.nan])) 

505 False 

506 """ 

507 left, right = np.asarray(left), np.asarray(right) 

508 

509 # shape compat 

510 if left.shape != right.shape: 

511 return False 

512 

513 if dtype_equal: 

514 # fastpath when we require that the dtypes match (Block.equals) 

515 if left.dtype.kind in "fc": 

516 return _array_equivalent_float(left, right) 

517 elif left.dtype.kind in "mM": 

518 return _array_equivalent_datetimelike(left, right) 

519 elif is_string_or_object_np_dtype(left.dtype): 

520 # TODO: fastpath for pandas' StringDtype 

521 return _array_equivalent_object(left, right, strict_nan) 

522 else: 

523 return np.array_equal(left, right) 

524 

525 # Slow path when we allow comparing different dtypes. 

526 # Object arrays can contain None, NaN and NaT. 

527 # string dtypes must be come to this path for NumPy 1.7.1 compat 

528 if left.dtype.kind in "OSU" or right.dtype.kind in "OSU": 

529 # Note: `in "OSU"` is non-trivially faster than `in ["O", "S", "U"]` 

530 # or `in ("O", "S", "U")` 

531 return _array_equivalent_object(left, right, strict_nan) 

532 

533 # NaNs can occur in float and complex arrays. 

534 if left.dtype.kind in "fc": 

535 if not (left.size and right.size): 

536 return True 

537 return ((left == right) | (isna(left) & isna(right))).all() 

538 

539 elif left.dtype.kind in "mM" or right.dtype.kind in "mM": 

540 # datetime64, timedelta64, Period 

541 if left.dtype != right.dtype: 

542 return False 

543 

544 left = left.view("i8") 

545 right = right.view("i8") 

546 

547 # if we have structured dtypes, compare first 

548 if ( 

549 left.dtype.type is np.void or right.dtype.type is np.void 

550 ) and left.dtype != right.dtype: 

551 return False 

552 

553 return np.array_equal(left, right) 

554 

555 

556def _array_equivalent_float(left: np.ndarray, right: np.ndarray) -> bool: 

557 return bool(((left == right) | (np.isnan(left) & np.isnan(right))).all()) 

558 

559 

560def _array_equivalent_datetimelike(left: np.ndarray, right: np.ndarray): 

561 return np.array_equal(left.view("i8"), right.view("i8")) 

562 

563 

564def _array_equivalent_object(left: np.ndarray, right: np.ndarray, strict_nan: bool): 

565 left = ensure_object(left) 

566 right = ensure_object(right) 

567 

568 mask: npt.NDArray[np.bool_] | None = None 

569 if strict_nan: 

570 mask = isna(left) & isna(right) 

571 if not mask.any(): 

572 mask = None 

573 

574 try: 

575 if mask is None: 

576 return lib.array_equivalent_object(left, right) 

577 if not lib.array_equivalent_object(left[~mask], right[~mask]): 

578 return False 

579 left_remaining = left[mask] 

580 right_remaining = right[mask] 

581 except ValueError: 

582 # can raise a ValueError if left and right cannot be 

583 # compared (e.g. nested arrays) 

584 left_remaining = left 

585 right_remaining = right 

586 

587 for left_value, right_value in zip(left_remaining, right_remaining): 

588 if left_value is NaT and right_value is not NaT: 

589 return False 

590 

591 elif left_value is libmissing.NA and right_value is not libmissing.NA: 

592 return False 

593 

594 elif isinstance(left_value, float) and np.isnan(left_value): 

595 if not isinstance(right_value, float) or not np.isnan(right_value): 

596 return False 

597 else: 

598 with warnings.catch_warnings(): 

599 # suppress numpy's "elementwise comparison failed" 

600 warnings.simplefilter("ignore", DeprecationWarning) 

601 try: 

602 if np.any(np.asarray(left_value != right_value)): 

603 return False 

604 except TypeError as err: 

605 if "boolean value of NA is ambiguous" in str(err): 

606 return False 

607 raise 

608 except ValueError: 

609 # numpy can raise a ValueError if left and right cannot be 

610 # compared (e.g. nested arrays) 

611 return False 

612 return True 

613 

614 

615def array_equals(left: ArrayLike, right: ArrayLike) -> bool: 

616 """ 

617 ExtensionArray-compatible implementation of array_equivalent. 

618 """ 

619 if left.dtype != right.dtype: 

620 return False 

621 elif isinstance(left, ABCExtensionArray): 

622 return left.equals(right) 

623 else: 

624 return array_equivalent(left, right, dtype_equal=True) 

625 

626 

627def infer_fill_value(val): 

628 """ 

629 infer the fill value for the nan/NaT from the provided 

630 scalar/ndarray/list-like if we are a NaT, return the correct dtyped 

631 element to provide proper block construction 

632 """ 

633 if not is_list_like(val): 

634 val = [val] 

635 val = np.asarray(val) 

636 if val.dtype.kind in "mM": 

637 return np.array("NaT", dtype=val.dtype) 

638 elif val.dtype == object: 

639 dtype = lib.infer_dtype(ensure_object(val), skipna=False) 

640 if dtype in ["datetime", "datetime64"]: 

641 return np.array("NaT", dtype=DT64NS_DTYPE) 

642 elif dtype in ["timedelta", "timedelta64"]: 

643 return np.array("NaT", dtype=TD64NS_DTYPE) 

644 return np.array(np.nan, dtype=object) 

645 elif val.dtype.kind == "U": 

646 return np.array(np.nan, dtype=val.dtype) 

647 return np.nan 

648 

649 

650def construct_1d_array_from_inferred_fill_value( 

651 value: object, length: int 

652) -> ArrayLike: 

653 # Find our empty_value dtype by constructing an array 

654 # from our value and doing a .take on it 

655 from pandas.core.algorithms import take_nd 

656 from pandas.core.construction import sanitize_array 

657 from pandas.core.indexes.base import Index 

658 

659 arr = sanitize_array(value, Index(range(1)), copy=False) 

660 taker = -1 * np.ones(length, dtype=np.intp) 

661 return take_nd(arr, taker) 

662 

663 

664def maybe_fill(arr: np.ndarray) -> np.ndarray: 

665 """ 

666 Fill numpy.ndarray with NaN, unless we have a integer or boolean dtype. 

667 """ 

668 if arr.dtype.kind not in "iub": 

669 arr.fill(np.nan) 

670 return arr 

671 

672 

673def na_value_for_dtype(dtype: DtypeObj, compat: bool = True): 

674 """ 

675 Return a dtype compat na value 

676 

677 Parameters 

678 ---------- 

679 dtype : string / dtype 

680 compat : bool, default True 

681 

682 Returns 

683 ------- 

684 np.dtype or a pandas dtype 

685 

686 Examples 

687 -------- 

688 >>> na_value_for_dtype(np.dtype('int64')) 

689 0 

690 >>> na_value_for_dtype(np.dtype('int64'), compat=False) 

691 nan 

692 >>> na_value_for_dtype(np.dtype('float64')) 

693 nan 

694 >>> na_value_for_dtype(np.dtype('bool')) 

695 False 

696 >>> na_value_for_dtype(np.dtype('datetime64[ns]')) 

697 numpy.datetime64('NaT') 

698 """ 

699 

700 if isinstance(dtype, ExtensionDtype): 

701 return dtype.na_value 

702 elif dtype.kind in "mM": 

703 unit = np.datetime_data(dtype)[0] 

704 return dtype.type("NaT", unit) 

705 elif dtype.kind == "f": 

706 return np.nan 

707 elif dtype.kind in "iu": 

708 if compat: 

709 return 0 

710 return np.nan 

711 elif dtype.kind == "b": 

712 if compat: 

713 return False 

714 return np.nan 

715 return np.nan 

716 

717 

718def remove_na_arraylike(arr: Series | Index | np.ndarray): 

719 """ 

720 Return array-like containing only true/non-NaN values, possibly empty. 

721 """ 

722 if isinstance(arr.dtype, ExtensionDtype): 

723 return arr[notna(arr)] 

724 else: 

725 return arr[notna(np.asarray(arr))] 

726 

727 

728def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool: 

729 """ 

730 isna check that excludes incompatible dtypes 

731 

732 Parameters 

733 ---------- 

734 obj : object 

735 dtype : np.datetime64, np.timedelta64, DatetimeTZDtype, or PeriodDtype 

736 

737 Returns 

738 ------- 

739 bool 

740 """ 

741 if not lib.is_scalar(obj) or not isna(obj): 

742 return False 

743 elif dtype.kind == "M": 

744 if isinstance(dtype, np.dtype): 

745 # i.e. not tzaware 

746 return not isinstance(obj, (np.timedelta64, Decimal)) 

747 # we have to rule out tznaive dt64("NaT") 

748 return not isinstance(obj, (np.timedelta64, np.datetime64, Decimal)) 

749 elif dtype.kind == "m": 

750 return not isinstance(obj, (np.datetime64, Decimal)) 

751 elif dtype.kind in "iufc": 

752 # Numeric 

753 return obj is not NaT and not isinstance(obj, (np.datetime64, np.timedelta64)) 

754 elif dtype.kind == "b": 

755 # We allow pd.NA, None, np.nan in BooleanArray (same as IntervalDtype) 

756 return lib.is_float(obj) or obj is None or obj is libmissing.NA 

757 

758 elif dtype == _dtype_str: 

759 # numpy string dtypes to avoid float np.nan 

760 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal, float)) 

761 

762 elif dtype == _dtype_object: 

763 # This is needed for Categorical, but is kind of weird 

764 return True 

765 

766 elif isinstance(dtype, PeriodDtype): 

767 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal)) 

768 

769 elif isinstance(dtype, IntervalDtype): 

770 return lib.is_float(obj) or obj is None or obj is libmissing.NA 

771 

772 elif isinstance(dtype, CategoricalDtype): 

773 return is_valid_na_for_dtype(obj, dtype.categories.dtype) 

774 

775 # fallback, default to allowing NaN, None, NA, NaT 

776 return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal)) 

777 

778 

779def isna_all(arr: ArrayLike) -> bool: 

780 """ 

781 Optimized equivalent to isna(arr).all() 

782 """ 

783 total_len = len(arr) 

784 

785 # Usually it's enough to check but a small fraction of values to see if 

786 # a block is NOT null, chunks should help in such cases. 

787 # parameters 1000 and 40 were chosen arbitrarily 

788 chunk_len = max(total_len // 40, 1000) 

789 

790 dtype = arr.dtype 

791 if lib.is_np_dtype(dtype, "f"): 

792 checker = nan_checker 

793 

794 elif (lib.is_np_dtype(dtype, "mM")) or isinstance( 

795 dtype, (DatetimeTZDtype, PeriodDtype) 

796 ): 

797 # error: Incompatible types in assignment (expression has type 

798 # "Callable[[Any], Any]", variable has type "ufunc") 

799 checker = lambda x: np.asarray(x.view("i8")) == iNaT # type: ignore[assignment] 

800 

801 else: 

802 # error: Incompatible types in assignment (expression has type "Callable[[Any], 

803 # Any]", variable has type "ufunc") 

804 checker = lambda x: _isna_array( # type: ignore[assignment] 

805 x, inf_as_na=INF_AS_NA 

806 ) 

807 

808 return all( 

809 checker(arr[i : i + chunk_len]).all() for i in range(0, total_len, chunk_len) 

810 )