Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Routines for casting.
3"""
5from __future__ import annotations
7import datetime as dt
8import functools
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 Literal,
13 TypeVar,
14 cast,
15 overload,
16)
17import warnings
19import numpy as np
21from pandas._config import (
22 is_nan_na,
23 using_python_scalars,
24 using_string_dtype,
25)
27from pandas._libs import (
28 Interval,
29 Period,
30 lib,
31)
32from pandas._libs.missing import (
33 NA,
34 NAType,
35 checknull,
36)
37from pandas._libs.tslibs import (
38 NaT,
39 OutOfBoundsDatetime,
40 OutOfBoundsTimedelta,
41 Timedelta,
42 Timestamp,
43 is_supported_dtype,
44)
45from pandas._libs.tslibs.timedeltas import array_to_timedelta64
46from pandas.errors import (
47 IntCastingNaNError,
48 LossySetitemError,
49)
51from pandas.core.dtypes.common import (
52 ensure_int8,
53 ensure_int16,
54 ensure_int32,
55 ensure_int64,
56 ensure_object,
57 ensure_str,
58 is_bool,
59 is_complex,
60 is_float,
61 is_integer,
62 is_object_dtype,
63 is_scalar,
64 is_string_dtype,
65 pandas_dtype as pandas_dtype_func,
66)
67from pandas.core.dtypes.dtypes import (
68 ArrowDtype,
69 BaseMaskedDtype,
70 CategoricalDtype,
71 DatetimeTZDtype,
72 ExtensionDtype,
73 IntervalDtype,
74 PandasExtensionDtype,
75 PeriodDtype,
76)
77from pandas.core.dtypes.generic import (
78 ABCExtensionArray,
79 ABCIndex,
80 ABCSeries,
81)
82from pandas.core.dtypes.inference import is_list_like
83from pandas.core.dtypes.missing import (
84 is_valid_na_for_dtype,
85 isna,
86 na_value_for_dtype,
87 notna,
88)
90from pandas.io._util import _arrow_dtype_mapping
92if TYPE_CHECKING:
93 from collections.abc import (
94 Collection,
95 Sequence,
96 )
98 from pandas._typing import (
99 ArrayLike,
100 Dtype,
101 DtypeObj,
102 NumpyIndexT,
103 Scalar,
104 TimeUnit,
105 )
107 from pandas import Index
108 from pandas.core.arrays import (
109 Categorical,
110 DatetimeArray,
111 ExtensionArray,
112 IntervalArray,
113 PeriodArray,
114 TimedeltaArray,
115 )
118_int8_max = np.iinfo(np.int8).max
119_int16_max = np.iinfo(np.int16).max
120_int32_max = np.iinfo(np.int32).max
122_dtype_obj = np.dtype(object)
124NumpyArrayT = TypeVar("NumpyArrayT", bound=np.ndarray)
127def is_nested_object(obj: object) -> bool:
128 """
129 return a boolean if we have a nested object, e.g. a Series with 1 or
130 more Series elements
132 This may not be necessarily be performant.
134 """
135 return bool(
136 isinstance(obj, ABCSeries)
137 and is_object_dtype(obj.dtype)
138 and any(isinstance(v, ABCSeries) for v in obj._values)
139 )
142def maybe_box_datetimelike(value: Scalar, dtype: Dtype | None = None) -> Scalar:
143 """
144 Cast scalar to Timestamp or Timedelta if scalar is datetime-like
145 and dtype is not object.
147 Parameters
148 ----------
149 value : scalar
150 dtype : Dtype, optional
152 Returns
153 -------
154 scalar
155 """
156 if dtype == _dtype_obj:
157 pass
158 elif isinstance(value, (np.datetime64, dt.datetime)):
159 value = Timestamp(value)
160 elif isinstance(value, (np.timedelta64, dt.timedelta)):
161 value = Timedelta(value)
163 return value
166def maybe_box_native(value: Scalar | NAType | None) -> Scalar | NAType | None:
167 """
168 If passed a scalar cast the scalar to a python native type.
170 Parameters
171 ----------
172 value : scalar or Series
174 Returns
175 -------
176 scalar or Series
177 """
178 if is_float(value):
179 value = float(value)
180 elif is_integer(value):
181 value = int(value)
182 elif is_bool(value):
183 value = bool(value)
184 elif isinstance(value, (np.datetime64, np.timedelta64)):
185 value = maybe_box_datetimelike(value)
186 elif value is NA:
187 value = None
188 return value
191def _maybe_unbox_datetimelike(value: Scalar, dtype: DtypeObj) -> Scalar:
192 """
193 Convert a Timedelta or Timestamp to timedelta64 or datetime64 for setting
194 into a numpy array. Failing to unbox would risk dropping nanoseconds.
196 Notes
197 -----
198 Caller is responsible for checking dtype.kind in "mM"
199 """
200 if is_valid_na_for_dtype(value, dtype):
201 # GH#36541: can't fill array directly with pd.NaT
202 # > np.empty(10, dtype="datetime64[ns]").fill(pd.NaT)
203 # ValueError: cannot convert float NaN to integer
204 value = dtype.type("NaT", "ns")
205 elif isinstance(value, Timestamp):
206 if value.tz is None:
207 value = value.to_datetime64()
208 elif not isinstance(dtype, DatetimeTZDtype):
209 raise TypeError("Cannot unbox tzaware Timestamp to tznaive dtype")
210 elif isinstance(value, Timedelta):
211 value = value.to_timedelta64()
213 _disallow_mismatched_datetimelike(value, dtype)
214 return value
217def _disallow_mismatched_datetimelike(value: Scalar, dtype: DtypeObj) -> None:
218 """
219 numpy allows np.array(dt64values, dtype="timedelta64[ns]") and
220 vice-versa, but we do not want to allow this, so we need to
221 check explicitly
222 """
223 vdtype = getattr(value, "dtype", None)
224 if vdtype is None:
225 return
226 elif (vdtype.kind == "m" and dtype.kind == "M") or (
227 vdtype.kind == "M" and dtype.kind == "m"
228 ):
229 raise TypeError(f"Cannot cast {value!r} to {dtype}")
232@overload
233def maybe_downcast_to_dtype(result: np.ndarray, dtype: np.dtype) -> np.ndarray: ...
236@overload
237def maybe_downcast_to_dtype(
238 result: ExtensionArray, dtype: np.dtype
239) -> ExtensionArray: ...
242def maybe_downcast_to_dtype(result: ArrayLike, dtype: np.dtype) -> ArrayLike:
243 """
244 try to cast to the specified dtype (e.g. convert back to bool/int
245 or could be an astype of float64->float32
246 """
247 if isinstance(result, ABCSeries):
248 result = result._values
249 do_round = False
251 if not isinstance(dtype, np.dtype):
252 # enforce our signature annotation
253 raise TypeError(dtype) # pragma: no cover
255 converted = maybe_downcast_numeric(result, dtype, do_round)
256 if converted is not result:
257 return converted
259 # a datetimelike
260 # GH12821, iNaT is cast to float
261 if dtype.kind in "mM" and result.dtype.kind in "if":
262 result = result.astype(dtype)
264 elif dtype.kind == "m" and result.dtype == _dtype_obj:
265 # test_where_downcast_to_td64
266 result = cast("np.ndarray", result)
267 result = array_to_timedelta64(result)
269 elif dtype == np.dtype("M8[ns]") and result.dtype == _dtype_obj:
270 result = cast("np.ndarray", result)
271 return np.asarray(maybe_cast_to_datetime(result, dtype=dtype))
273 return result
276def floats_fit_integer_dtype(values: np.ndarray, dtype: np.dtype) -> bool:
277 """
278 Whether every entry of float- or bool-dtype `values` is finite and within
279 the range that casts exactly to integer dtype `dtype`.
281 Casting a float outside that range to an integer dtype is undefined
282 behavior in C, so numpy's result varies by platform.
283 """
284 iinfo = np.iinfo(dtype)
285 # float(iinfo.max) rounds *up* for int64/uint64 (e.g. float64 has no
286 # 2**63 - 1), so compare against the exactly-representable power of two
287 # one past it, with a strict inequality.
288 lower = np.float64(iinfo.min)
289 upper = np.float64(int(iinfo.max) + 1)
290 # np.float64 scalars, unlike Python floats, promote a narrower `values`
291 # (e.g. float16) instead of downcasting the bounds and overflowing.
292 # NaN and +/-inf both fail these comparisons.
293 return bool(((values >= lower) & (values < upper)).all())
296@overload
297def maybe_downcast_numeric(
298 result: np.ndarray, dtype: np.dtype, do_round: bool = False
299) -> np.ndarray: ...
302@overload
303def maybe_downcast_numeric(
304 result: ExtensionArray, dtype: DtypeObj, do_round: bool = False
305) -> ArrayLike: ...
308def maybe_downcast_numeric(
309 result: ArrayLike, dtype: DtypeObj, do_round: bool = False
310) -> ArrayLike:
311 """
312 Subset of maybe_downcast_to_dtype restricted to numeric dtypes.
314 Parameters
315 ----------
316 result : ndarray or ExtensionArray
317 dtype : np.dtype or ExtensionDtype
318 do_round : bool
320 Returns
321 -------
322 ndarray or ExtensionArray
323 """
324 if not isinstance(dtype, np.dtype) or not isinstance(result.dtype, np.dtype):
325 # e.g. SparseDtype has no itemsize attr
326 return result
328 def trans(x: np.ndarray) -> np.ndarray:
329 if do_round:
330 return x.round()
331 return x
333 if dtype.kind == result.dtype.kind:
334 # don't allow upcasts here (except if empty)
335 if result.dtype.itemsize <= dtype.itemsize and result.size:
336 return result
338 if dtype.kind in "biu":
339 if not result.size:
340 # if we don't have any elements, just astype it
341 return trans(result).astype(dtype)
343 if isinstance(result, np.ndarray):
344 element = result.item(0)
345 else:
346 element = result.iloc[0]
347 if not isinstance(element, (np.integer, np.floating, int, float, bool)):
348 # a comparable, e.g. a Decimal may slip in here
349 return result
351 if (
352 issubclass(result.dtype.type, (np.object_, np.number))
353 and notna(result).all()
354 ):
355 new_result = trans(result)
356 if new_result.dtype.kind == "f" and dtype.kind in "iu":
357 # GH#66394 an out-of-range or non-finite float->int cast is
358 # platform-dependent (aarch64 saturates, x86 wraps), and the
359 # np.allclose check below cannot detect it, since e.g. 2**63
360 # and 2**63 - 1 are the same float64.
361 if not floats_fit_integer_dtype(new_result, dtype):
362 return result
363 try:
364 new_result = new_result.astype(dtype)
365 except (OverflowError, ValueError):
366 # GH#66394 object-dtype entries that don't fit `dtype`, e.g. a
367 # python int larger than int64 or an inf; not downcasting is
368 # the correct outcome, so don't let the error escape.
369 return result
370 if new_result.dtype.kind == "O" or result.dtype.kind == "O":
371 # np.allclose may raise TypeError on object-dtype
372 if (new_result == result).all():
373 return new_result
374 elif np.allclose(new_result, result, rtol=0):
375 return new_result
377 elif (
378 issubclass(dtype.type, np.floating)
379 and result.dtype.kind != "b"
380 and not is_string_dtype(result.dtype)
381 ):
382 with warnings.catch_warnings():
383 warnings.filterwarnings(
384 "ignore", "overflow encountered in cast", RuntimeWarning
385 )
386 new_result = result.astype(dtype)
388 # Adjust tolerances based on floating point size
389 size_tols = {4: 5e-4, 8: 5e-8, 16: 5e-16}
391 atol = size_tols.get(new_result.dtype.itemsize, 0.0)
393 # Check downcast float values are still equal within 7 digits when
394 # converting from float64 to float32
395 if np.allclose(new_result, result, equal_nan=True, rtol=0.0, atol=atol):
396 return new_result
398 elif dtype.kind == result.dtype.kind == "c":
399 new_result = result.astype(dtype)
401 if np.array_equal(new_result, result, equal_nan=True):
402 # TODO: use tolerance like we do for float?
403 return new_result
405 return result
408def maybe_upcast_numeric_to_64bit(arr: NumpyIndexT) -> NumpyIndexT:
409 """
410 If array is an int/uint/float bit size lower than 64 bit, upcast it to 64 bit.
412 Parameters
413 ----------
414 arr : ndarray or ExtensionArray
416 Returns
417 -------
418 ndarray or ExtensionArray
419 """
420 dtype = arr.dtype
421 if dtype.kind == "i" and dtype != np.int64:
422 return arr.astype(np.int64)
423 elif dtype.kind == "u" and dtype != np.uint64:
424 return arr.astype(np.uint64)
425 elif dtype.kind == "f" and dtype != np.float64:
426 return arr.astype(np.float64)
427 else:
428 return arr
431@overload
432def ensure_dtype_can_hold_na(dtype: np.dtype) -> np.dtype: ...
435@overload
436def ensure_dtype_can_hold_na(dtype: ExtensionDtype) -> ExtensionDtype: ...
439def ensure_dtype_can_hold_na(dtype: DtypeObj) -> DtypeObj:
440 """
441 If we have a dtype that cannot hold NA values, find the best match that can.
442 """
443 if isinstance(dtype, ExtensionDtype):
444 if dtype._can_hold_na:
445 return dtype
446 elif isinstance(dtype, IntervalDtype):
447 # TODO(GH#45349): don't special-case IntervalDtype, allow
448 # overriding instead of returning object below.
449 return IntervalDtype(np.float64, closed=dtype.closed)
450 return _dtype_obj
451 elif dtype.kind in "bSUV":
452 return _dtype_obj
453 elif dtype.kind in "iu":
454 return np.dtype(np.float64)
455 return dtype
458_canonical_nans = {
459 np.datetime64: np.datetime64("NaT", "ns"),
460 np.timedelta64: np.timedelta64("NaT", "ns"),
461 type(np.nan): np.nan,
462}
465def maybe_promote(dtype: np.dtype, fill_value: Any = np.nan) -> tuple[np.dtype, Any]:
466 """
467 Find the minimal dtype that can hold both the given dtype and fill_value.
469 Parameters
470 ----------
471 dtype : np.dtype
472 fill_value : scalar, default np.nan
474 Returns
475 -------
476 dtype
477 Upcasted from dtype argument if necessary.
478 fill_value
479 Upcasted from fill_value argument if necessary.
481 Raises
482 ------
483 ValueError
484 If fill_value is a non-scalar and dtype is not object.
485 """
486 orig = fill_value
487 orig_is_nat = False
488 if checknull(fill_value):
489 # https://github.com/pandas-dev/pandas/pull/39692#issuecomment-1441051740
490 # avoid cache misses with NaN/NaT values that are not singletons
491 if fill_value is not NA:
492 try:
493 orig_is_nat = np.isnat(fill_value)
494 except TypeError:
495 pass
497 fill_value = _canonical_nans.get(type(fill_value), fill_value)
499 # for performance, we are using a cached version of the actual implementation
500 # of the function in _maybe_promote. However, this doesn't always work (in case
501 # of non-hashable arguments), so we fallback to the actual implementation if needed
502 try:
503 # error: Argument 3 to "__call__" of "_lru_cache_wrapper" has incompatible type
504 # "Type[Any]"; expected "Hashable" [arg-type]
505 dtype, fill_value = _maybe_promote_cached(
506 dtype,
507 fill_value,
508 type(fill_value), # type: ignore[arg-type]
509 )
510 except TypeError:
511 # if fill_value is not hashable (required for caching)
512 dtype, fill_value = _maybe_promote(dtype, fill_value)
514 if (dtype == _dtype_obj and orig is not None) or (
515 orig_is_nat and np.datetime_data(orig)[0] != "ns"
516 ):
517 # GH#51592,53497 restore our potentially non-canonical fill_value
518 fill_value = orig
519 return dtype, fill_value
522@functools.lru_cache
523def _maybe_promote_cached(
524 dtype: np.dtype, fill_value: Any, fill_value_type: type
525) -> tuple[np.dtype, Any]:
526 # The cached version of _maybe_promote below
527 # This also use fill_value_type as (unused) argument to use this in the
528 # cache lookup -> to differentiate 1 and True
529 return _maybe_promote(dtype, fill_value)
532def _maybe_promote(dtype: np.dtype, fill_value: Any = np.nan) -> tuple[np.dtype, Any]:
533 # The actual implementation of the function, use `maybe_promote` above for
534 # a cached version.
535 if not is_scalar(fill_value):
536 # with object dtype there is nothing to promote, and the user can
537 # pass pretty much any weird fill_value they like
538 if dtype != object:
539 # with object dtype there is nothing to promote, and the user can
540 # pass pretty much any weird fill_value they like
541 raise ValueError("fill_value must be a scalar")
542 dtype = _dtype_obj
543 return dtype, fill_value
545 if is_valid_na_for_dtype(fill_value, dtype) and dtype.kind in "iufcmM":
546 dtype = ensure_dtype_can_hold_na(dtype)
547 fv = na_value_for_dtype(dtype)
548 return dtype, fv
550 elif isinstance(dtype, CategoricalDtype):
551 if fill_value in dtype.categories or isna(fill_value):
552 return dtype, fill_value
553 else:
554 return object, ensure_object(fill_value)
556 elif isna(fill_value):
557 dtype = _dtype_obj
558 if fill_value is None:
559 # but we retain e.g. pd.NA
560 fill_value = np.nan
561 return dtype, fill_value
563 # returns tuple of (dtype, fill_value)
564 if issubclass(dtype.type, np.datetime64):
565 inferred, fv = infer_dtype_from_scalar(fill_value)
566 if inferred == dtype:
567 return dtype, fv
569 from pandas.core.arrays import DatetimeArray
571 dta = DatetimeArray._from_sequence([], dtype="M8[ns]")
572 try:
573 fv = dta._validate_setitem_value(fill_value) # type: ignore[no-untyped-call]
574 return cast("np.dtype", dta.dtype), fv
575 except (ValueError, TypeError):
576 return _dtype_obj, fill_value
578 elif issubclass(dtype.type, np.timedelta64):
579 inferred, fv = infer_dtype_from_scalar(fill_value)
580 if inferred == dtype:
581 return dtype, fv
583 elif inferred.kind == "m":
584 # different unit, e.g. passed np.timedelta64(24, "h") with dtype=m8[ns]
585 # see if we can losslessly cast it to our dtype
586 unit = np.datetime_data(dtype)[0]
587 unit = cast("TimeUnit", unit)
588 try:
589 td = Timedelta(fill_value).as_unit(unit, round_ok=False)
590 except OutOfBoundsTimedelta:
591 return _dtype_obj, fill_value
592 else:
593 return dtype, td.asm8
595 return _dtype_obj, fill_value
597 elif is_float(fill_value):
598 if issubclass(dtype.type, np.bool_):
599 dtype = np.dtype(np.object_)
601 elif issubclass(dtype.type, np.integer):
602 dtype = np.dtype(np.float64)
604 elif dtype.kind == "f":
605 mst = np.min_scalar_type(fill_value)
606 if mst > dtype:
607 # e.g. mst is np.float64 and dtype is np.float32
608 dtype = mst
610 elif dtype.kind == "c":
611 mst = np.min_scalar_type(fill_value)
612 dtype = np.promote_types(dtype, mst)
614 elif is_bool(fill_value):
615 if not issubclass(dtype.type, np.bool_):
616 dtype = np.dtype(np.object_)
618 elif is_integer(fill_value):
619 if issubclass(dtype.type, np.bool_):
620 dtype = np.dtype(np.object_)
622 elif issubclass(dtype.type, np.integer):
623 if not np_can_cast_scalar(fill_value, dtype):
624 # upcast to prevent overflow
625 mst = np.min_scalar_type(fill_value)
626 dtype = np.promote_types(dtype, mst)
627 if dtype.kind == "f":
628 # Case where we disagree with numpy
629 dtype = np.dtype(np.object_)
631 elif is_complex(fill_value):
632 if issubclass(dtype.type, np.bool_):
633 dtype = np.dtype(np.object_)
635 elif issubclass(dtype.type, (np.integer, np.floating)):
636 mst = np.min_scalar_type(fill_value)
637 dtype = np.promote_types(dtype, mst)
639 elif dtype.kind == "c":
640 mst = np.min_scalar_type(fill_value)
641 if mst > dtype:
642 # e.g. mst is np.complex128 and dtype is np.complex64
643 dtype = mst
645 else:
646 dtype = np.dtype(np.object_)
648 # in case we have a string that looked like a number
649 if issubclass(dtype.type, (bytes, str)):
650 dtype = np.dtype(np.object_)
652 fill_value = _ensure_dtype_type(fill_value, dtype)
653 return dtype, fill_value
656def _ensure_dtype_type(value: Any, dtype: np.dtype) -> Any:
657 """
658 Ensure that the given value is an instance of the given dtype.
660 e.g. if out dtype is np.complex64_, we should have an instance of that
661 as opposed to a python complex object.
663 Parameters
664 ----------
665 value : object
666 dtype : np.dtype
668 Returns
669 -------
670 object
671 """
672 # Start with exceptions in which we do _not_ cast to numpy types
674 if dtype == _dtype_obj:
675 return value
677 # Note: before we get here we have already excluded isna(value)
678 return dtype.type(value)
681def infer_dtype_from(val: object) -> tuple[DtypeObj, Any]:
682 """
683 Interpret the dtype from a scalar or array.
685 Parameters
686 ----------
687 val : object
688 """
689 if not is_list_like(val):
690 return infer_dtype_from_scalar(val)
691 return infer_dtype_from_array(val)
694def infer_dtype_from_scalar(val: object) -> tuple[DtypeObj, Any]:
695 """
696 Interpret the dtype from a scalar.
698 Parameters
699 ----------
700 val : object
701 """
702 dtype: DtypeObj = _dtype_obj
704 # a 1-element ndarray
705 if isinstance(val, np.ndarray):
706 if val.ndim != 0:
707 msg = "invalid ndarray passed to infer_dtype_from_scalar"
708 raise ValueError(msg)
710 dtype = val.dtype
711 val = lib.item_from_zerodim(val)
713 elif isinstance(val, str):
714 # If we create an empty array using a string to infer
715 # the dtype, NumPy will only allocate one character per entry
716 # so this is kind of bad. Alternately we could use np.repeat
717 # instead of np.empty (but then you still don't want things
718 # coming out as np.str_!
720 dtype = _dtype_obj
721 if using_string_dtype():
722 from pandas.core.arrays.string_ import StringDtype
724 dtype = StringDtype(na_value=np.nan)
726 elif isinstance(val, (np.datetime64, dt.datetime)):
727 try:
728 val = Timestamp(val)
729 except OutOfBoundsDatetime:
730 return _dtype_obj, val
732 # error: Non-overlapping identity check (left operand type: "Timestamp",
733 # right operand type: "NaTType")
734 if val is NaT or val.tz is None: # type: ignore[comparison-overlap]
735 val = val.to_datetime64()
736 dtype = val.dtype
737 # TODO: test with datetime(2920, 10, 1) based on test_replace_dtypes
738 else:
739 dtype = DatetimeTZDtype(unit=val.unit, tz=val.tz)
741 elif isinstance(val, (np.timedelta64, dt.timedelta)):
742 try:
743 val = Timedelta(val)
744 except (OutOfBoundsTimedelta, OverflowError):
745 dtype = _dtype_obj
746 else:
747 # error: Non-overlapping identity check (left operand type: "Timedelta",
748 # right operand type: "NaTType")
749 if val is NaT: # type: ignore[comparison-overlap]
750 val = np.timedelta64("NaT", "ns")
751 else:
752 val = val.asm8
753 dtype = val.dtype
755 elif is_bool(val):
756 dtype = np.dtype(np.bool_)
758 elif is_integer(val):
759 if isinstance(val, np.integer):
760 dtype = np.dtype(type(val))
761 else:
762 dtype = np.dtype(np.int64)
764 try:
765 np.array(val, dtype=dtype)
766 except OverflowError:
767 dtype = np.array(val).dtype
769 elif is_float(val):
770 if isinstance(val, np.floating):
771 dtype = np.dtype(type(val))
772 else:
773 dtype = np.dtype(np.float64)
775 elif is_complex(val):
776 dtype = np.dtype(np.complex128)
778 if isinstance(val, Period):
779 dtype = PeriodDtype(freq=val.freq)
780 elif isinstance(val, Interval):
781 subtype = infer_dtype_from_scalar(val.left)[0]
782 dtype = IntervalDtype(subtype=subtype, closed=val.closed)
784 return dtype, val
787def dict_compat(d: dict[Scalar, Scalar]) -> dict[Scalar, Scalar]:
788 """
789 Convert datetimelike-keyed dicts to a Timestamp-keyed dict.
791 Parameters
792 ----------
793 d: dict-like object
795 Returns
796 -------
797 dict
798 """
799 return {maybe_box_datetimelike(key): value for key, value in d.items()}
802def infer_dtype_from_array(arr: Any) -> tuple[DtypeObj, ArrayLike]:
803 """
804 Infer the dtype from an array.
806 Parameters
807 ----------
808 arr : array
810 Returns
811 -------
812 tuple (pandas-compat dtype, array)
815 Examples
816 --------
817 >>> np.asarray([1, "1"])
818 array(['1', '1'], dtype='<U21')
820 >>> infer_dtype_from_array([1, "1"])
821 (dtype('O'), [1, '1'])
822 """
823 if isinstance(arr, np.ndarray):
824 return arr.dtype, arr
826 if not is_list_like(arr):
827 raise TypeError("'arr' must be list-like")
829 arr_dtype = getattr(arr, "dtype", None)
830 if isinstance(arr_dtype, ExtensionDtype):
831 return arr.dtype, arr
833 elif isinstance(arr, ABCSeries):
834 return arr.dtype, np.asarray(arr)
836 # don't force numpy coerce with nan's
837 inferred = lib.infer_dtype(arr, skipna=False)
838 if inferred in ["string", "bytes", "mixed", "mixed-integer"]:
839 return (np.dtype(np.object_), arr)
841 arr = np.asarray(arr)
842 return arr.dtype, arr
845def _maybe_infer_dtype_type(element: object) -> DtypeObj | None:
846 """
847 Try to infer an object's dtype, for use in arithmetic ops.
849 Uses `element.dtype` if that's available.
850 Objects implementing the iterator protocol are cast to a NumPy array,
851 and from there the array's type is used.
853 Parameters
854 ----------
855 element : object
856 Possibly has a `.dtype` attribute, and possibly the iterator
857 protocol.
859 Returns
860 -------
861 tipo : type
863 Examples
864 --------
865 >>> from collections import namedtuple
866 >>> Foo = namedtuple("Foo", "dtype")
867 >>> _maybe_infer_dtype_type(Foo(np.dtype("i8")))
868 dtype('int64')
869 """
870 tipo = None
871 if hasattr(element, "dtype"):
872 tipo = element.dtype
873 elif is_list_like(element):
874 element = np.asarray(element)
875 tipo = element.dtype
876 return tipo
879def coerce_indexer_dtype(indexer: np.ndarray, categories: Index) -> np.ndarray:
880 """coerce the indexer input array to the smallest dtype possible"""
881 length = len(categories)
882 if length < _int8_max:
883 return ensure_int8(indexer)
884 elif length < _int16_max:
885 return ensure_int16(indexer)
886 elif length < _int32_max:
887 return ensure_int32(indexer)
888 return ensure_int64(indexer)
891def convert_dtypes(
892 input_array: ArrayLike,
893 convert_string: bool = True,
894 convert_integer: bool = True,
895 convert_boolean: bool = True,
896 convert_floating: bool = True,
897 infer_objects: bool = False,
898 dtype_backend: Literal["numpy_nullable", "pyarrow"] = "numpy_nullable",
899) -> DtypeObj:
900 """
901 Convert objects to best possible type, and optionally,
902 to types supporting ``pd.NA``.
904 Parameters
905 ----------
906 input_array : ExtensionArray or np.ndarray
907 convert_string : bool, default True
908 Whether object dtypes should be converted to ``StringDtype()``.
909 convert_integer : bool, default True
910 Whether, if possible, conversion can be done to integer extension types.
911 convert_boolean : bool, defaults True
912 Whether object dtypes should be converted to ``BooleanDtypes()``.
913 convert_floating : bool, defaults True
914 Whether, if possible, conversion can be done to floating extension types.
915 If `convert_integer` is also True, preference will be give to integer
916 dtypes if the floats can be faithfully casted to integers.
917 infer_objects : bool, defaults False
918 Whether to also infer objects to float/int if possible. Is only hit if the
919 object array contains pd.NA.
920 dtype_backend : {'numpy_nullable', 'pyarrow'}, default 'numpy_nullable'
921 Back-end data type applied to the resultant :class:`DataFrame`
922 (still experimental). Behaviour is as follows:
924 * ``"numpy_nullable"``: returns nullable-dtype
925 * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype`
927 .. versionadded:: 2.0
929 Returns
930 -------
931 np.dtype, or ExtensionDtype
932 """
933 from pandas.core.arrays.string_ import StringDtype
935 inferred_dtype: str | DtypeObj
937 if (
938 convert_string or convert_integer or convert_boolean or convert_floating
939 ) and isinstance(input_array, np.ndarray):
940 if input_array.dtype.kind == "c":
941 return input_array.dtype
943 if input_array.dtype == object:
944 inferred_dtype = lib.infer_dtype(input_array)
945 else:
946 inferred_dtype = input_array.dtype
948 if is_string_dtype(inferred_dtype):
949 if not convert_string or inferred_dtype == "bytes":
950 inferred_dtype = input_array.dtype
951 else:
952 inferred_dtype = pandas_dtype_func("string")
954 if convert_integer:
955 target_int_dtype = pandas_dtype_func("Int64")
957 if input_array.dtype.kind in "iu":
958 from pandas.core.arrays.integer import NUMPY_INT_TO_DTYPE
960 inferred_dtype = NUMPY_INT_TO_DTYPE.get(
961 input_array.dtype, target_int_dtype
962 )
963 elif input_array.dtype.kind in "fb":
964 arr = input_array[notna(input_array)]
965 if len(arr) < len(input_array) and not is_nan_na():
966 # In the presence of NaNs, we cannot convert to IntegerDtype
967 pass
968 elif (
969 floats_fit_integer_dtype(arr, np.dtype(int))
970 and (arr.astype(int) == arr).all()
971 ):
972 inferred_dtype = target_int_dtype
973 else:
974 inferred_dtype = input_array.dtype
975 elif (
976 infer_objects
977 and input_array.dtype == object
978 and (isinstance(inferred_dtype, str) and inferred_dtype == "integer")
979 ):
980 # GH#66517 the values need not fit in int64; ask for the dtype
981 # that can actually hold them and retain object if there is none
982 maybe_casted = lib.maybe_convert_objects(
983 input_array.ravel(), convert_to_nullable_dtype=True
984 )
985 if isinstance(maybe_casted.dtype, BaseMaskedDtype):
986 inferred_dtype = maybe_casted.dtype
987 else:
988 inferred_dtype = input_array.dtype
990 if convert_floating:
991 if input_array.dtype.kind in "fb":
992 # i.e. numeric but not integer
993 from pandas.core.arrays.floating import NUMPY_FLOAT_TO_DTYPE
995 inferred_float_dtype: DtypeObj = NUMPY_FLOAT_TO_DTYPE.get(
996 input_array.dtype, pandas_dtype_func("Float64")
997 )
998 # if we could also convert to integer, check if all floats
999 # are actually integers
1000 if convert_integer:
1001 arr = input_array[notna(input_array)]
1002 if len(arr) < len(input_array) and not is_nan_na():
1003 # In the presence of NaNs, we can't convert to IntegerDtype
1004 inferred_dtype = inferred_float_dtype
1005 elif (
1006 floats_fit_integer_dtype(arr, np.dtype(int))
1007 and (arr.astype(int) == arr).all()
1008 ):
1009 inferred_dtype = pandas_dtype_func("Int64")
1010 else:
1011 inferred_dtype = inferred_float_dtype
1012 else:
1013 inferred_dtype = inferred_float_dtype
1014 elif (
1015 infer_objects
1016 and input_array.dtype == object
1017 and (isinstance(inferred_dtype, str) and inferred_dtype == "floating")
1018 ):
1019 inferred_dtype = pandas_dtype_func("Float64")
1021 if convert_boolean:
1022 if input_array.dtype.kind == "b":
1023 inferred_dtype = pandas_dtype_func("boolean")
1024 elif isinstance(inferred_dtype, str) and inferred_dtype == "boolean":
1025 inferred_dtype = pandas_dtype_func("boolean")
1027 if isinstance(inferred_dtype, str):
1028 # If we couldn't do anything else, then we retain the dtype
1029 inferred_dtype = input_array.dtype
1031 elif (
1032 convert_string
1033 and isinstance(input_array.dtype, StringDtype)
1034 and input_array.dtype.na_value is np.nan
1035 ):
1036 inferred_dtype = pandas_dtype_func("string")
1038 else:
1039 inferred_dtype = input_array.dtype
1041 if dtype_backend == "pyarrow" and not isinstance(inferred_dtype, ArrowDtype):
1042 from pandas.core.arrays.arrow.array import to_pyarrow_type
1043 from pandas.core.arrays.string_ import StringDtype
1045 assert not isinstance(inferred_dtype, str)
1047 if (
1048 (convert_integer and inferred_dtype.kind in "iu")
1049 or (convert_floating and inferred_dtype.kind in "f")
1050 or (convert_boolean and inferred_dtype.kind == "b")
1051 or (convert_string and isinstance(inferred_dtype, StringDtype))
1052 or (
1053 inferred_dtype.kind not in "iufb"
1054 and not isinstance(inferred_dtype, StringDtype)
1055 and not isinstance(inferred_dtype, CategoricalDtype)
1056 )
1057 ):
1058 if isinstance(inferred_dtype, PandasExtensionDtype) and not isinstance(
1059 inferred_dtype, DatetimeTZDtype
1060 ):
1061 base_dtype = inferred_dtype.base
1062 elif isinstance(inferred_dtype, (BaseMaskedDtype, ArrowDtype)):
1063 base_dtype = inferred_dtype.numpy_dtype
1064 elif isinstance(inferred_dtype, StringDtype):
1065 base_dtype = np.dtype(str)
1066 else:
1067 base_dtype = inferred_dtype
1068 if (
1069 base_dtype.kind == "O" # type: ignore[union-attr]
1070 and input_array.size > 0
1071 and isna(input_array).all()
1072 ):
1073 import pyarrow as pa
1075 pa_type = pa.null()
1076 else:
1077 pa_type = to_pyarrow_type(base_dtype)
1078 if pa_type is not None:
1079 inferred_dtype = ArrowDtype(pa_type)
1080 elif dtype_backend == "numpy_nullable" and isinstance(inferred_dtype, ArrowDtype):
1081 # GH 53648
1082 inferred_dtype = _arrow_dtype_mapping()[inferred_dtype.pyarrow_dtype]
1084 assert not isinstance(inferred_dtype, str)
1085 return inferred_dtype
1088def maybe_cast_to_datetime(
1089 value: np.ndarray | list[Any], dtype: np.dtype
1090) -> DatetimeArray | TimedeltaArray | np.ndarray:
1091 """
1092 try to cast the array/value to a datetimelike dtype, converting float
1093 nan to iNaT
1095 Caller is responsible for handling ExtensionDtype cases and non dt64/td64
1096 cases.
1097 """
1098 from pandas.core.arrays.datetimes import DatetimeArray
1099 from pandas.core.arrays.timedeltas import TimedeltaArray
1101 assert dtype.kind in "mM"
1102 if not is_list_like(value):
1103 raise TypeError("value must be listlike")
1105 # TODO: _from_sequence would raise ValueError in cases where
1106 # _ensure_nanosecond_dtype raises TypeError
1107 _ensure_nanosecond_dtype(dtype)
1109 if lib.is_np_dtype(dtype, "m"):
1110 res = TimedeltaArray._from_sequence(value, dtype=dtype)
1111 return res
1112 else:
1113 try:
1114 dta = DatetimeArray._from_sequence(value, dtype=dtype)
1115 except ValueError as err:
1116 # We can give a Series-specific exception message.
1117 if "cannot supply both a tz and a timezone-naive dtype" in str(err):
1118 raise ValueError(
1119 "Cannot convert timezone-aware data to "
1120 "timezone-naive dtype. Use "
1121 "pd.Series(values).dt.tz_localize(None) instead."
1122 ) from err
1123 raise
1125 return dta
1128def _ensure_nanosecond_dtype(dtype: DtypeObj) -> None:
1129 """
1130 Convert dtypes with granularity less than nanosecond to nanosecond
1132 >>> _ensure_nanosecond_dtype(np.dtype("M8[us]"))
1134 >>> _ensure_nanosecond_dtype(np.dtype("M8[D]"))
1135 Traceback (most recent call last):
1136 ...
1137 TypeError: dtype=datetime64[D] is not supported. Supported resolutions are 's', 'ms', 'us', and 'ns'
1139 >>> _ensure_nanosecond_dtype(np.dtype("m8[ps]"))
1140 Traceback (most recent call last):
1141 ...
1142 TypeError: dtype=timedelta64[ps] is not supported. Supported resolutions are 's', 'ms', 'us', and 'ns'
1143 """ # noqa: E501
1144 msg = (
1145 f"The '{dtype.name}' dtype has no unit. "
1146 f"Please pass in '{dtype.name}[ns]' instead."
1147 )
1149 # unpack e.g. SparseDtype
1150 dtype = getattr(dtype, "subtype", dtype)
1152 if not isinstance(dtype, np.dtype):
1153 # i.e. datetime64tz
1154 pass
1156 elif dtype.kind in "mM":
1157 if not is_supported_dtype(dtype):
1158 # pre-2.0 we would silently swap in nanos for lower-resolutions,
1159 # raise for above-nano resolutions
1160 if dtype.name in ["datetime64", "timedelta64"]:
1161 raise ValueError(msg)
1162 # TODO: ValueError or TypeError? existing test
1163 # test_constructor_generic_timestamp_bad_frequency expects TypeError
1164 raise TypeError(
1165 f"dtype={dtype} is not supported. Supported resolutions are 's', "
1166 "'ms', 'us', and 'ns'"
1167 )
1170# TODO: other value-dependent functions to standardize here include
1171# Index._find_common_type_compat
1172def find_result_type(left_dtype: DtypeObj, right: Any) -> DtypeObj:
1173 """
1174 Find the type/dtype for the result of an operation between objects.
1176 This is similar to find_common_type, but looks at the right object instead
1177 of just its dtype. This can be useful in particular when the right
1178 object does not have a `dtype`.
1180 Parameters
1181 ----------
1182 left_dtype : np.dtype or ExtensionDtype
1183 right : Any
1185 Returns
1186 -------
1187 np.dtype or ExtensionDtype
1189 See also
1190 --------
1191 find_common_type
1192 numpy.result_type
1193 """
1194 new_dtype: DtypeObj
1196 if (
1197 isinstance(left_dtype, np.dtype)
1198 and left_dtype.kind in "iuc"
1199 and (lib.is_integer(right) or lib.is_float(right))
1200 ):
1201 # e.g. with int8 dtype and right=512, we want to end up with
1202 # np.int16, whereas infer_dtype_from(512) gives np.int64,
1203 # which will make us upcast too far.
1204 if lib.is_float(right) and right.is_integer() and left_dtype.kind != "f":
1205 right = int(right)
1206 # After NEP 50, numpy won't inspect Python scalars
1207 # TODO: do we need to recreate numpy's inspection logic for floats too
1208 # (this breaks some tests)
1209 if isinstance(right, int) and not isinstance(right, np.integer):
1210 # This gives an unsigned type by default
1211 # (if our number is positive)
1213 # If our left dtype is signed, we might not want this since
1214 # this might give us 1 dtype too big
1215 # We should check if the corresponding int dtype (e.g. int64 for uint64)
1216 # can hold the number
1217 right_dtype = np.min_scalar_type(right)
1218 if right == 0:
1219 # Special case 0
1220 right = left_dtype
1221 elif (
1222 not np.issubdtype(left_dtype, np.unsignedinteger)
1223 and 0 < right <= np.iinfo(right_dtype).max
1224 ):
1225 # If left dtype isn't unsigned, check if it fits in the signed dtype
1226 right = np.dtype(f"i{right_dtype.itemsize}")
1227 else:
1228 right = right_dtype
1230 new_dtype = np.result_type(left_dtype, right)
1232 elif is_valid_na_for_dtype(right, left_dtype):
1233 # e.g. IntervalDtype[int] and None/np.nan
1234 new_dtype = ensure_dtype_can_hold_na(left_dtype)
1236 else:
1237 dtype, _ = infer_dtype_from(right)
1238 new_dtype = find_common_type([left_dtype, dtype])
1240 return new_dtype
1243def common_dtype_categorical_compat(
1244 objs: Sequence[Index | ArrayLike], dtype: DtypeObj
1245) -> DtypeObj:
1246 """
1247 Update the result of find_common_type to account for NAs in a Categorical.
1249 Parameters
1250 ----------
1251 objs : list[np.ndarray | ExtensionArray | Index]
1252 dtype : np.dtype or ExtensionDtype
1254 Returns
1255 -------
1256 np.dtype or ExtensionDtype
1257 """
1258 # GH#38240
1260 # TODO: more generally, could do `not can_hold_na(dtype)`
1261 if lib.is_np_dtype(dtype, "iu"):
1262 for obj in objs:
1263 # We don't want to accidentally allow e.g. "categorical" str here
1264 obj_dtype = getattr(obj, "dtype", None)
1265 if isinstance(obj_dtype, CategoricalDtype):
1266 if isinstance(obj, ABCIndex):
1267 # This check may already be cached
1268 hasnas = obj.hasnans
1269 else:
1270 # Categorical
1271 hasnas = cast("Categorical", obj)._hasna
1273 if hasnas:
1274 # see test_union_int_categorical_with_nan
1275 dtype = np.dtype(np.float64)
1276 break
1277 return dtype
1280def np_find_common_type(*dtypes: np.dtype) -> np.dtype:
1281 """
1282 np.find_common_type implementation pre-1.25 deprecation using np.result_type
1283 https://github.com/pandas-dev/pandas/pull/49569#issuecomment-1308300065
1285 Parameters
1286 ----------
1287 dtypes : np.dtypes
1289 Returns
1290 -------
1291 np.dtype
1292 """
1293 try:
1294 common_dtype = np.result_type(*dtypes)
1295 if common_dtype.kind in "mMSU":
1296 # NumPy promotion currently (1.25) misbehaves for times and strings,
1297 # so fall back to object (find_common_dtype did unless there
1298 # was only one dtype)
1299 common_dtype = np.dtype("O")
1301 except TypeError:
1302 common_dtype = np.dtype("O")
1303 return common_dtype
1306@overload
1307def find_common_type(types: list[np.dtype]) -> np.dtype: ...
1310@overload
1311def find_common_type(types: list[ExtensionDtype]) -> DtypeObj: ...
1314@overload
1315def find_common_type(types: list[DtypeObj]) -> DtypeObj: ...
1318def find_common_type(types: list[DtypeObj]) -> DtypeObj: # type: ignore[misc]
1319 """
1320 Find a common data type among the given dtypes.
1322 Parameters
1323 ----------
1324 types : list of dtypes
1326 Returns
1327 -------
1328 pandas extension or numpy dtype
1330 See Also
1331 --------
1332 numpy.find_common_type
1334 """
1335 if not types:
1336 raise ValueError("no types given")
1338 first = types[0]
1340 # workaround for find_common_type([np.dtype('datetime64[ns]')] * 2)
1341 # => object
1342 if lib.dtypes_all_equal(list(types)):
1343 return first
1345 # get unique types (dict.fromkeys is used as order-preserving set())
1346 types = list(dict.fromkeys(types).keys())
1348 if any(isinstance(t, ExtensionDtype) for t in types):
1349 for t in types:
1350 if isinstance(t, ExtensionDtype):
1351 res = t._get_common_dtype(types)
1352 if res is not None:
1353 return res
1354 return np.dtype("object")
1356 # At this point, all types are np.dtype (ExtensionDtype was handled above)
1357 np_types = cast("list[np.dtype]", types)
1359 # take lowest unit
1360 if all(lib.is_np_dtype(t, "M") for t in np_types):
1361 return np.dtype(max(np_types))
1362 if all(lib.is_np_dtype(t, "m") for t in np_types):
1363 return np.dtype(max(np_types))
1365 # don't mix bool / int or float or complex
1366 # this is different from numpy, which casts bool with float/int as int
1367 has_bools = any(t.kind == "b" for t in np_types)
1368 if has_bools:
1369 for t in np_types:
1370 if t.kind in "iufc":
1371 return np.dtype("object")
1373 return np_find_common_type(*np_types)
1376def construct_2d_arraylike_from_scalar(
1377 value: Scalar, length: int, width: int, dtype: np.dtype, copy: bool
1378) -> np.ndarray:
1379 shape = (length, width)
1381 if dtype.kind in "mM":
1382 value = _maybe_box_and_unbox_datetimelike(value, dtype)
1383 elif dtype == _dtype_obj:
1384 if isinstance(value, (np.timedelta64, np.datetime64)):
1385 # calling np.array below would cast to pytimedelta/pydatetime
1386 out = np.empty(shape, dtype=object)
1387 out.fill(value)
1388 return out
1390 # Attempt to coerce to a numpy array
1391 try:
1392 if not copy:
1393 arr = np.asarray(value, dtype=dtype)
1394 else:
1395 arr = np.array(value, dtype=dtype, copy=copy)
1396 except (ValueError, TypeError) as err:
1397 raise TypeError(
1398 f"DataFrame constructor called with incompatible data and dtype: {err}"
1399 ) from err
1401 if arr.ndim != 0:
1402 raise ValueError("DataFrame constructor not properly called!")
1404 return np.full(shape, arr)
1407def construct_1d_arraylike_from_scalar(
1408 value: Scalar, length: int, dtype: DtypeObj | None
1409) -> ArrayLike:
1410 """
1411 create an np.ndarray / pandas type of specified shape and dtype
1412 filled with values
1414 Parameters
1415 ----------
1416 value : scalar value
1417 length : int
1418 dtype : pandas_dtype or np.dtype
1420 Returns
1421 -------
1422 np.ndarray / pandas type of length, filled with value
1424 """
1426 if dtype is None:
1427 try:
1428 dtype, value = infer_dtype_from_scalar(value)
1429 except OutOfBoundsDatetime:
1430 dtype = _dtype_obj
1432 if isinstance(dtype, ExtensionDtype):
1433 cls = dtype.construct_array_type()
1434 seq = [] if length == 0 else [value]
1435 return cls._from_sequence(seq, dtype=dtype).repeat(length)
1437 if length and dtype.kind in "iu" and isna(value):
1438 # coerce if we have nan for an integer dtype
1439 dtype = np.dtype("float64")
1440 elif lib.is_np_dtype(dtype, "US"):
1441 # we need to coerce to object dtype to avoid
1442 # to allow numpy to take our string as a scalar value
1443 dtype = np.dtype("object")
1444 if not isna(value):
1445 value = ensure_str(value)
1446 elif dtype.kind in "mM":
1447 value = _maybe_box_and_unbox_datetimelike(value, dtype)
1449 subarr = np.empty(length, dtype=dtype)
1450 if length:
1451 # GH 47391: numpy > 1.24 will raise filling np.nan into int dtypes
1452 subarr.fill(value)
1454 return subarr
1457def maybe_unbox_numpy_scalar(value: Any, *, object_with_dtype: Any = None) -> Any:
1458 """
1459 Maybe convert a NumPy scalar to its Python equivalent.
1461 If future.python_scalars is disabled or ``value`` is not a NumPy scalar, ``value``
1462 is returned unchanged. ``np.datetime64`` and ``np.timedelta64`` values are
1463 converted to ``Timestamp`` and ``Timedelta`` respectively. Converting
1464 ``np.longdouble`` to ``float`` and ``np.clongdouble`` to ``complex`` can
1465 lose precision.
1467 Parameters
1468 ----------
1469 value : Any
1470 The value to unbox.
1471 object_with_dtype : object with a ``dtype`` attribute, optional
1472 The data (e.g. a Series or array) that ``value`` came from. Pass this
1473 whenever ``value`` is an element of the data or is derived from its
1474 elements: object dtype stores arbitrary user objects, so a NumPy
1475 scalar coming from object-dtype data is a stored value rather than a
1476 boxing artifact, and is returned unchanged. Omit for values whose
1477 type does not follow the data's dtype, e.g. positions, counts, and
1478 the results of any/all. The ``.dtype`` attribute is only accessed if
1479 unboxing is actually going to happen, so passing the object rather
1480 than its dtype avoids that cost on hot paths.
1482 Returns
1483 -------
1484 Any
1485 The equivalent Python scalar, or ``value`` unchanged.
1486 """
1487 result = value
1488 if using_python_scalars() and isinstance(value, np.generic):
1489 if object_with_dtype is not None and object_with_dtype.dtype == object:
1490 return result
1491 if isinstance(result, np.longdouble):
1492 result = float(result)
1493 elif isinstance(result, np.clongdouble):
1494 result = complex(result)
1495 elif isinstance(result, np.datetime64):
1496 result = Timestamp(result)
1497 elif isinstance(result, np.timedelta64):
1498 result = Timedelta(result)
1499 else:
1500 result = value.item()
1501 return result
1504def _maybe_box_and_unbox_datetimelike(value: Scalar, dtype: DtypeObj) -> Scalar:
1505 # Caller is responsible for checking dtype.kind in "mM"
1507 if isinstance(value, dt.datetime):
1508 # we dont want to box dt64, in particular datetime64("NaT")
1509 value = maybe_box_datetimelike(value, dtype)
1511 return _maybe_unbox_datetimelike(value, dtype)
1514def construct_1d_object_array_from_listlike(values: Collection[Any]) -> np.ndarray:
1515 """
1516 Transform any list-like object in a 1-dimensional numpy array of object
1517 dtype.
1519 Parameters
1520 ----------
1521 values : any iterable which has a len()
1523 Raises
1524 ------
1525 TypeError
1526 * If `values` does not have a len()
1528 Returns
1529 -------
1530 1-dimensional numpy array of dtype object
1531 """
1532 # numpy will try to interpret nested lists as further dimensions in np.array(),
1533 # hence explicitly making a 1D array using np.fromiter
1534 return np.fromiter(values, dtype="object", count=len(values))
1537def maybe_cast_to_integer_array(
1538 arr: list[Any] | np.ndarray, dtype: np.dtype
1539) -> np.ndarray:
1540 """
1541 Takes any dtype and returns the casted version, raising for when data is
1542 incompatible with integer/unsigned integer dtypes.
1544 Parameters
1545 ----------
1546 arr : np.ndarray or list
1547 The array to cast.
1548 dtype : np.dtype
1549 The integer dtype to cast the array to.
1551 Returns
1552 -------
1553 ndarray
1554 Array of integer or unsigned integer dtype.
1556 Raises
1557 ------
1558 OverflowError : the dtype is incompatible with the data
1559 ValueError : loss of precision has occurred during casting
1561 Examples
1562 --------
1563 If you try to coerce negative values to unsigned integers, it raises:
1565 >>> pd.Series([-1], dtype="uint64")
1566 Traceback (most recent call last):
1567 ...
1568 OverflowError: Trying to coerce negative values to unsigned integers
1570 Also, if you try to coerce float values to integers, it raises:
1572 >>> maybe_cast_to_integer_array([1, 2, 3.5], dtype=np.dtype("int64"))
1573 Traceback (most recent call last):
1574 ...
1575 ValueError: Trying to coerce float values to integers
1576 """
1577 assert dtype.kind in "iu"
1579 try:
1580 if not isinstance(arr, np.ndarray):
1581 with warnings.catch_warnings():
1582 # We already disallow dtype=uint w/ negative numbers
1583 # (test_constructor_coercion_signed_to_unsigned) so safe to ignore.
1584 warnings.filterwarnings(
1585 "ignore",
1586 "NumPy will stop allowing conversion of out-of-bound Python int",
1587 DeprecationWarning,
1588 )
1589 casted = np.asarray(arr, dtype=dtype)
1590 else:
1591 with warnings.catch_warnings():
1592 warnings.filterwarnings("ignore", category=RuntimeWarning)
1593 casted = arr.astype(dtype, copy=False)
1594 except OverflowError as err:
1595 raise OverflowError(
1596 "The elements provided in the data cannot all be "
1597 f"casted to the dtype {dtype}"
1598 ) from err
1600 if isinstance(arr, np.ndarray) and arr.dtype == dtype:
1601 # avoid expensive array_equal check
1602 return casted
1604 with warnings.catch_warnings():
1605 warnings.filterwarnings("ignore", category=RuntimeWarning)
1606 warnings.filterwarnings(
1607 "ignore", "elementwise comparison failed", FutureWarning
1608 )
1609 if np.array_equal(arr, casted):
1610 return casted
1612 # We do this casting to allow for proper
1613 # data and dtype checking.
1614 #
1615 # We didn't do this earlier because NumPy
1616 # doesn't handle `uint64` correctly.
1617 arr = np.asarray(arr)
1619 if np.issubdtype(arr.dtype, str):
1620 # TODO(numpy-2.0 min): This case will raise an OverflowError above
1621 if (casted.astype(str) == arr).all():
1622 return casted
1623 raise ValueError(f"string values cannot be losslessly cast to {dtype}")
1625 if dtype.kind == "u" and (arr < 0).any():
1626 # TODO: can this be hit anymore after numpy 2.0?
1627 raise OverflowError("Trying to coerce negative values to unsigned integers")
1629 if arr.dtype.kind == "f":
1630 if not np.isfinite(arr).all():
1631 raise IntCastingNaNError(
1632 "Cannot convert non-finite values (NA or inf) to integer"
1633 )
1634 raise ValueError("Trying to coerce float values to integers")
1635 if arr.dtype == object:
1636 raise ValueError("Trying to coerce object values to integers")
1638 if casted.dtype < arr.dtype:
1639 # TODO: Can this path be hit anymore with numpy > 2
1640 # GH#41734 e.g. [1, 200, 923442] and dtype="int8" -> overflows
1641 raise ValueError(
1642 f"Values are too large to be losslessly converted to {dtype}. "
1643 f"To cast anyway, use pd.Series(values).astype({dtype})"
1644 )
1646 if arr.dtype.kind in "mM":
1647 # test_constructor_maskedarray_nonfloat
1648 raise TypeError(
1649 f"Constructing a Series or DataFrame from {arr.dtype} values and "
1650 f"dtype={dtype} is not supported. Use values.view({dtype}) instead."
1651 )
1653 # No known cases that get here, but raising explicitly to cover our bases.
1654 raise ValueError(f"values cannot be losslessly cast to {dtype}")
1657def can_hold_element(arr: ArrayLike, element: Any) -> bool:
1658 """
1659 Can we do an inplace setitem with this element in an array with this dtype?
1661 Parameters
1662 ----------
1663 arr : np.ndarray or ExtensionArray
1664 element : Any
1666 Returns
1667 -------
1668 bool
1669 """
1670 dtype = arr.dtype
1671 if not isinstance(dtype, np.dtype) or dtype.kind in "mM":
1672 if isinstance(dtype, (PeriodDtype, IntervalDtype, DatetimeTZDtype, np.dtype)):
1673 # np.dtype here catches datetime64ns and timedelta64ns; we assume
1674 # in this case that we have DatetimeArray/TimedeltaArray
1675 arr = cast(
1676 "PeriodArray | DatetimeArray | TimedeltaArray | IntervalArray", arr
1677 )
1678 try:
1679 arr._validate_setitem_value(element) # type: ignore[no-untyped-call]
1680 return True
1681 except (ValueError, TypeError):
1682 return False
1684 if dtype == "string":
1685 try:
1686 arr._validate_setitem_value(element) # type: ignore[union-attr]
1687 return True
1688 except (ValueError, TypeError):
1689 return False
1691 # This is technically incorrect, but maintains the behavior of
1692 # ExtensionBlock._can_hold_element
1693 return True
1695 try:
1696 np_can_hold_element(dtype, element)
1697 return True
1698 except (TypeError, LossySetitemError):
1699 return False
1702def np_can_hold_element(dtype: np.dtype, element: Any) -> Any:
1703 """
1704 Raise if we cannot losslessly set this element into an ndarray with this dtype.
1706 Specifically about places where we disagree with numpy. i.e. there are
1707 cases where numpy will raise in doing the setitem that we do not check
1708 for here, e.g. setting str "X" into a numeric ndarray.
1710 Returns
1711 -------
1712 Any
1713 The element, potentially cast to the dtype.
1715 Raises
1716 ------
1717 ValueError : If we cannot losslessly store this element with this dtype.
1718 """
1719 if dtype == _dtype_obj:
1720 return element
1722 tipo = _maybe_infer_dtype_type(element)
1724 if dtype.kind in "iu":
1725 if isinstance(element, range):
1726 if _dtype_can_hold_range(element, dtype):
1727 return element
1728 raise LossySetitemError
1730 if is_integer(element) or (is_float(element) and element.is_integer()):
1731 # e.g. test_setitem_series_int8 if we have a python int 1
1732 # tipo may be np.int32, despite the fact that it will fit
1733 # in smaller int dtypes.
1734 info = np.iinfo(dtype)
1735 if info.min <= element <= info.max:
1736 return dtype.type(element)
1737 raise LossySetitemError
1739 if tipo is not None:
1740 if tipo.kind not in "iu":
1741 if isinstance(element, np.ndarray) and element.dtype.kind == "f":
1742 # If all can be losslessly cast to integers, then we can hold them
1743 with np.errstate(invalid="ignore"):
1744 # We check afterwards if cast was losslessly, so no need to show
1745 # the warning
1746 casted = element.astype(dtype)
1747 comp = casted == element
1748 if comp.all():
1749 # Return the casted values bc they can be passed to
1750 # np.putmask, whereas the raw values cannot.
1751 # see TestSetitemFloatNDarrayIntoIntegerSeries
1752 return casted
1753 raise LossySetitemError
1755 elif isinstance(element, ABCExtensionArray) and isinstance(
1756 element.dtype, CategoricalDtype
1757 ):
1758 # GH#52927 setting Categorical value into non-EA frame
1759 # TODO: general-case for EAs?
1760 try:
1761 casted = element.astype(dtype)
1762 except (ValueError, TypeError) as err:
1763 raise LossySetitemError from err
1764 # Check for cases of either
1765 # a) lossy overflow/rounding or
1766 # b) semantic changes like dt64->int64
1767 comp = casted == element
1768 if not comp.all():
1769 raise LossySetitemError
1770 return casted
1772 # Anything other than integer we cannot hold
1773 raise LossySetitemError
1774 if (
1775 dtype.kind == "u"
1776 and isinstance(element, np.ndarray)
1777 and element.dtype.kind == "i"
1778 ):
1779 # see test_where_uint64
1780 casted = element.astype(dtype)
1781 if (casted == element).all():
1782 # TODO: faster to check (element >=0).all()? potential
1783 # itemsize issues there?
1784 return casted
1785 raise LossySetitemError
1786 if dtype.itemsize < tipo.itemsize: # type: ignore[union-attr]
1787 raise LossySetitemError
1788 if not isinstance(tipo, np.dtype):
1789 # i.e. nullable IntegerDtype; we can put this into an ndarray
1790 # losslessly iff it has no NAs and the values themselves fit
1791 arr = (
1792 element._values
1793 if isinstance(element, (ABCIndex, ABCSeries))
1794 else element
1795 )
1796 if arr._hasna: # type: ignore[union-attr]
1797 raise LossySetitemError
1798 # GH#47776 re-run the ndarray guards on the NA-free values, e.g.
1799 # to reject a negative value going into an unsigned dtype.
1800 np_can_hold_element(dtype, np.asarray(arr))
1801 return element
1803 return element
1805 raise LossySetitemError
1807 if dtype.kind == "f":
1808 if lib.is_integer(element) or lib.is_float(element):
1809 casted = dtype.type(element)
1810 if np.isnan(casted) or casted == element:
1811 return casted
1812 # otherwise e.g. overflow see TestCoercionFloat32
1813 raise LossySetitemError
1815 if tipo is not None:
1816 # TODO: itemsize check?
1818 if isinstance(tipo, CategoricalDtype):
1819 # GH#56376
1820 if tipo.categories.dtype.kind not in "iuf":
1821 # Anything other than float/integer we cannot hold
1822 raise LossySetitemError
1823 casted = np.asarray(element, dtype=dtype)
1824 if np.array_equal(casted, element, equal_nan=True):
1825 return casted
1826 raise LossySetitemError
1828 if tipo.kind not in "iuf":
1829 # Anything other than float/integer we cannot hold
1830 raise LossySetitemError
1831 if not isinstance(tipo, np.dtype):
1832 # i.e. nullable IntegerDtype or FloatingDtype;
1833 # we can put this into an ndarray losslessly iff it has no NAs
1834 # and the values themselves fit
1835 arr = (
1836 element._values
1837 if isinstance(element, (ABCIndex, ABCSeries))
1838 else element
1839 )
1840 if arr._hasna: # type: ignore[union-attr]
1841 raise LossySetitemError
1842 # GH#47776 re-run the ndarray guards on the NA-free values, e.g.
1843 # to reject a value that overflows the target float dtype.
1844 np_can_hold_element(dtype, np.asarray(arr))
1845 return element
1846 elif tipo.itemsize > dtype.itemsize or tipo.kind != dtype.kind:
1847 if isinstance(element, np.ndarray):
1848 # e.g. TestDataFrameIndexingWhere::test_where_alignment
1849 with np.errstate(over="ignore", invalid="ignore"):
1850 # losslessness is checked below, so cast warnings
1851 # (e.g. overflow) are spurious
1852 casted = element.astype(dtype)
1853 if np.array_equal(casted, element, equal_nan=True):
1854 return casted
1855 raise LossySetitemError
1857 return element
1859 raise LossySetitemError
1861 if dtype.kind == "c":
1862 if lib.is_integer(element) or lib.is_complex(element) or lib.is_float(element):
1863 if np.isnan(element):
1864 # see test_where_complex GH#6345
1865 return dtype.type(element)
1867 with warnings.catch_warnings():
1868 warnings.filterwarnings("ignore")
1869 casted = dtype.type(element)
1870 if casted == element:
1871 return casted
1872 # otherwise e.g. overflow see test_32878_complex_itemsize
1873 raise LossySetitemError
1875 if tipo is not None:
1876 if tipo.kind in "iufc":
1877 return element
1878 raise LossySetitemError
1879 raise LossySetitemError
1881 if dtype.kind == "b":
1882 if tipo is not None:
1883 if tipo.kind == "b":
1884 if not isinstance(tipo, np.dtype):
1885 # i.e. we have a BooleanArray
1886 arr = (
1887 element._values
1888 if isinstance(element, (ABCIndex, ABCSeries))
1889 else element
1890 )
1891 if arr._hasna: # type: ignore[union-attr]
1892 # i.e. there are pd.NA elements
1893 raise LossySetitemError
1894 return element
1895 # GH 57338 check boolean array set as object type
1896 if tipo.kind == "O" and isinstance(element, np.ndarray):
1897 if lib.is_bool_array(element):
1898 return element.astype("bool")
1899 raise LossySetitemError
1900 if lib.is_bool(element):
1901 return element
1902 raise LossySetitemError
1904 if dtype.kind == "S":
1905 # TODO: test tests.frame.methods.test_replace tests get here,
1906 # need more targeted tests. xref phofl has a PR about this
1907 if tipo is not None:
1908 if tipo.kind == "S" and tipo.itemsize <= dtype.itemsize: # type: ignore[union-attr]
1909 return element
1910 raise LossySetitemError
1911 if isinstance(element, bytes) and len(element) <= dtype.itemsize:
1912 return element
1913 raise LossySetitemError
1915 if dtype.kind == "V":
1916 # i.e. np.void, which cannot hold _anything_
1917 raise LossySetitemError
1919 raise NotImplementedError(dtype)
1922def _dtype_can_hold_range(rng: range, dtype: np.dtype) -> bool:
1923 """
1924 _maybe_infer_dtype_type infers to int64 (and float64 for very large endpoints),
1925 but in many cases a range can be held by a smaller integer dtype.
1926 Check if this is one of those cases.
1927 """
1928 if not len(rng):
1929 return True
1930 return np_can_cast_scalar(rng.start, dtype) and np_can_cast_scalar(rng.stop, dtype)
1933def np_can_cast_scalar(element: Scalar, dtype: np.dtype) -> bool:
1934 """
1935 np.can_cast pandas-equivalent for pre 2-0 behavior that allowed scalar
1936 inference
1938 Parameters
1939 ----------
1940 element : Scalar
1941 dtype : np.dtype
1943 Returns
1944 -------
1945 bool
1946 """
1947 try:
1948 np_can_hold_element(dtype, element)
1949 return True
1950 except (LossySetitemError, NotImplementedError):
1951 return False