Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Routines for casting.
3"""
5from __future__ import annotations
7import datetime as dt
8import functools
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 Literal,
13 TypeVar,
14 cast,
15 overload,
16)
17import warnings
19import numpy as np
21from pandas._config import (
22 is_nan_na,
23 using_python_scalars,
24 using_string_dtype,
25)
27from pandas._libs import (
28 Interval,
29 Period,
30 lib,
31)
32from pandas._libs.missing import (
33 NA,
34 NAType,
35 checknull,
36)
37from pandas._libs.tslibs import (
38 NaT,
39 OutOfBoundsDatetime,
40 OutOfBoundsTimedelta,
41 Timedelta,
42 Timestamp,
43 is_supported_dtype,
44)
45from pandas._libs.tslibs.timedeltas import array_to_timedelta64
46from pandas.errors import (
47 IntCastingNaNError,
48 LossySetitemError,
49)
51from pandas.core.dtypes.common import (
52 ensure_int8,
53 ensure_int16,
54 ensure_int32,
55 ensure_int64,
56 ensure_object,
57 ensure_str,
58 is_bool,
59 is_complex,
60 is_float,
61 is_integer,
62 is_object_dtype,
63 is_scalar,
64 is_string_dtype,
65 pandas_dtype as pandas_dtype_func,
66)
67from pandas.core.dtypes.dtypes import (
68 ArrowDtype,
69 BaseMaskedDtype,
70 CategoricalDtype,
71 DatetimeTZDtype,
72 ExtensionDtype,
73 IntervalDtype,
74 PandasExtensionDtype,
75 PeriodDtype,
76)
77from pandas.core.dtypes.generic import (
78 ABCExtensionArray,
79 ABCIndex,
80 ABCSeries,
81)
82from pandas.core.dtypes.inference import is_list_like
83from pandas.core.dtypes.missing import (
84 is_valid_na_for_dtype,
85 isna,
86 na_value_for_dtype,
87 notna,
88)
90from pandas.io._util import _arrow_dtype_mapping
92if TYPE_CHECKING:
93 from collections.abc import (
94 Collection,
95 Sequence,
96 )
98 from pandas._typing import (
99 ArrayLike,
100 Dtype,
101 DtypeObj,
102 NumpyIndexT,
103 Scalar,
104 TimeUnit,
105 )
107 from pandas import Index
108 from pandas.core.arrays import (
109 Categorical,
110 DatetimeArray,
111 ExtensionArray,
112 IntervalArray,
113 PeriodArray,
114 TimedeltaArray,
115 )
118_int8_max = np.iinfo(np.int8).max
119_int16_max = np.iinfo(np.int16).max
120_int32_max = np.iinfo(np.int32).max
122_dtype_obj = np.dtype(object)
124NumpyArrayT = TypeVar("NumpyArrayT", bound=np.ndarray)
127def maybe_convert_platform(
128 values: list | tuple | range | np.ndarray | ExtensionArray,
129) -> ArrayLike:
130 """try to do platform conversion, allow ndarray or list here"""
131 arr: ArrayLike
133 if isinstance(values, (list, tuple, range)):
134 arr = construct_1d_object_array_from_listlike(values)
135 else:
136 # The caller is responsible for ensuring that we have np.ndarray
137 # or ExtensionArray here.
138 arr = values
140 if arr.dtype == _dtype_obj:
141 arr = cast(np.ndarray, arr)
142 arr = lib.maybe_convert_objects(arr)
144 return arr
147def is_nested_object(obj) -> bool:
148 """
149 return a boolean if we have a nested object, e.g. a Series with 1 or
150 more Series elements
152 This may not be necessarily be performant.
154 """
155 return bool(
156 isinstance(obj, ABCSeries)
157 and is_object_dtype(obj.dtype)
158 and any(isinstance(v, ABCSeries) for v in obj._values)
159 )
162def maybe_box_datetimelike(value: Scalar, dtype: Dtype | None = None) -> Scalar:
163 """
164 Cast scalar to Timestamp or Timedelta if scalar is datetime-like
165 and dtype is not object.
167 Parameters
168 ----------
169 value : scalar
170 dtype : Dtype, optional
172 Returns
173 -------
174 scalar
175 """
176 if dtype == _dtype_obj:
177 pass
178 elif isinstance(value, (np.datetime64, dt.datetime)):
179 value = Timestamp(value)
180 elif isinstance(value, (np.timedelta64, dt.timedelta)):
181 value = Timedelta(value)
183 return value
186def maybe_box_native(value: Scalar | None | NAType) -> Scalar | None | NAType:
187 """
188 If passed a scalar cast the scalar to a python native type.
190 Parameters
191 ----------
192 value : scalar or Series
194 Returns
195 -------
196 scalar or Series
197 """
198 if is_float(value):
199 value = float(value)
200 elif is_integer(value):
201 value = int(value)
202 elif is_bool(value):
203 value = bool(value)
204 elif isinstance(value, (np.datetime64, np.timedelta64)):
205 value = maybe_box_datetimelike(value)
206 elif value is NA:
207 value = None
208 return value
211def _maybe_unbox_datetimelike(value: Scalar, dtype: DtypeObj) -> Scalar:
212 """
213 Convert a Timedelta or Timestamp to timedelta64 or datetime64 for setting
214 into a numpy array. Failing to unbox would risk dropping nanoseconds.
216 Notes
217 -----
218 Caller is responsible for checking dtype.kind in "mM"
219 """
220 if is_valid_na_for_dtype(value, dtype):
221 # GH#36541: can't fill array directly with pd.NaT
222 # > np.empty(10, dtype="datetime64[ns]").fill(pd.NaT)
223 # ValueError: cannot convert float NaN to integer
224 value = dtype.type("NaT", "ns")
225 elif isinstance(value, Timestamp):
226 if value.tz is None:
227 value = value.to_datetime64()
228 elif not isinstance(dtype, DatetimeTZDtype):
229 raise TypeError("Cannot unbox tzaware Timestamp to tznaive dtype")
230 elif isinstance(value, Timedelta):
231 value = value.to_timedelta64()
233 _disallow_mismatched_datetimelike(value, dtype)
234 return value
237def _disallow_mismatched_datetimelike(value, dtype: DtypeObj) -> None:
238 """
239 numpy allows np.array(dt64values, dtype="timedelta64[ns]") and
240 vice-versa, but we do not want to allow this, so we need to
241 check explicitly
242 """
243 vdtype = getattr(value, "dtype", None)
244 if vdtype is None:
245 return
246 elif (vdtype.kind == "m" and dtype.kind == "M") or (
247 vdtype.kind == "M" and dtype.kind == "m"
248 ):
249 raise TypeError(f"Cannot cast {value!r} to {dtype}")
252@overload
253def maybe_downcast_to_dtype(result: np.ndarray, dtype: np.dtype) -> np.ndarray: ...
256@overload
257def maybe_downcast_to_dtype(
258 result: ExtensionArray, dtype: np.dtype
259) -> ExtensionArray: ...
262def maybe_downcast_to_dtype(result: ArrayLike, dtype: np.dtype) -> ArrayLike:
263 """
264 try to cast to the specified dtype (e.g. convert back to bool/int
265 or could be an astype of float64->float32
266 """
267 if isinstance(result, ABCSeries):
268 result = result._values
269 do_round = False
271 if not isinstance(dtype, np.dtype):
272 # enforce our signature annotation
273 raise TypeError(dtype) # pragma: no cover
275 converted = maybe_downcast_numeric(result, dtype, do_round)
276 if converted is not result:
277 return converted
279 # a datetimelike
280 # GH12821, iNaT is cast to float
281 if dtype.kind in "mM" and result.dtype.kind in "if":
282 result = result.astype(dtype)
284 elif dtype.kind == "m" and result.dtype == _dtype_obj:
285 # test_where_downcast_to_td64
286 result = cast(np.ndarray, result)
287 result = array_to_timedelta64(result)
289 elif dtype == np.dtype("M8[ns]") and result.dtype == _dtype_obj:
290 result = cast(np.ndarray, result)
291 return np.asarray(maybe_cast_to_datetime(result, dtype=dtype))
293 return result
296@overload
297def maybe_downcast_numeric(
298 result: np.ndarray, dtype: np.dtype, do_round: bool = False
299) -> np.ndarray: ...
302@overload
303def maybe_downcast_numeric(
304 result: ExtensionArray, dtype: DtypeObj, do_round: bool = False
305) -> ArrayLike: ...
308def maybe_downcast_numeric(
309 result: ArrayLike, dtype: DtypeObj, do_round: bool = False
310) -> ArrayLike:
311 """
312 Subset of maybe_downcast_to_dtype restricted to numeric dtypes.
314 Parameters
315 ----------
316 result : ndarray or ExtensionArray
317 dtype : np.dtype or ExtensionDtype
318 do_round : bool
320 Returns
321 -------
322 ndarray or ExtensionArray
323 """
324 if not isinstance(dtype, np.dtype) or not isinstance(result.dtype, np.dtype):
325 # e.g. SparseDtype has no itemsize attr
326 return result
328 def trans(x):
329 if do_round:
330 return x.round()
331 return x
333 if dtype.kind == result.dtype.kind:
334 # don't allow upcasts here (except if empty)
335 if result.dtype.itemsize <= dtype.itemsize and result.size:
336 return result
338 if dtype.kind in "biu":
339 if not result.size:
340 # if we don't have any elements, just astype it
341 return trans(result).astype(dtype)
343 if isinstance(result, np.ndarray):
344 element = result.item(0)
345 else:
346 element = result.iloc[0]
347 if not isinstance(element, (np.integer, np.floating, int, float, bool)):
348 # a comparable, e.g. a Decimal may slip in here
349 return result
351 if (
352 issubclass(result.dtype.type, (np.object_, np.number))
353 and notna(result).all()
354 ):
355 new_result = trans(result).astype(dtype)
356 if new_result.dtype.kind == "O" or result.dtype.kind == "O":
357 # np.allclose may raise TypeError on object-dtype
358 if (new_result == result).all():
359 return new_result
360 elif np.allclose(new_result, result, rtol=0):
361 return new_result
363 elif (
364 issubclass(dtype.type, np.floating)
365 and result.dtype.kind != "b"
366 and not is_string_dtype(result.dtype)
367 ):
368 with warnings.catch_warnings():
369 warnings.filterwarnings(
370 "ignore", "overflow encountered in cast", RuntimeWarning
371 )
372 new_result = result.astype(dtype)
374 # Adjust tolerances based on floating point size
375 size_tols = {4: 5e-4, 8: 5e-8, 16: 5e-16}
377 atol = size_tols.get(new_result.dtype.itemsize, 0.0)
379 # Check downcast float values are still equal within 7 digits when
380 # converting from float64 to float32
381 if np.allclose(new_result, result, equal_nan=True, rtol=0.0, atol=atol):
382 return new_result
384 elif dtype.kind == result.dtype.kind == "c":
385 new_result = result.astype(dtype)
387 if np.array_equal(new_result, result, equal_nan=True):
388 # TODO: use tolerance like we do for float?
389 return new_result
391 return result
394def maybe_upcast_numeric_to_64bit(arr: NumpyIndexT) -> NumpyIndexT:
395 """
396 If array is an int/uint/float bit size lower than 64 bit, upcast it to 64 bit.
398 Parameters
399 ----------
400 arr : ndarray or ExtensionArray
402 Returns
403 -------
404 ndarray or ExtensionArray
405 """
406 dtype = arr.dtype
407 if dtype.kind == "i" and dtype != np.int64:
408 return arr.astype(np.int64)
409 elif dtype.kind == "u" and dtype != np.uint64:
410 return arr.astype(np.uint64)
411 elif dtype.kind == "f" and dtype != np.float64:
412 return arr.astype(np.float64)
413 else:
414 return arr
417@overload
418def ensure_dtype_can_hold_na(dtype: np.dtype) -> np.dtype: ...
421@overload
422def ensure_dtype_can_hold_na(dtype: ExtensionDtype) -> ExtensionDtype: ...
425def ensure_dtype_can_hold_na(dtype: DtypeObj) -> DtypeObj:
426 """
427 If we have a dtype that cannot hold NA values, find the best match that can.
428 """
429 if isinstance(dtype, ExtensionDtype):
430 if dtype._can_hold_na:
431 return dtype
432 elif isinstance(dtype, IntervalDtype):
433 # TODO(GH#45349): don't special-case IntervalDtype, allow
434 # overriding instead of returning object below.
435 return IntervalDtype(np.float64, closed=dtype.closed)
436 return _dtype_obj
437 elif dtype.kind == "b":
438 return _dtype_obj
439 elif dtype.kind in "iu":
440 return np.dtype(np.float64)
441 return dtype
444_canonical_nans = {
445 np.datetime64: np.datetime64("NaT", "ns"),
446 np.timedelta64: np.timedelta64("NaT", "ns"),
447 type(np.nan): np.nan,
448}
451def maybe_promote(dtype: np.dtype, fill_value=np.nan):
452 """
453 Find the minimal dtype that can hold both the given dtype and fill_value.
455 Parameters
456 ----------
457 dtype : np.dtype
458 fill_value : scalar, default np.nan
460 Returns
461 -------
462 dtype
463 Upcasted from dtype argument if necessary.
464 fill_value
465 Upcasted from fill_value argument if necessary.
467 Raises
468 ------
469 ValueError
470 If fill_value is a non-scalar and dtype is not object.
471 """
472 orig = fill_value
473 orig_is_nat = False
474 if checknull(fill_value):
475 # https://github.com/pandas-dev/pandas/pull/39692#issuecomment-1441051740
476 # avoid cache misses with NaN/NaT values that are not singletons
477 if fill_value is not NA:
478 try:
479 orig_is_nat = np.isnat(fill_value)
480 except TypeError:
481 pass
483 fill_value = _canonical_nans.get(type(fill_value), fill_value)
485 # for performance, we are using a cached version of the actual implementation
486 # of the function in _maybe_promote. However, this doesn't always work (in case
487 # of non-hashable arguments), so we fallback to the actual implementation if needed
488 try:
489 # error: Argument 3 to "__call__" of "_lru_cache_wrapper" has incompatible type
490 # "Type[Any]"; expected "Hashable" [arg-type]
491 dtype, fill_value = _maybe_promote_cached(
492 dtype,
493 fill_value,
494 type(fill_value), # type: ignore[arg-type]
495 )
496 except TypeError:
497 # if fill_value is not hashable (required for caching)
498 dtype, fill_value = _maybe_promote(dtype, fill_value)
500 if (dtype == _dtype_obj and orig is not None) or (
501 orig_is_nat and np.datetime_data(orig)[0] != "ns"
502 ):
503 # GH#51592,53497 restore our potentially non-canonical fill_value
504 fill_value = orig
505 return dtype, fill_value
508@functools.lru_cache
509def _maybe_promote_cached(dtype, fill_value, fill_value_type):
510 # The cached version of _maybe_promote below
511 # This also use fill_value_type as (unused) argument to use this in the
512 # cache lookup -> to differentiate 1 and True
513 return _maybe_promote(dtype, fill_value)
516def _maybe_promote(dtype: np.dtype, fill_value=np.nan):
517 # The actual implementation of the function, use `maybe_promote` above for
518 # a cached version.
519 if not is_scalar(fill_value):
520 # with object dtype there is nothing to promote, and the user can
521 # pass pretty much any weird fill_value they like
522 if dtype != object:
523 # with object dtype there is nothing to promote, and the user can
524 # pass pretty much any weird fill_value they like
525 raise ValueError("fill_value must be a scalar")
526 dtype = _dtype_obj
527 return dtype, fill_value
529 if is_valid_na_for_dtype(fill_value, dtype) and dtype.kind in "iufcmM":
530 dtype = ensure_dtype_can_hold_na(dtype)
531 fv = na_value_for_dtype(dtype)
532 return dtype, fv
534 elif isinstance(dtype, CategoricalDtype):
535 if fill_value in dtype.categories or isna(fill_value):
536 return dtype, fill_value
537 else:
538 return object, ensure_object(fill_value)
540 elif isna(fill_value):
541 dtype = _dtype_obj
542 if fill_value is None:
543 # but we retain e.g. pd.NA
544 fill_value = np.nan
545 return dtype, fill_value
547 # returns tuple of (dtype, fill_value)
548 if issubclass(dtype.type, np.datetime64):
549 inferred, fv = infer_dtype_from_scalar(fill_value)
550 if inferred == dtype:
551 return dtype, fv
553 from pandas.core.arrays import DatetimeArray
555 dta = DatetimeArray._from_sequence([], dtype="M8[ns]")
556 try:
557 fv = dta._validate_setitem_value(fill_value)
558 return dta.dtype, fv
559 except (ValueError, TypeError):
560 return _dtype_obj, fill_value
562 elif issubclass(dtype.type, np.timedelta64):
563 inferred, fv = infer_dtype_from_scalar(fill_value)
564 if inferred == dtype:
565 return dtype, fv
567 elif inferred.kind == "m":
568 # different unit, e.g. passed np.timedelta64(24, "h") with dtype=m8[ns]
569 # see if we can losslessly cast it to our dtype
570 unit = np.datetime_data(dtype)[0]
571 unit = cast("TimeUnit", unit)
572 try:
573 td = Timedelta(fill_value).as_unit(unit, round_ok=False)
574 except OutOfBoundsTimedelta:
575 return _dtype_obj, fill_value
576 else:
577 return dtype, td.asm8
579 return _dtype_obj, fill_value
581 elif is_float(fill_value):
582 if issubclass(dtype.type, np.bool_):
583 dtype = np.dtype(np.object_)
585 elif issubclass(dtype.type, np.integer):
586 dtype = np.dtype(np.float64)
588 elif dtype.kind == "f":
589 mst = np.min_scalar_type(fill_value)
590 if mst > dtype:
591 # e.g. mst is np.float64 and dtype is np.float32
592 dtype = mst
594 elif dtype.kind == "c":
595 mst = np.min_scalar_type(fill_value)
596 dtype = np.promote_types(dtype, mst)
598 elif is_bool(fill_value):
599 if not issubclass(dtype.type, np.bool_):
600 dtype = np.dtype(np.object_)
602 elif is_integer(fill_value):
603 if issubclass(dtype.type, np.bool_):
604 dtype = np.dtype(np.object_)
606 elif issubclass(dtype.type, np.integer):
607 if not np_can_cast_scalar(fill_value, dtype):
608 # upcast to prevent overflow
609 mst = np.min_scalar_type(fill_value)
610 dtype = np.promote_types(dtype, mst)
611 if dtype.kind == "f":
612 # Case where we disagree with numpy
613 dtype = np.dtype(np.object_)
615 elif is_complex(fill_value):
616 if issubclass(dtype.type, np.bool_):
617 dtype = np.dtype(np.object_)
619 elif issubclass(dtype.type, (np.integer, np.floating)):
620 mst = np.min_scalar_type(fill_value)
621 dtype = np.promote_types(dtype, mst)
623 elif dtype.kind == "c":
624 mst = np.min_scalar_type(fill_value)
625 if mst > dtype:
626 # e.g. mst is np.complex128 and dtype is np.complex64
627 dtype = mst
629 else:
630 dtype = np.dtype(np.object_)
632 # in case we have a string that looked like a number
633 if issubclass(dtype.type, (bytes, str)):
634 dtype = np.dtype(np.object_)
636 fill_value = _ensure_dtype_type(fill_value, dtype)
637 return dtype, fill_value
640def _ensure_dtype_type(value, dtype: np.dtype):
641 """
642 Ensure that the given value is an instance of the given dtype.
644 e.g. if out dtype is np.complex64_, we should have an instance of that
645 as opposed to a python complex object.
647 Parameters
648 ----------
649 value : object
650 dtype : np.dtype
652 Returns
653 -------
654 object
655 """
656 # Start with exceptions in which we do _not_ cast to numpy types
658 if dtype == _dtype_obj:
659 return value
661 # Note: before we get here we have already excluded isna(value)
662 return dtype.type(value)
665def infer_dtype_from(val) -> tuple[DtypeObj, Any]:
666 """
667 Interpret the dtype from a scalar or array.
669 Parameters
670 ----------
671 val : object
672 """
673 if not is_list_like(val):
674 return infer_dtype_from_scalar(val)
675 return infer_dtype_from_array(val)
678def infer_dtype_from_scalar(val) -> tuple[DtypeObj, Any]:
679 """
680 Interpret the dtype from a scalar.
682 Parameters
683 ----------
684 val : object
685 """
686 dtype: DtypeObj = _dtype_obj
688 # a 1-element ndarray
689 if isinstance(val, np.ndarray):
690 if val.ndim != 0:
691 msg = "invalid ndarray passed to infer_dtype_from_scalar"
692 raise ValueError(msg)
694 dtype = val.dtype
695 val = lib.item_from_zerodim(val)
697 elif isinstance(val, str):
698 # If we create an empty array using a string to infer
699 # the dtype, NumPy will only allocate one character per entry
700 # so this is kind of bad. Alternately we could use np.repeat
701 # instead of np.empty (but then you still don't want things
702 # coming out as np.str_!
704 dtype = _dtype_obj
705 if using_string_dtype():
706 from pandas.core.arrays.string_ import StringDtype
708 dtype = StringDtype(na_value=np.nan)
710 elif isinstance(val, (np.datetime64, dt.datetime)):
711 try:
712 val = Timestamp(val)
713 except OutOfBoundsDatetime:
714 return _dtype_obj, val
716 if val is NaT or val.tz is None:
717 val = val.to_datetime64()
718 dtype = val.dtype
719 # TODO: test with datetime(2920, 10, 1) based on test_replace_dtypes
720 else:
721 dtype = DatetimeTZDtype(unit=val.unit, tz=val.tz)
723 elif isinstance(val, (np.timedelta64, dt.timedelta)):
724 try:
725 val = Timedelta(val)
726 except (OutOfBoundsTimedelta, OverflowError):
727 dtype = _dtype_obj
728 else:
729 if val is NaT:
730 val = np.timedelta64("NaT", "ns")
731 else:
732 val = val.asm8
733 dtype = val.dtype
735 elif is_bool(val):
736 dtype = np.dtype(np.bool_)
738 elif is_integer(val):
739 if isinstance(val, np.integer):
740 dtype = np.dtype(type(val))
741 else:
742 dtype = np.dtype(np.int64)
744 try:
745 np.array(val, dtype=dtype)
746 except OverflowError:
747 dtype = np.array(val).dtype
749 elif is_float(val):
750 if isinstance(val, np.floating):
751 dtype = np.dtype(type(val))
752 else:
753 dtype = np.dtype(np.float64)
755 elif is_complex(val):
756 dtype = np.dtype(np.complex128)
758 if isinstance(val, Period):
759 dtype = PeriodDtype(freq=val.freq)
760 elif isinstance(val, Interval):
761 subtype = infer_dtype_from_scalar(val.left)[0]
762 dtype = IntervalDtype(subtype=subtype, closed=val.closed)
764 return dtype, val
767def dict_compat(d: dict[Scalar, Scalar]) -> dict[Scalar, Scalar]:
768 """
769 Convert datetimelike-keyed dicts to a Timestamp-keyed dict.
771 Parameters
772 ----------
773 d: dict-like object
775 Returns
776 -------
777 dict
778 """
779 return {maybe_box_datetimelike(key): value for key, value in d.items()}
782def infer_dtype_from_array(arr) -> tuple[DtypeObj, ArrayLike]:
783 """
784 Infer the dtype from an array.
786 Parameters
787 ----------
788 arr : array
790 Returns
791 -------
792 tuple (pandas-compat dtype, array)
795 Examples
796 --------
797 >>> np.asarray([1, "1"])
798 array(['1', '1'], dtype='<U21')
800 >>> infer_dtype_from_array([1, "1"])
801 (dtype('O'), [1, '1'])
802 """
803 if isinstance(arr, np.ndarray):
804 return arr.dtype, arr
806 if not is_list_like(arr):
807 raise TypeError("'arr' must be list-like")
809 arr_dtype = getattr(arr, "dtype", None)
810 if isinstance(arr_dtype, ExtensionDtype):
811 return arr.dtype, arr
813 elif isinstance(arr, ABCSeries):
814 return arr.dtype, np.asarray(arr)
816 # don't force numpy coerce with nan's
817 inferred = lib.infer_dtype(arr, skipna=False)
818 if inferred in ["string", "bytes", "mixed", "mixed-integer"]:
819 return (np.dtype(np.object_), arr)
821 arr = np.asarray(arr)
822 return arr.dtype, arr
825def _maybe_infer_dtype_type(element):
826 """
827 Try to infer an object's dtype, for use in arithmetic ops.
829 Uses `element.dtype` if that's available.
830 Objects implementing the iterator protocol are cast to a NumPy array,
831 and from there the array's type is used.
833 Parameters
834 ----------
835 element : object
836 Possibly has a `.dtype` attribute, and possibly the iterator
837 protocol.
839 Returns
840 -------
841 tipo : type
843 Examples
844 --------
845 >>> from collections import namedtuple
846 >>> Foo = namedtuple("Foo", "dtype")
847 >>> _maybe_infer_dtype_type(Foo(np.dtype("i8")))
848 dtype('int64')
849 """
850 tipo = None
851 if hasattr(element, "dtype"):
852 tipo = element.dtype
853 elif is_list_like(element):
854 element = np.asarray(element)
855 tipo = element.dtype
856 return tipo
859def invalidate_string_dtypes(dtype_set: set[DtypeObj]) -> None:
860 """
861 Change string like dtypes to object for
862 ``DataFrame.select_dtypes()``.
863 """
864 # error: Argument 1 to <set> has incompatible type "Type[generic]"; expected
865 # "Union[dtype[Any], ExtensionDtype, None]"
866 # error: Argument 2 to <set> has incompatible type "Type[generic]"; expected
867 # "Union[dtype[Any], ExtensionDtype, None]"
868 non_string_dtypes = dtype_set - {
869 np.dtype("S").type, # type: ignore[arg-type]
870 np.dtype("<U").type, # type: ignore[arg-type]
871 }
872 if non_string_dtypes != dtype_set:
873 raise TypeError(
874 "numpy string dtypes are not allowed, use 'str' or 'object' instead"
875 )
878def coerce_indexer_dtype(indexer, categories) -> np.ndarray:
879 """coerce the indexer input array to the smallest dtype possible"""
880 length = len(categories)
881 if length < _int8_max:
882 return ensure_int8(indexer)
883 elif length < _int16_max:
884 return ensure_int16(indexer)
885 elif length < _int32_max:
886 return ensure_int32(indexer)
887 return ensure_int64(indexer)
890def convert_dtypes(
891 input_array: ArrayLike,
892 convert_string: bool = True,
893 convert_integer: bool = True,
894 convert_boolean: bool = True,
895 convert_floating: bool = True,
896 infer_objects: bool = False,
897 dtype_backend: Literal["numpy_nullable", "pyarrow"] = "numpy_nullable",
898) -> DtypeObj:
899 """
900 Convert objects to best possible type, and optionally,
901 to types supporting ``pd.NA``.
903 Parameters
904 ----------
905 input_array : ExtensionArray or np.ndarray
906 convert_string : bool, default True
907 Whether object dtypes should be converted to ``StringDtype()``.
908 convert_integer : bool, default True
909 Whether, if possible, conversion can be done to integer extension types.
910 convert_boolean : bool, defaults True
911 Whether object dtypes should be converted to ``BooleanDtypes()``.
912 convert_floating : bool, defaults True
913 Whether, if possible, conversion can be done to floating extension types.
914 If `convert_integer` is also True, preference will be give to integer
915 dtypes if the floats can be faithfully casted to integers.
916 infer_objects : bool, defaults False
917 Whether to also infer objects to float/int if possible. Is only hit if the
918 object array contains pd.NA.
919 dtype_backend : {'numpy_nullable', 'pyarrow'}, default 'numpy_nullable'
920 Back-end data type applied to the resultant :class:`DataFrame`
921 (still experimental). Behaviour is as follows:
923 * ``"numpy_nullable"``: returns nullable-dtype
924 * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype`
926 .. versionadded:: 2.0
928 Returns
929 -------
930 np.dtype, or ExtensionDtype
931 """
932 from pandas.core.arrays.string_ import StringDtype
934 inferred_dtype: str | DtypeObj
936 if (
937 convert_string or convert_integer or convert_boolean or convert_floating
938 ) and isinstance(input_array, np.ndarray):
939 if input_array.dtype.kind == "c":
940 return input_array.dtype
942 if input_array.dtype == object:
943 inferred_dtype = lib.infer_dtype(input_array)
944 else:
945 inferred_dtype = input_array.dtype
947 if is_string_dtype(inferred_dtype):
948 if not convert_string or inferred_dtype == "bytes":
949 inferred_dtype = input_array.dtype
950 else:
951 inferred_dtype = pandas_dtype_func("string")
953 if convert_integer:
954 target_int_dtype = pandas_dtype_func("Int64")
956 if input_array.dtype.kind in "iu":
957 from pandas.core.arrays.integer import NUMPY_INT_TO_DTYPE
959 inferred_dtype = NUMPY_INT_TO_DTYPE.get(
960 input_array.dtype, target_int_dtype
961 )
962 elif input_array.dtype.kind in "fb":
963 # TODO: de-dup with maybe_cast_to_integer_array?
964 arr = input_array[notna(input_array)]
965 if len(arr) < len(input_array) and not is_nan_na():
966 # In the presence of NaNs, we cannot convert to IntegerDtype
967 pass
968 elif (arr.astype(int) == arr).all():
969 inferred_dtype = target_int_dtype
970 else:
971 inferred_dtype = input_array.dtype
972 elif (
973 infer_objects
974 and input_array.dtype == object
975 and (isinstance(inferred_dtype, str) and inferred_dtype == "integer")
976 ):
977 inferred_dtype = target_int_dtype
979 if convert_floating:
980 if input_array.dtype.kind in "fb":
981 # i.e. numeric but not integer
982 from pandas.core.arrays.floating import NUMPY_FLOAT_TO_DTYPE
984 inferred_float_dtype: DtypeObj = NUMPY_FLOAT_TO_DTYPE.get(
985 input_array.dtype, pandas_dtype_func("Float64")
986 )
987 # if we could also convert to integer, check if all floats
988 # are actually integers
989 if convert_integer:
990 # TODO: de-dup with maybe_cast_to_integer_array?
991 arr = input_array[notna(input_array)]
992 if len(arr) < len(input_array) and not is_nan_na():
993 # In the presence of NaNs, we can't convert to IntegerDtype
994 inferred_dtype = inferred_float_dtype
995 elif (arr.astype(int) == arr).all():
996 inferred_dtype = pandas_dtype_func("Int64")
997 else:
998 inferred_dtype = inferred_float_dtype
999 else:
1000 inferred_dtype = inferred_float_dtype
1001 elif (
1002 infer_objects
1003 and input_array.dtype == object
1004 and (isinstance(inferred_dtype, str) and inferred_dtype == "floating")
1005 ):
1006 inferred_dtype = pandas_dtype_func("Float64")
1008 if convert_boolean:
1009 if input_array.dtype.kind == "b":
1010 inferred_dtype = pandas_dtype_func("boolean")
1011 elif isinstance(inferred_dtype, str) and inferred_dtype == "boolean":
1012 inferred_dtype = pandas_dtype_func("boolean")
1014 if isinstance(inferred_dtype, str):
1015 # If we couldn't do anything else, then we retain the dtype
1016 inferred_dtype = input_array.dtype
1018 elif (
1019 convert_string
1020 and isinstance(input_array.dtype, StringDtype)
1021 and input_array.dtype.na_value is np.nan
1022 ):
1023 inferred_dtype = pandas_dtype_func("string")
1025 else:
1026 inferred_dtype = input_array.dtype
1028 if dtype_backend == "pyarrow" and not isinstance(inferred_dtype, ArrowDtype):
1029 from pandas.core.arrays.arrow.array import to_pyarrow_type
1030 from pandas.core.arrays.string_ import StringDtype
1032 assert not isinstance(inferred_dtype, str)
1034 if (
1035 (convert_integer and inferred_dtype.kind in "iu")
1036 or (convert_floating and inferred_dtype.kind in "f")
1037 or (convert_boolean and inferred_dtype.kind == "b")
1038 or (convert_string and isinstance(inferred_dtype, StringDtype))
1039 or (
1040 inferred_dtype.kind not in "iufb"
1041 and not isinstance(inferred_dtype, StringDtype)
1042 and not isinstance(inferred_dtype, CategoricalDtype)
1043 )
1044 ):
1045 if isinstance(inferred_dtype, PandasExtensionDtype) and not isinstance(
1046 inferred_dtype, DatetimeTZDtype
1047 ):
1048 base_dtype = inferred_dtype.base
1049 elif isinstance(inferred_dtype, (BaseMaskedDtype, ArrowDtype)):
1050 base_dtype = inferred_dtype.numpy_dtype
1051 elif isinstance(inferred_dtype, StringDtype):
1052 base_dtype = np.dtype(str)
1053 else:
1054 base_dtype = inferred_dtype
1055 if (
1056 base_dtype.kind == "O" # type: ignore[union-attr]
1057 and input_array.size > 0
1058 and isna(input_array).all()
1059 ):
1060 import pyarrow as pa
1062 pa_type = pa.null()
1063 else:
1064 pa_type = to_pyarrow_type(base_dtype)
1065 if pa_type is not None:
1066 inferred_dtype = ArrowDtype(pa_type)
1067 elif dtype_backend == "numpy_nullable" and isinstance(inferred_dtype, ArrowDtype):
1068 # GH 53648
1069 inferred_dtype = _arrow_dtype_mapping()[inferred_dtype.pyarrow_dtype]
1071 # error: Incompatible return value type (got "Union[str, Union[dtype[Any],
1072 # ExtensionDtype]]", expected "Union[dtype[Any], ExtensionDtype]")
1073 return inferred_dtype # type: ignore[return-value]
1076def maybe_cast_to_datetime(
1077 value: np.ndarray | list, dtype: np.dtype
1078) -> DatetimeArray | TimedeltaArray | np.ndarray:
1079 """
1080 try to cast the array/value to a datetimelike dtype, converting float
1081 nan to iNaT
1083 Caller is responsible for handling ExtensionDtype cases and non dt64/td64
1084 cases.
1085 """
1086 from pandas.core.arrays.datetimes import DatetimeArray
1087 from pandas.core.arrays.timedeltas import TimedeltaArray
1089 assert dtype.kind in "mM"
1090 if not is_list_like(value):
1091 raise TypeError("value must be listlike")
1093 # TODO: _from_sequence would raise ValueError in cases where
1094 # _ensure_nanosecond_dtype raises TypeError
1095 _ensure_nanosecond_dtype(dtype)
1097 if lib.is_np_dtype(dtype, "m"):
1098 res = TimedeltaArray._from_sequence(value, dtype=dtype)
1099 return res
1100 else:
1101 try:
1102 dta = DatetimeArray._from_sequence(value, dtype=dtype)
1103 except ValueError as err:
1104 # We can give a Series-specific exception message.
1105 if "cannot supply both a tz and a timezone-naive dtype" in str(err):
1106 raise ValueError(
1107 "Cannot convert timezone-aware data to "
1108 "timezone-naive dtype. Use "
1109 "pd.Series(values).dt.tz_localize(None) instead."
1110 ) from err
1111 raise
1113 return dta
1116def _ensure_nanosecond_dtype(dtype: DtypeObj) -> None:
1117 """
1118 Convert dtypes with granularity less than nanosecond to nanosecond
1120 >>> _ensure_nanosecond_dtype(np.dtype("M8[us]"))
1122 >>> _ensure_nanosecond_dtype(np.dtype("M8[D]"))
1123 Traceback (most recent call last):
1124 ...
1125 TypeError: dtype=datetime64[D] is not supported. Supported resolutions are 's', 'ms', 'us', and 'ns'
1127 >>> _ensure_nanosecond_dtype(np.dtype("m8[ps]"))
1128 Traceback (most recent call last):
1129 ...
1130 TypeError: dtype=timedelta64[ps] is not supported. Supported resolutions are 's', 'ms', 'us', and 'ns'
1131 """ # noqa: E501
1132 msg = (
1133 f"The '{dtype.name}' dtype has no unit. "
1134 f"Please pass in '{dtype.name}[ns]' instead."
1135 )
1137 # unpack e.g. SparseDtype
1138 dtype = getattr(dtype, "subtype", dtype)
1140 if not isinstance(dtype, np.dtype):
1141 # i.e. datetime64tz
1142 pass
1144 elif dtype.kind in "mM":
1145 if not is_supported_dtype(dtype):
1146 # pre-2.0 we would silently swap in nanos for lower-resolutions,
1147 # raise for above-nano resolutions
1148 if dtype.name in ["datetime64", "timedelta64"]:
1149 raise ValueError(msg)
1150 # TODO: ValueError or TypeError? existing test
1151 # test_constructor_generic_timestamp_bad_frequency expects TypeError
1152 raise TypeError(
1153 f"dtype={dtype} is not supported. Supported resolutions are 's', "
1154 "'ms', 'us', and 'ns'"
1155 )
1158# TODO: other value-dependent functions to standardize here include
1159# Index._find_common_type_compat
1160def find_result_type(left_dtype: DtypeObj, right: Any) -> DtypeObj:
1161 """
1162 Find the type/dtype for the result of an operation between objects.
1164 This is similar to find_common_type, but looks at the right object instead
1165 of just its dtype. This can be useful in particular when the right
1166 object does not have a `dtype`.
1168 Parameters
1169 ----------
1170 left_dtype : np.dtype or ExtensionDtype
1171 right : Any
1173 Returns
1174 -------
1175 np.dtype or ExtensionDtype
1177 See also
1178 --------
1179 find_common_type
1180 numpy.result_type
1181 """
1182 new_dtype: DtypeObj
1184 if (
1185 isinstance(left_dtype, np.dtype)
1186 and left_dtype.kind in "iuc"
1187 and (lib.is_integer(right) or lib.is_float(right))
1188 ):
1189 # e.g. with int8 dtype and right=512, we want to end up with
1190 # np.int16, whereas infer_dtype_from(512) gives np.int64,
1191 # which will make us upcast too far.
1192 if lib.is_float(right) and right.is_integer() and left_dtype.kind != "f":
1193 right = int(right)
1194 # After NEP 50, numpy won't inspect Python scalars
1195 # TODO: do we need to recreate numpy's inspection logic for floats too
1196 # (this breaks some tests)
1197 if isinstance(right, int) and not isinstance(right, np.integer):
1198 # This gives an unsigned type by default
1199 # (if our number is positive)
1201 # If our left dtype is signed, we might not want this since
1202 # this might give us 1 dtype too big
1203 # We should check if the corresponding int dtype (e.g. int64 for uint64)
1204 # can hold the number
1205 right_dtype = np.min_scalar_type(right)
1206 if right == 0:
1207 # Special case 0
1208 right = left_dtype
1209 elif (
1210 not np.issubdtype(left_dtype, np.unsignedinteger)
1211 and 0 < right <= np.iinfo(right_dtype).max
1212 ):
1213 # If left dtype isn't unsigned, check if it fits in the signed dtype
1214 right = np.dtype(f"i{right_dtype.itemsize}")
1215 else:
1216 right = right_dtype
1218 new_dtype = np.result_type(left_dtype, right)
1220 elif is_valid_na_for_dtype(right, left_dtype):
1221 # e.g. IntervalDtype[int] and None/np.nan
1222 new_dtype = ensure_dtype_can_hold_na(left_dtype)
1224 else:
1225 dtype, _ = infer_dtype_from(right)
1226 new_dtype = find_common_type([left_dtype, dtype])
1228 return new_dtype
1231def common_dtype_categorical_compat(
1232 objs: Sequence[Index | ArrayLike], dtype: DtypeObj
1233) -> DtypeObj:
1234 """
1235 Update the result of find_common_type to account for NAs in a Categorical.
1237 Parameters
1238 ----------
1239 objs : list[np.ndarray | ExtensionArray | Index]
1240 dtype : np.dtype or ExtensionDtype
1242 Returns
1243 -------
1244 np.dtype or ExtensionDtype
1245 """
1246 # GH#38240
1248 # TODO: more generally, could do `not can_hold_na(dtype)`
1249 if lib.is_np_dtype(dtype, "iu"):
1250 for obj in objs:
1251 # We don't want to accidentally allow e.g. "categorical" str here
1252 obj_dtype = getattr(obj, "dtype", None)
1253 if isinstance(obj_dtype, CategoricalDtype):
1254 if isinstance(obj, ABCIndex):
1255 # This check may already be cached
1256 hasnas = obj.hasnans
1257 else:
1258 # Categorical
1259 hasnas = cast("Categorical", obj)._hasna
1261 if hasnas:
1262 # see test_union_int_categorical_with_nan
1263 dtype = np.dtype(np.float64)
1264 break
1265 return dtype
1268def np_find_common_type(*dtypes: np.dtype) -> np.dtype:
1269 """
1270 np.find_common_type implementation pre-1.25 deprecation using np.result_type
1271 https://github.com/pandas-dev/pandas/pull/49569#issuecomment-1308300065
1273 Parameters
1274 ----------
1275 dtypes : np.dtypes
1277 Returns
1278 -------
1279 np.dtype
1280 """
1281 try:
1282 common_dtype = np.result_type(*dtypes)
1283 if common_dtype.kind in "mMSU":
1284 # NumPy promotion currently (1.25) misbehaves for for times and strings,
1285 # so fall back to object (find_common_dtype did unless there
1286 # was only one dtype)
1287 common_dtype = np.dtype("O")
1289 except TypeError:
1290 common_dtype = np.dtype("O")
1291 return common_dtype
1294@overload
1295def find_common_type(types: list[np.dtype]) -> np.dtype: ...
1298@overload
1299def find_common_type(types: list[ExtensionDtype]) -> DtypeObj: ...
1302@overload
1303def find_common_type(types: list[DtypeObj]) -> DtypeObj: ...
1306def find_common_type(types):
1307 """
1308 Find a common data type among the given dtypes.
1310 Parameters
1311 ----------
1312 types : list of dtypes
1314 Returns
1315 -------
1316 pandas extension or numpy dtype
1318 See Also
1319 --------
1320 numpy.find_common_type
1322 """
1323 if not types:
1324 raise ValueError("no types given")
1326 first = types[0]
1328 # workaround for find_common_type([np.dtype('datetime64[ns]')] * 2)
1329 # => object
1330 if lib.dtypes_all_equal(list(types)):
1331 return first
1333 # get unique types (dict.fromkeys is used as order-preserving set())
1334 types = list(dict.fromkeys(types).keys())
1336 if any(isinstance(t, ExtensionDtype) for t in types):
1337 for t in types:
1338 if isinstance(t, ExtensionDtype):
1339 res = t._get_common_dtype(types)
1340 if res is not None:
1341 return res
1342 return np.dtype("object")
1344 # take lowest unit
1345 if all(lib.is_np_dtype(t, "M") for t in types):
1346 return np.dtype(max(types))
1347 if all(lib.is_np_dtype(t, "m") for t in types):
1348 return np.dtype(max(types))
1350 # don't mix bool / int or float or complex
1351 # this is different from numpy, which casts bool with float/int as int
1352 has_bools = any(t.kind == "b" for t in types)
1353 if has_bools:
1354 for t in types:
1355 if t.kind in "iufc":
1356 return np.dtype("object")
1358 return np_find_common_type(*types)
1361def construct_2d_arraylike_from_scalar(
1362 value: Scalar, length: int, width: int, dtype: np.dtype, copy: bool
1363) -> np.ndarray:
1364 shape = (length, width)
1366 if dtype.kind in "mM":
1367 value = _maybe_box_and_unbox_datetimelike(value, dtype)
1368 elif dtype == _dtype_obj:
1369 if isinstance(value, (np.timedelta64, np.datetime64)):
1370 # calling np.array below would cast to pytimedelta/pydatetime
1371 out = np.empty(shape, dtype=object)
1372 out.fill(value)
1373 return out
1375 # Attempt to coerce to a numpy array
1376 try:
1377 if not copy:
1378 arr = np.asarray(value, dtype=dtype)
1379 else:
1380 arr = np.array(value, dtype=dtype, copy=copy)
1381 except (ValueError, TypeError) as err:
1382 raise TypeError(
1383 f"DataFrame constructor called with incompatible data and dtype: {err}"
1384 ) from err
1386 if arr.ndim != 0:
1387 raise ValueError("DataFrame constructor not properly called!")
1389 return np.full(shape, arr)
1392def construct_1d_arraylike_from_scalar(
1393 value: Scalar, length: int, dtype: DtypeObj | None
1394) -> ArrayLike:
1395 """
1396 create an np.ndarray / pandas type of specified shape and dtype
1397 filled with values
1399 Parameters
1400 ----------
1401 value : scalar value
1402 length : int
1403 dtype : pandas_dtype or np.dtype
1405 Returns
1406 -------
1407 np.ndarray / pandas type of length, filled with value
1409 """
1411 if dtype is None:
1412 try:
1413 dtype, value = infer_dtype_from_scalar(value)
1414 except OutOfBoundsDatetime:
1415 dtype = _dtype_obj
1417 if isinstance(dtype, ExtensionDtype):
1418 cls = dtype.construct_array_type()
1419 seq = [] if length == 0 else [value]
1420 return cls._from_sequence(seq, dtype=dtype).repeat(length)
1422 if length and dtype.kind in "iu" and isna(value):
1423 # coerce if we have nan for an integer dtype
1424 dtype = np.dtype("float64")
1425 elif lib.is_np_dtype(dtype, "US"):
1426 # we need to coerce to object dtype to avoid
1427 # to allow numpy to take our string as a scalar value
1428 dtype = np.dtype("object")
1429 if not isna(value):
1430 value = ensure_str(value)
1431 elif dtype.kind in "mM":
1432 value = _maybe_box_and_unbox_datetimelike(value, dtype)
1434 subarr = np.empty(length, dtype=dtype)
1435 if length:
1436 # GH 47391: numpy > 1.24 will raise filling np.nan into int dtypes
1437 subarr.fill(value)
1439 return subarr
1442def maybe_unbox_numpy_scalar(value):
1443 result = value
1444 if using_python_scalars() and isinstance(value, np.generic):
1445 if isinstance(result, np.longdouble):
1446 result = float(result)
1447 elif isinstance(result, np.complex256):
1448 result = complex(result)
1449 elif isinstance(result, np.datetime64):
1450 result = Timestamp(result)
1451 elif isinstance(result, np.timedelta64):
1452 result = Timedelta(result)
1453 else:
1454 result = value.item()
1455 return result
1458def _maybe_box_and_unbox_datetimelike(value: Scalar, dtype: DtypeObj):
1459 # Caller is responsible for checking dtype.kind in "mM"
1461 if isinstance(value, dt.datetime):
1462 # we dont want to box dt64, in particular datetime64("NaT")
1463 value = maybe_box_datetimelike(value, dtype)
1465 return _maybe_unbox_datetimelike(value, dtype)
1468def construct_1d_object_array_from_listlike(values: Collection) -> np.ndarray:
1469 """
1470 Transform any list-like object in a 1-dimensional numpy array of object
1471 dtype.
1473 Parameters
1474 ----------
1475 values : any iterable which has a len()
1477 Raises
1478 ------
1479 TypeError
1480 * If `values` does not have a len()
1482 Returns
1483 -------
1484 1-dimensional numpy array of dtype object
1485 """
1486 # numpy will try to interpret nested lists as further dimensions in np.array(),
1487 # hence explicitly making a 1D array using np.fromiter
1488 return np.fromiter(values, dtype="object", count=len(values))
1491def maybe_cast_to_integer_array(arr: list | np.ndarray, dtype: np.dtype) -> np.ndarray:
1492 """
1493 Takes any dtype and returns the casted version, raising for when data is
1494 incompatible with integer/unsigned integer dtypes.
1496 Parameters
1497 ----------
1498 arr : np.ndarray or list
1499 The array to cast.
1500 dtype : np.dtype
1501 The integer dtype to cast the array to.
1503 Returns
1504 -------
1505 ndarray
1506 Array of integer or unsigned integer dtype.
1508 Raises
1509 ------
1510 OverflowError : the dtype is incompatible with the data
1511 ValueError : loss of precision has occurred during casting
1513 Examples
1514 --------
1515 If you try to coerce negative values to unsigned integers, it raises:
1517 >>> pd.Series([-1], dtype="uint64")
1518 Traceback (most recent call last):
1519 ...
1520 OverflowError: Trying to coerce negative values to unsigned integers
1522 Also, if you try to coerce float values to integers, it raises:
1524 >>> maybe_cast_to_integer_array([1, 2, 3.5], dtype=np.dtype("int64"))
1525 Traceback (most recent call last):
1526 ...
1527 ValueError: Trying to coerce float values to integers
1528 """
1529 assert dtype.kind in "iu"
1531 try:
1532 if not isinstance(arr, np.ndarray):
1533 with warnings.catch_warnings():
1534 # We already disallow dtype=uint w/ negative numbers
1535 # (test_constructor_coercion_signed_to_unsigned) so safe to ignore.
1536 warnings.filterwarnings(
1537 "ignore",
1538 "NumPy will stop allowing conversion of out-of-bound Python int",
1539 DeprecationWarning,
1540 )
1541 casted = np.asarray(arr, dtype=dtype)
1542 else:
1543 with warnings.catch_warnings():
1544 warnings.filterwarnings("ignore", category=RuntimeWarning)
1545 casted = arr.astype(dtype, copy=False)
1546 except OverflowError as err:
1547 raise OverflowError(
1548 "The elements provided in the data cannot all be "
1549 f"casted to the dtype {dtype}"
1550 ) from err
1552 if isinstance(arr, np.ndarray) and arr.dtype == dtype:
1553 # avoid expensive array_equal check
1554 return casted
1556 with warnings.catch_warnings():
1557 warnings.filterwarnings("ignore", category=RuntimeWarning)
1558 warnings.filterwarnings(
1559 "ignore", "elementwise comparison failed", FutureWarning
1560 )
1561 if np.array_equal(arr, casted):
1562 return casted
1564 # We do this casting to allow for proper
1565 # data and dtype checking.
1566 #
1567 # We didn't do this earlier because NumPy
1568 # doesn't handle `uint64` correctly.
1569 arr = np.asarray(arr)
1571 if np.issubdtype(arr.dtype, str):
1572 # TODO(numpy-2.0 min): This case will raise an OverflowError above
1573 if (casted.astype(str) == arr).all():
1574 return casted
1575 raise ValueError(f"string values cannot be losslessly cast to {dtype}")
1577 if dtype.kind == "u" and (arr < 0).any():
1578 # TODO: can this be hit anymore after numpy 2.0?
1579 raise OverflowError("Trying to coerce negative values to unsigned integers")
1581 if arr.dtype.kind == "f":
1582 if not np.isfinite(arr).all():
1583 raise IntCastingNaNError(
1584 "Cannot convert non-finite values (NA or inf) to integer"
1585 )
1586 raise ValueError("Trying to coerce float values to integers")
1587 if arr.dtype == object:
1588 raise ValueError("Trying to coerce object values to integers")
1590 if casted.dtype < arr.dtype:
1591 # TODO: Can this path be hit anymore with numpy > 2
1592 # GH#41734 e.g. [1, 200, 923442] and dtype="int8" -> overflows
1593 raise ValueError(
1594 f"Values are too large to be losslessly converted to {dtype}. "
1595 f"To cast anyway, use pd.Series(values).astype({dtype})"
1596 )
1598 if arr.dtype.kind in "mM":
1599 # test_constructor_maskedarray_nonfloat
1600 raise TypeError(
1601 f"Constructing a Series or DataFrame from {arr.dtype} values and "
1602 f"dtype={dtype} is not supported. Use values.view({dtype}) instead."
1603 )
1605 # No known cases that get here, but raising explicitly to cover our bases.
1606 raise ValueError(f"values cannot be losslessly cast to {dtype}")
1609def can_hold_element(arr: ArrayLike, element: Any) -> bool:
1610 """
1611 Can we do an inplace setitem with this element in an array with this dtype?
1613 Parameters
1614 ----------
1615 arr : np.ndarray or ExtensionArray
1616 element : Any
1618 Returns
1619 -------
1620 bool
1621 """
1622 dtype = arr.dtype
1623 if not isinstance(dtype, np.dtype) or dtype.kind in "mM":
1624 if isinstance(dtype, (PeriodDtype, IntervalDtype, DatetimeTZDtype, np.dtype)):
1625 # np.dtype here catches datetime64ns and timedelta64ns; we assume
1626 # in this case that we have DatetimeArray/TimedeltaArray
1627 arr = cast(
1628 "PeriodArray | DatetimeArray | TimedeltaArray | IntervalArray", arr
1629 )
1630 try:
1631 arr._validate_setitem_value(element)
1632 return True
1633 except (ValueError, TypeError):
1634 return False
1636 if dtype == "string":
1637 try:
1638 arr._maybe_convert_setitem_value(element) # type: ignore[union-attr]
1639 return True
1640 except (ValueError, TypeError):
1641 return False
1643 # This is technically incorrect, but maintains the behavior of
1644 # ExtensionBlock._can_hold_element
1645 return True
1647 try:
1648 np_can_hold_element(dtype, element)
1649 return True
1650 except (TypeError, LossySetitemError):
1651 return False
1654def np_can_hold_element(dtype: np.dtype, element: Any) -> Any:
1655 """
1656 Raise if we cannot losslessly set this element into an ndarray with this dtype.
1658 Specifically about places where we disagree with numpy. i.e. there are
1659 cases where numpy will raise in doing the setitem that we do not check
1660 for here, e.g. setting str "X" into a numeric ndarray.
1662 Returns
1663 -------
1664 Any
1665 The element, potentially cast to the dtype.
1667 Raises
1668 ------
1669 ValueError : If we cannot losslessly store this element with this dtype.
1670 """
1671 if dtype == _dtype_obj:
1672 return element
1674 tipo = _maybe_infer_dtype_type(element)
1676 if dtype.kind in "iu":
1677 if isinstance(element, range):
1678 if _dtype_can_hold_range(element, dtype):
1679 return element
1680 raise LossySetitemError
1682 if is_integer(element) or (is_float(element) and element.is_integer()):
1683 # e.g. test_setitem_series_int8 if we have a python int 1
1684 # tipo may be np.int32, despite the fact that it will fit
1685 # in smaller int dtypes.
1686 info = np.iinfo(dtype)
1687 if info.min <= element <= info.max:
1688 return dtype.type(element)
1689 raise LossySetitemError
1691 if tipo is not None:
1692 if tipo.kind not in "iu":
1693 if isinstance(element, np.ndarray) and element.dtype.kind == "f":
1694 # If all can be losslessly cast to integers, then we can hold them
1695 with np.errstate(invalid="ignore"):
1696 # We check afterwards if cast was losslessly, so no need to show
1697 # the warning
1698 casted = element.astype(dtype)
1699 comp = casted == element
1700 if comp.all():
1701 # Return the casted values bc they can be passed to
1702 # np.putmask, whereas the raw values cannot.
1703 # see TestSetitemFloatNDarrayIntoIntegerSeries
1704 return casted
1705 raise LossySetitemError
1707 elif isinstance(element, ABCExtensionArray) and isinstance(
1708 element.dtype, CategoricalDtype
1709 ):
1710 # GH#52927 setting Categorical value into non-EA frame
1711 # TODO: general-case for EAs?
1712 try:
1713 casted = element.astype(dtype)
1714 except (ValueError, TypeError) as err:
1715 raise LossySetitemError from err
1716 # Check for cases of either
1717 # a) lossy overflow/rounding or
1718 # b) semantic changes like dt64->int64
1719 comp = casted == element
1720 if not comp.all():
1721 raise LossySetitemError
1722 return casted
1724 # Anything other than integer we cannot hold
1725 raise LossySetitemError
1726 if (
1727 dtype.kind == "u"
1728 and isinstance(element, np.ndarray)
1729 and element.dtype.kind == "i"
1730 ):
1731 # see test_where_uint64
1732 casted = element.astype(dtype)
1733 if (casted == element).all():
1734 # TODO: faster to check (element >=0).all()? potential
1735 # itemsize issues there?
1736 return casted
1737 raise LossySetitemError
1738 if dtype.itemsize < tipo.itemsize:
1739 raise LossySetitemError
1740 if not isinstance(tipo, np.dtype):
1741 # i.e. nullable IntegerDtype; we can put this into an ndarray
1742 # losslessly iff it has no NAs
1743 arr = element._values if isinstance(element, ABCSeries) else element
1744 if arr._hasna:
1745 raise LossySetitemError
1746 return element
1748 return element
1750 raise LossySetitemError
1752 if dtype.kind == "f":
1753 if lib.is_integer(element) or lib.is_float(element):
1754 casted = dtype.type(element)
1755 if np.isnan(casted) or casted == element:
1756 return casted
1757 # otherwise e.g. overflow see TestCoercionFloat32
1758 raise LossySetitemError
1760 if tipo is not None:
1761 # TODO: itemsize check?
1763 if isinstance(tipo, CategoricalDtype):
1764 # GH#56376
1765 if tipo.categories.dtype.kind not in "iuf":
1766 # Anything other than float/integer we cannot hold
1767 raise LossySetitemError
1768 casted = np.asarray(element, dtype=dtype)
1769 if np.array_equal(casted, element, equal_nan=True):
1770 return casted
1771 raise LossySetitemError
1773 if tipo.kind not in "iuf":
1774 # Anything other than float/integer we cannot hold
1775 raise LossySetitemError
1776 if not isinstance(tipo, np.dtype):
1777 # i.e. nullable IntegerDtype or FloatingDtype;
1778 # we can put this into an ndarray losslessly iff it has no NAs
1779 if element._hasna:
1780 raise LossySetitemError
1781 return element
1782 elif tipo.itemsize > dtype.itemsize or tipo.kind != dtype.kind:
1783 if isinstance(element, np.ndarray):
1784 # e.g. TestDataFrameIndexingWhere::test_where_alignment
1785 casted = element.astype(dtype)
1786 if np.array_equal(casted, element, equal_nan=True):
1787 return casted
1788 raise LossySetitemError
1790 return element
1792 raise LossySetitemError
1794 if dtype.kind == "c":
1795 if lib.is_integer(element) or lib.is_complex(element) or lib.is_float(element):
1796 if np.isnan(element):
1797 # see test_where_complex GH#6345
1798 return dtype.type(element)
1800 with warnings.catch_warnings():
1801 warnings.filterwarnings("ignore")
1802 casted = dtype.type(element)
1803 if casted == element:
1804 return casted
1805 # otherwise e.g. overflow see test_32878_complex_itemsize
1806 raise LossySetitemError
1808 if tipo is not None:
1809 if tipo.kind in "iufc":
1810 return element
1811 raise LossySetitemError
1812 raise LossySetitemError
1814 if dtype.kind == "b":
1815 if tipo is not None:
1816 if tipo.kind == "b":
1817 if not isinstance(tipo, np.dtype):
1818 # i.e. we have a BooleanArray
1819 if element._hasna:
1820 # i.e. there are pd.NA elements
1821 raise LossySetitemError
1822 return element
1823 # GH 57338 check boolean array set as object type
1824 if tipo.kind == "O" and isinstance(element, np.ndarray):
1825 if lib.is_bool_array(element):
1826 return element.astype("bool")
1827 raise LossySetitemError
1828 if lib.is_bool(element):
1829 return element
1830 raise LossySetitemError
1832 if dtype.kind == "S":
1833 # TODO: test tests.frame.methods.test_replace tests get here,
1834 # need more targeted tests. xref phofl has a PR about this
1835 if tipo is not None:
1836 if tipo.kind == "S" and tipo.itemsize <= dtype.itemsize:
1837 return element
1838 raise LossySetitemError
1839 if isinstance(element, bytes) and len(element) <= dtype.itemsize:
1840 return element
1841 raise LossySetitemError
1843 if dtype.kind == "V":
1844 # i.e. np.void, which cannot hold _anything_
1845 raise LossySetitemError
1847 raise NotImplementedError(dtype)
1850def _dtype_can_hold_range(rng: range, dtype: np.dtype) -> bool:
1851 """
1852 _maybe_infer_dtype_type infers to int64 (and float64 for very large endpoints),
1853 but in many cases a range can be held by a smaller integer dtype.
1854 Check if this is one of those cases.
1855 """
1856 if not len(rng):
1857 return True
1858 return np_can_cast_scalar(rng.start, dtype) and np_can_cast_scalar(rng.stop, dtype)
1861def np_can_cast_scalar(element: Scalar, dtype: np.dtype) -> bool:
1862 """
1863 np.can_cast pandas-equivalent for pre 2-0 behavior that allowed scalar
1864 inference
1866 Parameters
1867 ----------
1868 element : Scalar
1869 dtype : np.dtype
1871 Returns
1872 -------
1873 bool
1874 """
1875 try:
1876 np_can_hold_element(dtype, element)
1877 return True
1878 except (LossySetitemError, NotImplementedError):
1879 return False