Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from collections import abc
4from datetime import date
5from functools import partial
6from itertools import islice
7from typing import (
8 TYPE_CHECKING,
9 TypeAlias,
10 TypedDict,
11 Union,
12 cast,
13 overload,
14)
15import warnings
17import numpy as np
19from pandas._libs import (
20 lib,
21 tslib,
22)
23from pandas._libs.tslibs import (
24 NaT,
25 OutOfBoundsDatetime,
26 OutOfBoundsTimedelta,
27 Timedelta,
28 Timestamp,
29 astype_overflowsafe,
30 iNaT,
31 is_supported_dtype,
32 periods_per_second,
33 timezones as libtimezones,
34)
35from pandas._libs.tslibs.conversion import (
36 cast_from_unit_vectorized,
37 datetime_from_fields,
38)
39from pandas._libs.tslibs.dtypes import get_default_reso
40from pandas._libs.tslibs.parsing import (
41 DateParseError,
42 guess_datetime_format,
43)
44from pandas._libs.tslibs.strptime import (
45 array_strptime,
46 format_is_iso,
47)
48from pandas._typing import (
49 AnyArrayLike,
50 ArrayLike,
51 DateTimeErrorChoices,
52)
53from pandas.util._decorators import set_module
54from pandas.util._exceptions import find_stack_level
56from pandas.core.dtypes.common import (
57 ensure_object,
58 is_bool_dtype,
59 is_float,
60 is_float_dtype,
61 is_integer,
62 is_integer_dtype,
63 is_list_like,
64 is_numeric_dtype,
65)
66from pandas.core.dtypes.dtypes import (
67 ArrowDtype,
68 DatetimeTZDtype,
69)
70from pandas.core.dtypes.generic import (
71 ABCDataFrame,
72 ABCSeries,
73)
75from pandas.arrays import (
76 DatetimeArray,
77 IntegerArray,
78 NumpyExtensionArray,
79)
80from pandas.core.algorithms import unique
81from pandas.core.arrays.base import ExtensionArray
82from pandas.core.arrays.datetimes import (
83 maybe_convert_dtype,
84 objects_to_datetime64,
85 tz_to_dtype,
86)
87from pandas.core.construction import extract_array
88from pandas.core.indexes.base import Index
89from pandas.core.indexes.datetimes import DatetimeIndex
90from pandas.core.tools.timedeltas import to_timedelta
92if TYPE_CHECKING:
93 from collections.abc import (
94 Callable,
95 Hashable,
96 )
98 from pandas._libs.tslibs.nattype import NaTType
99 from pandas._libs.tslibs.timedeltas import UnitChoices
100 from pandas._typing import TimeUnit
102 from pandas import (
103 DataFrame,
104 Series,
105 )
106 from pandas.core.arrays import (
107 ArrowExtensionArray,
108 TimedeltaArray,
109 )
111# ---------------------------------------------------------------------
112# types used in annotations
114ArrayConvertible: TypeAlias = list | tuple | AnyArrayLike
115Scalar: TypeAlias = float | str
116DatetimeScalar: TypeAlias = Scalar | date | np.datetime64
118DatetimeScalarOrArrayConvertible: TypeAlias = DatetimeScalar | ArrayConvertible
119DatetimeDictArg: TypeAlias = list[Scalar] | tuple[Scalar, ...] | AnyArrayLike
122class YearMonthDayDict(TypedDict, total=True):
123 year: DatetimeDictArg
124 month: DatetimeDictArg
125 day: DatetimeDictArg
128class FulldatetimeDict(YearMonthDayDict, total=False):
129 hour: DatetimeDictArg
130 hours: DatetimeDictArg
131 minute: DatetimeDictArg
132 minutes: DatetimeDictArg
133 second: DatetimeDictArg
134 seconds: DatetimeDictArg
135 ms: DatetimeDictArg
136 us: DatetimeDictArg
137 ns: DatetimeDictArg
140DictConvertible = Union[FulldatetimeDict, "DataFrame"]
141start_caching_at = 50
144# ---------------------------------------------------------------------
147def _guess_datetime_format_for_array(arr, dayfirst: bool | None = False) -> str | None:
148 # Try to guess the format based on the first non-NaN element, return None if can't
149 if (first_non_null := tslib.first_non_null(arr)) != -1:
150 if type(first_non_nan_element := arr[first_non_null]) is str:
151 # GH#32264 np.str_ object
152 guessed_format = guess_datetime_format(
153 first_non_nan_element, dayfirst=dayfirst
154 )
155 if guessed_format is not None:
156 return guessed_format
157 # If there are multiple non-null elements, warn about
158 # how parsing might not be consistent
159 if tslib.first_non_null(arr[first_non_null + 1 :]) != -1:
160 warnings.warn(
161 "Could not infer format, so each element will be parsed "
162 "individually, falling back to `dateutil`. To ensure parsing is "
163 "consistent and as-expected, please specify a format.",
164 UserWarning,
165 stacklevel=find_stack_level(),
166 )
167 return None
170def should_cache(
171 arg: ArrayConvertible,
172 unique_share: float = 0.7,
173 check_count: int | None = None,
174 format: str | None = None,
175 unit: str | None = None,
176) -> bool:
177 """
178 Decides whether to do caching.
180 If the percent of unique elements among `check_count` elements less
181 than `unique_share * 100` then we can do caching.
183 Parameters
184 ----------
185 arg: listlike, tuple, 1-d array, Series
186 unique_share: float, default=0.7, optional
187 0 < unique_share < 1
188 check_count: int, optional
189 0 <= check_count <= len(arg)
190 format : str or None, default None
191 Strftime format to parse time.
192 unit : str or None, default None
193 The unit of the arg (e.g. 's', 'ms').
195 Returns
196 -------
197 do_caching: bool
199 Notes
200 -----
201 By default for a sequence of less than 50 items in size, we don't do
202 caching; for the number of elements less than 5000, we take ten percent of
203 all elements to check for a uniqueness share; if the sequence size is more
204 than 5000, then we check only the first 500 elements.
205 All constants were chosen empirically by.
206 """
207 # GH#65380 O(1) bail for input shapes where caching cannot help: a
208 # numeric+unit cast, an already-datetime dtype, or an ISO 8601 format
209 # (parsed by the C parser rather than strptime) is a vectorized
210 # conversion, so deduplicating the input only adds overhead.
211 # NB: a non-ISO ``format`` is intentionally *not* a bail condition --
212 # strptime parsing of highly-duplicated strings is exactly where caching
213 # pays off (GH#65380 originally bailed on any format and regressed those).
214 if unit is not None:
215 return False
216 if format is not None and format_is_iso(format):
217 return False
218 arg_dtype = getattr(arg, "dtype", None)
219 if (
220 lib.is_np_dtype(arg_dtype, "M")
221 or isinstance(arg_dtype, DatetimeTZDtype)
222 or (isinstance(arg_dtype, ArrowDtype) and arg_dtype.type is Timestamp)
223 ):
224 return False
226 do_caching = True
228 # default realization
229 if check_count is None:
230 # in this case, the gain from caching is negligible
231 if len(arg) <= start_caching_at:
232 return False
234 if len(arg) <= 5000:
235 check_count = len(arg) // 10
236 else:
237 check_count = 500
238 else:
239 assert 0 <= check_count <= len(arg), (
240 "check_count must be in next bounds: [0; len(arg)]"
241 )
242 if check_count == 0:
243 return False
245 assert 0 < unique_share < 1, "unique_share must be in next bounds: (0; 1)"
247 try:
248 # We can't cache if the items are not hashable.
249 unique_elements = set(islice(arg, check_count))
250 except TypeError:
251 return False
252 if len(unique_elements) > check_count * unique_share:
253 do_caching = False
254 return do_caching
257def _maybe_cache(
258 arg: ArrayConvertible,
259 format: str | None,
260 cache: bool,
261 convert_listlike: Callable,
262 unit: str | None = None,
263) -> Series:
264 """
265 Create a cache of unique dates from an array of dates
267 Parameters
268 ----------
269 arg : listlike, tuple, 1-d array, Series
270 format : string
271 Strftime format to parse time
272 cache : bool
273 True attempts to create a cache of converted values
274 convert_listlike : function
275 Conversion function to apply on dates
276 unit : str, optional
277 The unit of the arg.
279 Returns
280 -------
281 cache_array : Series
282 Cache of converted, unique dates. Can be empty
283 """
284 from pandas import Series
286 cache_array = Series(dtype=object)
288 if cache:
289 # Perform a quicker unique check
290 if not should_cache(arg, format=format, unit=unit):
291 return cache_array
293 if not isinstance(arg, (np.ndarray, ExtensionArray, Index, ABCSeries)):
294 arg = np.array(arg)
296 unique_dates = unique(arg)
297 if len(unique_dates) < len(arg):
298 cache_dates = convert_listlike(unique_dates, format)
299 # GH#45319
300 try:
301 cache_array = Series(cache_dates, index=unique_dates, copy=False)
302 except OutOfBoundsDatetime:
303 return cache_array
304 # GH#39882 and GH#35888 in case of None and NaT we get duplicates
305 if not cache_array.index.is_unique:
306 cache_array = cache_array[~cache_array.index.duplicated()]
307 return cache_array
310def _box_as_indexlike(
311 dt_array: ArrayLike, utc: bool = False, name: Hashable | None = None
312) -> Index:
313 """
314 Properly boxes the ndarray of datetimes to DatetimeIndex
315 if it is possible or to generic Index instead
317 Parameters
318 ----------
319 dt_array: 1-d array
320 Array of datetimes to be wrapped in an Index.
321 utc : bool
322 Whether to convert/localize timestamps to UTC.
323 name : string, default None
324 Name for a resulting index
326 Returns
327 -------
328 result : datetime of converted dates
329 - DatetimeIndex if convertible to sole datetime64 type
330 - general Index otherwise
331 """
333 if lib.is_np_dtype(dt_array.dtype, "M"):
334 tz = "utc" if utc else None
335 return DatetimeIndex(dt_array, tz=tz, name=name)
336 return Index(dt_array, name=name, dtype=dt_array.dtype)
339def _convert_and_box_cache(
340 arg: DatetimeScalarOrArrayConvertible,
341 cache_array: Series,
342 name: Hashable | None = None,
343) -> Index:
344 """
345 Convert array of dates with a cache and wrap the result in an Index.
347 Parameters
348 ----------
349 arg : integer, float, string, datetime, list, tuple, 1-d array, Series
350 cache_array : Series
351 Cache of converted, unique dates
352 name : string, default None
353 Name for a DatetimeIndex
355 Returns
356 -------
357 result : Index-like of converted dates
358 """
359 from pandas import Series
361 # map can give back the input's own container, e.g. a Categorical, or object
362 # dtype, so cast to what the uncached conversion produced (GH#28629)
363 result = Series(arg, dtype=cache_array.index.dtype).map(cache_array)
364 result = result.astype(cache_array.dtype)
365 return _box_as_indexlike(result._values, utc=False, name=name)
368def _convert_listlike_datetimes(
369 arg,
370 format: str | None,
371 name: Hashable | None = None,
372 utc: bool = False,
373 unit: str | None = None,
374 errors: DateTimeErrorChoices = "raise",
375 dayfirst: bool | None = None,
376 yearfirst: bool | None = None,
377 exact: bool = True,
378):
379 """
380 Helper function for to_datetime. Performs the conversions of 1D listlike
381 of dates
383 Parameters
384 ----------
385 arg : list, tuple, ndarray, Series, Index
386 date to be parsed
387 name : object
388 None or string for the Index name
389 utc : bool
390 Whether to convert/localize timestamps to UTC.
391 unit : str
392 None or string of the frequency of the passed data
393 errors : str
394 error handing behaviors from to_datetime, 'raise', 'coerce'
395 dayfirst : bool
396 dayfirst parsing behavior from to_datetime
397 yearfirst : bool
398 yearfirst parsing behavior from to_datetime
399 exact : bool, default True
400 exact format matching behavior from to_datetime
402 Returns
403 -------
404 Index-like of parsed dates
405 """
406 if isinstance(arg, (list, tuple)):
407 arg = np.array(arg, dtype="O")
408 elif isinstance(arg, NumpyExtensionArray):
409 arg = np.array(arg)
411 arg_dtype = getattr(arg, "dtype", None)
412 # these are shortcutable
413 tz = "utc" if utc else None
414 if isinstance(arg_dtype, DatetimeTZDtype):
415 if not isinstance(arg, (DatetimeArray, DatetimeIndex)):
416 return DatetimeIndex(arg, tz=tz, name=name)
417 if utc:
418 arg = arg.tz_convert(None).tz_localize("utc")
419 return arg
421 elif isinstance(arg_dtype, ArrowDtype) and arg_dtype.type is Timestamp:
422 # TODO: Combine with above if DTI/DTA supports Arrow timestamps
423 if utc:
424 # pyarrow uses UTC, not lowercase utc
425 if isinstance(arg, Index):
426 arg_array = cast("ArrowExtensionArray", arg.array)
427 if arg_dtype.pyarrow_dtype.tz is not None:
428 arg_array = arg_array._dt_tz_convert("UTC")
429 else:
430 arg_array = arg_array._dt_tz_localize("UTC")
431 arg = Index(arg_array, copy=False)
432 # ArrowExtensionArray
433 elif arg_dtype.pyarrow_dtype.tz is not None:
434 arg = arg._dt_tz_convert("UTC")
435 else:
436 arg = arg._dt_tz_localize("UTC")
437 return arg
439 elif lib.is_np_dtype(arg_dtype, "M"):
440 if not is_supported_dtype(arg_dtype):
441 # We go to closest supported reso, i.e. "s"
442 arg = astype_overflowsafe(
443 np.asarray(arg),
444 np.dtype("M8[s]"),
445 is_coerce=errors == "coerce",
446 )
448 if not isinstance(arg, (DatetimeArray, DatetimeIndex)):
449 return DatetimeIndex(arg, tz=tz, name=name)
450 elif utc:
451 # DatetimeArray, DatetimeIndex
452 return arg.tz_localize("utc")
454 return arg
456 elif unit is not None:
457 if format is not None:
458 raise ValueError("cannot specify both format and unit")
459 return _to_datetime_with_unit(
460 arg, unit, name, utc, errors, dayfirst=dayfirst, yearfirst=yearfirst
461 )
462 elif getattr(arg, "ndim", 1) > 1:
463 raise TypeError(
464 "arg must be a string, datetime, list, tuple, 1-d array, or Series"
465 )
467 # warn if passing timedelta64, raise for PeriodDtype
468 # NB: this must come after unit transformation
469 try:
470 arg, _ = maybe_convert_dtype(arg, copy=False, tz=libtimezones.maybe_get_tz(tz))
471 except TypeError:
472 if errors == "coerce":
473 npvalues = np.full(len(arg), np.datetime64("NaT", "ns"))
474 return DatetimeIndex(npvalues, name=name)
475 raise
477 arg = ensure_object(arg)
479 if format is None:
480 format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
482 # `format` could be inferred, or user didn't ask for mixed-format parsing.
483 if format is not None and format != "mixed":
484 return _array_strptime_with_fallback(arg, name, utc, format, exact, errors)
486 result, tz_parsed = objects_to_datetime64(
487 arg,
488 dayfirst=dayfirst,
489 yearfirst=yearfirst,
490 utc=utc,
491 errors=errors,
492 )
494 if tz_parsed is not None:
495 # We can take a shortcut since the datetime64 numpy array
496 # is in UTC
497 out_unit = np.datetime_data(result.dtype)[0]
498 out_unit = cast("TimeUnit", out_unit)
499 dtype = tz_to_dtype(tz_parsed, out_unit)
500 dt64_values = result.view(f"M8[{dtype.unit}]")
501 dta = DatetimeArray._simple_new(dt64_values, dtype=dtype)
502 return DatetimeIndex._simple_new(dta, name=name)
504 return _box_as_indexlike(result, utc=utc, name=name)
507def _array_strptime_with_fallback(
508 arg,
509 name,
510 utc: bool,
511 fmt: str,
512 exact: bool,
513 errors: str,
514) -> Index:
515 """
516 Call array_strptime, with fallback behavior depending on 'errors'.
517 """
518 result, tz_out = array_strptime(arg, fmt, exact=exact, errors=errors, utc=utc)
519 if tz_out is not None:
520 unit = np.datetime_data(result.dtype)[0]
521 unit = cast("TimeUnit", unit)
522 dtype = DatetimeTZDtype(tz=tz_out, unit=unit)
523 dta = DatetimeArray._simple_new(result, dtype=dtype)
524 if utc:
525 dta = dta.tz_convert("UTC")
526 return Index(dta, name=name, copy=False)
527 elif result.dtype != object and utc:
528 unit = np.datetime_data(result.dtype)[0]
529 unit = cast("TimeUnit", unit)
530 res = Index(result, dtype=f"M8[{unit}, UTC]", name=name, copy=False)
531 return res
532 return Index(result, dtype=result.dtype, name=name, copy=False)
535def _to_datetime_with_unit(
536 arg,
537 unit,
538 name,
539 utc: bool,
540 errors: str,
541 dayfirst,
542 yearfirst,
543) -> Index:
544 """
545 to_datetime specalized to the case where a 'unit' is passed.
546 """
547 arg = extract_array(arg, extract_numpy=True)
549 # convert to an ndarray because tslib.array_to_datetime expects an ndarray argument
550 if isinstance(arg, IntegerArray):
551 mask = arg._mask if arg._hasna else None
552 arg = arg._data
553 if mask is not None:
554 # set to 0 to avoid OOB on masked values (setting iNaT only works for int64)
555 # will get converted to NaT later
556 arg = arg.copy()
557 arg[mask] = 0
558 else:
559 mask = None
560 arg = np.asarray(arg)
562 if arg.dtype.kind in "iu":
563 # Note we can't do "f" here because that could induce unwanted
564 # rounding GH#14156, GH#20445
566 # GH#60677 unsigned integers > int64 max overflow silently
567 # when cast to datetime64 (which is backed by int64)
568 if arg.dtype == np.dtype("uint64"):
569 mask_oob = arg > np.iinfo(np.int64).max
570 if mask_oob.any():
571 if errors == "raise":
572 raise OutOfBoundsDatetime(
573 f"cannot convert input with unit '{unit}'"
574 )
576 arg = arg.astype(object)
577 if mask is not None:
578 arg[mask] = None
579 return _to_datetime_with_unit(
580 arg, unit, name, utc, errors, dayfirst, yearfirst
581 )
582 arr = arg.astype(f"datetime64[{unit}]", copy=False)
583 if unit not in ["us", "ns"]:
584 out_unit = get_default_reso(unit)
585 try:
586 arr = astype_overflowsafe(arr, np.dtype(f"M8[{out_unit}]"), copy=False)
587 except OutOfBoundsDatetime:
588 if errors == "raise":
589 raise
590 arg = arg.astype(object)
591 if mask is not None:
592 arg[mask] = None
593 return _to_datetime_with_unit(
594 arg, unit, name, utc, errors, dayfirst, yearfirst
595 )
596 if mask is not None:
597 arr[mask] = iNaT
598 tz_parsed = None
600 elif arg.dtype.kind == "f":
601 mask = np.isnan(arg)
602 with np.errstate(invalid="ignore"):
603 int_values = arg.astype(np.int64)
604 # On ARM, float-to-int64 overflow saturates to INT64_MAX
605 # instead of wrapping, which makes the arg == int_values
606 # check pass incorrectly for OOB values like float(2**63).
607 # Exclude values outside the int64 domain from the check.
608 i64 = np.iinfo(np.int64)
609 in_int64_range = (arg >= np.float64(i64.min)) & (arg < np.float64(i64.max))
610 if (mask | (in_int64_range & (arg == int_values))).all():
611 # With all-round-or-NaN entries, we give the requested unit
612 # back like with integers
613 result = _to_datetime_with_unit(
614 int_values,
615 unit=unit,
616 name=name,
617 utc=utc,
618 errors=errors,
619 dayfirst=dayfirst,
620 yearfirst=yearfirst,
621 )
622 result._data[mask] = NaT
623 return result
625 arg = arg.astype("float64", copy=False)
626 with np.errstate(over="raise"):
627 try:
628 arr = cast_from_unit_vectorized(arg, unit=unit)
629 except OutOfBoundsDatetime as err:
630 if errors != "raise":
631 return _to_datetime_with_unit(
632 arg.astype(object),
633 unit,
634 name,
635 utc,
636 errors,
637 dayfirst,
638 yearfirst,
639 )
640 raise OutOfBoundsDatetime(
641 f"cannot convert input with unit '{unit}'"
642 ) from err
644 arr = arr.view("M8[ns]")
645 tz_parsed = None
646 else:
647 arg = arg.astype(object, copy=False)
648 arr, tz_parsed = tslib.array_to_datetime(
649 arg,
650 utc=utc,
651 errors=errors,
652 dayfirst=dayfirst,
653 yearfirst=yearfirst,
654 unit_for_numerics=unit,
655 )
657 result = DatetimeIndex(arr, name=name)
658 if not isinstance(result, DatetimeIndex):
659 return result
661 # GH#23758: We may still need to localize the result with tz
662 # GH#25546: Apply tz_parsed first (from arg), then tz (from caller)
663 # result will be naive but in UTC
664 result = result.tz_localize("UTC").tz_convert(tz_parsed)
666 if utc:
667 if result.tz is None:
668 result = result.tz_localize("utc")
669 else:
670 result = result.tz_convert("utc")
671 return result
674def _adjust_to_origin(arg, origin, unit, errors: DateTimeErrorChoices = "raise"):
675 """
676 Helper function for to_datetime.
677 Adjust input argument to the specified origin
679 Parameters
680 ----------
681 arg : list, tuple, ndarray, Series, Index
682 date to be adjusted
683 origin : 'julian' or Timestamp
684 origin offset for the arg
685 unit : str
686 passed unit from to_datetime, must be 'D'
687 errors : {'raise', 'coerce'}, default 'raise'
688 under ``"coerce"``, values for which ``origin + value`` overflows
689 datetime64 become ``NaT`` instead of raising.
691 Returns
692 -------
693 DatetimeArray, ndarray, or scalar of adjusted date(s)
694 """
695 if origin == "julian":
696 original = arg
697 j0 = Timestamp(0).to_julian_date()
698 if unit != "D":
699 raise ValueError("unit must be 'D' for origin='julian'")
700 try:
701 arg = arg - j0
702 except TypeError as err:
703 raise ValueError(
704 "incompatible 'arg' type for given 'origin'='julian'"
705 ) from err
707 # preemptively check this for a nice range
708 j_max = Timestamp.max.to_julian_date() - j0
709 j_min = Timestamp.min.to_julian_date() - j0
710 if np.any(arg > j_max) or np.any(arg < j_min):
711 raise OutOfBoundsDatetime(
712 f"{original} is Out of Bounds for origin='julian'"
713 )
714 else:
715 # arg must be numeric
716 if not (
717 (is_integer(arg) or is_float(arg)) or is_numeric_dtype(np.asarray(arg))
718 ):
719 raise ValueError(
720 f"'{arg}' is not compatible with origin='{origin}'; "
721 "it must be numeric with a unit specified"
722 )
724 # we are going to offset back to unix / epoch time
725 try:
726 if lib.is_integer(origin) or lib.is_float(origin):
727 offset = Timestamp(origin, unit=unit)
728 else:
729 offset = Timestamp(origin)
730 except OutOfBoundsDatetime as err:
731 raise OutOfBoundsDatetime(f"origin {origin} is Out of Bounds") from err
732 except ValueError as err:
733 raise ValueError(
734 f"origin {origin} cannot be converted to a Timestamp"
735 ) from err
737 if offset.tz is not None:
738 raise ValueError(f"origin offset {offset} must be tz-naive")
739 tda = extract_array(to_timedelta(arg, unit=unit, errors=errors))
740 try:
741 arg = offset + tda
742 except (OutOfBoundsDatetime, OutOfBoundsTimedelta, OverflowError) as err:
743 # GH#63419 the timedeltas fit their own resolution, but
744 # offset + tda overflows datetime64
745 if errors == "raise":
746 if isinstance(err, OutOfBoundsDatetime):
747 raise
748 raise OutOfBoundsDatetime(
749 f"cannot add values to origin {origin} without overflow"
750 ) from err
751 if isinstance(tda, Timedelta):
752 arg = NaT
753 else:
754 arg = _coerce_origin_overflow(offset, tda)
755 return arg
758def _coerce_origin_overflow(offset: Timestamp, tda: TimedeltaArray) -> DatetimeArray:
759 """
760 Redo ``offset + tda`` with entries that cannot be represented in the
761 resolution the addition would produce coerced to NaT.
762 """
763 res_unit = tda.unit if tda._creso >= offset._creso else offset.unit
764 pps_res = periods_per_second(max(tda._creso, offset._creso))
765 # exact conversion factors as Python ints to avoid intermediate overflow
766 td_factor = pps_res // periods_per_second(tda._creso)
767 off_factor = pps_res // periods_per_second(offset._creso)
768 offset_i8 = int(offset._value) * off_factor
770 i8max = np.iinfo(np.int64).max
771 if not -i8max <= offset_i8 <= i8max:
772 # the origin itself is not representable in the result resolution,
773 # so no entry can be computed
774 nat_vals = np.full(len(tda), iNaT).view(f"M8[{res_unit}]")
775 return DatetimeArray._simple_new(nat_vals, dtype=nat_vals.dtype)
777 # valid iff the sum stays within [-i8max, i8max] (-2**63 is the NaT
778 # sentinel) and the timedelta itself is castable to the result unit
779 hi = min((i8max - offset_i8) // td_factor, i8max // td_factor)
780 lo = max(-((i8max + offset_i8) // td_factor), -(i8max // td_factor))
782 i8vals = tda.asi8
783 mask = (i8vals < lo) | (i8vals > hi)
784 new_vals = np.where(mask, iNaT, i8vals).view(tda.dtype)
785 tda = type(tda)._simple_new(new_vals, dtype=tda.dtype)
786 return offset + tda
789@overload
790def to_datetime(
791 arg: DatetimeScalar,
792 errors: DateTimeErrorChoices = ...,
793 dayfirst: bool = ...,
794 yearfirst: bool = ...,
795 utc: bool = ...,
796 format: str | None = ...,
797 exact: bool = ...,
798 unit: str | None = ...,
799 origin=...,
800 cache: bool = ...,
801) -> Timestamp: ...
804@overload
805def to_datetime(
806 arg: Series | DictConvertible,
807 errors: DateTimeErrorChoices = ...,
808 dayfirst: bool = ...,
809 yearfirst: bool = ...,
810 utc: bool = ...,
811 format: str | None = ...,
812 exact: bool = ...,
813 unit: str | None = ...,
814 origin=...,
815 cache: bool = ...,
816) -> Series: ...
819@overload
820def to_datetime(
821 arg: list | tuple | Index | ArrayLike,
822 errors: DateTimeErrorChoices = ...,
823 dayfirst: bool = ...,
824 yearfirst: bool = ...,
825 utc: bool = ...,
826 format: str | None = ...,
827 exact: bool = ...,
828 unit: str | None = ...,
829 origin=...,
830 cache: bool = ...,
831) -> DatetimeIndex: ...
834@set_module("pandas")
835def to_datetime(
836 arg: DatetimeScalarOrArrayConvertible | DictConvertible,
837 errors: DateTimeErrorChoices = "raise",
838 dayfirst: bool = False,
839 yearfirst: bool = False,
840 utc: bool = False,
841 format: str | None = None,
842 exact: bool | lib.NoDefault = lib.no_default,
843 unit: str | None = None,
844 origin: str = "unix",
845 cache: bool = True,
846) -> DatetimeIndex | Series | DatetimeScalar | NaTType:
847 """
848 Convert argument to datetime.
850 This function converts a scalar, array-like, :class:`Series` or
851 :class:`DataFrame`/dict-like to a pandas datetime object.
853 Parameters
854 ----------
855 arg : int, float, str, datetime, list, tuple, 1-d array, Series, DataFrame/dict-like
856 The object to convert to a datetime. If a :class:`DataFrame` is provided, the
857 method expects minimally the following columns: :const:`"year"`,
858 :const:`"month"`, :const:`"day"`. The column "year"
859 must be specified in 4-digit format.
860 errors : {'raise', 'coerce'}, default 'raise'
861 - If :const:`'raise'`, then invalid parsing will raise an exception.
862 - If :const:`'coerce'`, then invalid parsing will be set as :const:`NaT`.
863 dayfirst : bool, default False
864 Specify a date parse order if `arg` is str or is list-like.
865 If :const:`True`, parses dates with the day first, e.g. :const:`"10/11/12"`
866 is parsed as :const:`2012-11-10`.
868 .. warning::
870 ``dayfirst=True`` is not strict, but will prefer to parse
871 with day first.
873 yearfirst : bool, default False
874 Specify a date parse order if `arg` is str or is list-like.
876 - If :const:`True` parses dates with the year first, e.g.
877 :const:`"10/11/12"` is parsed as :const:`2010-11-12`.
878 - If both `dayfirst` and `yearfirst` are :const:`True`, `yearfirst` is
879 preceded (same as ``dateutil``).
881 .. warning::
883 ``yearfirst=True`` is not strict, but will prefer to parse
884 with year first.
886 utc : bool, default False
887 Control timezone-related parsing, localization and conversion.
889 - If :const:`True`, the function *always* returns a timezone-aware
890 UTC-localized :class:`Timestamp`, :class:`Series` or
891 :class:`DatetimeIndex`. To do this, timezone-naive inputs are
892 *localized* as UTC, while timezone-aware inputs are *converted* to UTC.
894 - If :const:`False` (default), inputs will not be coerced to UTC.
895 Timezone-naive inputs will remain naive, while timezone-aware ones
896 will keep their time offsets. Limitations exist for mixed
897 offsets (typically, daylight savings), see :ref:`Examples
898 <to_datetime_tz_examples>` section for details.
900 See also: pandas general documentation about `timezone conversion and
901 localization
902 <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
903 #time-zone-handling>`_.
905 format : str, default None
906 The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See
907 `strftime documentation
908 <https://docs.python.org/3/library/datetime.html
909 #strftime-and-strptime-behavior>`_ for more information on choices, though
910 note that :const:`"%f"` will parse all the way up to nanoseconds.
911 You can also pass:
913 - "ISO8601", to parse any `ISO8601 <https://en.wikipedia.org/wiki/ISO_8601>`_
914 time string (not necessarily in exactly the same format);
915 - "mixed", to infer the format for each element individually. This is risky,
916 and you should probably use it along with `dayfirst`.
918 .. deprecated:: 3.1.0
920 Passing integer or float values together with ``format`` is
921 deprecated; cast them to strings first. In a future version numeric
922 values will be interpreted as epochs via ``unit`` instead, and
923 ``format`` will only apply to string input.
925 .. note::
927 If a :class:`DataFrame` is passed, then `format` has no effect.
929 .. note::
931 When using ``format``, any datetime components not present in the
932 format string default to ``1900-01-01 00:00:00``, consistent with
933 Python's :meth:`datetime.datetime.strptime` behavior. For example,
934 ``to_datetime(["1", "2"], format="%d")`` returns dates in January 1900.
935 The ``origin`` parameter does not affect string parsing via ``format``;
936 it only applies to numeric input interpreted through ``unit``.
938 exact : bool, default True
939 Control how `format` is used:
941 - If :const:`True`, require an exact `format` match.
942 - If :const:`False`, allow the `format` to match anywhere in the target
943 string.
945 Cannot be used alongside ``format='ISO8601'`` or ``format='mixed'``.
946 unit : str, default 'ns'
947 The unit of the numeric arg (Y, M, W, D, h, m, s, ms, us, ns, ps,
948 fs, as). Specifies the unit of the input values when `arg` is numeric
949 (int or float), interpreted relative to ``origin``.
950 For example, with ``unit='ms'`` and ``origin='unix'``, the input values
951 are treated as millisecond offsets from the Unix epoch (1970-01-01).
953 This does not truncate or round datetime-like inputs to the given unit.
954 To change the resolution of the result, use :meth:`Series.dt.as_unit`.
955 To truncate datetime values, use :meth:`Series.dt.floor` or
956 :meth:`Series.dt.normalize`.
958 Only applicable to numeric input; has no effect on datetime-like input
959 or when ``format`` is specified.
960 origin : scalar, default 'unix'
961 Define the reference date. The numeric values would be parsed as number
962 of units (defined by ``unit``) since this reference date.
964 - If :const:`'unix'` (or POSIX) time; origin is set to 1970-01-01.
965 - If :const:`'julian'`, unit must be :const:`'D'`, and origin is set to
966 beginning of Julian Calendar. Julian day number :const:`0` is assigned
967 to the day starting at noon on January 1, 4713 BC.
968 - If Timestamp convertible (Timestamp, dt.datetime, np.datetime64 or date
969 string), origin is set to Timestamp identified by origin.
970 - If a float or integer, origin is the difference
971 (in units determined by the ``unit`` argument) relative to 1970-01-01.
973 .. note::
975 This parameter only affects numeric input used with ``unit``.
976 It does not affect string parsing via ``format``. See the ``format``
977 parameter for how defaults are handled during string parsing.
978 cache : bool, default True
979 If :const:`True`, use a cache of unique, converted dates to apply the
980 datetime conversion. May produce significant speed-up when parsing
981 duplicate date strings, especially ones with timezone offsets. The cache
982 is only used when there are at least 50 values. The presence of
983 out-of-bounds values will render the cache unusable and may slow down
984 parsing.
986 Returns
987 -------
988 datetime
989 If parsing succeeded.
990 Return type depends on input (types in parenthesis correspond to
991 fallback in case of unsuccessful timezone or out-of-range timestamp
992 parsing):
994 - scalar: :class:`Timestamp` (or :class:`datetime.datetime`)
995 - array-like: :class:`DatetimeIndex` (or :class:`Series` with
996 :class:`object` dtype containing :class:`datetime.datetime`)
997 - Series: :class:`Series` of :class:`datetime64` dtype (or
998 :class:`Series` of :class:`object` dtype containing
999 :class:`datetime.datetime`)
1000 - DataFrame: :class:`Series` of :class:`datetime64` dtype (or
1001 :class:`Series` of :class:`object` dtype containing
1002 :class:`datetime.datetime`)
1004 Raises
1005 ------
1006 ParserError
1007 When parsing a date from string fails.
1008 ValueError
1009 When another datetime conversion error happens. For example when one
1010 of 'year', 'month', day' columns is missing in a :class:`DataFrame`, or
1011 when a Timezone-aware :class:`datetime.datetime` is found in an array-like
1012 of mixed time offsets, and ``utc=False``, or when parsing datetimes
1013 with mixed time zones unless ``utc=True``. If parsing datetimes with mixed
1014 time zones, please specify ``utc=True``.
1016 See Also
1017 --------
1018 DataFrame.astype : Cast argument to a specified dtype.
1019 to_timedelta : Convert argument to timedelta.
1020 convert_dtypes : Convert dtypes.
1022 Notes
1023 -----
1025 Many input types are supported, and lead to different output types:
1027 - **scalars** can be int, float, str, datetime object (from stdlib :mod:`datetime`
1028 module or :mod:`numpy`). They are converted to :class:`Timestamp` when
1029 possible, otherwise they are converted to :class:`datetime.datetime`.
1030 None/NaN/null scalars are converted to :const:`NaT`.
1032 - **array-like** can contain int, float, str, datetime objects. They are
1033 converted to :class:`DatetimeIndex` when possible, otherwise they are
1034 converted to :class:`Index` with :class:`object` dtype, containing
1035 :class:`datetime.datetime`. None/NaN/null entries are converted to
1036 :const:`NaT` in both cases.
1038 - **Series** are converted to :class:`Series` with :class:`datetime64`
1039 dtype when possible, otherwise they are converted to :class:`Series` with
1040 :class:`object` dtype, containing :class:`datetime.datetime`. None/NaN/null
1041 entries are converted to :const:`NaT` in both cases.
1043 - **DataFrame/dict-like** are converted to :class:`Series` with
1044 :class:`datetime64` dtype. For each row a datetime is created from assembling
1045 the various dataframe columns. Column keys can be common abbreviations
1046 like ['year', 'month', 'day', 'minute', 'second', 'ms', 'us', 'ns']) or
1047 plurals of the same.
1049 The following causes are responsible for :class:`datetime.datetime` objects
1050 being returned (possibly inside an :class:`Index` or a :class:`Series` with
1051 :class:`object` dtype) instead of a proper pandas designated type
1052 (:class:`Timestamp`, :class:`DatetimeIndex` or :class:`Series`
1053 with :class:`datetime64` dtype):
1055 - when any input element is before :const:`Timestamp.min` or after
1056 :const:`Timestamp.max`, see `timestamp limitations
1057 <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
1058 #timeseries-timestamp-limits>`_.
1060 - when ``utc=False`` (default) and the input is an array-like or
1061 :class:`Series` containing mixed naive/aware datetime, or aware with mixed
1062 time offsets. Note that this happens in the (quite frequent) situation when
1063 the timezone has a daylight savings policy. In that case you may wish to
1064 use ``utc=True``.
1066 Examples
1067 --------
1069 **Handling various input formats**
1071 Assembling a datetime from multiple columns of a :class:`DataFrame`. The keys
1072 can be common abbreviations like ['year', 'month', 'day', 'minute', 'second',
1073 'ms', 'us', 'ns']) or plurals of the same
1075 >>> df = pd.DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
1076 >>> pd.to_datetime(df)
1077 0 2015-02-04
1078 1 2016-03-05
1079 dtype: datetime64[us]
1081 Using a unix epoch time
1083 >>> pd.to_datetime(1490195805, unit="s")
1084 Timestamp('2017-03-22 15:16:45')
1085 >>> pd.to_datetime(1490195805433502912, unit="ns")
1086 Timestamp('2017-03-22 15:16:45.433502912')
1088 .. warning:: For float arg, precision rounding might happen. To prevent
1089 unexpected behavior use a fixed-width exact type.
1091 Using a non-unix epoch origin
1093 >>> pd.to_datetime([1, 2, 3], unit="D", origin=pd.Timestamp("1960-01-01"))
1094 DatetimeIndex(['1960-01-02', '1960-01-03', '1960-01-04'],
1095 dtype='datetime64[us]', freq=None)
1097 **Differences with strptime behavior**
1099 :const:`"%f"` will parse all the way up to nanoseconds.
1101 >>> pd.to_datetime("2018-10-26 12:00:00.0000000011", format="%Y-%m-%d %H:%M:%S.%f")
1102 Timestamp('2018-10-26 12:00:00.000000001')
1104 **Non-convertible date/times**
1106 Passing ``errors='coerce'`` will force an out-of-bounds date to :const:`NaT`,
1107 in addition to forcing non-dates (or non-parseable dates) to :const:`NaT`.
1109 >>> pd.to_datetime("invalid for Ymd", format="%Y%m%d", errors="coerce")
1110 NaT
1112 .. _to_datetime_tz_examples:
1114 **Timezones and time offsets**
1116 The default behaviour (``utc=False``) is as follows:
1118 - Timezone-naive inputs are converted to timezone-naive :class:`DatetimeIndex`:
1120 >>> pd.to_datetime(["2018-10-26 12:00:00", "2018-10-26 13:00:15"])
1121 DatetimeIndex(['2018-10-26 12:00:00', '2018-10-26 13:00:15'],
1122 dtype='datetime64[us]', freq=None)
1124 - Timezone-aware inputs *with constant time offset* are converted to
1125 timezone-aware :class:`DatetimeIndex`:
1127 >>> pd.to_datetime(["2018-10-26 12:00 -0500", "2018-10-26 13:00 -0500"])
1128 DatetimeIndex(['2018-10-26 12:00:00-05:00', '2018-10-26 13:00:00-05:00'],
1129 dtype='datetime64[us, UTC-05:00]', freq=None)
1131 - However, timezone-aware inputs *with mixed time offsets* (for example
1132 issued from a timezone with daylight savings, such as Europe/Paris)
1133 are **not successfully converted** to a :class:`DatetimeIndex`.
1134 Parsing datetimes with mixed time zones will raise a ValueError unless
1135 ``utc=True``:
1137 >>> pd.to_datetime(
1138 ... ["2020-10-25 02:00 +0200", "2020-10-25 04:00 +0100"]
1139 ... ) # doctest: +SKIP
1140 ValueError: Mixed timezones detected. Pass utc=True in to_datetime
1141 or tz='UTC' in DatetimeIndex to convert to a common timezone.
1143 - To create a :class:`Series` with mixed offsets and ``object`` dtype, please use
1144 :meth:`Series.apply` and :func:`datetime.datetime.strptime`:
1146 >>> import datetime as dt
1147 >>> ser = pd.Series(["2020-10-25 02:00 +0200", "2020-10-25 04:00 +0100"])
1148 >>> ser.apply(lambda x: dt.datetime.strptime(x, "%Y-%m-%d %H:%M %z"))
1149 0 2020-10-25 02:00:00+02:00
1150 1 2020-10-25 04:00:00+01:00
1151 dtype: object
1153 - A mix of timezone-aware and timezone-naive inputs will also raise a ValueError
1154 unless ``utc=True``:
1156 >>> from datetime import datetime
1157 >>> pd.to_datetime(
1158 ... ["2020-01-01 01:00:00-01:00", datetime(2020, 1, 1, 3, 0)]
1159 ... ) # doctest: +SKIP
1160 ValueError: Mixed timezones detected. Pass utc=True in to_datetime
1161 or tz='UTC' in DatetimeIndex to convert to a common timezone.
1163 |
1165 Setting ``utc=True`` solves most of the above issues:
1167 - Timezone-naive inputs are *localized* as UTC
1169 >>> pd.to_datetime(["2018-10-26 12:00", "2018-10-26 13:00"], utc=True)
1170 DatetimeIndex(['2018-10-26 12:00:00+00:00', '2018-10-26 13:00:00+00:00'],
1171 dtype='datetime64[us, UTC]', freq=None)
1173 - Timezone-aware inputs are *converted* to UTC (the output represents the
1174 exact same datetime, but viewed from the UTC time offset `+00:00`).
1176 >>> pd.to_datetime(["2018-10-26 12:00 -0530", "2018-10-26 12:00 -0500"], utc=True)
1177 DatetimeIndex(['2018-10-26 17:30:00+00:00', '2018-10-26 17:00:00+00:00'],
1178 dtype='datetime64[us, UTC]', freq=None)
1180 - Inputs can contain both string or datetime, the above
1181 rules still apply
1183 >>> pd.to_datetime(["2018-10-26 12:00", datetime(2020, 1, 1, 18)], utc=True)
1184 DatetimeIndex(['2018-10-26 12:00:00+00:00', '2020-01-01 18:00:00+00:00'],
1185 dtype='datetime64[us, UTC]', freq=None)
1186 """
1187 if errors not in ("raise", "coerce"):
1188 raise ValueError("errors must be one of 'raise', or 'coerce'.")
1189 if exact is not lib.no_default and format in {"mixed", "ISO8601"}:
1190 raise ValueError("Cannot use 'exact' when 'format' is 'mixed' or 'ISO8601'")
1191 if arg is None:
1192 return NaT
1194 if origin != "unix":
1195 # Capture name/index before _adjust_to_origin replaces arg with a
1196 # DatetimeArray (or Timestamp scalar) for non-julian origins.
1197 arg_name = getattr(arg, "name", None)
1198 arg_index = arg.index if isinstance(arg, ABCSeries) else None
1199 is_series = isinstance(arg, ABCSeries)
1200 arg = _adjust_to_origin(arg, origin, unit, errors=errors)
1201 if origin != "julian":
1202 # GH#63419 _adjust_to_origin already produced the final datetime
1203 # result; localize and re-wrap into the input's container type.
1204 if utc:
1205 arg = arg.tz_localize("utc") # type: ignore[union-attr]
1206 if is_series:
1207 from pandas import Series
1209 return Series(arg, index=arg_index, name=arg_name)
1210 if is_list_like(arg):
1211 return DatetimeIndex(arg, name=arg_name)
1212 else:
1213 return arg # type: ignore[return-value]
1215 convert_listlike = partial(
1216 _convert_listlike_datetimes,
1217 utc=utc,
1218 unit=unit,
1219 dayfirst=dayfirst,
1220 yearfirst=yearfirst,
1221 errors=errors,
1222 exact=exact, # type: ignore[arg-type]
1223 )
1224 result: Timestamp | NaTType | Series | Index
1226 if isinstance(arg, Timestamp):
1227 result = arg
1228 if utc:
1229 if arg.tz is not None:
1230 result = arg.tz_convert("utc")
1231 else:
1232 result = arg.tz_localize("utc")
1233 elif isinstance(arg, ABCSeries):
1234 cache_array = _maybe_cache(arg, format, cache, convert_listlike, unit)
1235 if not cache_array.empty:
1236 # see _convert_and_box_cache re: astype
1237 result = arg.map(cache_array).astype(cache_array.dtype)
1238 else:
1239 values = convert_listlike(arg._values, format)
1240 result = arg._constructor(values, index=arg.index, name=arg.name)
1241 elif isinstance(arg, (ABCDataFrame, abc.MutableMapping)):
1242 result = _assemble_from_unit_mappings(arg, errors, utc)
1243 elif isinstance(arg, Index):
1244 cache_array = _maybe_cache(arg, format, cache, convert_listlike, unit)
1245 if not cache_array.empty:
1246 result = _convert_and_box_cache(arg, cache_array, name=arg.name)
1247 else:
1248 result = convert_listlike(arg, format, name=arg.name)
1249 elif is_list_like(arg):
1250 try:
1251 # error: Argument 1 to "_maybe_cache" has incompatible type
1252 # "Union[float, str, datetime, List[Any], Tuple[Any, ...], ExtensionArray,
1253 # ndarray[Any, Any], Series]"; expected "Union[List[Any], Tuple[Any, ...],
1254 # Union[Union[ExtensionArray, ndarray[Any, Any]], Index, Series], Series]"
1255 argc = cast(
1256 "list | tuple | ExtensionArray | np.ndarray | Series | Index", arg
1257 )
1258 cache_array = _maybe_cache(argc, format, cache, convert_listlike, unit)
1259 except OutOfBoundsDatetime:
1260 # caching attempts to create a DatetimeIndex, which may raise
1261 # an OOB. If that's the desired behavior, then just reraise...
1262 if errors == "raise":
1263 raise
1264 # ... otherwise, continue without the cache.
1265 from pandas import Series
1267 cache_array = Series([], dtype=object) # just an empty array
1268 if not cache_array.empty:
1269 result = _convert_and_box_cache(argc, cache_array)
1270 else:
1271 result = convert_listlike(argc, format)
1272 else:
1273 result = convert_listlike(np.array([arg]), format)[0]
1274 if isinstance(arg, bool) and isinstance(result, np.bool_):
1275 result = bool(result) # TODO: avoid this kludge.
1277 # error: Incompatible return value type (got "Union[Timestamp, NaTType,
1278 # Series, Index]", expected "Union[DatetimeIndex, Series, float, str,
1279 # NaTType, None]")
1280 return result # type: ignore[return-value]
1283# mappings for assembling units
1284_unit_map = {
1285 "year": "year",
1286 "years": "year",
1287 "month": "month",
1288 "months": "month",
1289 "day": "day",
1290 "days": "day",
1291 "hour": "h",
1292 "hours": "h",
1293 "minute": "m",
1294 "minutes": "m",
1295 "second": "s",
1296 "seconds": "s",
1297 "ms": "ms",
1298 "millisecond": "ms",
1299 "milliseconds": "ms",
1300 "us": "us",
1301 "microsecond": "us",
1302 "microseconds": "us",
1303 "ns": "ns",
1304 "nanosecond": "ns",
1305 "nanoseconds": "ns",
1306}
1309def stringify_numeric_column(values: Series) -> Series:
1310 """
1311 Cast a numeric column to object-dtype strings so that to_datetime with a
1312 ``format`` doesn't have to infer how to interpret numeric input. GH#55663
1314 Floats that can't be represented as int64 (e.g. +/-inf) get their plain
1315 ``str`` form, which fails to parse downstream: raise for errors="raise",
1316 NaT for errors="coerce".
1317 """
1318 notna_mask = values.notna().to_numpy()
1319 int_mask = notna_mask
1320 if is_float_dtype(values.dtype):
1321 iinfo = np.iinfo(np.int64)
1322 # float64(iinfo.max) rounds up to 2**63, just out of range, so
1323 # exclude the right endpoint
1324 in_range = values.between(iinfo.min, iinfo.max, inclusive="left")
1325 int_mask = notna_mask & in_range.to_numpy(dtype=bool, na_value=False)
1327 result = np.full(len(values), np.nan, dtype=object)
1328 oob_mask = notna_mask & ~int_mask
1329 if oob_mask.any():
1330 result[oob_mask] = np.asarray(values[oob_mask].astype(str), dtype=object)
1331 if int_mask.any():
1332 result[int_mask] = np.asarray(
1333 values[int_mask].astype("int64").astype(str), dtype=object
1334 )
1335 return values._constructor(
1336 result, index=values.index, name=values.name, dtype=object
1337 )
1340def _assemble_from_unit_mappings(
1341 arg, errors: DateTimeErrorChoices, utc: bool
1342) -> Series:
1343 """
1344 assemble the unit specified fields from the arg (DataFrame)
1345 Return a Series for actual parsing
1347 Parameters
1348 ----------
1349 arg : DataFrame
1350 errors : {'raise', 'coerce'}, default 'raise'
1352 - If :const:`'raise'`, then invalid parsing will raise an exception
1353 - If :const:`'coerce'`, then invalid parsing will be set as :const:`NaT`
1354 utc : bool
1355 Whether to convert/localize timestamps to UTC.
1357 Returns
1358 -------
1359 Series
1360 """
1361 from pandas import (
1362 DataFrame,
1363 Series,
1364 to_numeric,
1365 to_timedelta,
1366 )
1368 arg = DataFrame(arg)
1369 if not arg.columns.is_unique:
1370 raise ValueError("cannot assemble with duplicate keys")
1372 # replace passed unit with _unit_map
1373 def f(value):
1374 if value in _unit_map:
1375 return _unit_map[value]
1377 # m is case significant
1378 if value.lower() in _unit_map:
1379 return _unit_map[value.lower()]
1381 return value
1383 unit = {k: f(k) for k in arg.keys()}
1384 unit_rev = {v: k for k, v in unit.items()}
1386 # we require at least Ymd
1387 required = ["year", "month", "day"]
1388 req = set(required) - set(unit_rev.keys())
1389 if len(req):
1390 _required = ",".join(sorted(req))
1391 raise ValueError(
1392 "to assemble mappings requires at least that "
1393 f"[year, month, day] be specified: [{_required}] is missing"
1394 )
1396 # keys we don't recognize
1397 excess = set(unit_rev.keys()) - set(_unit_map.values())
1398 if len(excess):
1399 _excess = ",".join(sorted(excess))
1400 raise ValueError(
1401 f"extra keys have been passed to the datetime assemblage: [{_excess}]"
1402 )
1404 def coerce(values):
1405 # we allow coercion to if errors allows
1406 values = to_numeric(values, errors=errors)
1408 # prevent precision issues in case of float32 # GH#60506
1409 if is_float_dtype(values.dtype):
1410 values = values.astype("float64")
1412 # prevent overflow in case of int8 or int16
1413 if is_integer_dtype(values.dtype):
1414 values = values.astype("int64")
1415 return values
1417 # Convert field values to int64 arrays, tracking rows where a
1418 # year/month/day column has NaN (e.g. from errors="coerce" parsing)
1419 nan_mask = np.zeros(len(arg), dtype=bool)
1421 field_spec = [
1422 ("year", 2000),
1423 ("month", 1),
1424 ("day", 1),
1425 ("h", 0),
1426 ("m", 0),
1427 ("s", 0),
1428 ]
1429 i32info = np.iinfo(np.int32)
1430 field_arrs = []
1431 # hour/minute/second columns that cannot go through datetime_from_fields
1432 # (fractional e.g. hour=1.5, NaN, bool, or out-of-int32 values); added
1433 # via to_timedelta below, keeping the column-wise semantics of the
1434 # non-vectorized implementation
1435 td_units: list[tuple[UnitChoices, AnyArrayLike]] = []
1436 for field, default in field_spec:
1437 col_name = unit_rev.get(field)
1438 if col_name is None:
1439 field_arrs.append(np.zeros(len(arg), dtype=np.int64))
1440 continue
1441 vals = coerce(arg[col_name])
1442 arr = np.asarray(vals)
1444 if field in ("h", "m", "s"):
1445 # the npy_datetimestruct time fields are int32
1446 if is_integer_dtype(arr.dtype):
1447 fits_struct = len(arr) == 0 or (
1448 arr.min() >= i32info.min and arr.max() <= i32info.max
1449 )
1450 elif is_float_dtype(arr.dtype):
1451 fits_struct = bool(
1452 (
1453 (arr == np.floor(arr))
1454 & (arr >= i32info.min)
1455 & (arr <= i32info.max)
1456 ).all()
1457 )
1458 else:
1459 fits_struct = False
1460 if fits_struct:
1461 field_arrs.append(arr.astype(np.int64, copy=False))
1462 else:
1463 td_units.append((cast("UnitChoices", field), vals))
1464 field_arrs.append(np.zeros(len(arg), dtype=np.int64))
1465 continue
1467 # year/month/day
1468 if is_bool_dtype(vals.dtype):
1469 if errors == "raise":
1470 raise ValueError(
1471 f"cannot assemble the datetimes: column {col_name!r} has dtype bool"
1472 )
1473 nan_mask[:] = True
1474 field_arrs.append(np.full(len(arg), default, dtype=np.int64))
1475 continue
1476 if not is_float_dtype(arr.dtype):
1477 field_arrs.append(arr.astype(np.int64, copy=False))
1478 continue
1479 isnan = np.isnan(arr)
1480 fractional = (~isnan) & (arr != np.floor(arr))
1481 if fractional.any() and errors == "raise":
1482 raise ValueError(
1483 f"cannot assemble the datetimes: column {col_name!r} "
1484 f"contains fractional values"
1485 )
1486 # +/-inf and values beyond int64 range cannot be cast meaningfully
1487 out_of_range = (arr >= 2**63) | (arr < -(2**63))
1488 if out_of_range.any() and errors == "raise":
1489 raise ValueError(
1490 f"cannot assemble the datetimes: column {col_name!r} "
1491 f"contains out-of-bounds values"
1492 )
1493 bad = isnan | fractional | out_of_range
1494 if bad.any():
1495 nan_mask[bad] = True
1496 arr = np.where(bad, default, arr)
1497 field_arrs.append(arr.astype(np.int64))
1499 # Construct datetime64[us] directly from fields, avoiding the
1500 # object-dtype round-trip through format="%Y%m%d" string parsing.
1501 # Rows with NaN in a year/month/day column get valid placeholders in
1502 # every field and NaT at the end; the Cython function writes iNaT for
1503 # invalid or out-of-bounds dates.
1504 if nan_mask.any():
1505 for idx, (_, default) in enumerate(field_spec):
1506 field_arrs[idx] = np.where(nan_mask, default, field_arrs[idx])
1508 year_arr, month_arr, day_arr, hour_arr, minute_arr, second_arr = field_arrs
1510 usecs, first_invalid = datetime_from_fields(
1511 year_arr,
1512 month_arr,
1513 day_arr,
1514 hour_arr,
1515 minute_arr,
1516 second_arr,
1517 )
1518 if first_invalid >= 0 and errors == "raise":
1519 bad_val = (
1520 f"{year_arr[first_invalid]}-{month_arr[first_invalid]:02d}"
1521 f"-{day_arr[first_invalid]:02d}"
1522 )
1523 if (
1524 hour_arr[first_invalid]
1525 or minute_arr[first_invalid]
1526 or second_arr[first_invalid]
1527 ):
1528 bad_val += (
1529 f" {hour_arr[first_invalid]:02d}:{minute_arr[first_invalid]:02d}"
1530 f":{second_arr[first_invalid]:02d}"
1531 )
1532 raise ValueError(
1533 f'cannot assemble the datetimes: invalid or out-of-bounds date "{bad_val}"'
1534 )
1535 # errors="coerce": invalid entries already have iNaT from Cython
1536 if nan_mask.any():
1537 usecs[nan_mask] = iNaT
1539 dt64_values = usecs.view("M8[us]")
1541 if utc:
1542 dta = DatetimeArray._simple_new(
1543 dt64_values, dtype=DatetimeTZDtype(tz="UTC", unit="us")
1544 )
1545 values = Series(dta, index=arg.index, copy=False)
1546 else:
1547 values = Series(dt64_values, index=arg.index, copy=False)
1549 # Add hour/minute/second columns that couldn't go through the
1550 # vectorized path
1551 for u, vals in td_units:
1552 try:
1553 values += to_timedelta(vals, unit=u, errors=errors)
1554 except (TypeError, ValueError) as err:
1555 raise ValueError(
1556 f"cannot assemble the datetimes [{unit_rev[u]}]: {err}"
1557 ) from err
1559 # Add sub-second components as timedeltas
1560 units: list[UnitChoices] = ["ms", "us", "ns"]
1561 for u in units:
1562 value = unit_rev.get(u)
1563 if value is not None and value in arg:
1564 try:
1565 values += to_timedelta(coerce(arg[value]), unit=u, errors=errors)
1566 except (TypeError, ValueError) as err:
1567 raise ValueError(
1568 f"cannot assemble the datetimes [{value}]: {err}"
1569 ) from err
1570 return values
1573__all__ = [
1574 "DateParseError",
1575 "should_cache",
1576 "to_datetime",
1577]