Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from collections import abc
4from datetime import date
5from functools import partial
6from itertools import islice
7from typing import (
8 TYPE_CHECKING,
9 TypeAlias,
10 TypedDict,
11 Union,
12 cast,
13 overload,
14)
15import warnings
17import numpy as np
19from pandas._libs import (
20 lib,
21 tslib,
22)
23from pandas._libs.tslibs import (
24 NaT,
25 OutOfBoundsDatetime,
26 Timedelta,
27 Timestamp,
28 astype_overflowsafe,
29 get_supported_dtype,
30 is_supported_dtype,
31 timezones as libtimezones,
32)
33from pandas._libs.tslibs.conversion import cast_from_unit_vectorized
34from pandas._libs.tslibs.parsing import (
35 DateParseError,
36 guess_datetime_format,
37)
38from pandas._libs.tslibs.strptime import array_strptime
39from pandas._typing import (
40 AnyArrayLike,
41 ArrayLike,
42 DateTimeErrorChoices,
43)
44from pandas.util._decorators import set_module
45from pandas.util._exceptions import find_stack_level
47from pandas.core.dtypes.common import (
48 ensure_object,
49 is_float,
50 is_float_dtype,
51 is_integer,
52 is_integer_dtype,
53 is_list_like,
54 is_numeric_dtype,
55)
56from pandas.core.dtypes.dtypes import (
57 ArrowDtype,
58 DatetimeTZDtype,
59)
60from pandas.core.dtypes.generic import (
61 ABCDataFrame,
62 ABCSeries,
63)
65from pandas.arrays import (
66 DatetimeArray,
67 IntegerArray,
68 NumpyExtensionArray,
69)
70from pandas.core.algorithms import unique
71from pandas.core.arrays import ArrowExtensionArray
72from pandas.core.arrays.base import ExtensionArray
73from pandas.core.arrays.datetimes import (
74 maybe_convert_dtype,
75 objects_to_datetime64,
76 tz_to_dtype,
77)
78from pandas.core.construction import extract_array
79from pandas.core.indexes.base import Index
80from pandas.core.indexes.datetimes import DatetimeIndex
82if TYPE_CHECKING:
83 from collections.abc import (
84 Callable,
85 Hashable,
86 )
88 from pandas._libs.tslibs.nattype import NaTType
89 from pandas._libs.tslibs.timedeltas import UnitChoices
90 from pandas._typing import TimeUnit
92 from pandas import (
93 DataFrame,
94 Series,
95 )
97# ---------------------------------------------------------------------
98# types used in annotations
100ArrayConvertible: TypeAlias = list | tuple | AnyArrayLike
101Scalar: TypeAlias = float | str
102DatetimeScalar: TypeAlias = Scalar | date | np.datetime64
104DatetimeScalarOrArrayConvertible: TypeAlias = DatetimeScalar | ArrayConvertible
105DatetimeDictArg: TypeAlias = list[Scalar] | tuple[Scalar, ...] | AnyArrayLike
108class YearMonthDayDict(TypedDict, total=True):
109 year: DatetimeDictArg
110 month: DatetimeDictArg
111 day: DatetimeDictArg
114class FulldatetimeDict(YearMonthDayDict, total=False):
115 hour: DatetimeDictArg
116 hours: DatetimeDictArg
117 minute: DatetimeDictArg
118 minutes: DatetimeDictArg
119 second: DatetimeDictArg
120 seconds: DatetimeDictArg
121 ms: DatetimeDictArg
122 us: DatetimeDictArg
123 ns: DatetimeDictArg
126DictConvertible = Union[FulldatetimeDict, "DataFrame"]
127start_caching_at = 50
130# ---------------------------------------------------------------------
133def _guess_datetime_format_for_array(arr, dayfirst: bool | None = False) -> str | None:
134 # Try to guess the format based on the first non-NaN element, return None if can't
135 if (first_non_null := tslib.first_non_null(arr)) != -1:
136 if type(first_non_nan_element := arr[first_non_null]) is str:
137 # GH#32264 np.str_ object
138 guessed_format = guess_datetime_format(
139 first_non_nan_element, dayfirst=dayfirst
140 )
141 if guessed_format is not None:
142 return guessed_format
143 # If there are multiple non-null elements, warn about
144 # how parsing might not be consistent
145 if tslib.first_non_null(arr[first_non_null + 1 :]) != -1:
146 warnings.warn(
147 "Could not infer format, so each element will be parsed "
148 "individually, falling back to `dateutil`. To ensure parsing is "
149 "consistent and as-expected, please specify a format.",
150 UserWarning,
151 stacklevel=find_stack_level(),
152 )
153 return None
156def should_cache(
157 arg: ArrayConvertible, unique_share: float = 0.7, check_count: int | None = None
158) -> bool:
159 """
160 Decides whether to do caching.
162 If the percent of unique elements among `check_count` elements less
163 than `unique_share * 100` then we can do caching.
165 Parameters
166 ----------
167 arg: listlike, tuple, 1-d array, Series
168 unique_share: float, default=0.7, optional
169 0 < unique_share < 1
170 check_count: int, optional
171 0 <= check_count <= len(arg)
173 Returns
174 -------
175 do_caching: bool
177 Notes
178 -----
179 By default for a sequence of less than 50 items in size, we don't do
180 caching; for the number of elements less than 5000, we take ten percent of
181 all elements to check for a uniqueness share; if the sequence size is more
182 than 5000, then we check only the first 500 elements.
183 All constants were chosen empirically by.
184 """
185 do_caching = True
187 # default realization
188 if check_count is None:
189 # in this case, the gain from caching is negligible
190 if len(arg) <= start_caching_at:
191 return False
193 if len(arg) <= 5000:
194 check_count = len(arg) // 10
195 else:
196 check_count = 500
197 else:
198 assert 0 <= check_count <= len(arg), (
199 "check_count must be in next bounds: [0; len(arg)]"
200 )
201 if check_count == 0:
202 return False
204 assert 0 < unique_share < 1, "unique_share must be in next bounds: (0; 1)"
206 try:
207 # We can't cache if the items are not hashable.
208 unique_elements = set(islice(arg, check_count))
209 except TypeError:
210 return False
211 if len(unique_elements) > check_count * unique_share:
212 do_caching = False
213 return do_caching
216def _maybe_cache(
217 arg: ArrayConvertible,
218 format: str | None,
219 cache: bool,
220 convert_listlike: Callable,
221) -> Series:
222 """
223 Create a cache of unique dates from an array of dates
225 Parameters
226 ----------
227 arg : listlike, tuple, 1-d array, Series
228 format : string
229 Strftime format to parse time
230 cache : bool
231 True attempts to create a cache of converted values
232 convert_listlike : function
233 Conversion function to apply on dates
235 Returns
236 -------
237 cache_array : Series
238 Cache of converted, unique dates. Can be empty
239 """
240 from pandas import Series
242 cache_array = Series(dtype=object)
244 if cache:
245 # Perform a quicker unique check
246 if not should_cache(arg):
247 return cache_array
249 if not isinstance(arg, (np.ndarray, ExtensionArray, Index, ABCSeries)):
250 arg = np.array(arg)
252 unique_dates = unique(arg)
253 if len(unique_dates) < len(arg):
254 cache_dates = convert_listlike(unique_dates, format)
255 # GH#45319
256 try:
257 cache_array = Series(cache_dates, index=unique_dates, copy=False)
258 except OutOfBoundsDatetime:
259 return cache_array
260 # GH#39882 and GH#35888 in case of None and NaT we get duplicates
261 if not cache_array.index.is_unique:
262 cache_array = cache_array[~cache_array.index.duplicated()]
263 return cache_array
266def _box_as_indexlike(
267 dt_array: ArrayLike, utc: bool = False, name: Hashable | None = None
268) -> Index:
269 """
270 Properly boxes the ndarray of datetimes to DatetimeIndex
271 if it is possible or to generic Index instead
273 Parameters
274 ----------
275 dt_array: 1-d array
276 Array of datetimes to be wrapped in an Index.
277 utc : bool
278 Whether to convert/localize timestamps to UTC.
279 name : string, default None
280 Name for a resulting index
282 Returns
283 -------
284 result : datetime of converted dates
285 - DatetimeIndex if convertible to sole datetime64 type
286 - general Index otherwise
287 """
289 if lib.is_np_dtype(dt_array.dtype, "M"):
290 tz = "utc" if utc else None
291 return DatetimeIndex(dt_array, tz=tz, name=name)
292 return Index(dt_array, name=name, dtype=dt_array.dtype)
295def _convert_and_box_cache(
296 arg: DatetimeScalarOrArrayConvertible,
297 cache_array: Series,
298 name: Hashable | None = None,
299) -> Index:
300 """
301 Convert array of dates with a cache and wrap the result in an Index.
303 Parameters
304 ----------
305 arg : integer, float, string, datetime, list, tuple, 1-d array, Series
306 cache_array : Series
307 Cache of converted, unique dates
308 name : string, default None
309 Name for a DatetimeIndex
311 Returns
312 -------
313 result : Index-like of converted dates
314 """
315 from pandas import Series
317 result = Series(arg, dtype=cache_array.index.dtype).map(cache_array)
318 return _box_as_indexlike(result._values, utc=False, name=name)
321def _convert_listlike_datetimes(
322 arg,
323 format: str | None,
324 name: Hashable | None = None,
325 utc: bool = False,
326 unit: str | None = None,
327 errors: DateTimeErrorChoices = "raise",
328 dayfirst: bool | None = None,
329 yearfirst: bool | None = None,
330 exact: bool = True,
331):
332 """
333 Helper function for to_datetime. Performs the conversions of 1D listlike
334 of dates
336 Parameters
337 ----------
338 arg : list, tuple, ndarray, Series, Index
339 date to be parsed
340 name : object
341 None or string for the Index name
342 utc : bool
343 Whether to convert/localize timestamps to UTC.
344 unit : str
345 None or string of the frequency of the passed data
346 errors : str
347 error handing behaviors from to_datetime, 'raise', 'coerce'
348 dayfirst : bool
349 dayfirst parsing behavior from to_datetime
350 yearfirst : bool
351 yearfirst parsing behavior from to_datetime
352 exact : bool, default True
353 exact format matching behavior from to_datetime
355 Returns
356 -------
357 Index-like of parsed dates
358 """
359 if isinstance(arg, (list, tuple)):
360 arg = np.array(arg, dtype="O")
361 elif isinstance(arg, NumpyExtensionArray):
362 arg = np.array(arg)
364 arg_dtype = getattr(arg, "dtype", None)
365 # these are shortcutable
366 tz = "utc" if utc else None
367 if isinstance(arg_dtype, DatetimeTZDtype):
368 if not isinstance(arg, (DatetimeArray, DatetimeIndex)):
369 return DatetimeIndex(arg, tz=tz, name=name)
370 if utc:
371 arg = arg.tz_convert(None).tz_localize("utc")
372 return arg
374 elif isinstance(arg_dtype, ArrowDtype) and arg_dtype.type is Timestamp:
375 # TODO: Combine with above if DTI/DTA supports Arrow timestamps
376 if utc:
377 # pyarrow uses UTC, not lowercase utc
378 if isinstance(arg, Index):
379 arg_array = cast(ArrowExtensionArray, arg.array)
380 if arg_dtype.pyarrow_dtype.tz is not None:
381 arg_array = arg_array._dt_tz_convert("UTC")
382 else:
383 arg_array = arg_array._dt_tz_localize("UTC")
384 arg = Index(arg_array, copy=False)
385 # ArrowExtensionArray
386 elif arg_dtype.pyarrow_dtype.tz is not None:
387 arg = arg._dt_tz_convert("UTC")
388 else:
389 arg = arg._dt_tz_localize("UTC")
390 return arg
392 elif lib.is_np_dtype(arg_dtype, "M"):
393 if not is_supported_dtype(arg_dtype):
394 # We go to closest supported reso, i.e. "s"
395 arg = astype_overflowsafe(
396 np.asarray(arg),
397 np.dtype("M8[s]"),
398 is_coerce=errors == "coerce",
399 )
401 if not isinstance(arg, (DatetimeArray, DatetimeIndex)):
402 return DatetimeIndex(arg, tz=tz, name=name)
403 elif utc:
404 # DatetimeArray, DatetimeIndex
405 return arg.tz_localize("utc")
407 return arg
409 elif unit is not None:
410 if format is not None:
411 raise ValueError("cannot specify both format and unit")
412 return _to_datetime_with_unit(arg, unit, name, utc, errors)
413 elif getattr(arg, "ndim", 1) > 1:
414 raise TypeError(
415 "arg must be a string, datetime, list, tuple, 1-d array, or Series"
416 )
418 # warn if passing timedelta64, raise for PeriodDtype
419 # NB: this must come after unit transformation
420 try:
421 arg, _ = maybe_convert_dtype(arg, copy=False, tz=libtimezones.maybe_get_tz(tz))
422 except TypeError:
423 if errors == "coerce":
424 npvalues = np.full(len(arg), np.datetime64("NaT", "ns"))
425 return DatetimeIndex(npvalues, name=name)
426 raise
428 arg = ensure_object(arg)
430 if format is None:
431 format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
433 # `format` could be inferred, or user didn't ask for mixed-format parsing.
434 if format is not None and format != "mixed":
435 return _array_strptime_with_fallback(arg, name, utc, format, exact, errors)
437 result, tz_parsed = objects_to_datetime64(
438 arg,
439 dayfirst=dayfirst,
440 yearfirst=yearfirst,
441 utc=utc,
442 errors=errors,
443 allow_object=True,
444 )
446 if tz_parsed is not None:
447 # We can take a shortcut since the datetime64 numpy array
448 # is in UTC
449 out_unit = np.datetime_data(result.dtype)[0]
450 out_unit = cast("TimeUnit", out_unit)
451 dtype = tz_to_dtype(tz_parsed, out_unit)
452 dt64_values = result.view(f"M8[{dtype.unit}]")
453 dta = DatetimeArray._simple_new(dt64_values, dtype=dtype)
454 return DatetimeIndex._simple_new(dta, name=name)
456 return _box_as_indexlike(result, utc=utc, name=name)
459def _array_strptime_with_fallback(
460 arg,
461 name,
462 utc: bool,
463 fmt: str,
464 exact: bool,
465 errors: str,
466) -> Index:
467 """
468 Call array_strptime, with fallback behavior depending on 'errors'.
469 """
470 result, tz_out = array_strptime(arg, fmt, exact=exact, errors=errors, utc=utc)
471 if tz_out is not None:
472 unit = np.datetime_data(result.dtype)[0]
473 unit = cast("TimeUnit", unit)
474 dtype = DatetimeTZDtype(tz=tz_out, unit=unit)
475 dta = DatetimeArray._simple_new(result, dtype=dtype)
476 if utc:
477 dta = dta.tz_convert("UTC")
478 return Index(dta, name=name, copy=False)
479 elif result.dtype != object and utc:
480 unit = np.datetime_data(result.dtype)[0]
481 unit = cast("TimeUnit", unit)
482 res = Index(result, dtype=f"M8[{unit}, UTC]", name=name, copy=False)
483 return res
484 return Index(result, dtype=result.dtype, name=name, copy=False)
487def _to_datetime_with_unit(arg, unit, name, utc: bool, errors: str) -> Index:
488 """
489 to_datetime specalized to the case where a 'unit' is passed.
490 """
491 arg = extract_array(arg, extract_numpy=True)
493 # GH#30050 pass an ndarray to tslib.array_to_datetime
494 # because it expects an ndarray argument
495 if isinstance(arg, IntegerArray):
496 arr = arg.astype(f"datetime64[{unit}]")
497 tz_parsed = None
498 else:
499 arg = np.asarray(arg)
501 if arg.dtype.kind in "iu":
502 # Note we can't do "f" here because that could induce unwanted
503 # rounding GH#14156, GH#20445
504 arr = arg.astype(f"datetime64[{unit}]", copy=False)
505 dtype = get_supported_dtype(arr.dtype)
506 try:
507 arr = astype_overflowsafe(arr, dtype, copy=False)
508 except OutOfBoundsDatetime:
509 if errors == "raise":
510 raise
511 arg = arg.astype(object)
512 return _to_datetime_with_unit(arg, unit, name, utc, errors)
513 tz_parsed = None
515 elif arg.dtype.kind == "f":
516 with np.errstate(invalid="ignore"):
517 int_values = arg.astype(np.int64)
518 mask = np.isnan(arg)
519 if (mask | (arg == int_values)).all():
520 # With all-round-or-NaN entries, we give the requested unit
521 # back like with integers
522 result = _to_datetime_with_unit(
523 int_values, unit=unit, name=name, utc=utc, errors=errors
524 )
525 result._data[mask] = NaT
526 return result
527 with np.errstate(over="raise"):
528 try:
529 arr = cast_from_unit_vectorized(arg, unit=unit)
530 except OutOfBoundsDatetime as err:
531 if errors != "raise":
532 return _to_datetime_with_unit(
533 arg.astype(object), unit, name, utc, errors
534 )
535 raise OutOfBoundsDatetime(
536 f"cannot convert input with unit '{unit}'"
537 ) from err
539 arr = arr.view("M8[ns]")
540 tz_parsed = None
541 else:
542 arg = arg.astype(object, copy=False)
543 arr, tz_parsed = tslib.array_to_datetime(
544 arg,
545 utc=utc,
546 errors=errors,
547 unit_for_numerics=unit,
548 )
550 result = DatetimeIndex(arr, name=name)
551 if not isinstance(result, DatetimeIndex):
552 return result
554 # GH#23758: We may still need to localize the result with tz
555 # GH#25546: Apply tz_parsed first (from arg), then tz (from caller)
556 # result will be naive but in UTC
557 result = result.tz_localize("UTC").tz_convert(tz_parsed)
559 if utc:
560 if result.tz is None:
561 result = result.tz_localize("utc")
562 else:
563 result = result.tz_convert("utc")
564 return result
567def _adjust_to_origin(arg, origin, unit):
568 """
569 Helper function for to_datetime.
570 Adjust input argument to the specified origin
572 Parameters
573 ----------
574 arg : list, tuple, ndarray, Series, Index
575 date to be adjusted
576 origin : 'julian' or Timestamp
577 origin offset for the arg
578 unit : str
579 passed unit from to_datetime, must be 'D'
581 Returns
582 -------
583 ndarray or scalar of adjusted date(s)
584 """
585 if origin == "julian":
586 original = arg
587 j0 = Timestamp(0).to_julian_date()
588 if unit != "D":
589 raise ValueError("unit must be 'D' for origin='julian'")
590 try:
591 arg = arg - j0
592 except TypeError as err:
593 raise ValueError(
594 "incompatible 'arg' type for given 'origin'='julian'"
595 ) from err
597 # preemptively check this for a nice range
598 j_max = Timestamp.max.to_julian_date() - j0
599 j_min = Timestamp.min.to_julian_date() - j0
600 if np.any(arg > j_max) or np.any(arg < j_min):
601 raise OutOfBoundsDatetime(
602 f"{original} is Out of Bounds for origin='julian'"
603 )
604 else:
605 # arg must be numeric
606 if not (
607 (is_integer(arg) or is_float(arg)) or is_numeric_dtype(np.asarray(arg))
608 ):
609 raise ValueError(
610 f"'{arg}' is not compatible with origin='{origin}'; "
611 "it must be numeric with a unit specified"
612 )
614 # we are going to offset back to unix / epoch time
615 try:
616 if lib.is_integer(origin) or lib.is_float(origin):
617 offset = Timestamp(origin, unit=unit)
618 else:
619 offset = Timestamp(origin)
620 except OutOfBoundsDatetime as err:
621 raise OutOfBoundsDatetime(f"origin {origin} is Out of Bounds") from err
622 except ValueError as err:
623 raise ValueError(
624 f"origin {origin} cannot be converted to a Timestamp"
625 ) from err
627 if offset.tz is not None:
628 raise ValueError(f"origin offset {offset} must be tz-naive")
629 td_offset = offset - Timestamp(0)
631 # convert the offset to the unit of the arg
632 # this should be lossless in terms of precision
633 ioffset = td_offset // Timedelta(1, unit=unit)
635 # scalars & ndarray-like can handle the addition
636 if is_list_like(arg) and not isinstance(arg, (ABCSeries, Index, np.ndarray)):
637 arg = np.asarray(arg)
638 arg = arg + ioffset
639 return arg
642@overload
643def to_datetime(
644 arg: DatetimeScalar,
645 errors: DateTimeErrorChoices = ...,
646 dayfirst: bool = ...,
647 yearfirst: bool = ...,
648 utc: bool = ...,
649 format: str | None = ...,
650 exact: bool = ...,
651 unit: str | None = ...,
652 origin=...,
653 cache: bool = ...,
654) -> Timestamp: ...
657@overload
658def to_datetime(
659 arg: Series | DictConvertible,
660 errors: DateTimeErrorChoices = ...,
661 dayfirst: bool = ...,
662 yearfirst: bool = ...,
663 utc: bool = ...,
664 format: str | None = ...,
665 exact: bool = ...,
666 unit: str | None = ...,
667 origin=...,
668 cache: bool = ...,
669) -> Series: ...
672@overload
673def to_datetime(
674 arg: list | tuple | Index | ArrayLike,
675 errors: DateTimeErrorChoices = ...,
676 dayfirst: bool = ...,
677 yearfirst: bool = ...,
678 utc: bool = ...,
679 format: str | None = ...,
680 exact: bool = ...,
681 unit: str | None = ...,
682 origin=...,
683 cache: bool = ...,
684) -> DatetimeIndex: ...
687@set_module("pandas")
688def to_datetime(
689 arg: DatetimeScalarOrArrayConvertible | DictConvertible,
690 errors: DateTimeErrorChoices = "raise",
691 dayfirst: bool = False,
692 yearfirst: bool = False,
693 utc: bool = False,
694 format: str | None = None,
695 exact: bool | lib.NoDefault = lib.no_default,
696 unit: str | None = None,
697 origin: str = "unix",
698 cache: bool = True,
699) -> DatetimeIndex | Series | DatetimeScalar | NaTType:
700 """
701 Convert argument to datetime.
703 This function converts a scalar, array-like, :class:`Series` or
704 :class:`DataFrame`/dict-like to a pandas datetime object.
706 Parameters
707 ----------
708 arg : int, float, str, datetime, list, tuple, 1-d array, Series, DataFrame/dict-like
709 The object to convert to a datetime. If a :class:`DataFrame` is provided, the
710 method expects minimally the following columns: :const:`"year"`,
711 :const:`"month"`, :const:`"day"`. The column "year"
712 must be specified in 4-digit format.
713 errors : {'raise', 'coerce'}, default 'raise'
714 - If :const:`'raise'`, then invalid parsing will raise an exception.
715 - If :const:`'coerce'`, then invalid parsing will be set as :const:`NaT`.
716 dayfirst : bool, default False
717 Specify a date parse order if `arg` is str or is list-like.
718 If :const:`True`, parses dates with the day first, e.g. :const:`"10/11/12"`
719 is parsed as :const:`2012-11-10`.
721 .. warning::
723 ``dayfirst=True`` is not strict, but will prefer to parse
724 with day first.
726 yearfirst : bool, default False
727 Specify a date parse order if `arg` is str or is list-like.
729 - If :const:`True` parses dates with the year first, e.g.
730 :const:`"10/11/12"` is parsed as :const:`2010-11-12`.
731 - If both `dayfirst` and `yearfirst` are :const:`True`, `yearfirst` is
732 preceded (same as :mod:`dateutil`).
734 .. warning::
736 ``yearfirst=True`` is not strict, but will prefer to parse
737 with year first.
739 utc : bool, default False
740 Control timezone-related parsing, localization and conversion.
742 - If :const:`True`, the function *always* returns a timezone-aware
743 UTC-localized :class:`Timestamp`, :class:`Series` or
744 :class:`DatetimeIndex`. To do this, timezone-naive inputs are
745 *localized* as UTC, while timezone-aware inputs are *converted* to UTC.
747 - If :const:`False` (default), inputs will not be coerced to UTC.
748 Timezone-naive inputs will remain naive, while timezone-aware ones
749 will keep their time offsets. Limitations exist for mixed
750 offsets (typically, daylight savings), see :ref:`Examples
751 <to_datetime_tz_examples>` section for details.
753 See also: pandas general documentation about `timezone conversion and
754 localization
755 <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
756 #time-zone-handling>`_.
758 format : str, default None
759 The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See
760 `strftime documentation
761 <https://docs.python.org/3/library/datetime.html
762 #strftime-and-strptime-behavior>`_ for more information on choices, though
763 note that :const:`"%f"` will parse all the way up to nanoseconds.
764 You can also pass:
766 - "ISO8601", to parse any `ISO8601 <https://en.wikipedia.org/wiki/ISO_8601>`_
767 time string (not necessarily in exactly the same format);
768 - "mixed", to infer the format for each element individually. This is risky,
769 and you should probably use it along with `dayfirst`.
771 .. note::
773 If a :class:`DataFrame` is passed, then `format` has no effect.
775 exact : bool, default True
776 Control how `format` is used:
778 - If :const:`True`, require an exact `format` match.
779 - If :const:`False`, allow the `format` to match anywhere in the target
780 string.
782 Cannot be used alongside ``format='ISO8601'`` or ``format='mixed'``.
783 unit : str, default 'ns'
784 The unit of the arg (D,s,ms,us,ns) denote the unit, which is an
785 integer or float number. This will be based off the origin.
786 Example, with ``unit='ms'`` and ``origin='unix'``, this would calculate
787 the number of milliseconds to the unix epoch start.
788 origin : scalar, default 'unix'
789 Define the reference date. The numeric values would be parsed as number
790 of units (defined by `unit`) since this reference date.
792 - If :const:`'unix'` (or POSIX) time; origin is set to 1970-01-01.
793 - If :const:`'julian'`, unit must be :const:`'D'`, and origin is set to
794 beginning of Julian Calendar. Julian day number :const:`0` is assigned
795 to the day starting at noon on January 1, 4713 BC.
796 - If Timestamp convertible (Timestamp, dt.datetime, np.datetimt64 or date
797 string), origin is set to Timestamp identified by origin.
798 - If a float or integer, origin is the difference
799 (in units determined by the ``unit`` argument) relative to 1970-01-01.
800 cache : bool, default True
801 If :const:`True`, use a cache of unique, converted dates to apply the
802 datetime conversion. May produce significant speed-up when parsing
803 duplicate date strings, especially ones with timezone offsets. The cache
804 is only used when there are at least 50 values. The presence of
805 out-of-bounds values will render the cache unusable and may slow down
806 parsing.
808 Returns
809 -------
810 datetime
811 If parsing succeeded.
812 Return type depends on input (types in parenthesis correspond to
813 fallback in case of unsuccessful timezone or out-of-range timestamp
814 parsing):
816 - scalar: :class:`Timestamp` (or :class:`datetime.datetime`)
817 - array-like: :class:`DatetimeIndex` (or :class:`Series` with
818 :class:`object` dtype containing :class:`datetime.datetime`)
819 - Series: :class:`Series` of :class:`datetime64` dtype (or
820 :class:`Series` of :class:`object` dtype containing
821 :class:`datetime.datetime`)
822 - DataFrame: :class:`Series` of :class:`datetime64` dtype (or
823 :class:`Series` of :class:`object` dtype containing
824 :class:`datetime.datetime`)
826 Raises
827 ------
828 ParserError
829 When parsing a date from string fails.
830 ValueError
831 When another datetime conversion error happens. For example when one
832 of 'year', 'month', day' columns is missing in a :class:`DataFrame`, or
833 when a Timezone-aware :class:`datetime.datetime` is found in an array-like
834 of mixed time offsets, and ``utc=False``, or when parsing datetimes
835 with mixed time zones unless ``utc=True``. If parsing datetimes with mixed
836 time zones, please specify ``utc=True``.
838 See Also
839 --------
840 DataFrame.astype : Cast argument to a specified dtype.
841 to_timedelta : Convert argument to timedelta.
842 convert_dtypes : Convert dtypes.
844 Notes
845 -----
847 Many input types are supported, and lead to different output types:
849 - **scalars** can be int, float, str, datetime object (from stdlib :mod:`datetime`
850 module or :mod:`numpy`). They are converted to :class:`Timestamp` when
851 possible, otherwise they are converted to :class:`datetime.datetime`.
852 None/NaN/null scalars are converted to :const:`NaT`.
854 - **array-like** can contain int, float, str, datetime objects. They are
855 converted to :class:`DatetimeIndex` when possible, otherwise they are
856 converted to :class:`Index` with :class:`object` dtype, containing
857 :class:`datetime.datetime`. None/NaN/null entries are converted to
858 :const:`NaT` in both cases.
860 - **Series** are converted to :class:`Series` with :class:`datetime64`
861 dtype when possible, otherwise they are converted to :class:`Series` with
862 :class:`object` dtype, containing :class:`datetime.datetime`. None/NaN/null
863 entries are converted to :const:`NaT` in both cases.
865 - **DataFrame/dict-like** are converted to :class:`Series` with
866 :class:`datetime64` dtype. For each row a datetime is created from assembling
867 the various dataframe columns. Column keys can be common abbreviations
868 like ['year', 'month', 'day', 'minute', 'second', 'ms', 'us', 'ns']) or
869 plurals of the same.
871 The following causes are responsible for :class:`datetime.datetime` objects
872 being returned (possibly inside an :class:`Index` or a :class:`Series` with
873 :class:`object` dtype) instead of a proper pandas designated type
874 (:class:`Timestamp`, :class:`DatetimeIndex` or :class:`Series`
875 with :class:`datetime64` dtype):
877 - when any input element is before :const:`Timestamp.min` or after
878 :const:`Timestamp.max`, see `timestamp limitations
879 <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
880 #timeseries-timestamp-limits>`_.
882 - when ``utc=False`` (default) and the input is an array-like or
883 :class:`Series` containing mixed naive/aware datetime, or aware with mixed
884 time offsets. Note that this happens in the (quite frequent) situation when
885 the timezone has a daylight savings policy. In that case you may wish to
886 use ``utc=True``.
888 Examples
889 --------
891 **Handling various input formats**
893 Assembling a datetime from multiple columns of a :class:`DataFrame`. The keys
894 can be common abbreviations like ['year', 'month', 'day', 'minute', 'second',
895 'ms', 'us', 'ns']) or plurals of the same
897 >>> df = pd.DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
898 >>> pd.to_datetime(df)
899 0 2015-02-04
900 1 2016-03-05
901 dtype: datetime64[us]
903 Using a unix epoch time
905 >>> pd.to_datetime(1490195805, unit="s")
906 Timestamp('2017-03-22 15:16:45')
907 >>> pd.to_datetime(1490195805433502912, unit="ns")
908 Timestamp('2017-03-22 15:16:45.433502912')
910 .. warning:: For float arg, precision rounding might happen. To prevent
911 unexpected behavior use a fixed-width exact type.
913 Using a non-unix epoch origin
915 >>> pd.to_datetime([1, 2, 3], unit="D", origin=pd.Timestamp("1960-01-01"))
916 DatetimeIndex(['1960-01-02', '1960-01-03', '1960-01-04'],
917 dtype='datetime64[s]', freq=None)
919 **Differences with strptime behavior**
921 :const:`"%f"` will parse all the way up to nanoseconds.
923 >>> pd.to_datetime("2018-10-26 12:00:00.0000000011", format="%Y-%m-%d %H:%M:%S.%f")
924 Timestamp('2018-10-26 12:00:00.000000001')
926 **Non-convertible date/times**
928 Passing ``errors='coerce'`` will force an out-of-bounds date to :const:`NaT`,
929 in addition to forcing non-dates (or non-parseable dates) to :const:`NaT`.
931 >>> pd.to_datetime("invalid for Ymd", format="%Y%m%d", errors="coerce")
932 NaT
934 .. _to_datetime_tz_examples:
936 **Timezones and time offsets**
938 The default behaviour (``utc=False``) is as follows:
940 - Timezone-naive inputs are converted to timezone-naive :class:`DatetimeIndex`:
942 >>> pd.to_datetime(["2018-10-26 12:00:00", "2018-10-26 13:00:15"])
943 DatetimeIndex(['2018-10-26 12:00:00', '2018-10-26 13:00:15'],
944 dtype='datetime64[us]', freq=None)
946 - Timezone-aware inputs *with constant time offset* are converted to
947 timezone-aware :class:`DatetimeIndex`:
949 >>> pd.to_datetime(["2018-10-26 12:00 -0500", "2018-10-26 13:00 -0500"])
950 DatetimeIndex(['2018-10-26 12:00:00-05:00', '2018-10-26 13:00:00-05:00'],
951 dtype='datetime64[us, UTC-05:00]', freq=None)
953 - However, timezone-aware inputs *with mixed time offsets* (for example
954 issued from a timezone with daylight savings, such as Europe/Paris)
955 are **not successfully converted** to a :class:`DatetimeIndex`.
956 Parsing datetimes with mixed time zones will raise a ValueError unless
957 ``utc=True``:
959 >>> pd.to_datetime(
960 ... ["2020-10-25 02:00 +0200", "2020-10-25 04:00 +0100"]
961 ... ) # doctest: +SKIP
962 ValueError: Mixed timezones detected. Pass utc=True in to_datetime
963 or tz='UTC' in DatetimeIndex to convert to a common timezone.
965 - To create a :class:`Series` with mixed offsets and ``object`` dtype, please use
966 :meth:`Series.apply` and :func:`datetime.datetime.strptime`:
968 >>> import datetime as dt
969 >>> ser = pd.Series(["2020-10-25 02:00 +0200", "2020-10-25 04:00 +0100"])
970 >>> ser.apply(lambda x: dt.datetime.strptime(x, "%Y-%m-%d %H:%M %z"))
971 0 2020-10-25 02:00:00+02:00
972 1 2020-10-25 04:00:00+01:00
973 dtype: object
975 - A mix of timezone-aware and timezone-naive inputs will also raise a ValueError
976 unless ``utc=True``:
978 >>> from datetime import datetime
979 >>> pd.to_datetime(
980 ... ["2020-01-01 01:00:00-01:00", datetime(2020, 1, 1, 3, 0)]
981 ... ) # doctest: +SKIP
982 ValueError: Mixed timezones detected. Pass utc=True in to_datetime
983 or tz='UTC' in DatetimeIndex to convert to a common timezone.
985 |
987 Setting ``utc=True`` solves most of the above issues:
989 - Timezone-naive inputs are *localized* as UTC
991 >>> pd.to_datetime(["2018-10-26 12:00", "2018-10-26 13:00"], utc=True)
992 DatetimeIndex(['2018-10-26 12:00:00+00:00', '2018-10-26 13:00:00+00:00'],
993 dtype='datetime64[us, UTC]', freq=None)
995 - Timezone-aware inputs are *converted* to UTC (the output represents the
996 exact same datetime, but viewed from the UTC time offset `+00:00`).
998 >>> pd.to_datetime(["2018-10-26 12:00 -0530", "2018-10-26 12:00 -0500"], utc=True)
999 DatetimeIndex(['2018-10-26 17:30:00+00:00', '2018-10-26 17:00:00+00:00'],
1000 dtype='datetime64[us, UTC]', freq=None)
1002 - Inputs can contain both string or datetime, the above
1003 rules still apply
1005 >>> pd.to_datetime(["2018-10-26 12:00", datetime(2020, 1, 1, 18)], utc=True)
1006 DatetimeIndex(['2018-10-26 12:00:00+00:00', '2020-01-01 18:00:00+00:00'],
1007 dtype='datetime64[us, UTC]', freq=None)
1008 """
1009 if exact is not lib.no_default and format in {"mixed", "ISO8601"}:
1010 raise ValueError("Cannot use 'exact' when 'format' is 'mixed' or 'ISO8601'")
1011 if arg is None:
1012 return NaT
1014 if origin != "unix":
1015 arg = _adjust_to_origin(arg, origin, unit)
1017 convert_listlike = partial(
1018 _convert_listlike_datetimes,
1019 utc=utc,
1020 unit=unit,
1021 dayfirst=dayfirst,
1022 yearfirst=yearfirst,
1023 errors=errors,
1024 exact=exact, # type: ignore[arg-type]
1025 )
1026 result: Timestamp | NaTType | Series | Index
1028 if isinstance(arg, Timestamp):
1029 result = arg
1030 if utc:
1031 if arg.tz is not None:
1032 result = arg.tz_convert("utc")
1033 else:
1034 result = arg.tz_localize("utc")
1035 elif isinstance(arg, ABCSeries):
1036 cache_array = _maybe_cache(arg, format, cache, convert_listlike)
1037 if not cache_array.empty:
1038 result = arg.map(cache_array)
1039 else:
1040 values = convert_listlike(arg._values, format)
1041 result = arg._constructor(values, index=arg.index, name=arg.name)
1042 elif isinstance(arg, (ABCDataFrame, abc.MutableMapping)):
1043 result = _assemble_from_unit_mappings(arg, errors, utc)
1044 elif isinstance(arg, Index):
1045 cache_array = _maybe_cache(arg, format, cache, convert_listlike)
1046 if not cache_array.empty:
1047 result = _convert_and_box_cache(arg, cache_array, name=arg.name)
1048 else:
1049 result = convert_listlike(arg, format, name=arg.name)
1050 elif is_list_like(arg):
1051 try:
1052 # error: Argument 1 to "_maybe_cache" has incompatible type
1053 # "Union[float, str, datetime, List[Any], Tuple[Any, ...], ExtensionArray,
1054 # ndarray[Any, Any], Series]"; expected "Union[List[Any], Tuple[Any, ...],
1055 # Union[Union[ExtensionArray, ndarray[Any, Any]], Index, Series], Series]"
1056 argc = cast(
1057 Union[list, tuple, ExtensionArray, np.ndarray, "Series", Index], arg
1058 )
1059 cache_array = _maybe_cache(argc, format, cache, convert_listlike)
1060 except OutOfBoundsDatetime:
1061 # caching attempts to create a DatetimeIndex, which may raise
1062 # an OOB. If that's the desired behavior, then just reraise...
1063 if errors == "raise":
1064 raise
1065 # ... otherwise, continue without the cache.
1066 from pandas import Series
1068 cache_array = Series([], dtype=object) # just an empty array
1069 if not cache_array.empty:
1070 result = _convert_and_box_cache(argc, cache_array)
1071 else:
1072 result = convert_listlike(argc, format)
1073 else:
1074 result = convert_listlike(np.array([arg]), format)[0]
1075 if isinstance(arg, bool) and isinstance(result, np.bool_):
1076 result = bool(result) # TODO: avoid this kludge.
1078 # error: Incompatible return value type (got "Union[Timestamp, NaTType,
1079 # Series, Index]", expected "Union[DatetimeIndex, Series, float, str,
1080 # NaTType, None]")
1081 return result # type: ignore[return-value]
1084# mappings for assembling units
1085_unit_map = {
1086 "year": "year",
1087 "years": "year",
1088 "month": "month",
1089 "months": "month",
1090 "day": "day",
1091 "days": "day",
1092 "hour": "h",
1093 "hours": "h",
1094 "minute": "m",
1095 "minutes": "m",
1096 "second": "s",
1097 "seconds": "s",
1098 "ms": "ms",
1099 "millisecond": "ms",
1100 "milliseconds": "ms",
1101 "us": "us",
1102 "microsecond": "us",
1103 "microseconds": "us",
1104 "ns": "ns",
1105 "nanosecond": "ns",
1106 "nanoseconds": "ns",
1107}
1110def _assemble_from_unit_mappings(
1111 arg, errors: DateTimeErrorChoices, utc: bool
1112) -> Series:
1113 """
1114 assemble the unit specified fields from the arg (DataFrame)
1115 Return a Series for actual parsing
1117 Parameters
1118 ----------
1119 arg : DataFrame
1120 errors : {'raise', 'coerce'}, default 'raise'
1122 - If :const:`'raise'`, then invalid parsing will raise an exception
1123 - If :const:`'coerce'`, then invalid parsing will be set as :const:`NaT`
1124 utc : bool
1125 Whether to convert/localize timestamps to UTC.
1127 Returns
1128 -------
1129 Series
1130 """
1131 from pandas import (
1132 DataFrame,
1133 to_numeric,
1134 to_timedelta,
1135 )
1137 arg = DataFrame(arg)
1138 if not arg.columns.is_unique:
1139 raise ValueError("cannot assemble with duplicate keys")
1141 # replace passed unit with _unit_map
1142 def f(value):
1143 if value in _unit_map:
1144 return _unit_map[value]
1146 # m is case significant
1147 if value.lower() in _unit_map:
1148 return _unit_map[value.lower()]
1150 return value
1152 unit = {k: f(k) for k in arg.keys()}
1153 unit_rev = {v: k for k, v in unit.items()}
1155 # we require at least Ymd
1156 required = ["year", "month", "day"]
1157 req = set(required) - set(unit_rev.keys())
1158 if len(req):
1159 _required = ",".join(sorted(req))
1160 raise ValueError(
1161 "to assemble mappings requires at least that "
1162 f"[year, month, day] be specified: [{_required}] is missing"
1163 )
1165 # keys we don't recognize
1166 excess = set(unit_rev.keys()) - set(_unit_map.values())
1167 if len(excess):
1168 _excess = ",".join(sorted(excess))
1169 raise ValueError(
1170 f"extra keys have been passed to the datetime assemblage: [{_excess}]"
1171 )
1173 def coerce(values):
1174 # we allow coercion to if errors allows
1175 values = to_numeric(values, errors=errors)
1177 # prevent prevision issues in case of float32 # GH#60506
1178 if is_float_dtype(values.dtype):
1179 values = values.astype("float64")
1181 # prevent overflow in case of int8 or int16
1182 if is_integer_dtype(values.dtype):
1183 values = values.astype("int64")
1184 return values
1186 values = (
1187 coerce(arg[unit_rev["year"]]) * 10000
1188 + coerce(arg[unit_rev["month"]]) * 100
1189 + coerce(arg[unit_rev["day"]])
1190 )
1191 try:
1192 values = to_datetime(values, format="%Y%m%d", errors=errors, utc=utc)
1193 except (TypeError, ValueError) as err:
1194 raise ValueError(f"cannot assemble the datetimes: {err}") from err
1196 units: list[UnitChoices] = ["h", "m", "s", "ms", "us", "ns"]
1197 for u in units:
1198 value = unit_rev.get(u)
1199 if value is not None and value in arg:
1200 try:
1201 values += to_timedelta(coerce(arg[value]), unit=u, errors=errors)
1202 except (TypeError, ValueError) as err:
1203 raise ValueError(
1204 f"cannot assemble the datetimes [{value}]: {err}"
1205 ) from err
1206 return values
1209__all__ = [
1210 "DateParseError",
1211 "should_cache",
1212 "to_datetime",
1213]