Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/generic.py: 23%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# pyright: reportPropertyTypeMismatch=false
2from __future__ import annotations
4import collections
5from copy import deepcopy
6import datetime as dt
7from functools import partial
8from json import loads
9import operator
10import pickle
11import re
12import sys
13from typing import (
14 TYPE_CHECKING,
15 Any,
16 ClassVar,
17 Concatenate,
18 Literal,
19 NoReturn,
20 Self,
21 cast,
22 final,
23 overload,
24)
25import warnings
27import numpy as np
29from pandas._config import config
31from pandas._libs import lib
32from pandas._libs.lib import is_range_indexer
33from pandas._libs.tslibs import (
34 Period,
35 Timestamp,
36 to_offset,
37)
38from pandas._typing import (
39 AlignJoin,
40 AnyArrayLike,
41 ArrayLike,
42 Axes,
43 Axis,
44 AxisInt,
45 CompressionOptions,
46 DtypeArg,
47 DtypeBackend,
48 DtypeObj,
49 FilePath,
50 FillnaOptions,
51 FloatFormatType,
52 FormattersType,
53 Frequency,
54 IgnoreRaise,
55 IndexKeyFunc,
56 IndexLabel,
57 InterpolateOptions,
58 IntervalClosedType,
59 JSONSerializable,
60 Level,
61 ListLike,
62 Manager,
63 NaPosition,
64 NDFrameT,
65 OpenFileErrors,
66 RandomState,
67 ReindexMethod,
68 Renamer,
69 Scalar,
70 SequenceNotStr,
71 SortKind,
72 StorageOptions,
73 Suffixes,
74 T,
75 TimeAmbiguous,
76 TimedeltaConvertibleTypes,
77 TimeNonexistent,
78 TimestampConvertibleTypes,
79 TimeUnit,
80 ValueKeyFunc,
81 WriteBuffer,
82 WriteExcelBuffer,
83 npt,
84)
85from pandas.compat import CHAINED_WARNING_DISABLED
86from pandas.compat._constants import (
87 REF_COUNT_METHOD,
88)
89from pandas.compat._optional import import_optional_dependency
90from pandas.compat.numpy import function as nv
91from pandas.errors import (
92 AbstractMethodError,
93 ChainedAssignmentError,
94 InvalidIndexError,
95 Pandas4Warning,
96)
97from pandas.errors.cow import _chained_assignment_method_msg
98from pandas.util._decorators import (
99 deprecate_kwarg,
100 doc,
101)
102from pandas.util._exceptions import find_stack_level
103from pandas.util._validators import (
104 check_dtype_backend,
105 validate_ascending,
106 validate_bool_kwarg,
107 validate_inclusive,
108)
110from pandas.core.dtypes.astype import astype_is_view
111from pandas.core.dtypes.cast import can_hold_element
112from pandas.core.dtypes.common import (
113 ensure_object,
114 ensure_platform_int,
115 ensure_str,
116 is_bool,
117 is_bool_dtype,
118 is_dict_like,
119 is_extension_array_dtype,
120 is_list_like,
121 is_number,
122 is_numeric_dtype,
123 is_re_compilable,
124 is_scalar,
125 pandas_dtype,
126)
127from pandas.core.dtypes.dtypes import (
128 DatetimeTZDtype,
129 ExtensionDtype,
130 PeriodDtype,
131)
132from pandas.core.dtypes.generic import (
133 ABCDataFrame,
134 ABCSeries,
135)
136from pandas.core.dtypes.inference import (
137 is_hashable,
138 is_nested_list_like,
139)
140from pandas.core.dtypes.missing import (
141 isna,
142 notna,
143)
145from pandas.core import (
146 algorithms as algos,
147 arraylike,
148 common,
149 indexing,
150 missing,
151 nanops,
152 sample,
153)
154from pandas.core.array_algos.replace import should_use_regex
155from pandas.core.arrays import ExtensionArray
156from pandas.core.base import PandasObject
157from pandas.core.construction import extract_array
158from pandas.core.flags import Flags
159from pandas.core.indexes.api import (
160 DatetimeIndex,
161 Index,
162 MultiIndex,
163 PeriodIndex,
164 default_index,
165 ensure_index,
166)
167from pandas.core.internals import BlockManager
168from pandas.core.methods.describe import describe_ndframe
169from pandas.core.missing import (
170 clean_fill_method,
171 clean_reindex_fill_method,
172 find_valid_index,
173)
174from pandas.core.reshape.concat import concat
175from pandas.core.shared_docs import _shared_docs
176from pandas.core.sorting import get_indexer_indexer
177from pandas.core.window import (
178 Expanding,
179 ExponentialMovingWindow,
180 Rolling,
181 Window,
182)
184from pandas.io.formats.format import (
185 DataFrameFormatter,
186 DataFrameRenderer,
187)
188from pandas.io.formats.printing import pprint_thing
190if TYPE_CHECKING:
191 from collections.abc import (
192 Callable,
193 Hashable,
194 Iterator,
195 Mapping,
196 Sequence,
197 )
199 from pandas._libs.tslibs import BaseOffset
200 from pandas._typing import P
202 from pandas import (
203 DataFrame,
204 ExcelWriter,
205 HDFStore,
206 Series,
207 )
208 from pandas.core.indexers.objects import BaseIndexer
209 from pandas.core.resample import Resampler
212# goal is to be able to define the docs close to function, while still being
213# able to share
214_shared_docs = {**_shared_docs}
215_shared_doc_kwargs = {
216 "axes": "keywords for axes",
217 "klass": "Series/DataFrame",
218 "axes_single_arg": "{0 or 'index'} for Series, {0 or 'index', 1 or 'columns'} for DataFrame", # noqa: E501
219 "inplace": """
220 inplace : bool, default False
221 If True, performs operation inplace.""",
222 "optional_by": """
223 by : str or list of str
224 Name or list of names to sort by""",
225}
228class NDFrame(PandasObject, indexing.IndexingMixin):
229 """
230 N-dimensional analogue of DataFrame. Store multi-dimensional in a
231 size-mutable, labeled data structure
233 Parameters
234 ----------
235 data : BlockManager
236 axes : list
237 copy : bool, default False
238 """
240 _internal_names: list[str] = [
241 "_mgr",
242 "_cache",
243 "_name",
244 "_metadata",
245 "_flags",
246 ]
247 _internal_names_set: set[str] = set(_internal_names)
248 _accessors: set[str] = set()
249 _hidden_attrs: frozenset[str] = frozenset([])
250 _metadata: list[str] = []
251 _mgr: Manager
252 _attrs: dict[Hashable, Any]
253 _typ: str
255 # ----------------------------------------------------------------------
256 # Constructors
258 def __init__(self, data: Manager) -> None:
259 object.__setattr__(self, "_mgr", data)
260 object.__setattr__(self, "_attrs", {})
261 object.__setattr__(self, "_flags", Flags(self, allows_duplicate_labels=True))
263 @final
264 @classmethod
265 def _init_mgr(
266 cls,
267 mgr: Manager,
268 axes: dict[Literal["index", "columns"], Axes | None],
269 dtype: DtypeObj | None = None,
270 copy: bool = False,
271 ) -> Manager:
272 """passed a manager and a axes dict"""
273 for a, axe in axes.items():
274 if axe is not None:
275 axe = ensure_index(axe)
276 bm_axis = cls._get_block_manager_axis(a)
277 mgr = mgr.reindex_axis(axe, axis=bm_axis)
279 # make a copy if explicitly requested
280 if copy:
281 mgr = mgr.copy(deep=True)
282 if dtype is not None:
283 # avoid further copies if we can
284 if (
285 isinstance(mgr, BlockManager)
286 and len(mgr.blocks) == 1
287 and mgr.blocks[0].values.dtype == dtype
288 ):
289 pass
290 else:
291 mgr = mgr.astype(dtype=dtype)
292 return mgr
294 @final
295 @classmethod
296 def _from_mgr(cls, mgr: Manager, axes: list[Index]) -> Self:
297 """
298 Construct a new object of this type from a Manager object and axes.
300 Parameters
301 ----------
302 mgr : Manager
303 Must have the same ndim as cls.
304 axes : list[Index]
306 Notes
307 -----
308 The axes must match mgr.axes, but are required for future-proofing
309 in the event that axes are refactored out of the Manager objects.
310 """
311 obj = cls.__new__(cls)
312 NDFrame.__init__(obj, mgr)
313 return obj
315 # ----------------------------------------------------------------------
316 # attrs and flags
318 @property
319 def attrs(self) -> dict[Hashable, Any]:
320 """
321 Dictionary of global attributes of this dataset.
323 .. warning::
325 attrs is experimental and may change without warning.
327 See Also
328 --------
329 DataFrame.flags : Global flags applying to this object.
331 Notes
332 -----
333 Many operations that create new datasets will copy ``attrs``. Copies
334 are always deep so that changing ``attrs`` will only affect the
335 present dataset. :func:`pandas.concat` and :func:`pandas.merge` will
336 only copy ``attrs`` if all input datasets have the same ``attrs``.
338 Examples
339 --------
340 For Series:
342 >>> ser = pd.Series([1, 2, 3])
343 >>> ser.attrs = {"A": [10, 20, 30]}
344 >>> ser.attrs
345 {'A': [10, 20, 30]}
347 For DataFrame:
349 >>> df = pd.DataFrame({"A": [1, 2], "B": [3, 4]})
350 >>> df.attrs = {"A": [10, 20, 30]}
351 >>> df.attrs
352 {'A': [10, 20, 30]}
353 """
354 return self._attrs
356 @attrs.setter
357 def attrs(self, value: Mapping[Hashable, Any]) -> None:
358 self._attrs = dict(value)
360 @final
361 @property
362 def flags(self) -> Flags:
363 """
364 Get the properties associated with this pandas object.
366 The available flags are
368 * :attr:`Flags.allows_duplicate_labels`
370 See Also
371 --------
372 Flags : Flags that apply to pandas objects.
373 DataFrame.attrs : Global metadata applying to this dataset.
375 Notes
376 -----
377 "Flags" differ from "metadata". Flags reflect properties of the
378 pandas object (the Series or DataFrame). Metadata refer to properties
379 of the dataset, and should be stored in :attr:`DataFrame.attrs`.
381 Examples
382 --------
383 >>> df = pd.DataFrame({"A": [1, 2]})
384 >>> df.flags
385 <Flags(allows_duplicate_labels=True)>
387 Flags can be get or set using ``.``
389 >>> df.flags.allows_duplicate_labels
390 True
391 >>> df.flags.allows_duplicate_labels = False
393 Or by slicing with a key
395 >>> df.flags["allows_duplicate_labels"]
396 False
397 >>> df.flags["allows_duplicate_labels"] = True
398 """
399 return self._flags
401 @final
402 def set_flags(
403 self,
404 *,
405 copy: bool | lib.NoDefault = lib.no_default,
406 allows_duplicate_labels: bool | None = None,
407 ) -> Self:
408 """
409 Return a new object with updated flags.
411 This method creates a shallow copy of the original object, preserving its
412 underlying data while modifying its global flags. In particular, it allows
413 you to update properties such as whether duplicate labels are permitted. This
414 behavior is especially useful in method chains, where one wishes to
415 adjust DataFrame or Series characteristics without altering the original object.
417 Parameters
418 ----------
419 copy : bool, default False
420 This keyword is now ignored; changing its value will have no
421 impact on the method.
423 .. deprecated:: 3.0.0
425 This keyword is ignored and will be removed in pandas 4.0. Since
426 pandas 3.0, this method always returns a new object using a lazy
427 copy mechanism that defers copies until necessary
428 (Copy-on-Write). See the `user guide on Copy-on-Write
429 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
430 for more details.
432 allows_duplicate_labels : bool, optional
433 Whether the returned object allows duplicate labels.
435 Returns
436 -------
437 Series or DataFrame
438 The same type as the caller.
440 See Also
441 --------
442 DataFrame.attrs : Global metadata applying to this dataset.
443 DataFrame.flags : Global flags applying to this object.
445 Notes
446 -----
447 This method returns a new object that's a view on the same data
448 as the input. Mutating the input or the output values will be reflected
449 in the other.
451 This method is intended to be used in method chains.
453 "Flags" differ from "metadata". Flags reflect properties of the
454 pandas object (the Series or DataFrame). Metadata refer to properties
455 of the dataset, and should be stored in :attr:`DataFrame.attrs`.
457 Examples
458 --------
459 >>> df = pd.DataFrame({"A": [1, 2]})
460 >>> df.flags.allows_duplicate_labels
461 True
462 >>> df2 = df.set_flags(allows_duplicate_labels=False)
463 >>> df2.flags.allows_duplicate_labels
464 False
465 """
466 self._check_copy_deprecation(copy)
467 df = self.copy(deep=False)
468 if allows_duplicate_labels is not None:
469 df.flags["allows_duplicate_labels"] = allows_duplicate_labels
470 return df
472 @final
473 @classmethod
474 def _validate_dtype(cls, dtype) -> DtypeObj | None:
475 """validate the passed dtype"""
476 if dtype is not None:
477 dtype = pandas_dtype(dtype)
479 # a compound dtype
480 if dtype.kind == "V" and not isinstance(dtype, ExtensionDtype):
481 raise NotImplementedError(
482 "compound dtypes are not implemented "
483 f"in the {cls.__name__} constructor"
484 )
486 return dtype
488 # ----------------------------------------------------------------------
489 # Construction
491 # error: Signature of "_constructor" incompatible with supertype "PandasObject"
492 @property
493 def _constructor(self) -> Callable[..., Self]: # type: ignore[override]
494 """
495 Used when a manipulation result has the same dimensions as the
496 original.
497 """
498 raise AbstractMethodError(self)
500 # ----------------------------------------------------------------------
501 # Axis
502 _AXIS_ORDERS: list[Literal["index", "columns"]]
503 _AXIS_TO_AXIS_NUMBER: dict[Axis, AxisInt] = {0: 0, "index": 0, "rows": 0}
504 _info_axis_number: int
505 _info_axis_name: Literal["index", "columns"]
506 _AXIS_LEN: int
508 @final
509 def _construct_axes_dict(
510 self, axes: Sequence[Axis] | None = None, **kwargs: AxisInt
511 ) -> dict:
512 """Return an axes dictionary for myself."""
513 d = {a: self._get_axis(a) for a in (axes or self._AXIS_ORDERS)}
514 # error: Argument 1 to "update" of "MutableMapping" has incompatible type
515 # "Dict[str, Any]"; expected "SupportsKeysAndGetItem[Union[int, str], Any]"
516 d.update(kwargs) # type: ignore[arg-type]
517 return d
519 @final
520 @classmethod
521 def _get_axis_number(cls, axis: Axis) -> AxisInt:
522 try:
523 return cls._AXIS_TO_AXIS_NUMBER[axis]
524 except KeyError as err:
525 raise ValueError(
526 f"No axis named {axis} for object type {cls.__name__}"
527 ) from err
529 @final
530 @classmethod
531 def _get_axis_name(cls, axis: Axis) -> Literal["index", "columns"]:
532 axis_number = cls._get_axis_number(axis)
533 return cls._AXIS_ORDERS[axis_number]
535 @final
536 def _get_axis(self, axis: Axis) -> Index:
537 axis_number = self._get_axis_number(axis)
538 assert axis_number in {0, 1}
539 return self.index if axis_number == 0 else self.columns
541 @final
542 @classmethod
543 def _get_block_manager_axis(cls, axis: Axis) -> AxisInt:
544 """Map the axis to the block_manager axis."""
545 axis = cls._get_axis_number(axis)
546 ndim = cls._AXIS_LEN
547 if ndim == 2:
548 # i.e. DataFrame
549 return 1 - axis
550 return axis
552 @final
553 def _get_axis_resolvers(self, axis: str) -> dict[str, Series | MultiIndex]:
554 # index or columns
555 axis_index = getattr(self, axis)
556 d = {}
557 prefix = axis[0]
559 for i, name in enumerate(axis_index.names):
560 if name is not None:
561 key = level = name
562 else:
563 # prefix with 'i' or 'c' depending on the input axis
564 # e.g., you must do ilevel_0 for the 0th level of an unnamed
565 # multiiindex
566 key = f"{prefix}level_{i}"
567 level = i
569 level_values = axis_index.get_level_values(level)
570 s = level_values.to_series()
571 s.index = axis_index
572 d[key] = s
574 # put the index/columns itself in the dict
575 if isinstance(axis_index, MultiIndex):
576 dindex = axis_index
577 else:
578 dindex = axis_index.to_series()
580 d[axis] = dindex
581 return d
583 @final
584 def _get_index_resolvers(self) -> dict[Hashable, Series | MultiIndex]:
585 from pandas.core.computation.parsing import clean_column_name
587 d: dict[str, Series | MultiIndex] = {}
588 for axis_name in self._AXIS_ORDERS:
589 d.update(self._get_axis_resolvers(axis_name))
591 return {clean_column_name(k): v for k, v in d.items() if not isinstance(k, int)}
593 @final
594 def _get_cleaned_column_resolvers(self) -> dict[Hashable, Series]:
595 """
596 Return the special character free column resolvers of a DataFrame.
598 Column names with special characters are 'cleaned up' so that they can
599 be referred to by backtick quoting.
600 Used in :meth:`DataFrame.eval`.
601 """
602 from pandas.core.computation.parsing import clean_column_name
604 if isinstance(self, ABCSeries):
605 return {clean_column_name(self.name): self}
607 return {clean_column_name(k): v for k, v in self.items()}
609 @final
610 @property
611 def _info_axis(self) -> Index:
612 return getattr(self, self._info_axis_name)
614 @property
615 def shape(self) -> tuple[int, ...]:
616 """
617 Return a tuple of axis dimensions
618 """
619 return tuple(len(self._get_axis(a)) for a in self._AXIS_ORDERS)
621 @property
622 def axes(self) -> list[Index]:
623 """
624 Return index label(s) of the internal NDFrame
625 """
626 # we do it this way because if we have reversed axes, then
627 # the block manager shows then reversed
628 return [self._get_axis(a) for a in self._AXIS_ORDERS]
630 @final
631 @property
632 def ndim(self) -> int:
633 """
634 Return an int representing the number of axes / array dimensions.
636 Return 1 if Series. Otherwise return 2 if DataFrame.
638 See Also
639 --------
640 numpy.ndarray.ndim : Number of array dimensions.
642 Examples
643 --------
644 >>> s = pd.Series({"a": 1, "b": 2, "c": 3})
645 >>> s.ndim
646 1
648 >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
649 >>> df.ndim
650 2
651 """
652 return self._mgr.ndim
654 @final
655 @property
656 def size(self) -> int:
657 """
658 Return an int representing the number of elements in this object.
660 Return the number of rows if Series. Otherwise return the number of
661 rows times number of columns if DataFrame.
663 See Also
664 --------
665 numpy.ndarray.size : Number of elements in the array.
667 Examples
668 --------
669 >>> s = pd.Series({"a": 1, "b": 2, "c": 3})
670 >>> s.size
671 3
673 >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
674 >>> df.size
675 4
676 """
678 return int(np.prod(self.shape))
680 def set_axis(
681 self,
682 labels,
683 *,
684 axis: Axis = 0,
685 copy: bool | lib.NoDefault = lib.no_default,
686 ) -> Self:
687 """
688 Assign desired index to given axis.
690 Indexes for%(extended_summary_sub)s row labels can be changed by assigning
691 a list-like or Index.
693 Parameters
694 ----------
695 labels : list-like, Index
696 The values for the new index.
698 axis : %(axes_single_arg)s, default 0
699 The axis to update. The value 0 identifies the rows. For `Series`
700 this parameter is unused and defaults to 0.
702 copy : bool, default False
703 This keyword is now ignored; changing its value will have no
704 impact on the method.
706 .. deprecated:: 3.0.0
708 This keyword is ignored and will be removed in pandas 4.0. Since
709 pandas 3.0, this method always returns a new object using a lazy
710 copy mechanism that defers copies until necessary
711 (Copy-on-Write). See the `user guide on Copy-on-Write
712 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
713 for more details.
715 Returns
716 -------
717 %(klass)s
718 An object of type %(klass)s.
720 See Also
721 --------
722 %(klass)s.rename_axis : Alter the name of the index%(see_also_sub)s.
723 """
724 self._check_copy_deprecation(copy)
725 return self._set_axis_nocheck(labels, axis, inplace=False)
727 @overload
728 def _set_axis_nocheck(
729 self, labels, axis: Axis, inplace: Literal[False]
730 ) -> Self: ...
732 @overload
733 def _set_axis_nocheck(self, labels, axis: Axis, inplace: Literal[True]) -> None: ...
735 @overload
736 def _set_axis_nocheck(self, labels, axis: Axis, inplace: bool) -> Self | None: ...
738 @final
739 def _set_axis_nocheck(self, labels, axis: Axis, inplace: bool) -> Self | None:
740 if inplace:
741 setattr(self, self._get_axis_name(axis), labels)
742 return None
743 obj = self.copy(deep=False)
744 setattr(obj, obj._get_axis_name(axis), labels)
745 return obj
747 @final
748 def _set_axis(self, axis: AxisInt, labels: AnyArrayLike | list) -> None:
749 """
750 This is called from the cython code when we set the `index` attribute
751 directly, e.g. `series.index = [1, 2, 3]`.
752 """
753 labels = ensure_index(labels)
754 self._mgr.set_axis(axis, labels)
756 @final
757 def droplevel(self, level: IndexLabel, axis: Axis = 0) -> Self:
758 """
759 Return Series/DataFrame with requested index / column level(s) removed.
761 Parameters
762 ----------
763 level : int, str, or list-like
764 If a string is given, must be the name of a level
765 If list-like, elements must be names or positional indexes
766 of levels.
768 axis : {0 or 'index', 1 or 'columns'}, default 0
769 Axis along which the level(s) is removed:
771 * 0 or 'index': remove level(s) in column.
772 * 1 or 'columns': remove level(s) in row.
774 For `Series` this parameter is unused and defaults to 0.
776 Returns
777 -------
778 Series/DataFrame
779 Series/DataFrame with requested index / column level(s) removed.
781 See Also
782 --------
783 DataFrame.replace : Replace values given in `to_replace` with `value`.
784 DataFrame.pivot : Return reshaped DataFrame organized by given
785 index / column values.
787 Examples
788 --------
789 >>> df = (
790 ... pd.DataFrame([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])
791 ... .set_index([0, 1])
792 ... .rename_axis(["a", "b"])
793 ... )
795 >>> df.columns = pd.MultiIndex.from_tuples(
796 ... [("c", "e"), ("d", "f")], names=["level_1", "level_2"]
797 ... )
799 >>> df
800 level_1 c d
801 level_2 e f
802 a b
803 1 2 3 4
804 5 6 7 8
805 9 10 11 12
807 >>> df.droplevel("a")
808 level_1 c d
809 level_2 e f
810 b
811 2 3 4
812 6 7 8
813 10 11 12
815 >>> df.droplevel("level_2", axis=1)
816 level_1 c d
817 a b
818 1 2 3 4
819 5 6 7 8
820 9 10 11 12
821 """
822 labels = self._get_axis(axis)
823 new_labels = labels.droplevel(level)
824 return self.set_axis(new_labels, axis=axis)
826 def pop(self, item: Hashable) -> Series | Any:
827 result = self[item]
828 del self[item]
830 return result
832 @final
833 def squeeze(self, axis: Axis | None = None) -> Scalar | Series | DataFrame:
834 """
835 Squeeze 1 dimensional axis objects into scalars.
837 Series or DataFrames with a single element are squeezed to a scalar.
838 DataFrames with a single column or a single row are squeezed to a
839 Series. Otherwise the object is unchanged.
841 This method is most useful when you don't know if your
842 object is a Series or DataFrame, but you do know it has just a single
843 column. In that case you can safely call `squeeze` to ensure you have a
844 Series.
846 Parameters
847 ----------
848 axis : {0 or 'index', 1 or 'columns', None}, default None
849 A specific axis to squeeze. By default, all length-1 axes are
850 squeezed. For `Series` this parameter is unused and defaults to `None`.
852 Returns
853 -------
854 DataFrame, Series, or scalar
855 The projection after squeezing `axis` or all the axes.
857 See Also
858 --------
859 Series.iloc : Integer-location based indexing for selecting scalars.
860 DataFrame.iloc : Integer-location based indexing for selecting Series.
861 Series.to_frame : Inverse of DataFrame.squeeze for a
862 single-column DataFrame.
864 Examples
865 --------
866 >>> primes = pd.Series([2, 3, 5, 7])
868 Slicing might produce a Series with a single value:
870 >>> even_primes = primes[primes % 2 == 0]
871 >>> even_primes
872 0 2
873 dtype: int64
875 >>> even_primes.squeeze()
876 np.int64(2)
878 Squeezing objects with more than one value in every axis does nothing:
880 >>> odd_primes = primes[primes % 2 == 1]
881 >>> odd_primes
882 1 3
883 2 5
884 3 7
885 dtype: int64
887 >>> odd_primes.squeeze()
888 1 3
889 2 5
890 3 7
891 dtype: int64
893 Squeezing is even more effective when used with DataFrames.
895 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["a", "b"])
896 >>> df
897 a b
898 0 1 2
899 1 3 4
901 Slicing a single column will produce a DataFrame with the columns
902 having only one value:
904 >>> df_a = df[["a"]]
905 >>> df_a
906 a
907 0 1
908 1 3
910 So the columns can be squeezed down, resulting in a Series:
912 >>> df_a.squeeze("columns")
913 0 1
914 1 3
915 Name: a, dtype: int64
917 Slicing a single row from a single column will produce a single
918 scalar DataFrame:
920 >>> df_0a = df.loc[df.index < 1, ["a"]]
921 >>> df_0a
922 a
923 0 1
925 Squeezing the rows produces a single scalar Series:
927 >>> df_0a.squeeze("rows")
928 a 1
929 Name: 0, dtype: int64
931 Squeezing all axes will project directly into a scalar:
933 >>> df_0a.squeeze()
934 np.int64(1)
935 """
936 axes = range(self._AXIS_LEN) if axis is None else (self._get_axis_number(axis),)
937 result = self.iloc[
938 tuple(
939 0 if i in axes and len(a) == 1 else slice(None)
940 for i, a in enumerate(self.axes)
941 )
942 ]
943 if isinstance(result, NDFrame):
944 result = result.__finalize__(self, method="squeeze")
945 return result
947 # ----------------------------------------------------------------------
948 # Rename
950 @overload
951 def _rename(
952 self,
953 mapper: Renamer | None = ...,
954 *,
955 index: Renamer | None = ...,
956 columns: Renamer | None = ...,
957 axis: Axis | None = ...,
958 inplace: Literal[False] = ...,
959 level: Level | None = ...,
960 errors: str = ...,
961 ) -> Self: ...
963 @overload
964 def _rename(
965 self,
966 mapper: Renamer | None = ...,
967 *,
968 index: Renamer | None = ...,
969 columns: Renamer | None = ...,
970 axis: Axis | None = ...,
971 inplace: Literal[True],
972 level: Level | None = ...,
973 errors: str = ...,
974 ) -> None: ...
976 @overload
977 def _rename(
978 self,
979 mapper: Renamer | None = ...,
980 *,
981 index: Renamer | None = ...,
982 columns: Renamer | None = ...,
983 axis: Axis | None = ...,
984 inplace: bool,
985 level: Level | None = ...,
986 errors: str = ...,
987 ) -> Self | None: ...
989 @final
990 def _rename(
991 self,
992 mapper: Renamer | None = None,
993 *,
994 index: Renamer | None = None,
995 columns: Renamer | None = None,
996 axis: Axis | None = None,
997 inplace: bool = False,
998 level: Level | None = None,
999 errors: str = "ignore",
1000 ) -> Self | None:
1001 # called by Series.rename and DataFrame.rename
1003 if mapper is None and index is None and columns is None:
1004 raise TypeError("must pass an index to rename")
1006 if index is not None or columns is not None:
1007 if axis is not None:
1008 raise TypeError(
1009 "Cannot specify both 'axis' and any of 'index' or 'columns'"
1010 )
1011 if mapper is not None:
1012 raise TypeError(
1013 "Cannot specify both 'mapper' and any of 'index' or 'columns'"
1014 )
1015 # use the mapper argument
1016 elif axis and self._get_axis_number(axis) == 1:
1017 columns = mapper
1018 else:
1019 index = mapper
1021 self._check_inplace_and_allows_duplicate_labels(inplace)
1022 result = self if inplace else self.copy(deep=False)
1024 for axis_no, replacements in enumerate((index, columns)):
1025 if replacements is None:
1026 continue
1028 ax = self._get_axis(axis_no)
1029 f = common.get_rename_function(replacements)
1031 if level is not None:
1032 level = ax._get_level_number(level)
1034 if isinstance(replacements, ABCSeries) and not replacements.index.is_unique:
1035 # GH#58621
1036 raise ValueError("Cannot rename with a Series with non-unique index.")
1038 # GH 13473
1039 if not callable(replacements):
1040 if ax._is_multi and level is not None:
1041 indexer = ax.get_level_values(level).get_indexer_for(replacements)
1042 else:
1043 indexer = ax.get_indexer_for(replacements)
1045 if errors == "raise" and len(indexer[indexer == -1]):
1046 missing_labels = [
1047 label
1048 for index, label in enumerate(replacements)
1049 if indexer[index] == -1
1050 ]
1051 raise KeyError(f"{missing_labels} not found in axis")
1053 new_index = ax._transform_index(f, level=level)
1054 result._set_axis_nocheck(new_index, axis=axis_no, inplace=True)
1056 if inplace:
1057 self._update_inplace(result)
1058 return None
1059 else:
1060 return result.__finalize__(self, method="rename")
1062 @overload
1063 def rename_axis(
1064 self,
1065 mapper: IndexLabel | lib.NoDefault = ...,
1066 *,
1067 index=...,
1068 columns=...,
1069 axis: Axis = ...,
1070 copy: bool | lib.NoDefault = lib.no_default,
1071 inplace: Literal[False] = ...,
1072 ) -> Self: ...
1074 @overload
1075 def rename_axis(
1076 self,
1077 mapper: IndexLabel | lib.NoDefault = ...,
1078 *,
1079 index=...,
1080 columns=...,
1081 axis: Axis = ...,
1082 copy: bool | lib.NoDefault = lib.no_default,
1083 inplace: Literal[True],
1084 ) -> None: ...
1086 @overload
1087 def rename_axis(
1088 self,
1089 mapper: IndexLabel | lib.NoDefault = ...,
1090 *,
1091 index=...,
1092 columns=...,
1093 axis: Axis = ...,
1094 copy: bool | lib.NoDefault = lib.no_default,
1095 inplace: bool = ...,
1096 ) -> Self | None: ...
1098 def rename_axis(
1099 self,
1100 mapper: IndexLabel | lib.NoDefault = lib.no_default,
1101 *,
1102 index=lib.no_default,
1103 columns=lib.no_default,
1104 axis: Axis = 0,
1105 copy: bool | lib.NoDefault = lib.no_default,
1106 inplace: bool = False,
1107 ) -> Self | None:
1108 """
1109 Set the name of the axis for the index or columns.
1111 Parameters
1112 ----------
1113 mapper : scalar, list-like, optional
1114 Value to set the axis name attribute.
1116 Use either ``mapper`` and ``axis`` to
1117 specify the axis to target with ``mapper``, or ``index``
1118 and/or ``columns``.
1119 index : scalar, list-like, dict-like or function, optional
1120 A scalar, list-like, dict-like or functions transformations to
1121 apply to that axis' values.
1122 columns : scalar, list-like, dict-like or function, optional
1123 A scalar, list-like, dict-like or functions transformations to
1124 apply to that axis' values.
1125 axis : {0 or 'index', 1 or 'columns'}, default 0
1126 The axis to rename.
1127 copy : bool, default False
1128 This keyword is now ignored; changing its value will have no
1129 impact on the method.
1131 .. deprecated:: 3.0.0
1133 This keyword is ignored and will be removed in pandas 4.0. Since
1134 pandas 3.0, this method always returns a new object using a lazy
1135 copy mechanism that defers copies until necessary
1136 (Copy-on-Write). See the `user guide on Copy-on-Write
1137 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
1138 for more details.
1140 inplace : bool, default False
1141 Modifies the object directly, instead of creating a new Series
1142 or DataFrame.
1144 Returns
1145 -------
1146 DataFrame, or None
1147 The same type as the caller or None if ``inplace=True``.
1149 See Also
1150 --------
1151 Series.rename : Alter Series index labels or name.
1152 DataFrame.rename : Alter DataFrame index labels or name.
1153 Index.rename : Set new names on index.
1155 Notes
1156 -----
1157 ``DataFrame.rename_axis`` supports two calling conventions
1159 * ``(index=index_mapper, columns=columns_mapper, ...)``
1160 * ``(mapper, axis={'index', 'columns'}, ...)``
1162 The first calling convention will only modify the names of
1163 the index and/or the names of the Index object that is the columns.
1164 In this case, the parameter ``copy`` is ignored.
1166 The second calling convention will modify the names of the
1167 corresponding index if mapper is a list or a scalar.
1168 However, if mapper is dict-like or a function, it will use the
1169 deprecated behavior of modifying the axis *labels*.
1171 We *highly* recommend using keyword arguments to clarify your
1172 intent.
1174 Examples
1175 --------
1176 **DataFrame**
1178 >>> df = pd.DataFrame(
1179 ... {"num_legs": [4, 4, 2], "num_arms": [0, 0, 2]}, ["dog", "cat", "monkey"]
1180 ... )
1181 >>> df
1182 num_legs num_arms
1183 dog 4 0
1184 cat 4 0
1185 monkey 2 2
1186 >>> df = df.rename_axis("animal")
1187 >>> df
1188 num_legs num_arms
1189 animal
1190 dog 4 0
1191 cat 4 0
1192 monkey 2 2
1193 >>> df = df.rename_axis("limbs", axis="columns")
1194 >>> df
1195 limbs num_legs num_arms
1196 animal
1197 dog 4 0
1198 cat 4 0
1199 monkey 2 2
1201 **MultiIndex**
1203 >>> df.index = pd.MultiIndex.from_product(
1204 ... [["mammal"], ["dog", "cat", "monkey"]], names=["type", "name"]
1205 ... )
1206 >>> df
1207 limbs num_legs num_arms
1208 type name
1209 mammal dog 4 0
1210 cat 4 0
1211 monkey 2 2
1213 >>> df.rename_axis(index={"type": "class"})
1214 limbs num_legs num_arms
1215 class name
1216 mammal dog 4 0
1217 cat 4 0
1218 monkey 2 2
1220 >>> df.rename_axis(columns=str.upper)
1221 LIMBS num_legs num_arms
1222 type name
1223 mammal dog 4 0
1224 cat 4 0
1225 monkey 2 2
1226 """
1227 self._check_copy_deprecation(copy)
1228 axes = {"index": index, "columns": columns}
1230 if axis is not None:
1231 axis = self._get_axis_number(axis)
1233 inplace = validate_bool_kwarg(inplace, "inplace")
1235 if mapper is not lib.no_default:
1236 # Use v0.23 behavior if a scalar or list
1237 non_mapper = is_scalar(mapper) or (
1238 is_list_like(mapper) and not is_dict_like(mapper)
1239 )
1240 if non_mapper:
1241 return self._set_axis_name(mapper, axis=axis, inplace=inplace)
1242 else:
1243 raise ValueError("Use `.rename` to alter labels with a mapper.")
1244 else:
1245 # Use new behavior. Means that index and/or columns
1246 # is specified
1247 result = self if inplace else self.copy(deep=False)
1249 for axis in range(self._AXIS_LEN):
1250 v = axes.get(self._get_axis_name(axis))
1251 if v is lib.no_default:
1252 continue
1253 non_mapper = is_scalar(v) or (is_list_like(v) and not is_dict_like(v))
1254 if non_mapper:
1255 newnames = v
1256 else:
1257 f = common.get_rename_function(v)
1258 curnames = self._get_axis(axis).names
1259 newnames = [f(name) for name in curnames]
1260 result._set_axis_name(newnames, axis=axis, inplace=True)
1261 if not inplace:
1262 return result
1263 return None
1265 @overload
1266 def _set_axis_name(
1267 self, name, axis: Axis = ..., *, inplace: Literal[False] = ...
1268 ) -> Self: ...
1270 @overload
1271 def _set_axis_name(
1272 self, name, axis: Axis = ..., *, inplace: Literal[True]
1273 ) -> None: ...
1275 @overload
1276 def _set_axis_name(
1277 self, name, axis: Axis = ..., *, inplace: bool
1278 ) -> Self | None: ...
1280 @final
1281 def _set_axis_name(
1282 self, name, axis: Axis = 0, *, inplace: bool = False
1283 ) -> Self | None:
1284 """
1285 Set the name(s) of the axis.
1287 Parameters
1288 ----------
1289 name : str or list of str
1290 Name(s) to set.
1291 axis : {0 or 'index', 1 or 'columns'}, default 0
1292 The axis to set the label. The value 0 or 'index' specifies index,
1293 and the value 1 or 'columns' specifies columns.
1294 inplace : bool, default False
1295 If `True`, do operation inplace and return None.
1297 Returns
1298 -------
1299 Series, DataFrame, or None
1300 The same type as the caller or `None` if `inplace` is `True`.
1302 See Also
1303 --------
1304 DataFrame.rename : Alter the axis labels of :class:`DataFrame`.
1305 Series.rename : Alter the index labels or set the index name
1306 of :class:`Series`.
1307 Index.rename : Set the name of :class:`Index` or :class:`MultiIndex`.
1309 Examples
1310 --------
1311 >>> df = pd.DataFrame({"num_legs": [4, 4, 2]}, ["dog", "cat", "monkey"])
1312 >>> df
1313 num_legs
1314 dog 4
1315 cat 4
1316 monkey 2
1317 >>> df._set_axis_name("animal")
1318 num_legs
1319 animal
1320 dog 4
1321 cat 4
1322 monkey 2
1323 >>> df.index = pd.MultiIndex.from_product(
1324 ... [["mammal"], ["dog", "cat", "monkey"]]
1325 ... )
1326 >>> df._set_axis_name(["type", "name"])
1327 num_legs
1328 type name
1329 mammal dog 4
1330 cat 4
1331 monkey 2
1332 """
1333 axis = self._get_axis_number(axis)
1334 idx = self._get_axis(axis).set_names(name)
1336 inplace = validate_bool_kwarg(inplace, "inplace")
1337 renamed = self if inplace else self.copy(deep=False)
1338 if axis == 0:
1339 renamed.index = idx
1340 else:
1341 renamed.columns = idx
1343 if not inplace:
1344 return renamed
1345 return None
1347 # ----------------------------------------------------------------------
1348 # Comparison Methods
1350 @final
1351 def _indexed_same(self, other) -> bool:
1352 return all(
1353 self._get_axis(a).equals(other._get_axis(a)) for a in self._AXIS_ORDERS
1354 )
1356 @final
1357 def equals(self, other: object) -> bool:
1358 """
1359 Test whether two objects contain the same elements.
1361 This function allows two Series or DataFrames to be compared against
1362 each other to see if they have the same shape and elements. NaNs in
1363 the same location are considered equal.
1365 The row/column index do not need to have the same type, as long
1366 as the values are considered equal. Corresponding columns and
1367 index must be of the same dtype.
1369 Parameters
1370 ----------
1371 other : Series or DataFrame
1372 The other Series or DataFrame to be compared with the first.
1374 Returns
1375 -------
1376 bool
1377 True if all elements are the same in both objects, False
1378 otherwise.
1380 See Also
1381 --------
1382 Series.eq : Compare two Series objects of the same length
1383 and return a Series where each element is True if the element
1384 in each Series is equal, False otherwise.
1385 DataFrame.eq : Compare two DataFrame objects of the same shape and
1386 return a DataFrame where each element is True if the respective
1387 element in each DataFrame is equal, False otherwise.
1388 testing.assert_series_equal : Raises an AssertionError if left and
1389 right are not equal. Provides an easy interface to ignore
1390 inequality in dtypes, indexes and precision among others.
1391 testing.assert_frame_equal : Like assert_series_equal, but targets
1392 DataFrames.
1393 numpy.array_equal : Return True if two arrays have the same shape
1394 and elements, False otherwise.
1396 Examples
1397 --------
1398 >>> df = pd.DataFrame({1: [10], 2: [20]})
1399 >>> df
1400 1 2
1401 0 10 20
1403 DataFrames df and exactly_equal have the same types and values for
1404 their elements and column labels, which will return True.
1406 >>> exactly_equal = pd.DataFrame({1: [10], 2: [20]})
1407 >>> exactly_equal
1408 1 2
1409 0 10 20
1410 >>> df.equals(exactly_equal)
1411 True
1413 DataFrames df and different_column_type have the same element
1414 types and values, but have different types for the column labels,
1415 which will still return True.
1417 >>> different_column_type = pd.DataFrame({1.0: [10], 2.0: [20]})
1418 >>> different_column_type
1419 1.0 2.0
1420 0 10 20
1421 >>> df.equals(different_column_type)
1422 True
1424 DataFrames df and different_data_type have different types for the
1425 same values for their elements, and will return False even though
1426 their column labels are the same values and types.
1428 >>> different_data_type = pd.DataFrame({1: [10.0], 2: [20.0]})
1429 >>> different_data_type
1430 1 2
1431 0 10.0 20.0
1432 >>> df.equals(different_data_type)
1433 False
1435 DataFrames with NaN in the same locations compare equal.
1437 >>> df_nan1 = pd.DataFrame({"a": [1, np.nan], "b": [3, np.nan]})
1438 >>> df_nan2 = pd.DataFrame({"a": [1, np.nan], "b": [3, np.nan]})
1439 >>> df_nan1.equals(df_nan2)
1440 True
1442 If the NaN values are not in the same locations, they compare unequal.
1444 >>> df_nan3 = pd.DataFrame({"a": [1, np.nan], "b": [3, 4]})
1445 >>> df_nan1.equals(df_nan3)
1446 False
1447 """
1448 if not (isinstance(other, type(self)) or isinstance(self, type(other))):
1449 return False
1450 other = cast(NDFrame, other)
1451 return self._mgr.equals(other._mgr)
1453 # -------------------------------------------------------------------------
1454 # Unary Methods
1456 @final
1457 def __neg__(self) -> Self:
1458 def blk_func(values: ArrayLike):
1459 if is_bool_dtype(values.dtype):
1460 # error: Argument 1 to "inv" has incompatible type "Union
1461 # [ExtensionArray, ndarray[Any, Any]]"; expected
1462 # "_SupportsInversion[ndarray[Any, dtype[bool_]]]"
1463 return operator.inv(values) # type: ignore[arg-type]
1464 else:
1465 # error: Argument 1 to "neg" has incompatible type "Union
1466 # [ExtensionArray, ndarray[Any, Any]]"; expected
1467 # "_SupportsNeg[ndarray[Any, dtype[Any]]]"
1468 return operator.neg(values) # type: ignore[arg-type]
1470 new_data = self._mgr.apply(blk_func)
1471 res = self._constructor_from_mgr(new_data, axes=new_data.axes)
1472 return res.__finalize__(self, method="__neg__")
1474 @final
1475 def __pos__(self) -> Self:
1476 def blk_func(values: ArrayLike):
1477 if is_bool_dtype(values.dtype):
1478 return values.copy()
1479 else:
1480 # error: Argument 1 to "pos" has incompatible type "Union
1481 # [ExtensionArray, ndarray[Any, Any]]"; expected
1482 # "_SupportsPos[ndarray[Any, dtype[Any]]]"
1483 return operator.pos(values) # type: ignore[arg-type]
1485 new_data = self._mgr.apply(blk_func)
1486 res = self._constructor_from_mgr(new_data, axes=new_data.axes)
1487 return res.__finalize__(self, method="__pos__")
1489 @final
1490 def __invert__(self) -> Self:
1491 if not self.size:
1492 # inv fails with 0 len
1493 return self.copy(deep=False)
1495 new_data = self._mgr.apply(operator.invert)
1496 res = self._constructor_from_mgr(new_data, axes=new_data.axes)
1497 return res.__finalize__(self, method="__invert__")
1499 @final
1500 def __bool__(self) -> NoReturn:
1501 raise ValueError(
1502 f"The truth value of a {type(self).__name__} is ambiguous. "
1503 "Use a.empty, a.bool(), a.item(), a.any() or a.all()."
1504 )
1506 @final
1507 def abs(self) -> Self:
1508 """
1509 Return a Series/DataFrame with absolute numeric value of each element.
1511 This function only applies to elements that are all numeric.
1513 Returns
1514 -------
1515 abs
1516 Series/DataFrame containing the absolute value of each element.
1518 See Also
1519 --------
1520 numpy.absolute : Calculate the absolute value element-wise.
1522 Notes
1523 -----
1524 For ``complex`` inputs, ``1.2 + 1j``, the absolute value is
1525 :math:`\\sqrt{ a^2 + b^2 }`.
1527 Examples
1528 --------
1529 Absolute numeric values in a Series.
1531 >>> s = pd.Series([-1.10, 2, -3.33, 4])
1532 >>> s.abs()
1533 0 1.10
1534 1 2.00
1535 2 3.33
1536 3 4.00
1537 dtype: float64
1539 Absolute numeric values in a Series with complex numbers.
1541 >>> s = pd.Series([1.2 + 1j])
1542 >>> s.abs()
1543 0 1.56205
1544 dtype: float64
1546 Absolute numeric values in a Series with a Timedelta element.
1548 >>> s = pd.Series([pd.Timedelta("1 days")])
1549 >>> s.abs()
1550 0 1 days
1551 dtype: timedelta64[us]
1553 Select rows with data closest to certain value using argsort (from
1554 `StackOverflow <https://stackoverflow.com/a/17758115>`__).
1556 >>> df = pd.DataFrame(
1557 ... {"a": [4, 5, 6, 7], "b": [10, 20, 30, 40], "c": [100, 50, -30, -50]}
1558 ... )
1559 >>> df
1560 a b c
1561 0 4 10 100
1562 1 5 20 50
1563 2 6 30 -30
1564 3 7 40 -50
1565 >>> df.loc[(df.c - 43).abs().argsort()]
1566 a b c
1567 1 5 20 50
1568 0 4 10 100
1569 2 6 30 -30
1570 3 7 40 -50
1571 """
1572 res_mgr = self._mgr.apply(np.abs)
1573 return self._constructor_from_mgr(res_mgr, axes=res_mgr.axes).__finalize__(
1574 self, name="abs"
1575 )
1577 @final
1578 def __abs__(self) -> Self:
1579 return self.abs()
1581 @final
1582 def __round__(self, decimals: int = 0) -> Self:
1583 return self.round(decimals).__finalize__(self, method="__round__")
1585 # -------------------------------------------------------------------------
1586 # Label or Level Combination Helpers
1587 #
1588 # A collection of helper methods for DataFrame/Series operations that
1589 # accept a combination of column/index labels and levels. All such
1590 # operations should utilize/extend these methods when possible so that we
1591 # have consistent precedence and validation logic throughout the library.
1593 @final
1594 def _is_level_reference(self, key: Level, axis: Axis = 0) -> bool:
1595 """
1596 Test whether a key is a level reference for a given axis.
1598 To be considered a level reference, `key` must be a string that:
1599 - (axis=0): Matches the name of an index level and does NOT match
1600 a column label.
1601 - (axis=1): Matches the name of a column level and does NOT match
1602 an index label.
1604 Parameters
1605 ----------
1606 key : Hashable
1607 Potential level name for the given axis
1608 axis : int, default 0
1609 Axis that levels are associated with (0 for index, 1 for columns)
1611 Returns
1612 -------
1613 is_level : bool
1614 """
1615 axis_int = self._get_axis_number(axis)
1617 return (
1618 key is not None
1619 and is_hashable(key)
1620 and key in self.axes[axis_int].names
1621 and not self._is_label_reference(key, axis=axis_int)
1622 )
1624 @final
1625 def _is_label_reference(self, key: Level, axis: Axis = 0) -> bool:
1626 """
1627 Test whether a key is a label reference for a given axis.
1629 To be considered a label reference, `key` must be a string that:
1630 - (axis=0): Matches a column label
1631 - (axis=1): Matches an index label
1633 Parameters
1634 ----------
1635 key : Hashable
1636 Potential label name, i.e. Index entry.
1637 axis : int, default 0
1638 Axis perpendicular to the axis that labels are associated with
1639 (0 means search for column labels, 1 means search for index labels)
1641 Returns
1642 -------
1643 is_label: bool
1644 """
1645 axis_int = self._get_axis_number(axis)
1646 other_axes = (ax for ax in range(self._AXIS_LEN) if ax != axis_int)
1648 return is_hashable(key) and any(key in self.axes[ax] for ax in other_axes)
1650 @final
1651 def _is_label_or_level_reference(self, key: Level, axis: AxisInt = 0) -> bool:
1652 """
1653 Test whether a key is a label or level reference for a given axis.
1655 To be considered either a label or a level reference, `key` must be a
1656 string that:
1657 - (axis=0): Matches a column label or an index level
1658 - (axis=1): Matches an index label or a column level
1660 Parameters
1661 ----------
1662 key : Hashable
1663 Potential label or level name
1664 axis : int, default 0
1665 Axis that levels are associated with (0 for index, 1 for columns)
1667 Returns
1668 -------
1669 bool
1670 """
1671 return self._is_level_reference(key, axis=axis) or self._is_label_reference(
1672 key, axis=axis
1673 )
1675 @final
1676 def _check_label_or_level_ambiguity(self, key: Level, axis: Axis = 0) -> None:
1677 """
1678 Check whether `key` is ambiguous.
1680 By ambiguous, we mean that it matches both a level of the input
1681 `axis` and a label of the other axis.
1683 Parameters
1684 ----------
1685 key : Hashable
1686 Label or level name.
1687 axis : int, default 0
1688 Axis that levels are associated with (0 for index, 1 for columns).
1690 Raises
1691 ------
1692 ValueError: `key` is ambiguous
1693 """
1695 axis_int = self._get_axis_number(axis)
1696 other_axes = (ax for ax in range(self._AXIS_LEN) if ax != axis_int)
1698 if (
1699 key is not None
1700 and is_hashable(key)
1701 and key in self.axes[axis_int].names
1702 and any(key in self.axes[ax] for ax in other_axes)
1703 ):
1704 # Build an informative and grammatical warning
1705 level_article, level_type = (
1706 ("an", "index") if axis_int == 0 else ("a", "column")
1707 )
1709 label_article, label_type = (
1710 ("a", "column") if axis_int == 0 else ("an", "index")
1711 )
1713 msg = (
1714 f"'{key}' is both {level_article} {level_type} level and "
1715 f"{label_article} {label_type} label, which is ambiguous."
1716 )
1717 raise ValueError(msg)
1719 @final
1720 def _get_label_or_level_values(self, key: Level, axis: AxisInt = 0) -> ArrayLike:
1721 """
1722 Return a 1-D array of values associated with `key`, a label or level
1723 from the given `axis`.
1725 Retrieval logic:
1726 - (axis=0): Return column values if `key` matches a column label.
1727 Otherwise return index level values if `key` matches an index
1728 level.
1729 - (axis=1): Return row values if `key` matches an index label.
1730 Otherwise return column level values if 'key' matches a column
1731 level
1733 Parameters
1734 ----------
1735 key : Hashable
1736 Label or level name.
1737 axis : int, default 0
1738 Axis that levels are associated with (0 for index, 1 for columns)
1740 Returns
1741 -------
1742 np.ndarray or ExtensionArray
1744 Raises
1745 ------
1746 KeyError
1747 if `key` matches neither a label nor a level
1748 ValueError
1749 if `key` matches multiple labels
1750 """
1751 axis = self._get_axis_number(axis)
1752 first_other_axes = next(
1753 (ax for ax in range(self._AXIS_LEN) if ax != axis), None
1754 )
1756 if self._is_label_reference(key, axis=axis):
1757 self._check_label_or_level_ambiguity(key, axis=axis)
1758 if first_other_axes is None:
1759 raise ValueError("axis matched all axes")
1760 values = self.xs(key, axis=first_other_axes)._values
1761 elif self._is_level_reference(key, axis=axis):
1762 values = self.axes[axis].get_level_values(key)._values
1763 else:
1764 raise KeyError(key)
1766 # Check for duplicates
1767 if values.ndim > 1:
1768 if first_other_axes is not None and isinstance(
1769 self._get_axis(first_other_axes), MultiIndex
1770 ):
1771 multi_message = (
1772 "\n"
1773 "For a multi-index, the label must be a "
1774 "tuple with elements corresponding to each level."
1775 )
1776 else:
1777 multi_message = ""
1779 label_axis_name = "column" if axis == 0 else "index"
1780 raise ValueError(
1781 f"The {label_axis_name} label '{key}' is not unique.{multi_message}"
1782 )
1784 return values
1786 @final
1787 def _drop_labels_or_levels(self, keys, axis: AxisInt = 0):
1788 """
1789 Drop labels and/or levels for the given `axis`.
1791 For each key in `keys`:
1792 - (axis=0): If key matches a column label then drop the column.
1793 Otherwise if key matches an index level then drop the level.
1794 - (axis=1): If key matches an index label then drop the row.
1795 Otherwise if key matches a column level then drop the level.
1797 Parameters
1798 ----------
1799 keys : str or list of str
1800 labels or levels to drop
1801 axis : int, default 0
1802 Axis that levels are associated with (0 for index, 1 for columns)
1804 Returns
1805 -------
1806 dropped: DataFrame
1808 Raises
1809 ------
1810 ValueError
1811 if any `keys` match neither a label nor a level
1812 """
1813 axis = self._get_axis_number(axis)
1815 # Validate keys
1816 keys = common.maybe_make_list(keys)
1817 invalid_keys = [
1818 k for k in keys if not self._is_label_or_level_reference(k, axis=axis)
1819 ]
1821 if invalid_keys:
1822 raise ValueError(
1823 "The following keys are not valid labels or "
1824 f"levels for axis {axis}: {invalid_keys}"
1825 )
1827 # Compute levels and labels to drop
1828 levels_to_drop = [k for k in keys if self._is_level_reference(k, axis=axis)]
1830 labels_to_drop = [k for k in keys if not self._is_level_reference(k, axis=axis)]
1832 # Perform copy upfront and then use inplace operations below.
1833 # This ensures that we always perform exactly one copy.
1834 # ``copy`` and/or ``inplace`` options could be added in the future.
1835 dropped = self.copy(deep=False)
1837 if axis == 0:
1838 # Handle dropping index levels
1839 if levels_to_drop:
1840 dropped.reset_index(levels_to_drop, drop=True, inplace=True)
1842 # Handle dropping columns labels
1843 if labels_to_drop:
1844 dropped.drop(labels_to_drop, axis=1, inplace=True)
1845 else:
1846 # Handle dropping column levels
1847 if levels_to_drop:
1848 if isinstance(dropped.columns, MultiIndex):
1849 # Drop the specified levels from the MultiIndex
1850 dropped.columns = dropped.columns.droplevel(levels_to_drop)
1851 else:
1852 # Drop the last level of Index by replacing with
1853 # a RangeIndex
1854 dropped.columns = default_index(dropped.columns.size)
1856 # Handle dropping index labels
1857 if labels_to_drop:
1858 dropped.drop(labels_to_drop, axis=0, inplace=True)
1860 return dropped
1862 # ----------------------------------------------------------------------
1863 # Iteration
1865 # https://github.com/python/typeshed/issues/2148#issuecomment-520783318
1866 # Incompatible types in assignment (expression has type "None", base class
1867 # "object" defined the type as "Callable[[object], int]")
1868 __hash__: ClassVar[None] # type: ignore[assignment]
1870 def __iter__(self) -> Iterator:
1871 """
1872 Iterate over info axis.
1874 Returns
1875 -------
1876 iterator
1877 Info axis as iterator.
1879 See Also
1880 --------
1881 DataFrame.items : Iterate over (column name, Series) pairs.
1882 DataFrame.itertuples : Iterate over DataFrame rows as namedtuples.
1884 Examples
1885 --------
1886 >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
1887 >>> for x in df:
1888 ... print(x)
1889 A
1890 B
1891 """
1892 return iter(self._info_axis)
1894 # can we get a better explanation of this?
1895 def keys(self) -> Index:
1896 """
1897 Get the 'info axis' (see Indexing for more).
1899 This is index for Series, columns for DataFrame.
1901 Returns
1902 -------
1903 Index
1904 Info axis.
1906 See Also
1907 --------
1908 DataFrame.index : The index (row labels) of the DataFrame.
1909 DataFrame.columns: The column labels of the DataFrame.
1911 Examples
1912 --------
1913 >>> d = pd.DataFrame(
1914 ... data={"A": [1, 2, 3], "B": [0, 4, 8]}, index=["a", "b", "c"]
1915 ... )
1916 >>> d
1917 A B
1918 a 1 0
1919 b 2 4
1920 c 3 8
1921 >>> d.keys()
1922 Index(['A', 'B'], dtype='str')
1923 """
1924 return self._info_axis
1926 def items(self):
1927 """
1928 Iterate over (label, values) on info axis
1930 This is index for Series and columns for DataFrame.
1932 Returns
1933 -------
1934 Generator
1935 """
1936 for h in self._info_axis:
1937 yield h, self[h]
1939 def __len__(self) -> int:
1940 """Returns length of info axis"""
1941 return len(self._info_axis)
1943 @final
1944 def __contains__(self, key) -> bool:
1945 """True if the key is in the info axis"""
1946 return key in self._info_axis
1948 @property
1949 def empty(self) -> bool:
1950 """
1951 Indicator whether Series/DataFrame is empty.
1953 True if Series/DataFrame is entirely empty (no items), meaning any of the
1954 axes are of length 0.
1956 Returns
1957 -------
1958 bool
1959 If Series/DataFrame is empty, return True, if not return False.
1961 See Also
1962 --------
1963 Series.dropna : Return series without null values.
1964 DataFrame.dropna : Return DataFrame with labels on given axis omitted
1965 where (all or any) data are missing.
1967 Notes
1968 -----
1969 If Series/DataFrame contains only NaNs, it is still not considered empty. See
1970 the example below.
1972 Examples
1973 --------
1974 An example of an actual empty DataFrame. Notice the index is empty:
1976 >>> df_empty = pd.DataFrame({"A": []})
1977 >>> df_empty
1978 Empty DataFrame
1979 Columns: [A]
1980 Index: []
1981 >>> df_empty.empty
1982 True
1984 If we only have NaNs in our DataFrame, it is not considered empty! We
1985 will need to drop the NaNs to make the DataFrame empty:
1987 >>> df = pd.DataFrame({"A": [np.nan]})
1988 >>> df
1989 A
1990 0 NaN
1991 >>> df.empty
1992 False
1993 >>> df.dropna().empty
1994 True
1996 >>> ser_empty = pd.Series({"A": []})
1997 >>> ser_empty
1998 A []
1999 dtype: object
2000 >>> ser_empty.empty
2001 False
2002 >>> ser_empty = pd.Series()
2003 >>> ser_empty.empty
2004 True
2005 """
2006 return any(len(self._get_axis(a)) == 0 for a in self._AXIS_ORDERS)
2008 # ----------------------------------------------------------------------
2009 # Array Interface
2011 # This is also set in IndexOpsMixin
2012 # GH#23114 Ensure ndarray.__op__(DataFrame) returns NotImplemented
2013 __array_priority__: int = 1000
2015 def __array__(
2016 self, dtype: npt.DTypeLike | None = None, copy: bool | None = None
2017 ) -> np.ndarray:
2018 if copy is False and not self._mgr.is_single_block and not self.empty:
2019 # check this manually, otherwise ._values will already return a copy
2020 # and np.array(values, copy=False) will not raise an error
2021 raise ValueError(
2022 "Unable to avoid copy while creating an array as requested."
2023 )
2024 values = self._values
2025 if copy is None:
2026 # Note: branch avoids `copy=None` for NumPy 1.x support
2027 arr = np.asarray(values, dtype=dtype)
2028 else:
2029 arr = np.array(values, dtype=dtype, copy=copy)
2031 if (
2032 copy is not True
2033 and astype_is_view(values.dtype, arr.dtype)
2034 and self._mgr.is_single_block
2035 ):
2036 # Check if both conversions can be done without a copy
2037 if astype_is_view(self.dtypes.iloc[0], values.dtype) and astype_is_view(
2038 values.dtype, arr.dtype
2039 ):
2040 arr = arr.view()
2041 arr.flags.writeable = False
2042 return arr
2044 @final
2045 def __array_ufunc__(
2046 self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any
2047 ):
2048 return arraylike.array_ufunc(self, ufunc, method, *inputs, **kwargs)
2050 # ----------------------------------------------------------------------
2051 # Picklability
2053 @final
2054 def __getstate__(self) -> dict[str, Any]:
2055 meta = {k: getattr(self, k, None) for k in self._metadata}
2056 return {
2057 "_mgr": self._mgr,
2058 "_typ": self._typ,
2059 "_metadata": self._metadata,
2060 "attrs": self.attrs,
2061 "_flags": {k: self.flags[k] for k in self.flags._keys},
2062 **meta,
2063 }
2065 @final
2066 def __setstate__(self, state) -> None:
2067 if isinstance(state, BlockManager):
2068 self._mgr = state
2069 elif isinstance(state, dict):
2070 if "_data" in state and "_mgr" not in state:
2071 # compat for older pickles
2072 state["_mgr"] = state.pop("_data")
2073 typ = state.get("_typ")
2074 if typ is not None:
2075 attrs = state.get("_attrs", {})
2076 if attrs is None: # should not happen, but better be on the safe side
2077 attrs = {}
2078 object.__setattr__(self, "_attrs", attrs)
2079 flags = state.get("_flags", {"allows_duplicate_labels": True})
2080 object.__setattr__(self, "_flags", Flags(self, **flags))
2082 # set in the order of internal names
2083 # to avoid definitional recursion
2084 # e.g. say fill_value needing _mgr to be
2085 # defined
2086 meta = set(self._internal_names + self._metadata)
2087 for k in meta:
2088 if k in state and k != "_flags":
2089 v = state[k]
2090 object.__setattr__(self, k, v)
2092 for k, v in state.items():
2093 if k not in meta:
2094 object.__setattr__(self, k, v)
2096 else:
2097 raise NotImplementedError("Pre-0.12 pickles are no longer supported")
2098 elif len(state) == 2:
2099 raise NotImplementedError("Pre-0.12 pickles are no longer supported")
2101 # ----------------------------------------------------------------------
2102 # Rendering Methods
2104 def __repr__(self) -> str:
2105 # string representation based upon iterating over self
2106 # (since, by definition, `PandasContainers` are iterable)
2107 prepr = f"[{','.join(map(pprint_thing, self))}]"
2108 return f"{type(self).__name__}({prepr})"
2110 @final
2111 def _repr_latex_(self):
2112 """
2113 Returns a LaTeX representation for a particular object.
2114 Mainly for use with nbconvert (jupyter notebook conversion to pdf).
2115 """
2116 if config.get_option("styler.render.repr") == "latex":
2117 return self.to_latex()
2118 else:
2119 return None
2121 @final
2122 def _repr_data_resource_(self):
2123 """
2124 Not a real Jupyter special repr method, but we use the same
2125 naming convention.
2126 """
2127 if config.get_option("display.html.table_schema"):
2128 data = self.head(config.get_option("display.max_rows"))
2130 as_json = data.to_json(orient="table")
2131 as_json = cast(str, as_json)
2132 return loads(as_json, object_pairs_hook=collections.OrderedDict)
2134 # ----------------------------------------------------------------------
2135 # I/O Methods
2137 @final
2138 def to_excel(
2139 self,
2140 excel_writer: FilePath | WriteExcelBuffer | ExcelWriter,
2141 *,
2142 sheet_name: str = "Sheet1",
2143 na_rep: str = "",
2144 float_format: str | None = None,
2145 columns: Sequence[Hashable] | None = None,
2146 header: Sequence[Hashable] | bool = True,
2147 index: bool = True,
2148 index_label: IndexLabel | None = None,
2149 startrow: int = 0,
2150 startcol: int = 0,
2151 engine: Literal["openpyxl", "xlsxwriter"] | None = None,
2152 merge_cells: bool = True,
2153 inf_rep: str = "inf",
2154 freeze_panes: tuple[int, int] | None = None,
2155 storage_options: StorageOptions | None = None,
2156 engine_kwargs: dict[str, Any] | None = None,
2157 autofilter: bool = False,
2158 ) -> None:
2159 """
2160 Write object to an Excel sheet.
2162 To write a single object to an Excel .xlsx file it is only necessary to
2163 specify a target file name. To write to multiple sheets it is necessary to
2164 create an `ExcelWriter` object with a target file name, and specify a sheet
2165 in the file to write to.
2167 Multiple sheets may be written to by specifying unique `sheet_name`.
2168 With all data written to the file it is necessary to save the changes.
2169 Note that creating an `ExcelWriter` object with a file name that already exists
2170 will overwrite the existing file because the default mode is write.
2172 Parameters
2173 ----------
2174 excel_writer : path-like, file-like, or ExcelWriter object
2175 File path or existing ExcelWriter.
2176 sheet_name : str, default 'Sheet1'
2177 Name of sheet which will contain DataFrame.
2178 na_rep : str, default ''
2179 Missing data representation.
2180 float_format : str, optional
2181 Format string for floating point numbers. For example
2182 ``float_format="%.2f"`` will format 0.1234 to 0.12.
2183 columns : sequence or list of str, optional
2184 Columns to write.
2185 header : bool or list of str, default True
2186 Write out the column names. If a list of string is given it is
2187 assumed to be aliases for the column names.
2188 index : bool, default True
2189 Write row names (index).
2190 index_label : str or sequence, optional
2191 Column label for index column(s) if desired. If not specified, and
2192 `header` and `index` are True, then the index names are used. A
2193 sequence should be given if the DataFrame uses MultiIndex.
2194 startrow : int, default 0
2195 Upper left cell row to dump data frame.
2196 startcol : int, default 0
2197 Upper left cell column to dump data frame.
2198 engine : str, optional
2199 Write engine to use, 'openpyxl' or 'xlsxwriter'. You can also set this
2200 via the options ``io.excel.xlsx.writer`` or
2201 ``io.excel.xlsm.writer``.
2202 merge_cells : bool or 'columns', default False
2203 If True, write MultiIndex index and columns as merged cells.
2204 If 'columns', merge MultiIndex column cells only.
2205 inf_rep : str, default 'inf'
2206 Representation for infinity (there is no native representation for
2207 infinity in Excel).
2208 freeze_panes : tuple of int (length 2), optional
2209 Specifies the one-based bottommost row and rightmost column that
2210 is to be frozen.
2211 storage_options : dict, optional
2212 Extra options that make sense for a particular storage connection, e.g.
2213 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
2214 are forwarded to ``urllib.request.Request`` as header options. For other
2215 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
2216 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
2217 details, and for more examples on storage options refer `here
2218 <https://pandas.pydata.org/docs/user_guide/io.html?
2219 highlight=storage_options#reading-writing-remote-files>`_.
2221 engine_kwargs : dict, optional
2222 Arbitrary keyword arguments passed to excel engine.
2223 autofilter : bool, default False
2224 If True, add automatic filters to all columns.
2226 See Also
2227 --------
2228 to_csv : Write DataFrame to a comma-separated values (csv) file.
2229 ExcelWriter : Class for writing DataFrame objects into excel sheets.
2230 read_excel : Read an Excel file into a pandas DataFrame.
2231 read_csv : Read a comma-separated values (csv) file into DataFrame.
2232 io.formats.style.Styler.to_excel : Add styles to Excel sheet.
2234 Notes
2235 -----
2236 For compatibility with :meth:`~DataFrame.to_csv`,
2237 to_excel serializes lists and dicts to strings before writing.
2239 Once a workbook has been saved it is not possible to write further
2240 data without rewriting the whole workbook.
2242 pandas will check the number of rows, columns,
2243 and cell character count does not exceed Excel's limitations.
2244 All other limitations must be checked by the user.
2246 Examples
2247 --------
2249 Create, write to and save a workbook:
2251 >>> df1 = pd.DataFrame(
2252 ... [["a", "b"], ["c", "d"]],
2253 ... index=["row 1", "row 2"],
2254 ... columns=["col 1", "col 2"],
2255 ... )
2256 >>> df1.to_excel("output.xlsx") # doctest: +SKIP
2258 To specify the sheet name:
2260 >>> df1.to_excel("output.xlsx", sheet_name="Sheet_name_1") # doctest: +SKIP
2262 If you wish to write to more than one sheet in the workbook, it is
2263 necessary to specify an ExcelWriter object:
2265 >>> df2 = df1.copy()
2266 >>> with pd.ExcelWriter("output.xlsx") as writer: # doctest: +SKIP
2267 ... df1.to_excel(writer, sheet_name="Sheet_name_1")
2268 ... df2.to_excel(writer, sheet_name="Sheet_name_2")
2270 ExcelWriter can also be used to append to an existing Excel file:
2272 >>> with pd.ExcelWriter("output.xlsx", mode="a") as writer: # doctest: +SKIP
2273 ... df1.to_excel(writer, sheet_name="Sheet_name_3")
2275 To set the library that is used to write the Excel file,
2276 you can pass the `engine` keyword (the default engine is
2277 automatically chosen depending on the file extension):
2279 >>> df1.to_excel("output1.xlsx", engine="xlsxwriter") # doctest: +SKIP
2280 """
2281 if engine_kwargs is None:
2282 engine_kwargs = {}
2284 df = self if isinstance(self, ABCDataFrame) else self.to_frame()
2286 from pandas.io.formats.excel import ExcelFormatter
2288 formatter = ExcelFormatter(
2289 df,
2290 na_rep=na_rep,
2291 cols=columns,
2292 header=header,
2293 float_format=float_format,
2294 index=index,
2295 index_label=index_label,
2296 merge_cells=merge_cells,
2297 inf_rep=inf_rep,
2298 autofilter=autofilter,
2299 )
2300 formatter.write(
2301 excel_writer,
2302 sheet_name=sheet_name,
2303 startrow=startrow,
2304 startcol=startcol,
2305 freeze_panes=freeze_panes,
2306 engine=engine,
2307 storage_options=storage_options,
2308 engine_kwargs=engine_kwargs,
2309 )
2311 @final
2312 def to_json(
2313 self,
2314 path_or_buf: FilePath | WriteBuffer[bytes] | WriteBuffer[str] | None = None,
2315 *,
2316 orient: Literal["split", "records", "index", "table", "columns", "values"]
2317 | None = None,
2318 date_format: str | None = None,
2319 double_precision: int = 10,
2320 force_ascii: bool = True,
2321 date_unit: TimeUnit = "ms",
2322 default_handler: Callable[[Any], JSONSerializable] | None = None,
2323 lines: bool = False,
2324 compression: CompressionOptions = "infer",
2325 index: bool | None = None,
2326 indent: int | None = None,
2327 storage_options: StorageOptions | None = None,
2328 mode: Literal["a", "w"] = "w",
2329 ) -> str | None:
2330 """
2331 Convert the object to a JSON string.
2333 Note NaN's and None will be converted to null and datetime objects
2334 will be converted to UNIX timestamps.
2336 Parameters
2337 ----------
2338 path_or_buf : str, path object, file-like object, or None, default None
2339 String, path object (implementing os.PathLike[str]), or file-like
2340 object implementing a write() function. If None, the result is
2341 returned as a string.
2342 orient : str
2343 Indication of expected JSON string format.
2345 * Series:
2347 - default is 'index'
2348 - allowed values are: {'split', 'records', 'index', 'table'}.
2350 * DataFrame:
2352 - default is 'columns'
2353 - allowed values are: {'split', 'records', 'index', 'columns',
2354 'values', 'table'}.
2356 * The format of the JSON string:
2358 - 'split' : dict like {'index' -> [index], 'columns' -> [columns],
2359 'data' -> [values]}
2360 - 'records' : list like [{column -> value}, ... , {column -> value}]
2361 - 'index' : dict like {index -> {column -> value}}
2362 - 'columns' : dict like {column -> {index -> value}}
2363 - 'values' : just the values array
2364 - 'table' : dict like {'schema': {schema}, 'data': {data}}
2366 Describing the data, where data component is like ``orient='records'``.
2368 date_format : {None, 'epoch', 'iso'}
2369 Type of date conversion. 'epoch' = epoch milliseconds,
2370 'iso' = ISO8601. The default depends on the `orient`. For
2371 ``orient='table'``, the default is 'iso'. For all other orients,
2372 the default is 'epoch'.
2374 .. deprecated:: 3.0.0
2375 'epoch' date format is deprecated and will be removed in a future
2376 version, please use 'iso' instead.
2378 double_precision : int, default 10
2379 The number of decimal places to use when encoding
2380 floating point values. The possible maximal value is 15.
2381 Passing double_precision greater than 15 will raise a ValueError.
2382 force_ascii : bool, default True
2383 Force encoded string to be ASCII.
2384 date_unit : str, default 'ms' (milliseconds)
2385 The time unit to encode to, governs timestamp and ISO8601
2386 precision. One of 's', 'ms', 'us', 'ns' for second, millisecond,
2387 microsecond, and nanosecond respectively.
2388 default_handler : callable, default None
2389 Handler to call if object cannot otherwise be converted to a
2390 suitable format for JSON. Should receive a single argument which is
2391 the object to convert and return a serialisable object.
2392 lines : bool, default False
2393 If 'orient' is 'records' write out line-delimited json format. Will
2394 throw ValueError if incorrect 'orient' since others are not
2395 list-like.
2397 compression : str or dict, default 'infer'
2398 For on-the-fly compression of the output data. If 'infer' and
2399 'path_or_buf' is path-like, then detect compression from the following
2400 extensions: '.gz',
2401 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
2402 (otherwise no compression).
2403 Set to ``None`` for no compression.
2404 Can also be a dict with key ``'method'`` set to one of
2405 {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``} and
2406 other key-value pairs are forwarded to
2407 ``zipfile.ZipFile``, ``gzip.GzipFile``,
2408 ``bz2.BZ2File``, ``zstandard.ZstdCompressor``, ``lzma.LZMAFile`` or
2409 ``tarfile.TarFile``, respectively.
2410 As an example, the following could be passed for faster compression and
2411 to create a reproducible gzip archive:
2412 ``compression={'method': 'gzip', 'compresslevel': 1, 'mtime': 1}``.
2414 index : bool or None, default None
2415 The index is only used when 'orient' is 'split', 'index', 'column',
2416 or 'table'. Of these, 'index' and 'column' do not support
2417 `index=False`. The string 'index' as a column name with empty :class:`Index`
2418 or if it is 'index' will raise a ``ValueError``.
2420 indent : int, optional
2421 Length of whitespace used to indent each record.
2423 storage_options : dict, optional
2424 Extra options that make sense for a particular storage connection, e.g.
2425 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
2426 are forwarded to ``urllib.request.Request`` as header options. For other
2427 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
2428 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
2429 details, and for more examples on storage options refer `here
2430 <https://pandas.pydata.org/docs/user_guide/io.html?
2431 highlight=storage_options#reading-writing-remote-files>`_.
2433 mode : str, default 'w' (writing)
2434 Specify the IO mode for output when supplying a path_or_buf.
2435 Accepted args are 'w' (writing) and 'a' (append) only.
2436 mode='a' is only supported when lines is True and orient is 'records'.
2438 Returns
2439 -------
2440 None or str
2441 If path_or_buf is None, returns the resulting json format as a
2442 string. Otherwise returns None.
2444 See Also
2445 --------
2446 read_json : Convert a JSON string to pandas object.
2448 Notes
2449 -----
2450 The behavior of ``indent=0`` varies from the stdlib, which does not
2451 indent the output but does insert newlines. Currently, ``indent=0``
2452 and the default ``indent=None`` are equivalent in pandas, though this
2453 may change in a future release.
2455 ``orient='table'`` contains a 'pandas_version' field under 'schema'.
2456 This stores the version of `pandas` used in the latest revision of the
2457 schema.
2459 Examples
2460 --------
2461 >>> from json import loads, dumps
2462 >>> df = pd.DataFrame(
2463 ... [["a", "b"], ["c", "d"]],
2464 ... index=["row 1", "row 2"],
2465 ... columns=["col 1", "col 2"],
2466 ... )
2468 >>> result = df.to_json(orient="split")
2469 >>> parsed = loads(result)
2470 >>> dumps(parsed, indent=4) # doctest: +SKIP
2471 {
2472 "columns": [
2473 "col 1",
2474 "col 2"
2475 ],
2476 "index": [
2477 "row 1",
2478 "row 2"
2479 ],
2480 "data": [
2481 [
2482 "a",
2483 "b"
2484 ],
2485 [
2486 "c",
2487 "d"
2488 ]
2489 ]
2490 }
2492 Encoding/decoding a Dataframe using ``'records'`` formatted JSON.
2493 Note that index labels are not preserved with this encoding.
2495 >>> result = df.to_json(orient="records")
2496 >>> parsed = loads(result)
2497 >>> dumps(parsed, indent=4) # doctest: +SKIP
2498 [
2499 {
2500 "col 1": "a",
2501 "col 2": "b"
2502 },
2503 {
2504 "col 1": "c",
2505 "col 2": "d"
2506 }
2507 ]
2509 Encoding/decoding a Dataframe using ``'index'`` formatted JSON:
2511 >>> result = df.to_json(orient="index")
2512 >>> parsed = loads(result)
2513 >>> dumps(parsed, indent=4) # doctest: +SKIP
2514 {
2515 "row 1": {
2516 "col 1": "a",
2517 "col 2": "b"
2518 },
2519 "row 2": {
2520 "col 1": "c",
2521 "col 2": "d"
2522 }
2523 }
2525 Encoding/decoding a Dataframe using ``'columns'`` formatted JSON:
2527 >>> result = df.to_json(orient="columns")
2528 >>> parsed = loads(result)
2529 >>> dumps(parsed, indent=4) # doctest: +SKIP
2530 {
2531 "col 1": {
2532 "row 1": "a",
2533 "row 2": "c"
2534 },
2535 "col 2": {
2536 "row 1": "b",
2537 "row 2": "d"
2538 }
2539 }
2541 Encoding/decoding a Dataframe using ``'values'`` formatted JSON:
2543 >>> result = df.to_json(orient="values")
2544 >>> parsed = loads(result)
2545 >>> dumps(parsed, indent=4) # doctest: +SKIP
2546 [
2547 [
2548 "a",
2549 "b"
2550 ],
2551 [
2552 "c",
2553 "d"
2554 ]
2555 ]
2557 Encoding with Table Schema:
2559 >>> result = df.to_json(orient="table")
2560 >>> parsed = loads(result)
2561 >>> dumps(parsed, indent=4) # doctest: +SKIP
2562 {
2563 "schema": {
2564 "fields": [
2565 {
2566 "name": "index",
2567 "type": "string"
2568 },
2569 {
2570 "name": "col 1",
2571 "type": "string"
2572 },
2573 {
2574 "name": "col 2",
2575 "type": "string"
2576 }
2577 ],
2578 "primaryKey": [
2579 "index"
2580 ],
2581 "pandas_version": "1.4.0"
2582 },
2583 "data": [
2584 {
2585 "index": "row 1",
2586 "col 1": "a",
2587 "col 2": "b"
2588 },
2589 {
2590 "index": "row 2",
2591 "col 1": "c",
2592 "col 2": "d"
2593 }
2594 ]
2595 }
2596 """
2597 from pandas.io import json
2599 if date_format is None and orient == "table":
2600 date_format = "iso"
2601 elif date_format is None:
2602 date_format = "epoch"
2603 dtypes = self.dtypes if self.ndim == 2 else [self.dtype]
2604 if any(dtype.kind in "mM" for dtype in dtypes):
2605 warnings.warn(
2606 "The default 'epoch' date format is deprecated and will be removed "
2607 "in a future version, please use 'iso' date format instead.",
2608 Pandas4Warning,
2609 stacklevel=find_stack_level(),
2610 )
2611 elif date_format == "epoch":
2612 # GH#57063
2613 warnings.warn(
2614 "'epoch' date format is deprecated and will be removed in a future "
2615 "version, please use 'iso' date format instead.",
2616 Pandas4Warning,
2617 stacklevel=find_stack_level(),
2618 )
2620 config.is_nonnegative_int(indent)
2621 indent = indent or 0
2623 return json.to_json(
2624 path_or_buf=path_or_buf,
2625 obj=self,
2626 orient=orient,
2627 date_format=date_format,
2628 double_precision=double_precision,
2629 force_ascii=force_ascii,
2630 date_unit=date_unit,
2631 default_handler=default_handler,
2632 lines=lines,
2633 compression=compression,
2634 index=index,
2635 indent=indent,
2636 storage_options=storage_options,
2637 mode=mode,
2638 )
2640 @final
2641 def to_hdf(
2642 self,
2643 path_or_buf: FilePath | HDFStore,
2644 *,
2645 key: str,
2646 mode: Literal["a", "w", "r+"] = "a",
2647 complevel: int | None = None,
2648 complib: Literal["zlib", "lzo", "bzip2", "blosc"] | None = None,
2649 append: bool = False,
2650 format: Literal["fixed", "table"] | None = None,
2651 index: bool = True,
2652 min_itemsize: int | dict[str, int] | None = None,
2653 nan_rep=None,
2654 dropna: bool | None = None,
2655 data_columns: Literal[True] | list[str] | None = None,
2656 errors: OpenFileErrors = "strict",
2657 encoding: str = "UTF-8",
2658 ) -> None:
2659 """
2660 Write the contained data to an HDF5 file using HDFStore.
2662 Hierarchical Data Format (HDF) is self-describing, allowing an
2663 application to interpret the structure and contents of a file with
2664 no outside information. One HDF file can hold a mix of related objects
2665 which can be accessed as a group or as individual objects.
2667 In order to add another DataFrame or Series to an existing HDF file
2668 please use append mode and a different a key.
2670 .. warning::
2672 One can store a subclass of ``DataFrame`` or ``Series`` to HDF5,
2673 but the type of the subclass is lost upon storing.
2675 For more information see the :ref:`user guide <io.hdf5>`.
2677 Parameters
2678 ----------
2679 path_or_buf : str or pandas.HDFStore
2680 File path or HDFStore object.
2681 key : str
2682 Identifier for the group in the store.
2683 mode : {'a', 'w', 'r+'}, default 'a'
2684 Mode to open file:
2686 - 'w': write, a new file is created (an existing file with
2687 the same name would be deleted).
2688 - 'a': append, an existing file is opened for reading and
2689 writing, and if the file does not exist it is created.
2690 - 'r+': similar to 'a', but the file must already exist.
2691 complevel : {0-9}, default None
2692 Specifies a compression level for data.
2693 A value of 0 or None disables compression.
2694 complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'
2695 Specifies the compression library to be used.
2696 These additional compressors for Blosc are supported
2697 (default if no compressor specified: 'blosc:blosclz'):
2698 {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',
2699 'blosc:zlib', 'blosc:zstd'}.
2700 Specifying a compression library which is not available issues
2701 a ValueError.
2702 append : bool, default False
2703 For Table formats, append the input data to the existing.
2704 format : {'fixed', 'table', None}, default 'fixed'
2705 Possible values:
2707 - 'fixed': Fixed format. Fast writing/reading. Not-appendable,
2708 nor searchable.
2709 - 'table': Table format. Write as a PyTables Table structure
2710 which may perform worse but allow more flexible operations
2711 like searching / selecting subsets of the data.
2712 - If None, pd.get_option('io.hdf.default_format') is checked,
2713 followed by fallback to "fixed".
2714 index : bool, default True
2715 Write DataFrame index as a column.
2716 min_itemsize : dict or int, optional
2717 Map column names to minimum string sizes for columns.
2718 nan_rep : Any, optional
2719 How to represent null values as str.
2720 Not allowed with append=True.
2721 dropna : bool, default False, optional
2722 Remove missing values.
2723 data_columns : list of columns or True, optional
2724 List of columns to create as indexed data columns for on-disk
2725 queries, or True to use all columns. By default only the axes
2726 of the object are indexed. See
2727 :ref:`Query via data columns<io.hdf5-query-data-columns>`. for
2728 more information.
2729 Applicable only to format='table'.
2730 errors : str, default 'strict'
2731 Specifies how encoding and decoding errors are to be handled.
2732 See the errors argument for :func:`open` for a full list
2733 of options.
2734 encoding : str, default "UTF-8"
2735 Set character encoding.
2737 See Also
2738 --------
2739 read_hdf : Read from HDF file.
2740 DataFrame.to_orc : Write a DataFrame to the binary orc format.
2741 DataFrame.to_parquet : Write a DataFrame to the binary parquet format.
2742 DataFrame.to_sql : Write to a SQL table.
2743 DataFrame.to_feather : Write out feather-format for DataFrames.
2744 DataFrame.to_csv : Write out to a csv file.
2746 Examples
2747 --------
2748 >>> df = pd.DataFrame(
2749 ... {"A": [1, 2, 3], "B": [4, 5, 6]}, index=["a", "b", "c"]
2750 ... ) # doctest: +SKIP
2751 >>> df.to_hdf("data.h5", key="df", mode="w") # doctest: +SKIP
2753 We can add another object to the same file:
2755 >>> s = pd.Series([1, 2, 3, 4]) # doctest: +SKIP
2756 >>> s.to_hdf("data.h5", key="s") # doctest: +SKIP
2758 Reading from HDF file:
2760 >>> pd.read_hdf("data.h5", "df") # doctest: +SKIP
2761 A B
2762 a 1 4
2763 b 2 5
2764 c 3 6
2765 >>> pd.read_hdf("data.h5", "s") # doctest: +SKIP
2766 0 1
2767 1 2
2768 2 3
2769 3 4
2770 dtype: int64
2771 """
2772 from pandas.io import pytables
2774 # Argument 3 to "to_hdf" has incompatible type "NDFrame"; expected
2775 # "Union[DataFrame, Series]" [arg-type]
2776 pytables.to_hdf(
2777 path_or_buf,
2778 key,
2779 self, # type: ignore[arg-type]
2780 mode=mode,
2781 complevel=complevel,
2782 complib=complib,
2783 append=append,
2784 format=format,
2785 index=index,
2786 min_itemsize=min_itemsize,
2787 nan_rep=nan_rep,
2788 dropna=dropna,
2789 data_columns=data_columns,
2790 errors=errors,
2791 encoding=encoding,
2792 )
2794 @final
2795 def to_sql(
2796 self,
2797 name: str,
2798 con,
2799 *,
2800 schema: str | None = None,
2801 if_exists: Literal["fail", "replace", "append", "delete_rows"] = "fail",
2802 index: bool = True,
2803 index_label: IndexLabel | None = None,
2804 chunksize: int | None = None,
2805 dtype: DtypeArg | None = None,
2806 method: Literal["multi"] | Callable | None = None,
2807 ) -> int | None:
2808 """
2809 Write records stored in a DataFrame to a SQL database.
2811 Databases supported by SQLAlchemy [1]_ are supported. Tables can be
2812 newly created, appended to, or overwritten.
2814 .. warning::
2815 The pandas library does not attempt to sanitize inputs provided via a to_sql call.
2816 Please refer to the documentation for the underlying database driver to see if it
2817 will properly prevent injection, or alternatively be advised of a security risk when
2818 executing arbitrary commands in a to_sql call.
2820 Parameters
2821 ----------
2822 name : str
2823 Name of SQL table.
2824 con : ADBC connection, sqlalchemy.engine.(Engine or Connection) or sqlite3.Connection
2825 ADBC provides high performance I/O with native type support, where available.
2826 Using SQLAlchemy makes it possible to use any DB supported by that
2827 library. Legacy support is provided for sqlite3.Connection objects. The user
2828 is responsible for engine disposal and connection closure for the SQLAlchemy
2829 connectable. See `here \
2830 <https://docs.sqlalchemy.org/en/20/core/connections.html>`_.
2831 If passing a sqlalchemy.engine.Connection which is already in a transaction,
2832 the transaction will not be committed. If passing a sqlite3.Connection,
2833 it will not be possible to roll back the record insertion.
2835 schema : str, optional
2836 Specify the schema (if database flavor supports this). If None, use
2837 default schema.
2838 if_exists : {'fail', 'replace', 'append', 'delete_rows'}, default 'fail'
2839 How to behave if the table already exists.
2841 * fail: Raise a ValueError.
2842 * replace: Drop the table before inserting new values.
2843 * append: Insert new values to the existing table.
2844 * delete_rows: If a table exists, delete all records and insert data.
2846 index : bool, default True
2847 Write DataFrame index as a column. Uses `index_label` as the column
2848 name in the table. Creates a table index for this column.
2849 index_label : str or sequence, default None
2850 Column label for index column(s). If None is given (default) and
2851 `index` is True, then the index names are used.
2852 A sequence should be given if the DataFrame uses MultiIndex.
2853 chunksize : int, optional
2854 Specify the number of rows in each batch to be written to the database connection at a time.
2855 By default, all rows will be written at once. Also see the method keyword.
2856 dtype : dict or scalar, optional
2857 Specifying the datatype for columns. If a dictionary is used, the
2858 keys should be the column names and the values should be the
2859 SQLAlchemy types or strings for the sqlite3 legacy mode. If a
2860 scalar is provided, it will be applied to all columns.
2861 method : {None, 'multi', callable}, optional
2862 Controls the SQL insertion clause used:
2864 * None : Uses standard SQL ``INSERT`` clause (one per row).
2865 * 'multi': Pass multiple values in a single ``INSERT`` clause.
2866 * callable with signature ``(pd_table, conn, keys, data_iter)``.
2868 Details and a sample callable implementation can be found in the
2869 section :ref:`insert method <io.sql.method>`.
2871 Returns
2872 -------
2873 None or int
2874 Number of rows affected by to_sql. None is returned if the callable
2875 passed into ``method`` does not return an integer number of rows.
2877 The number of returned rows affected is the sum of the ``rowcount``
2878 attribute of ``sqlite3.Cursor`` or SQLAlchemy connectable which may not
2879 reflect the exact number of written rows as stipulated in the
2880 `sqlite3 <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.rowcount>`__ or
2881 `SQLAlchemy <https://docs.sqlalchemy.org/en/20/core/connections.html#sqlalchemy.engine.CursorResult.rowcount>`__.
2883 Raises
2884 ------
2885 ValueError
2886 When the table already exists and `if_exists` is 'fail' (the
2887 default).
2889 See Also
2890 --------
2891 read_sql : Read a DataFrame from a table.
2893 Notes
2894 -----
2895 Timezone aware datetime columns will be written as
2896 ``Timestamp with timezone`` type with SQLAlchemy if supported by the
2897 database. Otherwise, the datetimes will be stored as timezone unaware
2898 timestamps local to the original timezone.
2900 Not all datastores support ``method="multi"``. Oracle, for example,
2901 does not support multi-value insert.
2903 References
2904 ----------
2905 .. [1] https://docs.sqlalchemy.org
2906 .. [2] https://www.python.org/dev/peps/pep-0249/
2908 Examples
2909 --------
2910 Create an in-memory SQLite database.
2912 >>> from sqlalchemy import create_engine
2913 >>> engine = create_engine('sqlite://', echo=False)
2915 Create a table from scratch with 3 rows.
2917 >>> df = pd.DataFrame({'name' : ['User 1', 'User 2', 'User 3']})
2918 >>> df
2919 name
2920 0 User 1
2921 1 User 2
2922 2 User 3
2924 >>> df.to_sql(name='users', con=engine)
2925 3
2926 >>> from sqlalchemy import text
2927 >>> with engine.connect() as conn:
2928 ... conn.execute(text("SELECT * FROM users")).fetchall()
2929 [(0, 'User 1'), (1, 'User 2'), (2, 'User 3')]
2931 An `sqlalchemy.engine.Connection` can also be passed to `con`:
2933 >>> with engine.begin() as connection:
2934 ... df1 = pd.DataFrame({'name' : ['User 4', 'User 5']})
2935 ... df1.to_sql(name='users', con=connection, if_exists='append')
2936 2
2938 This is allowed to support operations that require that the same
2939 DBAPI connection is used for the entire operation.
2941 >>> df2 = pd.DataFrame({'name' : ['User 6', 'User 7']})
2942 >>> df2.to_sql(name='users', con=engine, if_exists='append')
2943 2
2944 >>> with engine.connect() as conn:
2945 ... conn.execute(text("SELECT * FROM users")).fetchall()
2946 [(0, 'User 1'), (1, 'User 2'), (2, 'User 3'),
2947 (0, 'User 4'), (1, 'User 5'), (0, 'User 6'),
2948 (1, 'User 7')]
2950 Overwrite the table with just ``df2``.
2952 >>> df2.to_sql(name='users', con=engine, if_exists='replace',
2953 ... index_label='id')
2954 2
2955 >>> with engine.connect() as conn:
2956 ... conn.execute(text("SELECT * FROM users")).fetchall()
2957 [(0, 'User 6'), (1, 'User 7')]
2959 Delete all rows before inserting new records with ``df3``
2961 >>> df3 = pd.DataFrame({"name": ['User 8', 'User 9']})
2962 >>> df3.to_sql(name='users', con=engine, if_exists='delete_rows',
2963 ... index_label='id')
2964 2
2965 >>> with engine.connect() as conn:
2966 ... conn.execute(text("SELECT * FROM users")).fetchall()
2967 [(0, 'User 8'), (1, 'User 9')]
2969 Use ``method`` to define a callable insertion method to do nothing
2970 if there's a primary key conflict on a table in a PostgreSQL database.
2972 >>> from sqlalchemy.dialects.postgresql import insert
2973 >>> def insert_on_conflict_nothing(table, conn, keys, data_iter):
2974 ... # "a" is the primary key in "conflict_table"
2975 ... data = [dict(zip(keys, row)) for row in data_iter]
2976 ... stmt = insert(table.table).values(data).on_conflict_do_nothing(index_elements=["a"])
2977 ... result = conn.execute(stmt)
2978 ... return result.rowcount
2979 >>> df_conflict.to_sql(name="conflict_table", con=conn, if_exists="append", # noqa: F821
2980 ... method=insert_on_conflict_nothing) # doctest: +SKIP
2981 0
2983 For MySQL, a callable to update columns ``b`` and ``c`` if there's a conflict
2984 on a primary key.
2986 >>> from sqlalchemy.dialects.mysql import insert # noqa: F811
2987 >>> def insert_on_conflict_update(table, conn, keys, data_iter):
2988 ... # update columns "b" and "c" on primary key conflict
2989 ... data = [dict(zip(keys, row)) for row in data_iter]
2990 ... stmt = (
2991 ... insert(table.table)
2992 ... .values(data)
2993 ... )
2994 ... stmt = stmt.on_duplicate_key_update(b=stmt.inserted.b, c=stmt.inserted.c)
2995 ... result = conn.execute(stmt)
2996 ... return result.rowcount
2997 >>> df_conflict.to_sql(name="conflict_table", con=conn, if_exists="append", # noqa: F821
2998 ... method=insert_on_conflict_update) # doctest: +SKIP
2999 2
3001 Specify the dtype (especially useful for integers with missing values).
3002 Notice that while pandas is forced to store the data as floating point,
3003 the database supports nullable integers. When fetching the data with
3004 Python, we get back integer scalars.
3006 >>> df = pd.DataFrame({"A": [1, None, 2]})
3007 >>> df
3008 A
3009 0 1.0
3010 1 NaN
3011 2 2.0
3013 >>> from sqlalchemy.types import Integer
3014 >>> df.to_sql(name='integers', con=engine, index=False,
3015 ... dtype={"A": Integer()})
3016 3
3018 >>> with engine.connect() as conn:
3019 ... conn.execute(text("SELECT * FROM integers")).fetchall()
3020 [(1,), (None,), (2,)]
3022 .. versionadded:: 2.2.0
3024 pandas now supports writing via ADBC drivers
3026 >>> df = pd.DataFrame({'name' : ['User 10', 'User 11', 'User 12']})
3027 >>> df
3028 name
3029 0 User 10
3030 1 User 11
3031 2 User 12
3033 >>> from adbc_driver_sqlite import dbapi # doctest:+SKIP
3034 >>> with dbapi.connect("sqlite://") as conn: # doctest:+SKIP
3035 ... df.to_sql(name="users", con=conn)
3036 3
3037 """ # noqa: E501
3038 from pandas.io import sql
3040 return sql.to_sql(
3041 self,
3042 name,
3043 con,
3044 schema=schema,
3045 if_exists=if_exists,
3046 index=index,
3047 index_label=index_label,
3048 chunksize=chunksize,
3049 dtype=dtype,
3050 method=method,
3051 )
3053 @final
3054 def to_pickle(
3055 self,
3056 path: FilePath | WriteBuffer[bytes],
3057 *,
3058 compression: CompressionOptions = "infer",
3059 protocol: int = pickle.HIGHEST_PROTOCOL,
3060 storage_options: StorageOptions | None = None,
3061 ) -> None:
3062 """
3063 Pickle (serialize) object to file.
3065 Parameters
3066 ----------
3067 path : str, path object, or file-like object
3068 String, path object (implementing ``os.PathLike[str]``), or file-like
3069 object implementing a binary ``write()`` function. File path where
3070 the pickled object will be stored.
3072 compression : str or dict, default 'infer'
3073 For on-the-fly compression of the output data. If 'infer' and
3074 'path_or_buf' is path-like, then detect compression from the following
3075 extensions: '.gz',
3076 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
3077 (otherwise no compression).
3078 Set to ``None`` for no compression.
3079 Can also be a dict with key ``'method'`` set to one of
3080 {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``} and
3081 other key-value pairs are forwarded to
3082 ``zipfile.ZipFile``, ``gzip.GzipFile``,
3083 ``bz2.BZ2File``, ``zstandard.ZstdCompressor``, ``lzma.LZMAFile`` or
3084 ``tarfile.TarFile``, respectively.
3085 As an example, the following could be passed for faster compression and
3086 to create a reproducible gzip archive:
3087 ``compression={'method': 'gzip', 'compresslevel': 1, 'mtime': 1}``.
3089 protocol : int
3090 Int which indicates which protocol should be used by the pickler,
3091 default HIGHEST_PROTOCOL (see [1]_ paragraph 12.1.2). The possible
3092 values are 0, 1, 2, 3, 4, 5. A negative value for the protocol
3093 parameter is equivalent to setting its value to HIGHEST_PROTOCOL.
3095 .. [1] https://docs.python.org/3/library/pickle.html.
3097 storage_options : dict, optional
3098 Extra options that make sense for a particular storage connection, e.g.
3099 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
3100 are forwarded to ``urllib.request.Request`` as header options. For other
3101 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
3102 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
3103 details, and for more examples on storage options refer `here
3104 <https://pandas.pydata.org/docs/user_guide/io.html?
3105 highlight=storage_options#reading-writing-remote-files>`_.
3107 See Also
3108 --------
3109 read_pickle : Load pickled pandas object (or any object) from file.
3110 DataFrame.to_hdf : Write DataFrame to an HDF5 file.
3111 DataFrame.to_sql : Write DataFrame to a SQL database.
3112 DataFrame.to_parquet : Write a DataFrame to the binary parquet format.
3114 Examples
3115 --------
3116 >>> original_df = pd.DataFrame(
3117 ... {"foo": range(5), "bar": range(5, 10)}
3118 ... ) # doctest: +SKIP
3119 >>> original_df # doctest: +SKIP
3120 foo bar
3121 0 0 5
3122 1 1 6
3123 2 2 7
3124 3 3 8
3125 4 4 9
3126 >>> original_df.to_pickle("./dummy.pkl") # doctest: +SKIP
3128 >>> unpickled_df = pd.read_pickle("./dummy.pkl") # doctest: +SKIP
3129 >>> unpickled_df # doctest: +SKIP
3130 foo bar
3131 0 0 5
3132 1 1 6
3133 2 2 7
3134 3 3 8
3135 4 4 9
3136 """
3137 from pandas.io.pickle import to_pickle
3139 to_pickle(
3140 self,
3141 path,
3142 compression=compression,
3143 protocol=protocol,
3144 storage_options=storage_options,
3145 )
3147 @final
3148 def to_clipboard(
3149 self, *, excel: bool = True, sep: str | None = None, **kwargs
3150 ) -> None:
3151 r"""
3152 Copy object to the system clipboard.
3154 Write a text representation of object to the system clipboard.
3155 This can be pasted into Excel, for example.
3157 Parameters
3158 ----------
3159 excel : bool, default True
3160 Produce output in a csv format for easy pasting into excel.
3162 - True, use the provided separator for csv pasting.
3163 - False, write a string representation of the object to the clipboard.
3165 sep : str, default ``'\t'``
3166 Field delimiter.
3167 **kwargs
3168 These parameters will be passed to DataFrame.to_csv.
3170 See Also
3171 --------
3172 DataFrame.to_csv : Write a DataFrame to a comma-separated values
3173 (csv) file.
3174 read_clipboard : Read text from clipboard and pass to read_csv.
3176 Notes
3177 -----
3178 Requirements for your platform.
3180 - Linux : `xclip`, or `xsel` (with `PyQt4` modules)
3181 - Windows : none
3182 - macOS : none
3184 This method uses the processes developed for the package `pyperclip`. A
3185 solution to render any output string format is given in the examples.
3187 Examples
3188 --------
3189 Copy the contents of a DataFrame to the clipboard.
3191 >>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=["A", "B", "C"])
3193 >>> df.to_clipboard(sep=",") # doctest: +SKIP
3194 ... # Wrote the following to the system clipboard:
3195 ... # ,A,B,C
3196 ... # 0,1,2,3
3197 ... # 1,4,5,6
3199 We can omit the index by passing the keyword `index` and setting
3200 it to false.
3202 >>> df.to_clipboard(sep=",", index=False) # doctest: +SKIP
3203 ... # Wrote the following to the system clipboard:
3204 ... # A,B,C
3205 ... # 1,2,3
3206 ... # 4,5,6
3208 Using the original `pyperclip` package for any string output format.
3210 .. code-block:: python
3212 import pyperclip
3214 html = df.style.to_html()
3215 pyperclip.copy(html)
3216 """
3217 from pandas.io import clipboards
3219 clipboards.to_clipboard(self, excel=excel, sep=sep, **kwargs)
3221 @final
3222 def to_xarray(self):
3223 """
3224 Return an xarray object from the pandas object.
3226 Returns
3227 -------
3228 xarray.DataArray or xarray.Dataset
3229 Data in the pandas structure converted to Dataset if the object is
3230 a DataFrame, or a DataArray if the object is a Series.
3232 See Also
3233 --------
3234 DataFrame.to_hdf : Write DataFrame to an HDF5 file.
3235 DataFrame.to_parquet : Write a DataFrame to the binary parquet format.
3237 Notes
3238 -----
3239 See the `xarray docs <https://xarray.pydata.org/en/stable/>`__
3241 Examples
3242 --------
3243 >>> df = pd.DataFrame(
3244 ... [
3245 ... ("falcon", "bird", 389.0, 2),
3246 ... ("parrot", "bird", 24.0, 2),
3247 ... ("lion", "mammal", 80.5, 4),
3248 ... ("monkey", "mammal", np.nan, 4),
3249 ... ],
3250 ... columns=["name", "class", "max_speed", "num_legs"],
3251 ... )
3252 >>> df
3253 name class max_speed num_legs
3254 0 falcon bird 389.0 2
3255 1 parrot bird 24.0 2
3256 2 lion mammal 80.5 4
3257 3 monkey mammal NaN 4
3259 >>> df.to_xarray() # doctest: +SKIP
3260 <xarray.Dataset>
3261 Dimensions: (index: 4)
3262 Coordinates:
3263 * index (index) int64 32B 0 1 2 3
3264 Data variables:
3265 name (index) object 32B 'falcon' 'parrot' 'lion' 'monkey'
3266 class (index) object 32B 'bird' 'bird' 'mammal' 'mammal'
3267 max_speed (index) float64 32B 389.0 24.0 80.5 nan
3268 num_legs (index) int64 32B 2 2 4 4
3270 >>> df["max_speed"].to_xarray() # doctest: +SKIP
3271 <xarray.DataArray 'max_speed' (index: 4)>
3272 array([389. , 24. , 80.5, nan])
3273 Coordinates:
3274 * index (index) int64 0 1 2 3
3276 >>> dates = pd.to_datetime(
3277 ... ["2018-01-01", "2018-01-01", "2018-01-02", "2018-01-02"]
3278 ... )
3279 >>> df_multiindex = pd.DataFrame(
3280 ... {
3281 ... "date": dates,
3282 ... "animal": ["falcon", "parrot", "falcon", "parrot"],
3283 ... "speed": [350, 18, 361, 15],
3284 ... }
3285 ... )
3286 >>> df_multiindex = df_multiindex.set_index(["date", "animal"])
3288 >>> df_multiindex
3289 speed
3290 date animal
3291 2018-01-01 falcon 350
3292 parrot 18
3293 2018-01-02 falcon 361
3294 parrot 15
3296 >>> df_multiindex.to_xarray() # doctest: +SKIP
3297 <xarray.Dataset>
3298 Dimensions: (date: 2, animal: 2)
3299 Coordinates:
3300 * date (date) datetime64[s] 2018-01-01 2018-01-02
3301 * animal (animal) object 'falcon' 'parrot'
3302 Data variables:
3303 speed (date, animal) int64 350 18 361 15
3304 """
3305 xarray = import_optional_dependency("xarray")
3307 if self.ndim == 1:
3308 return xarray.DataArray.from_series(self)
3309 else:
3310 return xarray.Dataset.from_dataframe(self)
3312 @overload
3313 def to_latex(
3314 self,
3315 buf: None = ...,
3316 *,
3317 columns: Sequence[Hashable] | None = ...,
3318 header: bool | SequenceNotStr[str] = ...,
3319 index: bool = ...,
3320 na_rep: str = ...,
3321 formatters: FormattersType | None = ...,
3322 float_format: FloatFormatType | None = ...,
3323 sparsify: bool | None = ...,
3324 index_names: bool = ...,
3325 bold_rows: bool = ...,
3326 column_format: str | None = ...,
3327 longtable: bool | None = ...,
3328 escape: bool | None = ...,
3329 encoding: str | None = ...,
3330 decimal: str = ...,
3331 multicolumn: bool | None = ...,
3332 multicolumn_format: str | None = ...,
3333 multirow: bool | None = ...,
3334 caption: str | tuple[str, str] | None = ...,
3335 label: str | None = ...,
3336 position: str | None = ...,
3337 ) -> str: ...
3339 @overload
3340 def to_latex(
3341 self,
3342 buf: FilePath | WriteBuffer[str],
3343 *,
3344 columns: Sequence[Hashable] | None = ...,
3345 header: bool | SequenceNotStr[str] = ...,
3346 index: bool = ...,
3347 na_rep: str = ...,
3348 formatters: FormattersType | None = ...,
3349 float_format: FloatFormatType | None = ...,
3350 sparsify: bool | None = ...,
3351 index_names: bool = ...,
3352 bold_rows: bool = ...,
3353 column_format: str | None = ...,
3354 longtable: bool | None = ...,
3355 escape: bool | None = ...,
3356 encoding: str | None = ...,
3357 decimal: str = ...,
3358 multicolumn: bool | None = ...,
3359 multicolumn_format: str | None = ...,
3360 multirow: bool | None = ...,
3361 caption: str | tuple[str, str] | None = ...,
3362 label: str | None = ...,
3363 position: str | None = ...,
3364 ) -> None: ...
3366 @final
3367 def to_latex(
3368 self,
3369 buf: FilePath | WriteBuffer[str] | None = None,
3370 *,
3371 columns: Sequence[Hashable] | None = None,
3372 header: bool | SequenceNotStr[str] = True,
3373 index: bool = True,
3374 na_rep: str = "NaN",
3375 formatters: FormattersType | None = None,
3376 float_format: FloatFormatType | None = None,
3377 sparsify: bool | None = None,
3378 index_names: bool = True,
3379 bold_rows: bool = False,
3380 column_format: str | None = None,
3381 longtable: bool | None = None,
3382 escape: bool | None = None,
3383 encoding: str | None = None,
3384 decimal: str = ".",
3385 multicolumn: bool | None = None,
3386 multicolumn_format: str | None = None,
3387 multirow: bool | None = None,
3388 caption: str | tuple[str, str] | None = None,
3389 label: str | None = None,
3390 position: str | None = None,
3391 ) -> str | None:
3392 r"""
3393 Render object to a LaTeX tabular, longtable, or nested table.
3395 Requires ``\usepackage{booktabs}``. The output can be copy/pasted
3396 into a main LaTeX document or read from an external file
3397 with ``\input{table.tex}``.
3399 .. versionchanged:: 2.0.0
3400 Refactored to use the Styler implementation via jinja2 templating.
3402 Parameters
3403 ----------
3404 buf : str, Path or StringIO-like, optional, default None
3405 Buffer to write to. If None, the output is returned as a string.
3406 columns : list of label, optional
3407 The subset of columns to write. Writes all columns by default.
3408 header : bool or list of str, default True
3409 Write out the column names. If a list of strings is given,
3410 it is assumed to be aliases for the column names. Braces must be escaped.
3411 index : bool, default True
3412 Write row names (index).
3413 na_rep : str, default 'NaN'
3414 Missing data representation.
3415 formatters : list of functions or dict of {str: function}, optional
3416 Formatter functions to apply to columns' elements by position or
3417 name. The result of each function must be a unicode string.
3418 List must be of length equal to the number of columns.
3419 float_format : one-parameter function or str, optional, default None
3420 Formatter for floating point numbers. For example
3421 ``float_format="%.2f"`` and ``float_format="{:0.2f}".format`` will
3422 both result in 0.1234 being formatted as 0.12.
3423 sparsify : bool, optional
3424 Set to False for a DataFrame with a hierarchical index to print
3425 every multiindex key at each row. By default, the value will be
3426 read from the config module.
3427 index_names : bool, default True
3428 Prints the names of the indexes.
3429 bold_rows : bool, default False
3430 Make the row labels bold in the output.
3431 column_format : str, optional
3432 The columns format as specified in `LaTeX table format
3433 <https://en.wikibooks.org/wiki/LaTeX/Tables>`__ e.g. 'rcl' for 3
3434 columns. By default, 'l' will be used for all columns except
3435 columns of numbers, which default to 'r'.
3436 longtable : bool, optional
3437 Use a longtable environment instead of tabular. Requires
3438 adding a \usepackage{longtable} to your LaTeX preamble.
3439 By default, the value will be read from the pandas config
3440 module, and set to `True` if the option ``styler.latex.environment`` is
3441 `"longtable"`.
3443 .. versionchanged:: 2.0.0
3444 The pandas option affecting this argument has changed.
3445 escape : bool, optional
3446 By default, the value will be read from the pandas config
3447 module and set to `True` if the option ``styler.format.escape`` is
3448 `"latex"`. When set to False prevents from escaping latex special
3449 characters in column names.
3451 .. versionchanged:: 2.0.0
3452 The pandas option affecting this argument has changed, as has the
3453 default value to `False`.
3454 encoding : str, optional
3455 A string representing the encoding to use in the output file,
3456 defaults to 'utf-8'.
3457 decimal : str, default '.'
3458 Character recognized as decimal separator, e.g. ',' in Europe.
3459 multicolumn : bool, default True
3460 Use \multicolumn to enhance MultiIndex columns.
3461 The default will be read from the config module, and is set
3462 as the option ``styler.sparse.columns``.
3464 .. versionchanged:: 2.0.0
3465 The pandas option affecting this argument has changed.
3466 multicolumn_format : str, default 'r'
3467 The alignment for multicolumns, similar to `column_format`
3468 The default will be read from the config module, and is set as the option
3469 ``styler.latex.multicol_align``.
3471 .. versionchanged:: 2.0.0
3472 The pandas option affecting this argument has changed, as has the
3473 default value to "r".
3474 multirow : bool, default True
3475 Use \multirow to enhance MultiIndex rows. Requires adding a
3476 \usepackage{multirow} to your LaTeX preamble. Will print
3477 centered labels (instead of top-aligned) across the contained
3478 rows, separating groups via clines. The default will be read
3479 from the pandas config module, and is set as the option
3480 ``styler.sparse.index``.
3482 .. versionchanged:: 2.0.0
3483 The pandas option affecting this argument has changed, as has the
3484 default value to `True`.
3485 caption : str or tuple, optional
3486 Tuple (full_caption, short_caption),
3487 which results in ``\caption[short_caption]{full_caption}``;
3488 if a single string is passed, no short caption will be set.
3489 label : str, optional
3490 The LaTeX label to be placed inside ``\label{}`` in the output.
3491 This is used with ``\ref{}`` in the main ``.tex`` file.
3493 position : str, optional
3494 The LaTeX positional argument for tables, to be placed after
3495 ``\begin{}`` in the output.
3497 Returns
3498 -------
3499 str or None
3500 If buf is None, returns the result as a string. Otherwise returns None.
3502 See Also
3503 --------
3504 io.formats.style.Styler.to_latex : Render a DataFrame to LaTeX
3505 with conditional formatting.
3506 DataFrame.to_string : Render a DataFrame to a console-friendly
3507 tabular output.
3508 DataFrame.to_html : Render a DataFrame as an HTML table.
3510 Notes
3511 -----
3512 As of v2.0.0 this method has changed to use the Styler implementation as
3513 part of :meth:`.Styler.to_latex` via ``jinja2`` templating. This means
3514 that ``jinja2`` is a requirement, and needs to be installed, for this method
3515 to function. It is advised that users switch to using Styler, since that
3516 implementation is more frequently updated and contains much more
3517 flexibility with the output.
3519 Examples
3520 --------
3521 Convert a general DataFrame to LaTeX with formatting:
3523 >>> df = pd.DataFrame(dict(name=['Raphael', 'Donatello'],
3524 ... age=[26, 45],
3525 ... height=[181.23, 177.65]))
3526 >>> print(df.to_latex(index=False,
3527 ... formatters={"name": str.upper},
3528 ... float_format="{:.1f}".format,
3529 ... )) # doctest: +SKIP
3530 \begin{tabular}{lrr}
3531 \toprule
3532 name & age & height \\
3533 \midrule
3534 RAPHAEL & 26 & 181.2 \\
3535 DONATELLO & 45 & 177.7 \\
3536 \bottomrule
3537 \end{tabular}
3538 """
3539 # Get defaults from the pandas config
3540 if self.ndim == 1:
3541 self = self.to_frame()
3542 if longtable is None:
3543 longtable = config.get_option("styler.latex.environment") == "longtable"
3544 if escape is None:
3545 escape = config.get_option("styler.format.escape") == "latex"
3546 if multicolumn is None:
3547 multicolumn = config.get_option("styler.sparse.columns")
3548 if multicolumn_format is None:
3549 multicolumn_format = config.get_option("styler.latex.multicol_align")
3550 if multirow is None:
3551 multirow = config.get_option("styler.sparse.index")
3553 if column_format is not None and not isinstance(column_format, str):
3554 raise ValueError("`column_format` must be str or unicode")
3555 length = len(self.columns) if columns is None else len(columns)
3556 if isinstance(header, (list, tuple)) and len(header) != length:
3557 raise ValueError(f"Writing {length} cols but got {len(header)} aliases")
3559 # Refactor formatters/float_format/decimal/na_rep/escape to Styler structure
3560 base_format_ = {
3561 "na_rep": na_rep,
3562 "escape": "latex" if escape else None,
3563 "decimal": decimal,
3564 }
3565 index_format_: dict[str, Any] = {"axis": 0, **base_format_}
3566 column_format_: dict[str, Any] = {"axis": 1, **base_format_}
3568 if isinstance(float_format, str):
3569 float_format_: Callable | None = lambda x: float_format % x
3570 else:
3571 float_format_ = float_format
3573 def _wrap(x, alt_format_):
3574 if isinstance(x, (float, complex)) and float_format_ is not None:
3575 return float_format_(x)
3576 else:
3577 return alt_format_(x)
3579 formatters_: list | tuple | dict | Callable | None = None
3580 if isinstance(formatters, list):
3581 formatters_ = {
3582 c: partial(_wrap, alt_format_=formatters[i])
3583 for i, c in enumerate(self.columns)
3584 }
3585 elif isinstance(formatters, dict):
3586 index_formatter = formatters.pop("__index__", None)
3587 column_formatter = formatters.pop("__columns__", None)
3588 if index_formatter is not None:
3589 index_format_.update({"formatter": index_formatter})
3590 if column_formatter is not None:
3591 column_format_.update({"formatter": column_formatter})
3593 formatters_ = formatters
3594 float_columns = self.select_dtypes(include="float").columns
3595 for col in float_columns:
3596 if col not in formatters.keys():
3597 formatters_.update({col: float_format_})
3598 elif formatters is None and float_format is not None:
3599 formatters_ = partial(_wrap, alt_format_=lambda v: v)
3600 format_index_ = [index_format_, column_format_]
3601 format_index_names_ = [index_format_, column_format_]
3603 # Deal with hiding indexes and relabelling column names
3604 hide_: list[dict] = []
3605 relabel_index_: list[dict] = []
3606 if columns:
3607 hide_.append(
3608 {
3609 "subset": [c for c in self.columns if c not in columns],
3610 "axis": "columns",
3611 }
3612 )
3613 if header is False:
3614 hide_.append({"axis": "columns"})
3615 elif isinstance(header, (list, tuple)):
3616 relabel_index_.append({"labels": header, "axis": "columns"})
3617 format_index_ = [index_format_] # column_format is overwritten
3619 if index is False:
3620 hide_.append({"axis": "index"})
3621 if index_names is False:
3622 hide_.append({"names": True, "axis": "index"})
3624 render_kwargs_ = {
3625 "hrules": True,
3626 "sparse_index": sparsify,
3627 "sparse_columns": sparsify,
3628 "environment": "longtable" if longtable else None,
3629 "multicol_align": multicolumn_format
3630 if multicolumn
3631 else f"naive-{multicolumn_format}",
3632 "multirow_align": "t" if multirow else "naive",
3633 "encoding": encoding,
3634 "caption": caption,
3635 "label": label,
3636 "position": position,
3637 "column_format": column_format,
3638 "clines": "skip-last;data"
3639 if (multirow and isinstance(self.index, MultiIndex))
3640 else None,
3641 "bold_rows": bold_rows,
3642 }
3644 return self._to_latex_via_styler(
3645 buf,
3646 hide=hide_,
3647 relabel_index=relabel_index_,
3648 format={"formatter": formatters_, **base_format_},
3649 format_index=format_index_,
3650 format_index_names=format_index_names_,
3651 render_kwargs=render_kwargs_,
3652 )
3654 @final
3655 def _to_latex_via_styler(
3656 self,
3657 buf=None,
3658 *,
3659 hide: dict | list[dict] | None = None,
3660 relabel_index: dict | list[dict] | None = None,
3661 format: dict | list[dict] | None = None,
3662 format_index: dict | list[dict] | None = None,
3663 format_index_names: dict | list[dict] | None = None,
3664 render_kwargs: dict | None = None,
3665 ):
3666 """
3667 Render object to a LaTeX tabular, longtable, or nested table.
3669 Uses the ``Styler`` implementation with the following, ordered, method chaining:
3671 .. code-block:: python
3672 styler = Styler(DataFrame)
3673 styler.hide(**hide)
3674 styler.relabel_index(**relabel_index)
3675 styler.format(**format)
3676 styler.format_index(**format_index)
3677 styler.to_latex(buf=buf, **render_kwargs)
3679 Parameters
3680 ----------
3681 buf : str, Path or StringIO-like, optional, default None
3682 Buffer to write to. If None, the output is returned as a string.
3683 hide : dict, list of dict
3684 Keyword args to pass to the method call of ``Styler.hide``. If a list will
3685 call the method numerous times.
3686 relabel_index : dict, list of dict
3687 Keyword args to pass to the method of ``Styler.relabel_index``. If a list
3688 will call the method numerous times.
3689 format : dict, list of dict
3690 Keyword args to pass to the method call of ``Styler.format``. If a list will
3691 call the method numerous times.
3692 format_index : dict, list of dict
3693 Keyword args to pass to the method call of ``Styler.format_index``. If a
3694 list will call the method numerous times.
3695 render_kwargs : dict
3696 Keyword args to pass to the method call of ``Styler.to_latex``.
3698 Returns
3699 -------
3700 str or None
3701 If buf is None, returns the result as a string. Otherwise returns None.
3702 """
3703 from pandas.io.formats.style import Styler
3705 self = cast("DataFrame", self)
3706 styler = Styler(self, uuid="")
3708 for kw_name in [
3709 "hide",
3710 "relabel_index",
3711 "format",
3712 "format_index",
3713 "format_index_names",
3714 ]:
3715 kw = vars()[kw_name]
3716 if isinstance(kw, dict):
3717 getattr(styler, kw_name)(**kw)
3718 elif isinstance(kw, list):
3719 for sub_kw in kw:
3720 getattr(styler, kw_name)(**sub_kw)
3722 # bold_rows is not a direct kwarg of Styler.to_latex
3723 render_kwargs = {} if render_kwargs is None else render_kwargs
3724 if render_kwargs.pop("bold_rows"):
3725 styler.map_index(lambda v: "textbf:--rwrap;")
3727 return styler.to_latex(buf=buf, **render_kwargs)
3729 @overload
3730 def to_csv(
3731 self,
3732 path_or_buf: None = ...,
3733 *,
3734 sep: str = ...,
3735 na_rep: str = ...,
3736 float_format: str | Callable | None = ...,
3737 columns: Sequence[Hashable] | None = ...,
3738 header: bool | list[str] = ...,
3739 index: bool = ...,
3740 index_label: IndexLabel | None = ...,
3741 mode: str = ...,
3742 encoding: str | None = ...,
3743 compression: CompressionOptions = ...,
3744 quoting: int | None = ...,
3745 quotechar: str = ...,
3746 lineterminator: str | None = ...,
3747 chunksize: int | None = ...,
3748 date_format: str | None = ...,
3749 doublequote: bool = ...,
3750 escapechar: str | None = ...,
3751 decimal: str = ...,
3752 errors: OpenFileErrors = ...,
3753 storage_options: StorageOptions = ...,
3754 ) -> str: ...
3756 @overload
3757 def to_csv(
3758 self,
3759 path_or_buf: FilePath | WriteBuffer[bytes] | WriteBuffer[str],
3760 *,
3761 sep: str = ...,
3762 na_rep: str = ...,
3763 float_format: str | Callable | None = ...,
3764 columns: Sequence[Hashable] | None = ...,
3765 header: bool | list[str] = ...,
3766 index: bool = ...,
3767 index_label: IndexLabel | None = ...,
3768 mode: str = ...,
3769 encoding: str | None = ...,
3770 compression: CompressionOptions = ...,
3771 quoting: int | None = ...,
3772 quotechar: str = ...,
3773 lineterminator: str | None = ...,
3774 chunksize: int | None = ...,
3775 date_format: str | None = ...,
3776 doublequote: bool = ...,
3777 escapechar: str | None = ...,
3778 decimal: str = ...,
3779 errors: OpenFileErrors = ...,
3780 storage_options: StorageOptions = ...,
3781 ) -> None: ...
3783 @final
3784 def to_csv(
3785 self,
3786 path_or_buf: FilePath | WriteBuffer[bytes] | WriteBuffer[str] | None = None,
3787 *,
3788 sep: str = ",",
3789 na_rep: str = "",
3790 float_format: str | Callable | None = None,
3791 columns: Sequence[Hashable] | None = None,
3792 header: bool | list[str] = True,
3793 index: bool = True,
3794 index_label: IndexLabel | None = None,
3795 mode: str = "w",
3796 encoding: str | None = None,
3797 compression: CompressionOptions = "infer",
3798 quoting: int | None = None,
3799 quotechar: str = '"',
3800 lineterminator: str | None = None,
3801 chunksize: int | None = None,
3802 date_format: str | None = None,
3803 doublequote: bool = True,
3804 escapechar: str | None = None,
3805 decimal: str = ".",
3806 errors: OpenFileErrors = "strict",
3807 storage_options: StorageOptions | None = None,
3808 ) -> str | None:
3809 r"""
3810 Write object to a comma-separated values (csv) file.
3812 Parameters
3813 ----------
3814 path_or_buf : str, path object, file-like object, or None, default None
3815 String, path object (implementing os.PathLike[str]), or file-like
3816 object implementing a write() function. If None, the result is
3817 returned as a string. If a non-binary file object is passed, it should
3818 be opened with `newline=''`, disabling universal newlines. If a binary
3819 file object is passed, `mode` might need to contain a `'b'`.
3820 sep : str, default ','
3821 String of length 1. Field delimiter for the output file.
3822 na_rep : str, default ''
3823 Missing data representation.
3824 float_format : str, Callable, default None
3825 Format string for floating point numbers. If a Callable is given, it takes
3826 precedence over other numeric formatting parameters, like decimal.
3827 columns : sequence, optional
3828 Columns to write.
3829 header : bool or list of str, default True
3830 Write out the column names. If a list of strings is given it is
3831 assumed to be aliases for the column names.
3832 index : bool, default True
3833 Write row names (index).
3834 index_label : str or sequence, or False, default None
3835 Column label for index column(s) if desired. If None is given, and
3836 `header` and `index` are True, then the index names are used. A
3837 sequence should be given if the object uses MultiIndex. If
3838 False do not print fields for index names. Use index_label=False
3839 for easier importing in R.
3840 mode : {'w', 'x', 'a'}, default 'w'
3841 Forwarded to either `open(mode=)` or `fsspec.open(mode=)` to control
3842 the file opening. Typical values include:
3844 - 'w', truncate the file first.
3845 - 'x', exclusive creation, failing if the file already exists.
3846 - 'a', append to the end of file if it exists.
3848 encoding : str, optional
3849 A string representing the encoding to use in the output file,
3850 defaults to 'utf-8'. `encoding` is not supported if `path_or_buf`
3851 is a non-binary file object.
3853 compression : str or dict, default 'infer'
3854 For on-the-fly compression of the output data. If 'infer' and
3855 'path_or_buf' is path-like, then detect compression from the following
3856 extensions: '.gz',
3857 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
3858 (otherwise no compression).
3859 Set to ``None`` for no compression.
3860 Can also be a dict with key ``'method'`` set to one of
3861 {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``} and
3862 other key-value pairs are forwarded to
3863 ``zipfile.ZipFile``, ``gzip.GzipFile``,
3864 ``bz2.BZ2File``, ``zstandard.ZstdCompressor``, ``lzma.LZMAFile`` or
3865 ``tarfile.TarFile``, respectively.
3866 As an example, the following could be passed for faster compression and
3867 to create a reproducible gzip archive:
3868 ``compression={'method': 'gzip', 'compresslevel': 1, 'mtime': 1}``.
3870 May be a dict with key 'method' as compression mode
3871 and other entries as additional compression options if
3872 compression mode is 'zip'.
3874 Passing compression options as keys in dict is
3875 supported for compression modes 'gzip', 'bz2', 'zstd', and 'zip'.
3876 quoting : optional constant from csv module
3877 Defaults to csv.QUOTE_MINIMAL. If you have set a `float_format`
3878 then floats are converted to strings and thus csv.QUOTE_NONNUMERIC
3879 will treat them as non-numeric.
3880 quotechar : str, default '\"'
3881 String of length 1. Character used to quote fields.
3882 lineterminator : str, optional
3883 The newline character or character sequence to use in the output
3884 file. Defaults to `os.linesep`, which depends on the OS in which
3885 this method is called ('\\n' for linux, '\\r\\n' for Windows, i.e.).
3886 chunksize : int or None
3887 Rows to write at a time.
3888 date_format : str, default None
3889 Format string for datetime objects.
3890 doublequote : bool, default True
3891 Control quoting of `quotechar` inside a field.
3892 escapechar : str, default None
3893 String of length 1. Character used to escape `sep` and `quotechar`
3894 when appropriate.
3895 decimal : str, default '.'
3896 Character recognized as decimal separator. E.g. use ',' for
3897 European data.
3898 errors : str, default 'strict'
3899 Specifies how encoding and decoding errors are to be handled.
3900 See the errors argument for :func:`open` for a full list
3901 of options.
3903 storage_options : dict, optional
3904 Extra options that make sense for a particular storage connection, e.g.
3905 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
3906 are forwarded to ``urllib.request.Request`` as header options. For other
3907 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
3908 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
3909 details, and for more examples on storage options refer `here
3910 <https://pandas.pydata.org/docs/user_guide/io.html?
3911 highlight=storage_options#reading-writing-remote-files>`_.
3913 Returns
3914 -------
3915 None or str
3916 If path_or_buf is None, returns the resulting csv format as a
3917 string. Otherwise returns None.
3919 See Also
3920 --------
3921 read_csv : Load a CSV file into a DataFrame.
3922 to_excel : Write DataFrame to an Excel file.
3924 Examples
3925 --------
3926 Create 'out.csv' containing 'df' without indices
3928 >>> df = pd.DataFrame(
3929 ... [["Raphael", "red", "sai"], ["Donatello", "purple", "bo staff"]],
3930 ... columns=["name", "mask", "weapon"],
3931 ... )
3932 >>> df.to_csv("out.csv", index=False) # doctest: +SKIP
3934 Create 'out.zip' containing 'out.csv'
3936 >>> df.to_csv(index=False)
3937 'name,mask,weapon\nRaphael,red,sai\nDonatello,purple,bo staff\n'
3938 >>> compression_opts = dict(
3939 ... method="zip", archive_name="out.csv"
3940 ... ) # doctest: +SKIP
3941 >>> df.to_csv(
3942 ... "out.zip", index=False, compression=compression_opts
3943 ... ) # doctest: +SKIP
3945 To write a csv file to a new folder or nested folder you will first
3946 need to create it using either Pathlib or os:
3948 >>> from pathlib import Path # doctest: +SKIP
3949 >>> filepath = Path("folder/subfolder/out.csv") # doctest: +SKIP
3950 >>> filepath.parent.mkdir(parents=True, exist_ok=True) # doctest: +SKIP
3951 >>> df.to_csv(filepath) # doctest: +SKIP
3953 >>> import os # doctest: +SKIP
3954 >>> os.makedirs("folder/subfolder", exist_ok=True) # doctest: +SKIP
3955 >>> df.to_csv("folder/subfolder/out.csv") # doctest: +SKIP
3957 Format floats to two decimal places:
3959 >>> df.to_csv("out1.csv", float_format="%.2f") # doctest: +SKIP
3961 Format floats using scientific notation:
3963 >>> df.to_csv("out2.csv", float_format="{:.2e}".format) # doctest: +SKIP
3964 """
3965 df = self if isinstance(self, ABCDataFrame) else self.to_frame()
3967 formatter = DataFrameFormatter(
3968 frame=df,
3969 header=header,
3970 index=index,
3971 na_rep=na_rep,
3972 float_format=float_format,
3973 decimal=decimal,
3974 )
3976 return DataFrameRenderer(formatter).to_csv(
3977 path_or_buf,
3978 lineterminator=lineterminator,
3979 sep=sep,
3980 encoding=encoding,
3981 errors=errors,
3982 compression=compression,
3983 quoting=quoting,
3984 columns=columns,
3985 index_label=index_label,
3986 mode=mode,
3987 chunksize=chunksize,
3988 quotechar=quotechar,
3989 date_format=date_format,
3990 doublequote=doublequote,
3991 escapechar=escapechar,
3992 storage_options=storage_options,
3993 )
3995 # ----------------------------------------------------------------------
3996 # Indexing Methods
3998 @final
3999 def take(self, indices, axis: Axis = 0, **kwargs) -> Self:
4000 """
4001 Return the elements in the given *positional* indices along an axis.
4003 This means that we are not indexing according to actual values in
4004 the index attribute of the object. We are indexing according to the
4005 actual position of the element in the object.
4007 Parameters
4008 ----------
4009 indices : array-like
4010 An array of ints indicating which positions to take.
4011 axis : {0 or 'index', 1 or 'columns'}, default 0
4012 The axis on which to select elements. ``0`` means that we are
4013 selecting rows, ``1`` means that we are selecting columns.
4014 For `Series` this parameter is unused and defaults to 0.
4015 **kwargs
4016 For compatibility with :meth:`numpy.take`. Has no effect on the
4017 output.
4019 Returns
4020 -------
4021 same type as caller
4022 An array-like containing the elements taken from the object.
4024 See Also
4025 --------
4026 DataFrame.loc : Select a subset of a DataFrame by labels.
4027 DataFrame.iloc : Select a subset of a DataFrame by positions.
4028 numpy.take : Take elements from an array along an axis.
4030 Examples
4031 --------
4032 >>> df = pd.DataFrame(
4033 ... [
4034 ... ("falcon", "bird", 389.0),
4035 ... ("parrot", "bird", 24.0),
4036 ... ("lion", "mammal", 80.5),
4037 ... ("monkey", "mammal", np.nan),
4038 ... ],
4039 ... columns=["name", "class", "max_speed"],
4040 ... index=[0, 2, 3, 1],
4041 ... )
4042 >>> df
4043 name class max_speed
4044 0 falcon bird 389.0
4045 2 parrot bird 24.0
4046 3 lion mammal 80.5
4047 1 monkey mammal NaN
4049 Take elements at positions 0 and 3 along the axis 0 (default).
4051 Note how the actual indices selected (0 and 1) do not correspond to
4052 our selected indices 0 and 3. That's because we are selecting the 0th
4053 and 3rd rows, not rows whose indices equal 0 and 3.
4055 >>> df.take([0, 3])
4056 name class max_speed
4057 0 falcon bird 389.0
4058 1 monkey mammal NaN
4060 Take elements at indices 1 and 2 along the axis 1 (column selection).
4062 >>> df.take([1, 2], axis=1)
4063 class max_speed
4064 0 bird 389.0
4065 2 bird 24.0
4066 3 mammal 80.5
4067 1 mammal NaN
4069 We may take elements using negative integers for positive indices,
4070 starting from the end of the object, just like with Python lists.
4072 >>> df.take([-1, -2])
4073 name class max_speed
4074 1 monkey mammal NaN
4075 3 lion mammal 80.5
4076 """
4078 nv.validate_take((), kwargs)
4080 if isinstance(indices, slice):
4081 raise TypeError(
4082 f"{type(self).__name__}.take requires a sequence of integers, "
4083 "not slice."
4084 )
4085 indices = np.asarray(indices, dtype=np.intp)
4086 if axis == 0 and indices.ndim == 1 and is_range_indexer(indices, len(self)):
4087 return self.copy(deep=False)
4089 new_data = self._mgr.take(
4090 indices,
4091 axis=self._get_block_manager_axis(axis),
4092 verify=True,
4093 )
4094 return self._constructor_from_mgr(new_data, axes=new_data.axes).__finalize__(
4095 self, method="take"
4096 )
4098 @final
4099 def xs(
4100 self,
4101 key: IndexLabel,
4102 axis: Axis = 0,
4103 level: IndexLabel | None = None,
4104 drop_level: bool = True,
4105 ) -> Self:
4106 """
4107 Return cross-section from the Series/DataFrame.
4109 This method takes a `key` argument to select data at a particular
4110 level of a MultiIndex.
4112 Parameters
4113 ----------
4114 key : label or tuple of label
4115 Label contained in the index, or partially in a MultiIndex.
4116 axis : {0 or 'index', 1 or 'columns'}, default 0
4117 Axis to retrieve cross-section on.
4118 level : object, defaults to first n levels (n=1 or len(key))
4119 In case of a key partially contained in a MultiIndex, indicate
4120 which levels are used. Levels can be referred by label or position.
4121 drop_level : bool, default True
4122 If False, returns object with same levels as self.
4124 Returns
4125 -------
4126 Series or DataFrame
4127 Cross-section from the original Series or DataFrame
4128 corresponding to the selected index levels.
4130 See Also
4131 --------
4132 DataFrame.loc : Access a group of rows and columns
4133 by label(s) or a boolean array.
4134 DataFrame.iloc : Purely integer-location based indexing
4135 for selection by position.
4137 Notes
4138 -----
4139 `xs` can not be used to set values.
4141 MultiIndex Slicers is a generic way to get/set values on
4142 any level or levels.
4143 It is a superset of `xs` functionality, see
4144 :ref:`MultiIndex Slicers <advanced.mi_slicers>`.
4146 Examples
4147 --------
4148 >>> d = {
4149 ... "num_legs": [4, 4, 2, 2],
4150 ... "num_wings": [0, 0, 2, 2],
4151 ... "class": ["mammal", "mammal", "mammal", "bird"],
4152 ... "animal": ["cat", "dog", "bat", "penguin"],
4153 ... "locomotion": ["walks", "walks", "flies", "walks"],
4154 ... }
4155 >>> df = pd.DataFrame(data=d)
4156 >>> df = df.set_index(["class", "animal", "locomotion"])
4157 >>> df
4158 num_legs num_wings
4159 class animal locomotion
4160 mammal cat walks 4 0
4161 dog walks 4 0
4162 bat flies 2 2
4163 bird penguin walks 2 2
4165 Get values at specified index
4167 >>> df.xs("mammal")
4168 num_legs num_wings
4169 animal locomotion
4170 cat walks 4 0
4171 dog walks 4 0
4172 bat flies 2 2
4174 Get values at several indexes
4176 >>> df.xs(("mammal", "dog", "walks"))
4177 num_legs 4
4178 num_wings 0
4179 Name: (mammal, dog, walks), dtype: int64
4181 Get values at specified index and level
4183 >>> df.xs("cat", level=1)
4184 num_legs num_wings
4185 class locomotion
4186 mammal walks 4 0
4188 Get values at several indexes and levels
4190 >>> df.xs(("bird", "walks"), level=[0, "locomotion"])
4191 num_legs num_wings
4192 animal
4193 penguin 2 2
4195 Get values at specified column and axis
4197 >>> df.xs("num_wings", axis=1)
4198 class animal locomotion
4199 mammal cat walks 0
4200 dog walks 0
4201 bat flies 2
4202 bird penguin walks 2
4203 Name: num_wings, dtype: int64
4204 """
4205 axis = self._get_axis_number(axis)
4206 labels = self._get_axis(axis)
4208 if isinstance(key, list):
4209 raise TypeError("list keys are not supported in xs, pass a tuple instead")
4211 if level is not None:
4212 if not isinstance(labels, MultiIndex):
4213 raise TypeError("Index must be a MultiIndex")
4214 loc, new_ax = labels.get_loc_level(key, level=level, drop_level=drop_level)
4216 # create the tuple of the indexer
4217 _indexer = [slice(None)] * self.ndim
4218 _indexer[axis] = loc
4219 indexer = tuple(_indexer)
4221 result = self.iloc[indexer]
4222 setattr(result, result._get_axis_name(axis), new_ax)
4223 return result
4225 if axis == 1:
4226 if drop_level:
4227 return self[key]
4228 index = self.columns
4229 else:
4230 index = self.index
4232 if isinstance(index, MultiIndex):
4233 loc, new_index = index._get_loc_level(key, level=0)
4234 if not drop_level:
4235 if lib.is_integer(loc):
4236 # Slice index must be an integer or None
4237 new_index = index[loc : loc + 1]
4238 else:
4239 new_index = index[loc]
4240 else:
4241 loc = index.get_loc(key)
4243 if isinstance(loc, np.ndarray):
4244 if loc.dtype == np.bool_:
4245 (inds,) = loc.nonzero()
4246 return self.take(inds, axis=axis)
4247 else:
4248 return self.take(loc, axis=axis)
4250 if not is_scalar(loc):
4251 new_index = index[loc]
4253 if is_scalar(loc) and axis == 0:
4254 # In this case loc should be an integer
4255 if self.ndim == 1:
4256 # if we encounter an array-like and we only have 1 dim
4257 # that means that their are list/ndarrays inside the Series!
4258 # so just return them (GH 6394)
4259 return self._values[loc]
4261 new_mgr = self._mgr.fast_xs(loc)
4263 result = self._constructor_sliced_from_mgr(new_mgr, axes=new_mgr.axes)
4264 result._name = self.index[loc]
4265 result = result.__finalize__(self)
4266 elif is_scalar(loc):
4267 result = self.iloc[:, slice(loc, loc + 1)]
4268 elif axis == 1:
4269 result = self.iloc[:, loc]
4270 else:
4271 result = self.iloc[loc]
4272 result.index = new_index
4274 return result
4276 def __getitem__(self, item):
4277 raise AbstractMethodError(self)
4279 @final
4280 def _getitem_slice(self, key: slice) -> Self:
4281 """
4282 __getitem__ for the case where the key is a slice object.
4283 """
4284 # _convert_slice_indexer to determine if this slice is positional
4285 # or label based, and if the latter, convert to positional
4286 slobj = self.index._convert_slice_indexer(key, kind="getitem")
4287 if isinstance(slobj, np.ndarray):
4288 # reachable with DatetimeIndex
4289 indexer = lib.maybe_indices_to_slice(slobj.astype(np.intp), len(self))
4290 if isinstance(indexer, np.ndarray):
4291 # GH#43223 If we can not convert, use take
4292 return self.take(indexer, axis=0)
4293 slobj = indexer
4294 return self._slice(slobj)
4296 def _slice(self, slobj: slice, axis: AxisInt = 0) -> Self:
4297 """
4298 Construct a slice of this container.
4300 Slicing with this method is *always* positional.
4301 """
4302 assert isinstance(slobj, slice), type(slobj)
4303 axis = self._get_block_manager_axis(axis)
4304 new_mgr = self._mgr.get_slice(slobj, axis=axis)
4305 result = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
4306 result = result.__finalize__(self)
4307 return result
4309 @final
4310 def __delitem__(self, key) -> None:
4311 """
4312 Delete item
4313 """
4314 deleted = False
4316 maybe_shortcut = False
4317 if self.ndim == 2 and isinstance(self.columns, MultiIndex):
4318 try:
4319 # By using engine's __contains__ we effectively
4320 # restrict to same-length tuples
4321 maybe_shortcut = key not in self.columns._engine
4322 except TypeError:
4323 pass
4325 if maybe_shortcut:
4326 # Allow shorthand to delete all columns whose first len(key)
4327 # elements match key:
4328 if not isinstance(key, tuple):
4329 key = (key,)
4330 for col in self.columns:
4331 if isinstance(col, tuple) and col[: len(key)] == key:
4332 del self[col]
4333 deleted = True
4334 if not deleted:
4335 # If the above loop ran and didn't delete anything because
4336 # there was no match, this call should raise the appropriate
4337 # exception:
4338 loc = self.axes[-1].get_loc(key)
4339 self._mgr = self._mgr.idelete(loc)
4341 # ----------------------------------------------------------------------
4342 # Unsorted
4344 @final
4345 def _check_inplace_and_allows_duplicate_labels(self, inplace: bool) -> None:
4346 if inplace and not self.flags.allows_duplicate_labels:
4347 raise ValueError(
4348 "Cannot specify 'inplace=True' when "
4349 "'self.flags.allows_duplicate_labels' is False."
4350 )
4352 @final
4353 def get(self, key, default=None):
4354 """
4355 Get item from object for given key (ex: DataFrame column).
4357 Returns ``default`` value if not found.
4359 Parameters
4360 ----------
4361 key : object
4362 Key for which item should be returned.
4363 default : object, default None
4364 Default value to return if key is not found.
4366 Returns
4367 -------
4368 same type as items contained in object
4369 Item for given key or ``default`` value, if key is not found.
4371 See Also
4372 --------
4373 DataFrame.get : Get item from object for given key (ex: DataFrame column).
4374 Series.get : Get item from object for given key (ex: DataFrame column).
4376 Examples
4377 --------
4378 >>> df = pd.DataFrame(
4379 ... [
4380 ... [24.3, 75.7, "high"],
4381 ... [31, 87.8, "high"],
4382 ... [22, 71.6, "medium"],
4383 ... [35, 95, "medium"],
4384 ... ],
4385 ... columns=["temp_celsius", "temp_fahrenheit", "windspeed"],
4386 ... index=pd.date_range(start="2014-02-12", end="2014-02-15", freq="D"),
4387 ... )
4389 >>> df
4390 temp_celsius temp_fahrenheit windspeed
4391 2014-02-12 24.3 75.7 high
4392 2014-02-13 31.0 87.8 high
4393 2014-02-14 22.0 71.6 medium
4394 2014-02-15 35.0 95.0 medium
4396 >>> df.get(["temp_celsius", "windspeed"])
4397 temp_celsius windspeed
4398 2014-02-12 24.3 high
4399 2014-02-13 31.0 high
4400 2014-02-14 22.0 medium
4401 2014-02-15 35.0 medium
4403 >>> ser = df["windspeed"]
4404 >>> ser.get("2014-02-13")
4405 'high'
4407 If the key isn't found, the default value will be used.
4409 >>> df.get(["temp_celsius", "temp_kelvin"], default="default_value")
4410 'default_value'
4412 >>> ser.get("2014-02-10", "[unknown]")
4413 '[unknown]'
4414 """
4415 try:
4416 return self[key]
4417 except (KeyError, ValueError, IndexError):
4418 return default
4420 @staticmethod
4421 def _check_copy_deprecation(copy):
4422 if copy is not lib.no_default:
4423 warnings.warn(
4424 "The copy keyword is deprecated and will be removed in a future "
4425 "version. Copy-on-Write is active in pandas since 3.0 which utilizes "
4426 "a lazy copy mechanism that defers copies until necessary. Use "
4427 ".copy() to make an eager copy if necessary.",
4428 Pandas4Warning,
4429 stacklevel=find_stack_level(),
4430 )
4432 # issue 58667
4433 @deprecate_kwarg(Pandas4Warning, "method", new_arg_name=None)
4434 @final
4435 def reindex_like(
4436 self,
4437 other,
4438 method: Literal["backfill", "bfill", "pad", "ffill", "nearest"] | None = None,
4439 copy: bool | lib.NoDefault = lib.no_default,
4440 limit: int | None = None,
4441 tolerance=None,
4442 ) -> Self:
4443 """
4444 Return an object with matching indices as other object.
4446 Conform the object to the same index on all axes. Optional
4447 filling logic, placing NaN in locations having no value
4448 in the previous index. A new object is produced unless the
4449 new index is equivalent to the current one and copy=False.
4451 Parameters
4452 ----------
4453 other : Object of the same data type
4454 Its row and column indices are used to define the new indices
4455 of this object.
4456 method : {None, 'backfill'/'bfill', 'pad'/'ffill', 'nearest'}
4457 Method to use for filling holes in reindexed DataFrame.
4458 Please note: this is only applicable to DataFrames/Series with a
4459 monotonically increasing/decreasing index.
4461 .. deprecated:: 3.0.0
4463 * None (default): don't fill gaps
4464 * pad / ffill: propagate last valid observation forward to next
4465 valid
4466 * backfill / bfill: use next valid observation to fill gap
4467 * nearest: use nearest valid observations to fill gap.
4469 copy : bool, default False
4470 This keyword is now ignored; changing its value will have no
4471 impact on the method.
4473 .. deprecated:: 3.0.0
4475 This keyword is ignored and will be removed in pandas 4.0. Since
4476 pandas 3.0, this method always returns a new object using a lazy
4477 copy mechanism that defers copies until necessary
4478 (Copy-on-Write). See the `user guide on Copy-on-Write
4479 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
4480 for more details.
4482 limit : int, default None
4483 Maximum number of consecutive labels to fill for inexact matches.
4484 tolerance : optional
4485 Maximum distance between original and new labels for inexact
4486 matches. The values of the index at the matching locations must
4487 satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
4489 Tolerance may be a scalar value, which applies the same tolerance
4490 to all values, or list-like, which applies variable tolerance per
4491 element. List-like includes list, tuple, array, Series, and must be
4492 the same size as the index and its dtype must exactly match the
4493 index's type.
4495 Returns
4496 -------
4497 Series or DataFrame
4498 Same type as caller, but with changed indices on each axis.
4500 See Also
4501 --------
4502 DataFrame.set_index : Set row labels.
4503 DataFrame.reset_index : Remove row labels or move them to new columns.
4504 DataFrame.reindex : Change to new indices or expand indices.
4506 Notes
4507 -----
4508 Same as calling
4509 ``.reindex(index=other.index, columns=other.columns,...)``.
4511 Examples
4512 --------
4513 >>> df1 = pd.DataFrame(
4514 ... [
4515 ... [24.3, 75.7, "high"],
4516 ... [31, 87.8, "high"],
4517 ... [22, 71.6, "medium"],
4518 ... [35, 95, "medium"],
4519 ... ],
4520 ... columns=["temp_celsius", "temp_fahrenheit", "windspeed"],
4521 ... index=pd.date_range(start="2014-02-12", end="2014-02-15", freq="D"),
4522 ... )
4524 >>> df1
4525 temp_celsius temp_fahrenheit windspeed
4526 2014-02-12 24.3 75.7 high
4527 2014-02-13 31.0 87.8 high
4528 2014-02-14 22.0 71.6 medium
4529 2014-02-15 35.0 95.0 medium
4531 >>> df2 = pd.DataFrame(
4532 ... [[28, "low"], [30, "low"], [35.1, "medium"]],
4533 ... columns=["temp_celsius", "windspeed"],
4534 ... index=pd.DatetimeIndex(["2014-02-12", "2014-02-13", "2014-02-15"]),
4535 ... )
4537 >>> df2
4538 temp_celsius windspeed
4539 2014-02-12 28.0 low
4540 2014-02-13 30.0 low
4541 2014-02-15 35.1 medium
4543 >>> df2.reindex_like(df1)
4544 temp_celsius temp_fahrenheit windspeed
4545 2014-02-12 28.0 NaN low
4546 2014-02-13 30.0 NaN low
4547 2014-02-14 NaN NaN NaN
4548 2014-02-15 35.1 NaN medium
4549 """
4550 self._check_copy_deprecation(copy)
4551 d = other._construct_axes_dict(
4552 axes=self._AXIS_ORDERS,
4553 method=method,
4554 limit=limit,
4555 tolerance=tolerance,
4556 )
4558 return self.reindex(**d)
4560 @overload
4561 def drop(
4562 self,
4563 labels: IndexLabel | ListLike = ...,
4564 *,
4565 axis: Axis = ...,
4566 index: IndexLabel | ListLike = ...,
4567 columns: IndexLabel | ListLike = ...,
4568 level: Level | None = ...,
4569 inplace: Literal[True],
4570 errors: IgnoreRaise = ...,
4571 ) -> None: ...
4573 @overload
4574 def drop(
4575 self,
4576 labels: IndexLabel | ListLike = ...,
4577 *,
4578 axis: Axis = ...,
4579 index: IndexLabel | ListLike = ...,
4580 columns: IndexLabel | ListLike = ...,
4581 level: Level | None = ...,
4582 inplace: Literal[False] = ...,
4583 errors: IgnoreRaise = ...,
4584 ) -> Self: ...
4586 @overload
4587 def drop(
4588 self,
4589 labels: IndexLabel | ListLike = ...,
4590 *,
4591 axis: Axis = ...,
4592 index: IndexLabel | ListLike = ...,
4593 columns: IndexLabel | ListLike = ...,
4594 level: Level | None = ...,
4595 inplace: bool = ...,
4596 errors: IgnoreRaise = ...,
4597 ) -> Self | None: ...
4599 def drop(
4600 self,
4601 labels: IndexLabel | ListLike = None,
4602 *,
4603 axis: Axis = 0,
4604 index: IndexLabel | ListLike = None,
4605 columns: IndexLabel | ListLike = None,
4606 level: Level | None = None,
4607 inplace: bool = False,
4608 errors: IgnoreRaise = "raise",
4609 ) -> Self | None:
4610 inplace = validate_bool_kwarg(inplace, "inplace")
4612 if labels is not None:
4613 if index is not None or columns is not None:
4614 raise ValueError("Cannot specify both 'labels' and 'index'/'columns'")
4615 axis_name = self._get_axis_name(axis)
4616 axes = {axis_name: labels}
4617 elif index is not None or columns is not None:
4618 if axis == 1:
4619 raise ValueError("Cannot specify both 'axis' and 'index'/'columns'")
4620 axes = {"index": index}
4621 if self.ndim == 2:
4622 axes["columns"] = columns
4623 else:
4624 raise ValueError(
4625 "Need to specify at least one of 'labels', 'index' or 'columns'"
4626 )
4628 obj = self
4630 for axis, labels in axes.items():
4631 if labels is not None:
4632 obj = obj._drop_axis(labels, axis, level=level, errors=errors)
4634 if inplace:
4635 self._update_inplace(obj)
4636 return None
4637 else:
4638 return obj
4640 @final
4641 def _drop_axis(
4642 self,
4643 labels,
4644 axis,
4645 level=None,
4646 errors: IgnoreRaise = "raise",
4647 only_slice: bool = False,
4648 ) -> Self:
4649 """
4650 Drop labels from specified axis. Used in the ``drop`` method
4651 internally.
4653 Parameters
4654 ----------
4655 labels : single label or list-like
4656 axis : int or axis name
4657 level : int or level name, default None
4658 For MultiIndex
4659 errors : {'ignore', 'raise'}, default 'raise'
4660 If 'ignore', suppress error and existing labels are dropped.
4661 only_slice : bool, default False
4662 Whether indexing along columns should be view-only.
4664 """
4665 axis_num = self._get_axis_number(axis)
4666 axis = self._get_axis(axis)
4668 if axis.is_unique:
4669 if level is not None:
4670 if not isinstance(axis, MultiIndex):
4671 raise AssertionError("axis must be a MultiIndex")
4672 new_axis = axis.drop(labels, level=level, errors=errors)
4673 else:
4674 new_axis = axis.drop(labels, errors=errors)
4675 indexer = axis.get_indexer(new_axis)
4677 # Case for non-unique axis
4678 else:
4679 is_tuple_labels = is_nested_list_like(labels) or isinstance(labels, tuple)
4680 labels = ensure_object(common.index_labels_to_array(labels))
4681 if level is not None:
4682 if not isinstance(axis, MultiIndex):
4683 raise AssertionError("axis must be a MultiIndex")
4684 mask = ~axis.get_level_values(level).isin(labels)
4686 # GH 18561 MultiIndex.drop should raise if label is absent
4687 if errors == "raise" and mask.all():
4688 raise KeyError(f"{labels} not found in axis")
4689 elif (
4690 isinstance(axis, MultiIndex)
4691 and labels.dtype == "object"
4692 and not is_tuple_labels
4693 ):
4694 # Set level to zero in case of MultiIndex and label is string,
4695 # because isin can't handle strings for MultiIndexes GH#36293
4696 # In case of tuples we get dtype object but have to use isin GH#42771
4697 mask = ~axis.get_level_values(0).isin(labels)
4698 else:
4699 mask = ~axis.isin(labels)
4700 # Check if label doesn't exist along axis
4701 labels_missing = (axis.get_indexer_for(labels) == -1).any()
4702 if errors == "raise" and labels_missing:
4703 raise KeyError(f"{labels} not found in axis")
4705 if isinstance(mask.dtype, ExtensionDtype):
4706 # GH#45860
4707 mask = mask.to_numpy(dtype=bool)
4709 indexer = mask.nonzero()[0]
4710 new_axis = axis.take(indexer)
4712 bm_axis = self.ndim - axis_num - 1
4713 new_mgr = self._mgr.reindex_indexer(
4714 new_axis,
4715 indexer,
4716 axis=bm_axis,
4717 allow_dups=True,
4718 only_slice=only_slice,
4719 )
4720 result = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
4721 if self.ndim == 1:
4722 result._name = self.name
4724 return result.__finalize__(self)
4726 @final
4727 def _update_inplace(self, result) -> None:
4728 """
4729 Replace self internals with result.
4731 Parameters
4732 ----------
4733 result : same type as self
4734 """
4735 # NOTE: This does *not* call __finalize__ and that's an explicit
4736 # decision that we may revisit in the future.
4737 self._mgr = result._mgr
4739 @final
4740 def add_prefix(self, prefix: str, axis: Axis | None = None) -> Self:
4741 """
4742 Prefix labels with string `prefix`.
4744 For Series, the row labels are prefixed.
4745 For DataFrame, the column labels are prefixed.
4747 Parameters
4748 ----------
4749 prefix : str
4750 The string to add before each label.
4751 axis : {0 or 'index', 1 or 'columns', None}, default None
4752 Axis to add prefix on
4754 .. versionadded:: 2.0.0
4756 Returns
4757 -------
4758 Series or DataFrame
4759 New Series or DataFrame with updated labels.
4761 See Also
4762 --------
4763 Series.add_suffix: Suffix row labels with string `suffix`.
4764 DataFrame.add_suffix: Suffix column labels with string `suffix`.
4766 Examples
4767 --------
4768 >>> s = pd.Series([1, 2, 3, 4])
4769 >>> s
4770 0 1
4771 1 2
4772 2 3
4773 3 4
4774 dtype: int64
4776 >>> s.add_prefix("item_")
4777 item_0 1
4778 item_1 2
4779 item_2 3
4780 item_3 4
4781 dtype: int64
4783 >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [3, 4, 5, 6]})
4784 >>> df
4785 A B
4786 0 1 3
4787 1 2 4
4788 2 3 5
4789 3 4 6
4791 >>> df.add_prefix("col_")
4792 col_A col_B
4793 0 1 3
4794 1 2 4
4795 2 3 5
4796 3 4 6
4797 """
4798 f = lambda x: f"{prefix}{x}"
4800 axis_name = self._info_axis_name
4801 if axis is not None:
4802 axis_name = self._get_axis_name(axis)
4804 mapper = {axis_name: f}
4806 # error: Keywords must be strings
4807 # error: No overload variant of "_rename" of "NDFrame" matches
4808 # argument type "dict[Literal['index', 'columns'], Callable[[Any], str]]"
4809 return self._rename(**mapper) # type: ignore[call-overload, misc]
4811 @final
4812 def add_suffix(self, suffix: str, axis: Axis | None = None) -> Self:
4813 """
4814 Suffix labels with string `suffix`.
4816 For Series, the row labels are suffixed.
4817 For DataFrame, the column labels are suffixed.
4819 Parameters
4820 ----------
4821 suffix : str
4822 The string to add after each label.
4823 axis : {0 or 'index', 1 or 'columns', None}, default None
4824 Axis to add suffix on
4826 .. versionadded:: 2.0.0
4828 Returns
4829 -------
4830 Series or DataFrame
4831 New Series or DataFrame with updated labels.
4833 See Also
4834 --------
4835 Series.add_prefix: Prefix row labels with string `prefix`.
4836 DataFrame.add_prefix: Prefix column labels with string `prefix`.
4838 Examples
4839 --------
4840 >>> s = pd.Series([1, 2, 3, 4])
4841 >>> s
4842 0 1
4843 1 2
4844 2 3
4845 3 4
4846 dtype: int64
4848 >>> s.add_suffix("_item")
4849 0_item 1
4850 1_item 2
4851 2_item 3
4852 3_item 4
4853 dtype: int64
4855 >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [3, 4, 5, 6]})
4856 >>> df
4857 A B
4858 0 1 3
4859 1 2 4
4860 2 3 5
4861 3 4 6
4863 >>> df.add_suffix("_col")
4864 A_col B_col
4865 0 1 3
4866 1 2 4
4867 2 3 5
4868 3 4 6
4869 """
4870 f = lambda x: f"{x}{suffix}"
4872 axis_name = self._info_axis_name
4873 if axis is not None:
4874 axis_name = self._get_axis_name(axis)
4876 mapper = {axis_name: f}
4877 # error: Keywords must be strings
4878 # error: No overload variant of "_rename" of "NDFrame" matches argument
4879 # type "dict[Literal['index', 'columns'], Callable[[Any], str]]"
4880 return self._rename(**mapper) # type: ignore[call-overload, misc]
4882 @overload
4883 def sort_values(
4884 self,
4885 *,
4886 axis: Axis = ...,
4887 ascending: bool | Sequence[bool] = ...,
4888 inplace: Literal[False] = ...,
4889 kind: SortKind = ...,
4890 na_position: NaPosition = ...,
4891 ignore_index: bool = ...,
4892 key: ValueKeyFunc = ...,
4893 ) -> Self: ...
4895 @overload
4896 def sort_values(
4897 self,
4898 *,
4899 axis: Axis = ...,
4900 ascending: bool | Sequence[bool] = ...,
4901 inplace: Literal[True],
4902 kind: SortKind = ...,
4903 na_position: NaPosition = ...,
4904 ignore_index: bool = ...,
4905 key: ValueKeyFunc = ...,
4906 ) -> None: ...
4908 @overload
4909 def sort_values(
4910 self,
4911 *,
4912 axis: Axis = ...,
4913 ascending: bool | Sequence[bool] = ...,
4914 inplace: bool = ...,
4915 kind: SortKind = ...,
4916 na_position: NaPosition = ...,
4917 ignore_index: bool = ...,
4918 key: ValueKeyFunc = ...,
4919 ) -> Self | None: ...
4921 def sort_values(
4922 self,
4923 *,
4924 axis: Axis = 0,
4925 ascending: bool | Sequence[bool] = True,
4926 inplace: bool = False,
4927 kind: SortKind = "quicksort",
4928 na_position: NaPosition = "last",
4929 ignore_index: bool = False,
4930 key: ValueKeyFunc | None = None,
4931 ) -> Self | None:
4932 """
4933 Sort by the values along either axis.
4935 Parameters
4936 ----------%(optional_by)s
4937 axis : %(axes_single_arg)s, default 0
4938 Axis to be sorted.
4939 ascending : bool or list of bool, default True
4940 Sort ascending vs. descending. Specify list for multiple sort
4941 orders. If this is a list of bools, must match the length of
4942 the by.
4943 inplace : bool, default False
4944 If True, perform operation in-place.
4945 kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, default 'quicksort'
4946 Choice of sorting algorithm. See also :func:`numpy.sort` for more
4947 information. `mergesort` and `stable` are the only stable algorithms. For
4948 DataFrames, this option is only applied when sorting on a single
4949 column or label.
4950 na_position : {'first', 'last'}, default 'last'
4951 Puts NaNs at the beginning if `first`; `last` puts NaNs at the
4952 end.
4953 ignore_index : bool, default False
4954 If True, the resulting axis will be labeled 0, 1, …, n - 1.
4955 key : callable, optional
4956 Apply the key function to the values
4957 before sorting. This is similar to the `key` argument in the
4958 builtin :meth:`sorted` function, with the notable difference that
4959 this `key` function should be *vectorized*. It should expect a
4960 ``Series`` and return a Series with the same shape as the input.
4961 It will be applied to each column in `by` independently. The values in the
4962 returned Series will be used as the keys for sorting.
4964 Returns
4965 -------
4966 DataFrame or None
4967 DataFrame with sorted values or None if ``inplace=True``.
4969 See Also
4970 --------
4971 DataFrame.sort_index : Sort a DataFrame by the index.
4972 Series.sort_values : Similar method for a Series.
4974 Examples
4975 --------
4976 >>> df = pd.DataFrame(
4977 ... {
4978 ... "col1": ["A", "A", "B", np.nan, "D", "C"],
4979 ... "col2": [2, 1, 9, 8, 7, 4],
4980 ... "col3": [0, 1, 9, 4, 2, 3],
4981 ... "col4": ["a", "B", "c", "D", "e", "F"],
4982 ... }
4983 ... )
4984 >>> df
4985 col1 col2 col3 col4
4986 0 A 2 0 a
4987 1 A 1 1 B
4988 2 B 9 9 c
4989 3 NaN 8 4 D
4990 4 D 7 2 e
4991 5 C 4 3 F
4993 Sort by col1
4995 >>> df.sort_values(by=["col1"])
4996 col1 col2 col3 col4
4997 0 A 2 0 a
4998 1 A 1 1 B
4999 2 B 9 9 c
5000 5 C 4 3 F
5001 4 D 7 2 e
5002 3 NaN 8 4 D
5004 Sort by multiple columns
5006 >>> df.sort_values(by=["col1", "col2"])
5007 col1 col2 col3 col4
5008 1 A 1 1 B
5009 0 A 2 0 a
5010 2 B 9 9 c
5011 5 C 4 3 F
5012 4 D 7 2 e
5013 3 NaN 8 4 D
5015 Sort Descending
5017 >>> df.sort_values(by="col1", ascending=False)
5018 col1 col2 col3 col4
5019 4 D 7 2 e
5020 5 C 4 3 F
5021 2 B 9 9 c
5022 0 A 2 0 a
5023 1 A 1 1 B
5024 3 NaN 8 4 D
5026 Putting NAs first
5028 >>> df.sort_values(by="col1", ascending=False, na_position="first")
5029 col1 col2 col3 col4
5030 3 NaN 8 4 D
5031 4 D 7 2 e
5032 5 C 4 3 F
5033 2 B 9 9 c
5034 0 A 2 0 a
5035 1 A 1 1 B
5037 Sorting with a key function
5039 >>> df.sort_values(by="col4", key=lambda col: col.str.lower())
5040 col1 col2 col3 col4
5041 0 A 2 0 a
5042 1 A 1 1 B
5043 2 B 9 9 c
5044 3 NaN 8 4 D
5045 4 D 7 2 e
5046 5 C 4 3 F
5048 Natural sort with the key argument,
5049 using the `natsort <https://github.com/SethMMorton/natsort>` package.
5051 >>> df = pd.DataFrame(
5052 ... {
5053 ... "hours": ["0hr", "128hr", "0hr", "64hr", "64hr", "128hr"],
5054 ... "mins": [
5055 ... "10mins",
5056 ... "40mins",
5057 ... "40mins",
5058 ... "40mins",
5059 ... "10mins",
5060 ... "10mins",
5061 ... ],
5062 ... "value": [10, 20, 30, 40, 50, 60],
5063 ... }
5064 ... )
5065 >>> df
5066 hours mins value
5067 0 0hr 10mins 10
5068 1 128hr 40mins 20
5069 2 0hr 40mins 30
5070 3 64hr 40mins 40
5071 4 64hr 10mins 50
5072 5 128hr 10mins 60
5073 >>> from natsort import natsort_keygen
5074 >>> df.sort_values(
5075 ... by=["hours", "mins"],
5076 ... key=natsort_keygen(),
5077 ... )
5078 hours mins value
5079 0 0hr 10mins 10
5080 2 0hr 40mins 30
5081 4 64hr 10mins 50
5082 3 64hr 40mins 40
5083 5 128hr 10mins 60
5084 1 128hr 40mins 20
5085 """
5086 raise AbstractMethodError(self)
5088 @overload
5089 def sort_index(
5090 self,
5091 *,
5092 axis: Axis = ...,
5093 level: IndexLabel = ...,
5094 ascending: bool | Sequence[bool] = ...,
5095 inplace: Literal[True],
5096 kind: SortKind = ...,
5097 na_position: NaPosition = ...,
5098 sort_remaining: bool = ...,
5099 ignore_index: bool = ...,
5100 key: IndexKeyFunc = ...,
5101 ) -> None: ...
5103 @overload
5104 def sort_index(
5105 self,
5106 *,
5107 axis: Axis = ...,
5108 level: IndexLabel = ...,
5109 ascending: bool | Sequence[bool] = ...,
5110 inplace: Literal[False] = ...,
5111 kind: SortKind = ...,
5112 na_position: NaPosition = ...,
5113 sort_remaining: bool = ...,
5114 ignore_index: bool = ...,
5115 key: IndexKeyFunc = ...,
5116 ) -> Self: ...
5118 @overload
5119 def sort_index(
5120 self,
5121 *,
5122 axis: Axis = ...,
5123 level: IndexLabel = ...,
5124 ascending: bool | Sequence[bool] = ...,
5125 inplace: bool = ...,
5126 kind: SortKind = ...,
5127 na_position: NaPosition = ...,
5128 sort_remaining: bool = ...,
5129 ignore_index: bool = ...,
5130 key: IndexKeyFunc = ...,
5131 ) -> Self | None: ...
5133 def sort_index(
5134 self,
5135 *,
5136 axis: Axis = 0,
5137 level: IndexLabel | None = None,
5138 ascending: bool | Sequence[bool] = True,
5139 inplace: bool = False,
5140 kind: SortKind = "quicksort",
5141 na_position: NaPosition = "last",
5142 sort_remaining: bool = True,
5143 ignore_index: bool = False,
5144 key: IndexKeyFunc | None = None,
5145 ) -> Self | None:
5146 inplace = validate_bool_kwarg(inplace, "inplace")
5147 axis = self._get_axis_number(axis)
5148 ascending = validate_ascending(ascending)
5150 target = self._get_axis(axis)
5152 indexer = get_indexer_indexer(
5153 target, level, ascending, kind, na_position, sort_remaining, key
5154 )
5156 if indexer is None:
5157 if inplace:
5158 result = self
5159 else:
5160 result = self.copy(deep=False)
5162 if ignore_index:
5163 if axis == 1:
5164 result.columns = default_index(len(self.columns))
5165 else:
5166 result.index = default_index(len(self))
5167 if inplace:
5168 return None
5169 else:
5170 return result
5172 baxis = self._get_block_manager_axis(axis)
5173 new_data = self._mgr.take(indexer, axis=baxis, verify=False)
5175 # reconstruct axis if needed
5176 if not ignore_index:
5177 new_axis = new_data.axes[baxis]._sort_levels_monotonic()
5178 else:
5179 new_axis = default_index(len(indexer))
5180 new_data.set_axis(baxis, new_axis)
5182 result = self._constructor_from_mgr(new_data, axes=new_data.axes)
5184 if inplace:
5185 return self._update_inplace(result)
5186 else:
5187 return result.__finalize__(self, method="sort_index")
5189 def reindex(
5190 self,
5191 labels=None,
5192 *,
5193 index=None,
5194 columns=None,
5195 axis: Axis | None = None,
5196 method: ReindexMethod | None = None,
5197 copy: bool | lib.NoDefault = lib.no_default,
5198 level: Level | None = None,
5199 fill_value: Scalar | None = np.nan,
5200 limit: int | None = None,
5201 tolerance=None,
5202 ) -> Self:
5203 """
5204 Conform Series/DataFrame to new index with optional filling logic.
5206 Places NA/NaN in locations having no value in the previous index. A new object
5207 is produced unless the new index is equivalent to the current one and
5208 ``copy=False``.
5210 Parameters
5211 ----------
5212 method : {None, 'backfill'/'bfill', 'pad'/'ffill', 'nearest'}
5213 Method to use for filling holes in reindexed DataFrame.
5214 Please note: this is only applicable to DataFrames/Series with a
5215 monotonically increasing/decreasing index.
5217 * None (default): don't fill gaps
5218 * pad / ffill: Propagate last valid observation forward to next
5219 valid.
5220 * backfill / bfill: Use next valid observation to fill gap.
5221 * nearest: Use nearest valid observations to fill gap.
5223 copy : bool, default False
5224 This keyword is now ignored; changing its value will have no
5225 impact on the method.
5227 .. deprecated:: 3.0.0
5229 This keyword is ignored and will be removed in pandas 4.0. Since
5230 pandas 3.0, this method always returns a new object using a lazy
5231 copy mechanism that defers copies until necessary
5232 (Copy-on-Write). See the `user guide on Copy-on-Write
5233 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
5234 for more details.
5236 level : int or name
5237 Broadcast across a level, matching Index values on the
5238 passed MultiIndex level.
5239 fill_value : scalar, default np.nan
5240 Value to use for missing values. Defaults to NaN, but can be any
5241 "compatible" value.
5242 limit : int, default None
5243 Maximum number of consecutive elements to forward or backward fill.
5244 tolerance : optional
5245 Maximum distance between original and new labels for inexact
5246 matches. The values of the index at the matching locations most
5247 satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
5249 Tolerance may be a scalar value, which applies the same tolerance
5250 to all values, or list-like, which applies variable tolerance per
5251 element. List-like includes list, tuple, array, Series, and must be
5252 the same size as the index and its dtype must exactly match the
5253 index's type.
5255 Returns
5256 -------
5257 Series/DataFrame
5258 Series/DataFrame with changed index.
5260 See Also
5261 --------
5262 DataFrame.set_index : Set row labels.
5263 DataFrame.reset_index : Remove row labels or move them to new columns.
5264 DataFrame.reindex_like : Change to same indices as other DataFrame.
5266 Examples
5267 --------
5268 ``DataFrame.reindex`` supports two calling conventions
5270 * ``(index=index_labels, columns=column_labels, ...)``
5271 * ``(labels, axis={'index', 'columns'}, ...)``
5273 We *highly* recommend using keyword arguments to clarify your
5274 intent.
5276 Create a DataFrame with some fictional data.
5278 >>> index = ["Firefox", "Chrome", "Safari", "IE10", "Konqueror"]
5279 >>> columns = ["http_status", "response_time"]
5280 >>> df = pd.DataFrame(
5281 ... [[200, 0.04], [200, 0.02], [404, 0.07], [404, 0.08], [301, 1.0]],
5282 ... columns=columns,
5283 ... index=index,
5284 ... )
5285 >>> df
5286 http_status response_time
5287 Firefox 200 0.04
5288 Chrome 200 0.02
5289 Safari 404 0.07
5290 IE10 404 0.08
5291 Konqueror 301 1.00
5293 Create a new index and reindex the DataFrame. By default
5294 values in the new index that do not have corresponding
5295 records in the DataFrame are assigned ``NaN``.
5297 >>> new_index = ["Safari", "Iceweasel", "Comodo Dragon", "IE10", "Chrome"]
5298 >>> df.reindex(new_index)
5299 http_status response_time
5300 Safari 404.0 0.07
5301 Iceweasel NaN NaN
5302 Comodo Dragon NaN NaN
5303 IE10 404.0 0.08
5304 Chrome 200.0 0.02
5306 We can fill in the missing values by passing a value to
5307 the keyword ``fill_value``. Because the index is not monotonically
5308 increasing or decreasing, we cannot use arguments to the keyword
5309 ``method`` to fill the ``NaN`` values.
5311 >>> df.reindex(new_index, fill_value=0)
5312 http_status response_time
5313 Safari 404 0.07
5314 Iceweasel 0 0.00
5315 Comodo Dragon 0 0.00
5316 IE10 404 0.08
5317 Chrome 200 0.02
5319 >>> df.reindex(new_index, fill_value="missing")
5320 http_status response_time
5321 Safari 404 0.07
5322 Iceweasel missing missing
5323 Comodo Dragon missing missing
5324 IE10 404 0.08
5325 Chrome 200 0.02
5327 We can also reindex the columns.
5329 >>> df.reindex(columns=["http_status", "user_agent"])
5330 http_status user_agent
5331 Firefox 200 NaN
5332 Chrome 200 NaN
5333 Safari 404 NaN
5334 IE10 404 NaN
5335 Konqueror 301 NaN
5337 Or we can use "axis-style" keyword arguments
5339 >>> df.reindex(["http_status", "user_agent"], axis="columns")
5340 http_status user_agent
5341 Firefox 200 NaN
5342 Chrome 200 NaN
5343 Safari 404 NaN
5344 IE10 404 NaN
5345 Konqueror 301 NaN
5347 To further illustrate the filling functionality in
5348 ``reindex``, we will create a DataFrame with a
5349 monotonically increasing index (for example, a sequence
5350 of dates).
5352 >>> date_index = pd.date_range("1/1/2010", periods=6, freq="D")
5353 >>> df2 = pd.DataFrame(
5354 ... {"prices": [100, 101, np.nan, 100, 89, 88]}, index=date_index
5355 ... )
5356 >>> df2
5357 prices
5358 2010-01-01 100.0
5359 2010-01-02 101.0
5360 2010-01-03 NaN
5361 2010-01-04 100.0
5362 2010-01-05 89.0
5363 2010-01-06 88.0
5365 Suppose we decide to expand the DataFrame to cover a wider
5366 date range.
5368 >>> date_index2 = pd.date_range("12/29/2009", periods=10, freq="D")
5369 >>> df2.reindex(date_index2)
5370 prices
5371 2009-12-29 NaN
5372 2009-12-30 NaN
5373 2009-12-31 NaN
5374 2010-01-01 100.0
5375 2010-01-02 101.0
5376 2010-01-03 NaN
5377 2010-01-04 100.0
5378 2010-01-05 89.0
5379 2010-01-06 88.0
5380 2010-01-07 NaN
5382 The index entries that did not have a value in the original data frame
5383 (for example, '2009-12-29') are by default filled with ``NaN``.
5384 If desired, we can fill in the missing values using one of several
5385 options.
5387 For example, to back-propagate the last valid value to fill the ``NaN``
5388 values, pass ``bfill`` as an argument to the ``method`` keyword.
5390 >>> df2.reindex(date_index2, method="bfill")
5391 prices
5392 2009-12-29 100.0
5393 2009-12-30 100.0
5394 2009-12-31 100.0
5395 2010-01-01 100.0
5396 2010-01-02 101.0
5397 2010-01-03 NaN
5398 2010-01-04 100.0
5399 2010-01-05 89.0
5400 2010-01-06 88.0
5401 2010-01-07 NaN
5403 Please note that the ``NaN`` value present in the original DataFrame
5404 (at index value 2010-01-03) will not be filled by any of the
5405 value propagation schemes. This is because filling while reindexing
5406 does not look at DataFrame values, but only compares the original and
5407 desired indexes. If you do want to fill in the ``NaN`` values present
5408 in the original DataFrame, use the ``fillna()`` method.
5410 See the :ref:`user guide <basics.reindexing>` for more.
5411 """
5412 # TODO: Decide if we care about having different examples for different
5413 # kinds
5415 # Automatically detect matching level when reindexing from Index to MultiIndex.
5416 # This prevents values from being incorrectly set to NaN when the source index
5417 # name matches a index name in the target MultiIndex
5418 if (
5419 level is None
5420 and index is not None
5421 and isinstance(index, MultiIndex)
5422 and not isinstance(self.index, MultiIndex)
5423 and self.index.name in index.names
5424 ):
5425 level = self.index.name
5426 self._check_copy_deprecation(copy)
5428 if index is not None and columns is not None and labels is not None:
5429 raise TypeError("Cannot specify all of 'labels', 'index', 'columns'.")
5430 elif index is not None or columns is not None:
5431 if axis is not None:
5432 raise TypeError(
5433 "Cannot specify both 'axis' and any of 'index' or 'columns'"
5434 )
5435 if labels is not None:
5436 if index is not None:
5437 columns = labels
5438 else:
5439 index = labels
5440 elif axis and self._get_axis_number(axis) == 1:
5441 columns = labels
5442 else:
5443 index = labels
5444 axes: dict[Literal["index", "columns"], Any] = {
5445 "index": index,
5446 "columns": columns,
5447 }
5448 method = clean_reindex_fill_method(method)
5450 # if all axes that are requested to reindex are equal, then only copy
5451 # if indicated must have index names equal here as well as values
5452 if all(
5453 self._get_axis(axis_name).identical(ax)
5454 for axis_name, ax in axes.items()
5455 if ax is not None
5456 ):
5457 return self.copy(deep=False)
5459 # check if we are a multi reindex
5460 if self._needs_reindex_multi(axes, method, level):
5461 return self._reindex_multi(axes, fill_value)
5463 # perform the reindex on the axes
5464 return self._reindex_axes(
5465 axes, level, limit, tolerance, method, fill_value
5466 ).__finalize__(self, method="reindex")
5468 @final
5469 def _reindex_axes(
5470 self,
5471 axes,
5472 level: Level | None,
5473 limit: int | None,
5474 tolerance,
5475 method,
5476 fill_value: Scalar | None,
5477 ) -> Self:
5478 """Perform the reindex for all the axes."""
5479 obj = self
5480 for a in self._AXIS_ORDERS:
5481 labels = axes[a]
5482 if labels is None:
5483 continue
5485 ax = self._get_axis(a)
5486 new_index, indexer = ax.reindex(
5487 labels, level=level, limit=limit, tolerance=tolerance, method=method
5488 )
5490 axis = self._get_axis_number(a)
5491 obj = obj._reindex_with_indexers(
5492 {axis: [new_index, indexer]},
5493 fill_value=fill_value,
5494 allow_dups=False,
5495 )
5497 return obj
5499 def _needs_reindex_multi(self, axes, method, level: Level | None) -> bool:
5500 """Check if we do need a multi reindex."""
5501 return (
5502 (common.count_not_none(*axes.values()) == self._AXIS_LEN)
5503 and method is None
5504 and level is None
5505 # reindex_multi calls self.values, so we only want to go
5506 # down that path when doing so is cheap.
5507 and self._can_fast_transpose
5508 )
5510 def _reindex_multi(self, axes, fill_value):
5511 raise AbstractMethodError(self)
5513 @final
5514 def _reindex_with_indexers(
5515 self,
5516 reindexers,
5517 fill_value=None,
5518 allow_dups: bool = False,
5519 ) -> Self:
5520 """allow_dups indicates an internal call here"""
5521 # reindex doing multiple operations on different axes if indicated
5522 new_data = self._mgr
5523 for axis in sorted(reindexers.keys()):
5524 index, indexer = reindexers[axis]
5525 baxis = self._get_block_manager_axis(axis)
5527 if index is None:
5528 continue
5530 index = ensure_index(index)
5531 if indexer is not None:
5532 indexer = ensure_platform_int(indexer)
5534 # TODO: speed up on homogeneous DataFrame objects (see _reindex_multi)
5535 new_data = new_data.reindex_indexer(
5536 index,
5537 indexer,
5538 axis=baxis,
5539 fill_value=fill_value,
5540 allow_dups=allow_dups,
5541 )
5543 if new_data is self._mgr:
5544 new_data = new_data.copy(deep=False)
5546 return self._constructor_from_mgr(new_data, axes=new_data.axes).__finalize__(
5547 self
5548 )
5550 def filter(
5551 self,
5552 items=None,
5553 like: str | None = None,
5554 regex: str | None = None,
5555 axis: Axis | None = None,
5556 ) -> Self:
5557 """
5558 Subset the DataFrame or Series according to the specified index labels.
5560 For DataFrame, filter rows or columns depending on ``axis`` argument.
5561 Note that this routine does not filter based on content.
5562 The filter is applied to the labels of the index.
5564 Parameters
5565 ----------
5566 items : list-like
5567 Keep labels from axis which are in items.
5568 like : str
5569 Keep labels from axis for which "like in label == True".
5570 regex : str (regular expression)
5571 Keep labels from axis for which re.search(regex, label) == True.
5572 axis : {0 or 'index', 1 or 'columns', None}, default None
5573 The axis to filter on, expressed either as an index (int)
5574 or axis name (str). By default this is the info axis, 'columns' for
5575 ``DataFrame``. For ``Series`` this parameter is unused and defaults to
5576 ``None``.
5578 Returns
5579 -------
5580 Same type as caller
5581 The filtered subset of the DataFrame or Series.
5583 See Also
5584 --------
5585 DataFrame.loc : Access a group of rows and columns
5586 by label(s) or a boolean array.
5588 Notes
5589 -----
5590 The ``items``, ``like``, and ``regex`` parameters are
5591 enforced to be mutually exclusive.
5593 ``axis`` defaults to the info axis that is used when indexing
5594 with ``[]``.
5596 Examples
5597 --------
5598 >>> df = pd.DataFrame(
5599 ... np.array(([1, 2, 3], [4, 5, 6])),
5600 ... index=["mouse", "rabbit"],
5601 ... columns=["one", "two", "three"],
5602 ... )
5603 >>> df
5604 one two three
5605 mouse 1 2 3
5606 rabbit 4 5 6
5608 >>> # select columns by name
5609 >>> df.filter(items=["one", "three"])
5610 one three
5611 mouse 1 3
5612 rabbit 4 6
5614 >>> # select columns by regular expression
5615 >>> df.filter(regex="e$", axis=1)
5616 one three
5617 mouse 1 3
5618 rabbit 4 6
5620 >>> # select rows containing 'bbi'
5621 >>> df.filter(like="bbi", axis=0)
5622 one two three
5623 rabbit 4 5 6
5624 """
5625 nkw = common.count_not_none(items, like, regex)
5626 if nkw > 1:
5627 raise TypeError(
5628 "Keyword arguments `items`, `like`, or `regex` are mutually exclusive"
5629 )
5631 if axis is None:
5632 axis = self._info_axis_name
5633 labels = self._get_axis(axis)
5635 if items is not None:
5636 name = self._get_axis_name(axis)
5637 items = Index(items).intersection(labels)
5638 if len(items) == 0:
5639 # Keep the dtype of labels when we are empty
5640 items = items.astype(labels.dtype)
5641 # error: Keywords must be strings
5642 return self.reindex(**{name: items}) # type: ignore[misc]
5643 elif like:
5645 def f(x) -> bool:
5646 assert like is not None # needed for mypy
5647 return like in ensure_str(x)
5649 values = labels.map(f)
5650 return self.loc(axis=axis)[values]
5651 elif regex:
5653 def f(x) -> bool:
5654 return matcher.search(ensure_str(x)) is not None
5656 matcher = re.compile(regex)
5657 values = labels.map(f)
5658 return self.loc(axis=axis)[values]
5659 else:
5660 raise TypeError("Must pass either `items`, `like`, or `regex`")
5662 @final
5663 def head(self, n: int = 5) -> Self:
5664 """
5665 Return the first `n` rows.
5667 This function exhibits the same behavior as ``df[:n]``, returning the
5668 first ``n`` rows based on position. It is useful for quickly checking
5669 if your object has the right type of data in it.
5671 When ``n`` is positive, it returns the first ``n`` rows. For ``n`` equal to 0,
5672 it returns an empty object. When ``n`` is negative, it returns
5673 all rows except the last ``|n|`` rows, mirroring the behavior of ``df[:n]``.
5675 If ``n`` is larger than the number of rows, this function returns all rows.
5677 Parameters
5678 ----------
5679 n : int, default 5
5680 Number of rows to select.
5682 Returns
5683 -------
5684 same type as caller
5685 The first `n` rows of the caller object.
5687 See Also
5688 --------
5689 DataFrame.tail: Returns the last `n` rows.
5691 Examples
5692 --------
5693 >>> df = pd.DataFrame(
5694 ... {
5695 ... "animal": [
5696 ... "alligator",
5697 ... "bee",
5698 ... "falcon",
5699 ... "lion",
5700 ... "monkey",
5701 ... "parrot",
5702 ... "shark",
5703 ... "whale",
5704 ... "zebra",
5705 ... ]
5706 ... }
5707 ... )
5708 >>> df
5709 animal
5710 0 alligator
5711 1 bee
5712 2 falcon
5713 3 lion
5714 4 monkey
5715 5 parrot
5716 6 shark
5717 7 whale
5718 8 zebra
5720 Viewing the first 5 lines
5722 >>> df.head()
5723 animal
5724 0 alligator
5725 1 bee
5726 2 falcon
5727 3 lion
5728 4 monkey
5730 Viewing the first `n` lines (three in this case)
5732 >>> df.head(3)
5733 animal
5734 0 alligator
5735 1 bee
5736 2 falcon
5738 For negative values of `n`
5740 >>> df.head(-3)
5741 animal
5742 0 alligator
5743 1 bee
5744 2 falcon
5745 3 lion
5746 4 monkey
5747 5 parrot
5748 """
5749 return self.iloc[:n].copy()
5751 @final
5752 def tail(self, n: int = 5) -> Self:
5753 """
5754 Return the last `n` rows.
5756 This function returns last `n` rows from the object based on
5757 position. It is useful for quickly verifying data, for example,
5758 after sorting or appending rows.
5760 For negative values of `n`, this function returns all rows except
5761 the first `|n|` rows, equivalent to ``df[|n|:]``.
5763 If ``n`` is larger than the number of rows, this function returns all rows.
5765 Parameters
5766 ----------
5767 n : int, default 5
5768 Number of rows to select.
5770 Returns
5771 -------
5772 type of caller
5773 The last `n` rows of the caller object.
5775 See Also
5776 --------
5777 DataFrame.head : The first `n` rows of the caller object.
5779 Examples
5780 --------
5781 >>> df = pd.DataFrame(
5782 ... {
5783 ... "animal": [
5784 ... "alligator",
5785 ... "bee",
5786 ... "falcon",
5787 ... "lion",
5788 ... "monkey",
5789 ... "parrot",
5790 ... "shark",
5791 ... "whale",
5792 ... "zebra",
5793 ... ]
5794 ... }
5795 ... )
5796 >>> df
5797 animal
5798 0 alligator
5799 1 bee
5800 2 falcon
5801 3 lion
5802 4 monkey
5803 5 parrot
5804 6 shark
5805 7 whale
5806 8 zebra
5808 Viewing the last 5 lines
5810 >>> df.tail()
5811 animal
5812 4 monkey
5813 5 parrot
5814 6 shark
5815 7 whale
5816 8 zebra
5818 Viewing the last `n` lines (three in this case)
5820 >>> df.tail(3)
5821 animal
5822 6 shark
5823 7 whale
5824 8 zebra
5826 For negative values of `n`
5828 >>> df.tail(-3)
5829 animal
5830 3 lion
5831 4 monkey
5832 5 parrot
5833 6 shark
5834 7 whale
5835 8 zebra
5836 """
5837 if n == 0:
5838 return self.iloc[0:0].copy()
5839 return self.iloc[-n:].copy()
5841 @final
5842 def sample(
5843 self,
5844 n: int | None = None,
5845 frac: float | None = None,
5846 replace: bool = False,
5847 weights=None,
5848 random_state: RandomState | None = None,
5849 axis: Axis | None = None,
5850 ignore_index: bool = False,
5851 ) -> Self:
5852 """
5853 Return a random sample of items from an axis of object.
5855 You can use `random_state` for reproducibility.
5857 Parameters
5858 ----------
5859 n : int, optional
5860 Number of items from axis to return. Cannot be used with `frac`.
5861 Default = 1 if `frac` = None.
5862 frac : float, optional
5863 Fraction of axis items to return. Cannot be used with `n`.
5864 replace : bool, default False
5865 Allow or disallow sampling of the same row more than once.
5866 weights : str or ndarray-like, optional
5867 Default ``None`` results in equal probability weighting.
5868 If passed a Series, will align with target object on index. Index
5869 values in weights not found in sampled object will be ignored and
5870 index values in sampled object not in weights will be assigned
5871 weights of zero.
5872 If called on a DataFrame, will accept the name of a column
5873 when axis = 0.
5874 Unless weights are a Series, weights must be same length as axis
5875 being sampled.
5876 If weights do not sum to 1, they will be normalized to sum to 1.
5877 Missing values in the weights column will be treated as zero.
5878 Infinite values not allowed.
5879 When replace = False will not allow ``(n * max(weights) / sum(weights)) > 1``
5880 in order to avoid biased results. See the Notes below for more details.
5881 random_state : int, array-like, BitGenerator, np.random.RandomState, np.random.Generator, optional
5882 If int, array-like, or BitGenerator, seed for random number generator.
5883 If np.random.RandomState or np.random.Generator, use as given.
5884 Default ``None`` results in sampling with the current state of np.random.
5885 axis : {0 or 'index', 1 or 'columns', None}, default None
5886 Axis to sample. Accepts axis number or name. Default is stat axis
5887 for given data type. For `Series` this parameter is unused and defaults to `None`.
5888 ignore_index : bool, default False
5889 If True, the resulting index will be labeled 0, 1, …, n - 1.
5891 Returns
5892 -------
5893 Series or DataFrame
5894 A new object of same type as caller containing `n` items randomly
5895 sampled from the caller object.
5897 See Also
5898 --------
5899 DataFrameGroupBy.sample: Generates random samples from each group of a
5900 DataFrame object.
5901 SeriesGroupBy.sample: Generates random samples from each group of a
5902 Series object.
5903 numpy.random.choice: Generates a random sample from a given 1-D numpy
5904 array.
5906 Notes
5907 -----
5908 If `frac` > 1, `replacement` should be set to `True`.
5910 When replace = False will not allow ``(n * max(weights) / sum(weights)) > 1``,
5911 since that would cause results to be biased. E.g. sampling 2 items without replacement
5912 with weights [100, 1, 1] would yield two last items in 1/2 of cases, instead of 1/102.
5913 This is similar to specifying `n=4` without replacement on a Series with 3 elements.
5915 Examples
5916 --------
5917 >>> df = pd.DataFrame(
5918 ... {
5919 ... "num_legs": [2, 4, 8, 0],
5920 ... "num_wings": [2, 0, 0, 0],
5921 ... "num_specimen_seen": [10, 2, 1, 8],
5922 ... },
5923 ... index=["falcon", "dog", "spider", "fish"],
5924 ... )
5925 >>> df
5926 num_legs num_wings num_specimen_seen
5927 falcon 2 2 10
5928 dog 4 0 2
5929 spider 8 0 1
5930 fish 0 0 8
5932 Extract 3 random elements from the ``Series`` ``df['num_legs']``:
5933 Note that we use `random_state` to ensure the reproducibility of
5934 the examples.
5936 >>> df["num_legs"].sample(n=3, random_state=1)
5937 fish 0
5938 spider 8
5939 falcon 2
5940 Name: num_legs, dtype: int64
5942 A random 50% sample of the ``DataFrame`` with replacement:
5944 >>> df.sample(frac=0.5, replace=True, random_state=1)
5945 num_legs num_wings num_specimen_seen
5946 dog 4 0 2
5947 fish 0 0 8
5949 An upsample sample of the ``DataFrame`` with replacement:
5950 Note that `replace` parameter has to be `True` for `frac` parameter > 1.
5952 >>> df.sample(frac=2, replace=True, random_state=1)
5953 num_legs num_wings num_specimen_seen
5954 dog 4 0 2
5955 fish 0 0 8
5956 falcon 2 2 10
5957 falcon 2 2 10
5958 fish 0 0 8
5959 dog 4 0 2
5960 fish 0 0 8
5961 dog 4 0 2
5963 Using a DataFrame column as weights. Rows with larger value in the
5964 `num_specimen_seen` column are more likely to be sampled.
5966 >>> df.sample(n=2, weights="num_specimen_seen", random_state=1)
5967 num_legs num_wings num_specimen_seen
5968 falcon 2 2 10
5969 fish 0 0 8
5970 """ # noqa: E501
5971 if axis is None:
5972 axis = 0
5974 axis = self._get_axis_number(axis)
5975 obj_len = self.shape[axis]
5977 # Process random_state argument
5978 rs = common.random_state(random_state)
5980 size = sample.process_sampling_size(n, frac, replace)
5981 if size is None:
5982 assert frac is not None
5983 size = round(frac * obj_len)
5985 if weights is not None:
5986 weights = sample.preprocess_weights(self, weights, axis)
5988 sampled_indices = sample.sample(obj_len, size, replace, weights, rs)
5989 result = self.take(sampled_indices, axis=axis)
5991 if ignore_index:
5992 result.index = default_index(len(result))
5994 return result
5996 @overload
5997 def pipe(
5998 self,
5999 func: Callable[Concatenate[Self, P], T],
6000 *args: P.args,
6001 **kwargs: P.kwargs,
6002 ) -> T: ...
6004 @overload
6005 def pipe(
6006 self,
6007 func: tuple[Callable[..., T], str],
6008 *args: Any,
6009 **kwargs: Any,
6010 ) -> T: ...
6012 @final
6013 def pipe(
6014 self,
6015 func: Callable[Concatenate[Self, P], T] | tuple[Callable[..., T], str],
6016 *args: Any,
6017 **kwargs: Any,
6018 ) -> T:
6019 r"""
6020 Apply chainable functions that expect Series or DataFrames.
6022 Parameters
6023 ----------
6024 func : function
6025 Function to apply to the Series/DataFrame.
6026 ``args``, and ``kwargs`` are passed into ``func``.
6027 Alternatively a ``(callable, data_keyword)`` tuple where
6028 ``data_keyword`` is a string indicating the keyword of
6029 ``callable`` that expects the Series/DataFrame.
6030 *args : iterable, optional
6031 Positional arguments passed into ``func``.
6032 **kwargs : mapping, optional
6033 A dictionary of keyword arguments passed into ``func``.
6035 Returns
6036 -------
6037 The return type of ``func``.
6038 The result of applying ``func`` to the Series or DataFrame.
6040 See Also
6041 --------
6042 DataFrame.apply : Apply a function along input axis of DataFrame.
6043 DataFrame.map : Apply a function elementwise on a whole DataFrame.
6044 Series.map : Apply a mapping correspondence on a
6045 :class:`~pandas.Series`.
6047 Notes
6048 -----
6049 Use ``.pipe`` when chaining together functions that expect
6050 Series, DataFrames or GroupBy objects.
6052 Examples
6053 --------
6054 Constructing an income DataFrame from a dictionary.
6056 >>> data = [[8000, 1000], [9500, np.nan], [5000, 2000]]
6057 >>> df = pd.DataFrame(data, columns=["Salary", "Others"])
6058 >>> df
6059 Salary Others
6060 0 8000 1000.0
6061 1 9500 NaN
6062 2 5000 2000.0
6064 Functions that perform tax reductions on an income DataFrame.
6066 >>> def subtract_federal_tax(df):
6067 ... return df * 0.9
6068 >>> def subtract_state_tax(df, rate):
6069 ... return df * (1 - rate)
6070 >>> def subtract_national_insurance(df, rate, rate_increase):
6071 ... new_rate = rate + rate_increase
6072 ... return df * (1 - new_rate)
6074 Instead of writing
6076 >>> subtract_national_insurance(
6077 ... subtract_state_tax(subtract_federal_tax(df), rate=0.12),
6078 ... rate=0.05,
6079 ... rate_increase=0.02,
6080 ... ) # doctest: +SKIP
6082 You can write
6084 >>> (
6085 ... df.pipe(subtract_federal_tax)
6086 ... .pipe(subtract_state_tax, rate=0.12)
6087 ... .pipe(subtract_national_insurance, rate=0.05, rate_increase=0.02)
6088 ... )
6089 Salary Others
6090 0 5892.48 736.56
6091 1 6997.32 NaN
6092 2 3682.80 1473.12
6094 If you have a function that takes the data as (say) the second
6095 argument, pass a tuple indicating which keyword expects the
6096 data. For example, suppose ``national_insurance`` takes its data as ``df``
6097 in the second argument:
6099 >>> def subtract_national_insurance(rate, df, rate_increase):
6100 ... new_rate = rate + rate_increase
6101 ... return df * (1 - new_rate)
6102 >>> (
6103 ... df.pipe(subtract_federal_tax)
6104 ... .pipe(subtract_state_tax, rate=0.12)
6105 ... .pipe(
6106 ... (subtract_national_insurance, "df"), rate=0.05, rate_increase=0.02
6107 ... )
6108 ... )
6109 Salary Others
6110 0 5892.48 736.56
6111 1 6997.32 NaN
6112 2 3682.80 1473.12
6113 """
6114 return common.pipe(self.copy(deep=False), func, *args, **kwargs)
6116 # ----------------------------------------------------------------------
6117 # Attribute access
6119 @final
6120 def __finalize__(self, other, method: str | None = None, **kwargs) -> Self:
6121 """
6122 Propagate metadata from other to self.
6124 This is the default implementation. Subclasses may override this method to
6125 implement their own metadata handling.
6127 Parameters
6128 ----------
6129 other : the object from which to get the attributes that we are going
6130 to propagate. If ``other`` has an ``input_objs`` attribute, then
6131 this attribute must contain an iterable of objects, each with an
6132 ``attrs`` attribute.
6133 method : str, optional
6134 A passed method name providing context on where ``__finalize__``
6135 was called.
6137 .. warning::
6139 The value passed as `method` are not currently considered
6140 stable across pandas releases.
6142 Notes
6143 -----
6144 In case ``other`` has an ``input_objs`` attribute, this method only
6145 propagates its metadata if each object in ``input_objs`` has the exact
6146 same metadata as the others.
6147 """
6148 if isinstance(other, NDFrame):
6149 if other.attrs:
6150 # We want attrs propagation to have minimal performance
6151 # impact if attrs are not used; i.e. attrs is an empty dict.
6152 # One could make the deepcopy unconditionally, but a deepcopy
6153 # of an empty dict is 50x more expensive than the empty check.
6154 self.attrs = deepcopy(other.attrs)
6155 self.flags.allows_duplicate_labels = (
6156 self.flags.allows_duplicate_labels
6157 and other.flags.allows_duplicate_labels
6158 )
6159 # For subclasses using _metadata.
6160 for name in set(self._metadata) & set(other._metadata):
6161 assert isinstance(name, str)
6162 object.__setattr__(self, name, getattr(other, name, None))
6164 elif hasattr(other, "input_objs"):
6165 objs = other.input_objs
6166 # propagate attrs only if all inputs have the same attrs
6167 if all(bool(obj.attrs) for obj in objs):
6168 # all inputs have non-empty attrs
6169 attrs = objs[0].attrs
6170 have_same_attrs = all(obj.attrs == attrs for obj in objs[1:])
6171 if have_same_attrs:
6172 self.attrs = deepcopy(attrs)
6174 allows_duplicate_labels = all(x.flags.allows_duplicate_labels for x in objs)
6175 self.flags.allows_duplicate_labels = allows_duplicate_labels
6177 return self
6179 @final
6180 def __getattr__(self, name: str):
6181 """
6182 After regular attribute access, try looking up the name
6183 This allows simpler access to columns for interactive use.
6184 """
6185 # Note: obj.x will always call obj.__getattribute__('x') prior to
6186 # calling obj.__getattr__('x').
6187 if (
6188 name not in self._internal_names_set
6189 and name not in self._metadata
6190 and name not in self._accessors
6191 and self._info_axis._can_hold_identifiers_and_holds_name(name)
6192 ):
6193 return self[name]
6194 return object.__getattribute__(self, name)
6196 @final
6197 def __setattr__(self, name: str, value) -> None:
6198 """
6199 After regular attribute access, try setting the name
6200 This allows simpler access to columns for interactive use.
6201 """
6202 # first try regular attribute access via __getattribute__, so that
6203 # e.g. ``obj.x`` and ``obj.x = 4`` will always reference/modify
6204 # the same attribute.
6206 try:
6207 object.__getattribute__(self, name)
6208 return object.__setattr__(self, name, value)
6209 except AttributeError:
6210 pass
6212 # if this fails, go on to more involved attribute setting
6213 # (note that this matches __getattr__, above).
6214 if name in self._internal_names_set:
6215 object.__setattr__(self, name, value)
6216 elif name in self._metadata:
6217 object.__setattr__(self, name, value)
6218 else:
6219 try:
6220 existing = getattr(self, name)
6221 if isinstance(existing, Index):
6222 object.__setattr__(self, name, value)
6223 elif name in self._info_axis:
6224 self[name] = value
6225 else:
6226 object.__setattr__(self, name, value)
6227 except (AttributeError, TypeError):
6228 if isinstance(self, ABCDataFrame) and (is_list_like(value)):
6229 warnings.warn(
6230 "Pandas doesn't allow columns to be "
6231 "created via a new attribute name - see "
6232 "https://pandas.pydata.org/pandas-docs/"
6233 "stable/indexing.html#attribute-access",
6234 stacklevel=find_stack_level(),
6235 )
6236 object.__setattr__(self, name, value)
6238 @final
6239 def _dir_additions(self) -> set[str]:
6240 """
6241 add the string-like attributes from the info_axis.
6242 If info_axis is a MultiIndex, its first level values are used.
6243 """
6244 additions = super()._dir_additions()
6245 if self._info_axis._can_hold_strings:
6246 additions.update(self._info_axis._dir_additions_for_owner)
6247 return additions
6249 # ----------------------------------------------------------------------
6250 # Consolidation of internals
6252 @final
6253 def _consolidate_inplace(self) -> None:
6254 """Consolidate data in place and return None"""
6256 self._mgr = self._mgr.consolidate()
6258 @final
6259 def _consolidate(self):
6260 """
6261 Compute NDFrame with "consolidated" internals (data of each dtype
6262 grouped together in a single ndarray).
6264 Returns
6265 -------
6266 consolidated : same type as caller
6267 """
6268 cons_data = self._mgr.consolidate()
6269 return self._constructor_from_mgr(cons_data, axes=cons_data.axes).__finalize__(
6270 self
6271 )
6273 @final
6274 @property
6275 def _is_mixed_type(self) -> bool:
6276 if self._mgr.is_single_block:
6277 # Includes all Series cases
6278 return False
6280 if self._mgr.any_extension_types:
6281 # Even if they have the same dtype, we can't consolidate them,
6282 # so we pretend this is "mixed'"
6283 return True
6285 return self.dtypes.nunique() > 1
6287 @final
6288 def _get_numeric_data(self) -> Self:
6289 new_mgr = self._mgr.get_numeric_data()
6290 return self._constructor_from_mgr(new_mgr, axes=new_mgr.axes).__finalize__(self)
6292 @final
6293 def _get_bool_data(self):
6294 new_mgr = self._mgr.get_bool_data()
6295 return self._constructor_from_mgr(new_mgr, axes=new_mgr.axes).__finalize__(self)
6297 # ----------------------------------------------------------------------
6298 # Internal Interface Methods
6300 @property
6301 def values(self):
6302 raise AbstractMethodError(self)
6304 @property
6305 def _values(self) -> ArrayLike:
6306 """internal implementation"""
6307 raise AbstractMethodError(self)
6309 @property
6310 def dtypes(self):
6311 """
6312 Return the dtypes in the DataFrame.
6314 This returns a Series with the data type of each column.
6315 The result's index is the original DataFrame's columns. Columns
6316 with mixed types are stored with the ``object`` dtype. See
6317 :ref:`the User Guide <basics.dtypes>` for more.
6319 Returns
6320 -------
6321 pandas.Series
6322 The data type of each column.
6324 See Also
6325 --------
6326 Series.dtypes : Return the dtype object of the underlying data.
6328 Examples
6329 --------
6330 >>> df = pd.DataFrame(
6331 ... {
6332 ... "float": [1.0],
6333 ... "int": [1],
6334 ... "datetime": [pd.Timestamp("20180310")],
6335 ... "string": ["foo"],
6336 ... }
6337 ... )
6338 >>> df.dtypes
6339 float float64
6340 int int64
6341 datetime datetime64[us]
6342 string str
6343 dtype: object
6344 """
6345 data = self._mgr.get_dtypes()
6346 return self._constructor_sliced(data, index=self._info_axis, dtype=np.object_)
6348 @final
6349 def astype(
6350 self,
6351 dtype,
6352 copy: bool | lib.NoDefault = lib.no_default,
6353 errors: IgnoreRaise = "raise",
6354 ) -> Self:
6355 """
6356 Cast a pandas object to a specified dtype ``dtype``.
6358 This method allows the conversion of the data types of pandas objects,
6359 including DataFrames and Series, to the specified dtype. It supports casting
6360 entire objects to a single data type or applying different data types to
6361 individual columns using a mapping.
6363 Parameters
6364 ----------
6365 dtype : str, data type, Series or Mapping of column name -> data type
6366 Use a str, numpy.dtype, pandas.ExtensionDtype or Python type to
6367 cast entire pandas object to the same type. Alternatively, use a
6368 mapping, e.g. {col: dtype, ...}, where col is a column label and dtype is
6369 a numpy.dtype or Python type to cast one or more of the DataFrame's
6370 columns to column-specific types.
6371 copy : bool, default False
6372 This keyword is now ignored; changing its value will have no
6373 impact on the method.
6375 .. deprecated:: 3.0.0
6377 This keyword is ignored and will be removed in pandas 4.0. Since
6378 pandas 3.0, this method always returns a new object using a lazy
6379 copy mechanism that defers copies until necessary
6380 (Copy-on-Write). See the `user guide on Copy-on-Write
6381 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
6382 for more details.
6384 errors : {'raise', 'ignore'}, default 'raise'
6385 Control raising of exceptions on invalid data for provided dtype.
6387 - ``raise`` : allow exceptions to be raised
6388 - ``ignore`` : suppress exceptions. On error return original object.
6390 Returns
6391 -------
6392 same type as caller
6393 The pandas object casted to the specified ``dtype``.
6395 See Also
6396 --------
6397 to_datetime : Convert argument to datetime.
6398 to_timedelta : Convert argument to timedelta.
6399 to_numeric : Convert argument to a numeric type.
6400 numpy.ndarray.astype : Cast a numpy array to a specified type.
6402 Notes
6403 -----
6404 .. versionchanged:: 2.0.0
6406 Using ``astype`` to convert from timezone-naive dtype to
6407 timezone-aware dtype will raise an exception.
6408 Use :meth:`Series.dt.tz_localize` instead.
6410 Examples
6411 --------
6412 Create a DataFrame:
6414 >>> d = {"col1": [1, 2], "col2": [3, 4]}
6415 >>> df = pd.DataFrame(data=d)
6416 >>> df.dtypes
6417 col1 int64
6418 col2 int64
6419 dtype: object
6421 Cast all columns to int32:
6423 >>> df.astype("int32").dtypes
6424 col1 int32
6425 col2 int32
6426 dtype: object
6428 Cast col1 to int32 using a dictionary:
6430 >>> df.astype({"col1": "int32"}).dtypes
6431 col1 int32
6432 col2 int64
6433 dtype: object
6435 Create a series:
6437 >>> ser = pd.Series([1, 2], dtype="int32")
6438 >>> ser
6439 0 1
6440 1 2
6441 dtype: int32
6442 >>> ser.astype("int64")
6443 0 1
6444 1 2
6445 dtype: int64
6447 Convert to categorical type:
6449 >>> ser.astype("category")
6450 0 1
6451 1 2
6452 dtype: category
6453 Categories (2, int32): [1, 2]
6455 Convert to ordered categorical type with custom ordering:
6457 >>> from pandas.api.types import CategoricalDtype
6458 >>> cat_dtype = CategoricalDtype(categories=[2, 1], ordered=True)
6459 >>> ser.astype(cat_dtype)
6460 0 1
6461 1 2
6462 dtype: category
6463 Categories (2, int64): [2 < 1]
6465 Create a series of dates:
6467 >>> ser_date = pd.Series(pd.date_range("20200101", periods=3))
6468 >>> ser_date
6469 0 2020-01-01
6470 1 2020-01-02
6471 2 2020-01-03
6472 dtype: datetime64[us]
6473 """
6474 self._check_copy_deprecation(copy)
6475 if is_dict_like(dtype):
6476 if self.ndim == 1: # i.e. Series
6477 if len(dtype) > 1 or self.name not in dtype:
6478 raise KeyError(
6479 "Only the Series name can be used for "
6480 "the key in Series dtype mappings."
6481 )
6482 new_type = dtype[self.name]
6483 return self.astype(new_type, errors=errors)
6485 # GH#44417 cast to Series so we can use .iat below, which will be
6486 # robust in case we
6487 from pandas import Series
6489 dtype_ser = Series(dtype, dtype=object)
6491 for col_name in dtype_ser.index:
6492 if col_name not in self:
6493 raise KeyError(
6494 "Only a column name can be used for the "
6495 "key in a dtype mappings argument. "
6496 f"'{col_name}' not found in columns."
6497 )
6499 dtype_ser = dtype_ser.reindex(self.columns, fill_value=None)
6501 results = []
6502 for i, (col_name, col) in enumerate(self.items()):
6503 cdt = dtype_ser.iat[i]
6504 if isna(cdt):
6505 res_col = col.copy(deep=False)
6506 else:
6507 try:
6508 res_col = col.astype(dtype=cdt, errors=errors)
6509 except ValueError as ex:
6510 ex.args = (
6511 f"{ex}: Error while type casting for column '{col_name}'",
6512 )
6513 raise
6514 results.append(res_col)
6516 elif is_extension_array_dtype(dtype) and self.ndim > 1:
6517 # TODO(EA2D): special case not needed with 2D EAs
6518 dtype = pandas_dtype(dtype)
6519 if isinstance(dtype, ExtensionDtype) and all(
6520 block.values.dtype == dtype for block in self._mgr.blocks
6521 ):
6522 return self.copy(deep=False)
6523 # GH 18099/22869: columnwise conversion to extension dtype
6524 # GH 24704: self.items handles duplicate column names
6525 results = [ser.astype(dtype, errors=errors) for _, ser in self.items()]
6527 else:
6528 # else, only a single dtype is given
6529 new_data = self._mgr.astype(dtype=dtype, errors=errors)
6530 res = self._constructor_from_mgr(new_data, axes=new_data.axes)
6531 return res.__finalize__(self, method="astype")
6533 # GH 33113: handle empty frame or series
6534 if not results:
6535 return self.copy(deep=False)
6537 # GH 19920: retain column metadata after concat
6538 result = concat(results, axis=1)
6539 # GH#40810 retain subclass
6540 # error: Incompatible types in assignment
6541 # (expression has type "Self", variable has type "DataFrame")
6542 result = self._constructor(result) # type: ignore[assignment]
6543 result.columns = self.columns
6544 result = result.__finalize__(self, method="astype")
6545 # https://github.com/python/mypy/issues/8354
6546 return cast(Self, result)
6548 @final
6549 def copy(self, deep: bool = True) -> Self:
6550 """
6551 Make a copy of this object's indices and data.
6553 When ``deep=True`` (default), a new object will be created with a
6554 copy of the calling object's data and indices. Modifications to
6555 the data or indices of the copy will not be reflected in the
6556 original object (see notes below).
6558 When ``deep=False``, a new object will be created without copying
6559 the calling object's data or index (only references to the data
6560 and index are copied). With Copy-on-Write, changes to the original
6561 will *not* be reflected in the shallow copy (and vice versa). The
6562 shallow copy uses a lazy (deferred) copy mechanism that copies the
6563 data only when any changes to the original or shallow copy are made,
6564 ensuring memory efficiency while maintaining data integrity.
6566 .. note::
6567 In pandas versions prior to 3.0, the default behavior without
6568 Copy-on-Write was different: changes to the original *were* reflected
6569 in the shallow copy (and vice versa). See the :ref:`Copy-on-Write
6570 user guide <copy_on_write>` for more information.
6572 Parameters
6573 ----------
6574 deep : bool, default True
6575 Make a deep copy, including a copy of the data and the indices.
6576 With ``deep=False`` neither the indices nor the data are copied.
6578 Returns
6579 -------
6580 Series or DataFrame
6581 Object type matches caller.
6583 See Also
6584 --------
6585 copy.copy : Return a shallow copy of an object.
6586 copy.deepcopy : Return a deep copy of an object.
6588 Notes
6589 -----
6590 When ``deep=True``, data is copied but actual Python objects
6591 will not be copied recursively, only the reference to the object.
6592 This is in contrast to `copy.deepcopy` in the Standard Library,
6593 which recursively copies object data (see examples below).
6595 While ``Index`` objects are copied when ``deep=True``, the underlying
6596 numpy array is not copied for performance reasons. Since ``Index`` is
6597 immutable, the underlying data can be safely shared and a copy
6598 is not needed.
6600 Since pandas is not thread safe, see the
6601 :ref:`gotchas <gotchas.thread-safety>` when copying in a threading
6602 environment.
6604 Copy-on-Write protects shallow copies against accidental modifications.
6605 This means that any changes to the copied data would make a new copy
6606 of the data upon write (and vice versa). Changes made to either the
6607 original or copied variable would not be reflected in the counterpart.
6608 See :ref:`Copy_on_Write <copy_on_write>` for more information.
6610 Examples
6611 --------
6612 >>> s = pd.Series([1, 2], index=["a", "b"])
6613 >>> s
6614 a 1
6615 b 2
6616 dtype: int64
6618 >>> s_copy = s.copy(deep=True)
6619 >>> s_copy
6620 a 1
6621 b 2
6622 dtype: int64
6624 Due to Copy-on-Write, shallow copies still protect data modifications.
6625 Note shallow does not get modified below.
6627 >>> s = pd.Series([1, 2], index=["a", "b"])
6628 >>> shallow = s.copy(deep=False)
6629 >>> s.iloc[1] = 200
6630 >>> shallow
6631 a 1
6632 b 2
6633 dtype: int64
6635 When the data has object dtype, even a deep copy does not copy the
6636 underlying Python objects. Updating a nested data object will be
6637 reflected in the deep copy.
6639 >>> s = pd.Series([[1, 2], [3, 4]])
6640 >>> deep = s.copy()
6641 >>> s[0][0] = 10
6642 >>> s
6643 0 [10, 2]
6644 1 [3, 4]
6645 dtype: object
6646 >>> deep
6647 0 [10, 2]
6648 1 [3, 4]
6649 dtype: object
6650 """
6651 data = self._mgr.copy(deep=deep)
6652 return self._constructor_from_mgr(data, axes=data.axes).__finalize__(
6653 self, method="copy"
6654 )
6656 @final
6657 def __copy__(self) -> Self:
6658 return self.copy(deep=False)
6660 @final
6661 def __deepcopy__(self, memo=None) -> Self:
6662 """
6663 Parameters
6664 ----------
6665 memo, default None
6666 Standard signature. Unused
6667 """
6668 return self.copy(deep=True)
6670 @final
6671 def infer_objects(self, copy: bool | lib.NoDefault = lib.no_default) -> Self:
6672 """
6673 Attempt to infer better dtypes for object columns.
6675 Attempts soft conversion of object-dtyped
6676 columns, leaving non-object and unconvertible
6677 columns unchanged. The inference rules are the
6678 same as during normal Series/DataFrame construction.
6680 Parameters
6681 ----------
6682 copy : bool, default False
6683 This keyword is now ignored; changing its value will have no
6684 impact on the method.
6686 .. deprecated:: 3.0.0
6688 This keyword is ignored and will be removed in pandas 4.0. Since
6689 pandas 3.0, this method always returns a new object using a lazy
6690 copy mechanism that defers copies until necessary
6691 (Copy-on-Write). See the `user guide on Copy-on-Write
6692 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
6693 for more details.
6695 Returns
6696 -------
6697 same type as input object
6698 Returns an object of the same type as the input object.
6700 See Also
6701 --------
6702 to_datetime : Convert argument to datetime.
6703 to_timedelta : Convert argument to timedelta.
6704 to_numeric : Convert argument to numeric type.
6705 convert_dtypes : Convert argument to best possible dtype.
6707 Examples
6708 --------
6709 >>> df = pd.DataFrame({"A": ["a", 1, 2, 3]})
6710 >>> df = df.iloc[1:]
6711 >>> df
6712 A
6713 1 1
6714 2 2
6715 3 3
6717 >>> df.dtypes
6718 A object
6719 dtype: object
6721 >>> df.infer_objects().dtypes
6722 A int64
6723 dtype: object
6724 """
6725 self._check_copy_deprecation(copy)
6726 new_mgr = self._mgr.convert()
6727 res = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
6728 return res.__finalize__(self, method="infer_objects")
6730 @final
6731 def convert_dtypes(
6732 self,
6733 infer_objects: bool = True,
6734 convert_string: bool = True,
6735 convert_integer: bool = True,
6736 convert_boolean: bool = True,
6737 convert_floating: bool = True,
6738 dtype_backend: DtypeBackend = "numpy_nullable",
6739 ) -> Self:
6740 """
6741 Convert columns from numpy dtypes to the best dtypes that support ``pd.NA``.
6743 Parameters
6744 ----------
6745 infer_objects : bool, default True
6746 Whether object dtypes should be converted to the best possible types.
6747 convert_string : bool, default True
6748 Whether object dtypes should be converted to ``StringDtype()``.
6749 convert_integer : bool, default True
6750 Whether, if possible, conversion can be done to integer extension types.
6751 convert_boolean : bool, defaults True
6752 Whether object dtypes should be converted to ``BooleanDtypes()``.
6753 convert_floating : bool, defaults True
6754 Whether, if possible, conversion can be done to floating extension types.
6755 If `convert_integer` is also True, preference will be give to integer
6756 dtypes if the floats can be faithfully casted to integers.
6757 dtype_backend : {'numpy_nullable', 'pyarrow'}, default 'numpy_nullable'
6758 Back-end data type applied to the resultant :class:`DataFrame` or
6759 :class:`Series` (still experimental). Behaviour is as follows:
6761 * ``"numpy_nullable"``: returns nullable-dtype-backed
6762 :class:`DataFrame` or :class:`Serires`.
6763 * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype`
6764 :class:`DataFrame` or :class:`Series`.
6766 .. versionadded:: 2.0
6768 Returns
6769 -------
6770 Series or DataFrame
6771 Copy of input object with new dtype.
6773 See Also
6774 --------
6775 infer_objects : Infer dtypes of objects.
6776 to_datetime : Convert argument to datetime.
6777 to_timedelta : Convert argument to timedelta.
6778 to_numeric : Convert argument to a numeric type.
6780 Notes
6781 -----
6782 By default, ``convert_dtypes`` will attempt to convert a Series (or each
6783 Series in a DataFrame) to dtypes that support ``pd.NA``. By using the options
6784 ``convert_string``, ``convert_integer``, ``convert_boolean`` and
6785 ``convert_floating``, it is possible to turn off individual conversions
6786 to ``StringDtype``, the integer extension types, ``BooleanDtype``
6787 or floating extension types, respectively.
6789 For object-dtyped columns, if ``infer_objects`` is ``True``, use the inference
6790 rules as during normal Series/DataFrame construction. Then, if possible,
6791 convert to ``StringDtype``, ``BooleanDtype`` or an appropriate integer
6792 or floating extension type, otherwise leave as ``object``.
6794 If the dtype is integer, convert to an appropriate integer extension type.
6796 If the dtype is numeric, and consists of all integers, convert to an
6797 appropriate integer extension type. Otherwise, convert to an
6798 appropriate floating extension type.
6800 In the future, as new dtypes are added that support ``pd.NA``, the results
6801 of this method will change to support those new dtypes.
6803 Examples
6804 --------
6805 >>> df = pd.DataFrame(
6806 ... {
6807 ... "a": pd.Series([1, 2, 3], dtype=np.dtype("int32")),
6808 ... "b": pd.Series(["x", "y", "z"], dtype=np.dtype("O")),
6809 ... "c": pd.Series([True, False, np.nan], dtype=np.dtype("O")),
6810 ... "d": pd.Series(["h", "i", np.nan], dtype=np.dtype("O")),
6811 ... "e": pd.Series([10, np.nan, 20], dtype=np.dtype("float")),
6812 ... "f": pd.Series([np.nan, 100.5, 200], dtype=np.dtype("float")),
6813 ... }
6814 ... )
6816 Start with a DataFrame with default dtypes.
6818 >>> df
6819 a b c d e f
6820 0 1 x True h 10.0 NaN
6821 1 2 y False i NaN 100.5
6822 2 3 z NaN NaN 20.0 200.0
6824 >>> df.dtypes
6825 a int32
6826 b object
6827 c object
6828 d object
6829 e float64
6830 f float64
6831 dtype: object
6833 Convert the DataFrame to use best possible dtypes.
6835 >>> dfn = df.convert_dtypes()
6836 >>> dfn
6837 a b c d e f
6838 0 1 x True h 10 <NA>
6839 1 2 y False i <NA> 100.5
6840 2 3 z <NA> <NA> 20 200.0
6842 >>> dfn.dtypes
6843 a Int32
6844 b string
6845 c boolean
6846 d string
6847 e Int64
6848 f Float64
6849 dtype: object
6851 Start with a Series of strings and missing data represented by ``np.nan``.
6853 >>> s = pd.Series(["a", "b", np.nan])
6854 >>> s
6855 0 a
6856 1 b
6857 2 NaN
6858 dtype: str
6860 Obtain a Series with dtype ``StringDtype``.
6862 >>> s.convert_dtypes()
6863 0 a
6864 1 b
6865 2 <NA>
6866 dtype: string
6867 """
6868 check_dtype_backend(dtype_backend)
6869 new_mgr = self._mgr.convert_dtypes(
6870 infer_objects=infer_objects,
6871 convert_string=convert_string,
6872 convert_integer=convert_integer,
6873 convert_boolean=convert_boolean,
6874 convert_floating=convert_floating,
6875 dtype_backend=dtype_backend,
6876 )
6877 res = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
6878 return res.__finalize__(self, method="convert_dtypes")
6880 # ----------------------------------------------------------------------
6881 # Filling NA's
6883 @final
6884 def _pad_or_backfill(
6885 self,
6886 method: Literal["ffill", "bfill", "pad", "backfill"],
6887 *,
6888 axis: None | Axis = None,
6889 inplace: bool = False,
6890 limit: None | int = None,
6891 limit_area: Literal["inside", "outside"] | None = None,
6892 ):
6893 if axis is None:
6894 axis = 0
6895 axis = self._get_axis_number(axis)
6896 method = clean_fill_method(method)
6898 if axis == 1:
6899 if not self._mgr.is_single_block and inplace:
6900 raise NotImplementedError
6901 # e.g. test_align_fill_method
6902 result = self.T._pad_or_backfill(
6903 method=method, limit=limit, limit_area=limit_area
6904 ).T
6906 return result
6908 new_mgr = self._mgr.pad_or_backfill(
6909 method=method,
6910 limit=limit,
6911 limit_area=limit_area,
6912 inplace=inplace,
6913 )
6914 result = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
6915 if inplace:
6916 self._update_inplace(result)
6917 return self
6918 else:
6919 return result.__finalize__(self, method="fillna")
6921 @final
6922 def fillna(
6923 self,
6924 value: Hashable | Mapping | Series | DataFrame,
6925 *,
6926 axis: Axis | None = None,
6927 inplace: bool = False,
6928 limit: int | None = None,
6929 ) -> Self:
6930 """
6931 Fill NA/NaN values with `value`.
6933 Parameters
6934 ----------
6935 value : scalar, dict, Series, or DataFrame
6936 Value to use to fill holes (e.g. 0), alternately a
6937 dict/Series/DataFrame of values specifying which value to use for
6938 each index (for a Series) or column (for a DataFrame). Values not
6939 in the dict/Series/DataFrame will not be filled. This value cannot
6940 be a list.
6941 axis : {0 or 'index'} for Series, {0 or 'index', 1 or 'columns'} for DataFrame
6942 Axis along which to fill missing values. For `Series`
6943 this parameter is unused and defaults to 0.
6944 inplace : bool, default False
6945 If True, fill in-place. Note: this will modify any
6946 other views on this object (e.g., a no-copy slice for a column in a
6947 DataFrame).
6948 limit : int, default None
6949 This is the maximum number of entries along the entire axis
6950 where NaNs will be filled. Must be greater than 0 if not None.
6952 Returns
6953 -------
6954 Series/DataFrame
6955 Object with missing values filled.
6957 See Also
6958 --------
6959 ffill : Fill values by propagating the last valid observation to next valid.
6960 bfill : Fill values by using the next valid observation to fill the gap.
6961 interpolate : Fill NaN values using interpolation.
6962 reindex : Conform object to new index.
6963 asfreq : Convert TimeSeries to specified frequency.
6965 Notes
6966 -----
6967 For non-object dtype, ``value=None`` will use the NA value of the dtype.
6968 See more details in the :ref:`Filling missing data<missing_data.fillna>`
6969 section.
6971 Examples
6972 --------
6973 >>> df = pd.DataFrame(
6974 ... [
6975 ... [np.nan, 2, np.nan, 0],
6976 ... [3, 4, np.nan, 1],
6977 ... [np.nan, np.nan, np.nan, np.nan],
6978 ... [np.nan, 3, np.nan, 4],
6979 ... ],
6980 ... columns=list("ABCD"),
6981 ... )
6982 >>> df
6983 A B C D
6984 0 NaN 2.0 NaN 0.0
6985 1 3.0 4.0 NaN 1.0
6986 2 NaN NaN NaN NaN
6987 3 NaN 3.0 NaN 4.0
6989 Replace all NaN elements with 0s.
6991 >>> df.fillna(0)
6992 A B C D
6993 0 0.0 2.0 0.0 0.0
6994 1 3.0 4.0 0.0 1.0
6995 2 0.0 0.0 0.0 0.0
6996 3 0.0 3.0 0.0 4.0
6998 Replace all NaN elements in column 'A', 'B', 'C', and 'D', with 0, 1,
6999 2, and 3 respectively.
7001 >>> values = {"A": 0, "B": 1, "C": 2, "D": 3}
7002 >>> df.fillna(value=values)
7003 A B C D
7004 0 0.0 2.0 2.0 0.0
7005 1 3.0 4.0 2.0 1.0
7006 2 0.0 1.0 2.0 3.0
7007 3 0.0 3.0 2.0 4.0
7009 Only replace the first NaN element.
7011 >>> df.fillna(value=values, limit=1)
7012 A B C D
7013 0 0.0 2.0 2.0 0.0
7014 1 3.0 4.0 NaN 1.0
7015 2 NaN 1.0 NaN 3.0
7016 3 NaN 3.0 NaN 4.0
7018 When filling using a DataFrame, replacement happens along
7019 the same column names and same indices
7021 >>> df2 = pd.DataFrame(np.zeros((4, 4)), columns=list("ABCE"))
7022 >>> df.fillna(df2)
7023 A B C D
7024 0 0.0 2.0 0.0 0.0
7025 1 3.0 4.0 0.0 1.0
7026 2 0.0 0.0 0.0 NaN
7027 3 0.0 3.0 0.0 4.0
7029 Note that column D is not affected since it is not present in df2.
7030 """
7031 inplace = validate_bool_kwarg(inplace, "inplace")
7032 if inplace:
7033 if not CHAINED_WARNING_DISABLED:
7034 if sys.getrefcount(
7035 self
7036 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
7037 warnings.warn(
7038 _chained_assignment_method_msg,
7039 ChainedAssignmentError,
7040 stacklevel=2,
7041 )
7043 if isinstance(value, (list, tuple)):
7044 raise TypeError(
7045 '"value" parameter must be a scalar or dict, but '
7046 f'you passed a "{type(value).__name__}"'
7047 )
7049 # set the default here, so functions examining the signature
7050 # can detect if something was set (e.g. in groupby) (GH9221)
7051 if axis is None:
7052 axis = 0
7053 axis = self._get_axis_number(axis)
7055 if self.ndim == 1:
7056 if isinstance(value, (dict, ABCSeries)):
7057 if not len(value):
7058 # test_fillna_nonscalar
7059 return self if inplace else self.copy(deep=False)
7060 from pandas import Series
7062 value = Series(value)
7063 value = value.reindex(self.index)
7064 value = value._values
7065 elif not is_list_like(value):
7066 pass
7067 else:
7068 raise TypeError(
7069 '"value" parameter must be a scalar, dict '
7070 "or Series, but you passed a "
7071 f'"{type(value).__name__}"'
7072 )
7074 new_data = self._mgr.fillna(value=value, limit=limit, inplace=inplace)
7076 elif isinstance(value, (dict, ABCSeries)):
7077 result = self if inplace else self.copy(deep=False)
7078 if axis == 1:
7079 # Check that all columns in result have the same dtype
7080 # otherwise don't bother with fillna and losing accurate dtypes
7081 unique_dtypes = self._mgr.get_unique_dtypes()
7082 if len(unique_dtypes) > 1:
7083 raise ValueError(
7084 "All columns must have the same dtype, but got dtypes: "
7085 f"{list(unique_dtypes)}"
7086 )
7087 # Use the first column, which we have already validated has the
7088 # same dtypes as the other columns.
7089 if not can_hold_element(result.iloc[:, 0], value):
7090 frame_dtype = unique_dtypes.item()
7091 raise ValueError(
7092 f"{value} not a suitable type to fill into {frame_dtype}"
7093 )
7094 result = result.T.fillna(value=value).T
7095 if inplace:
7096 self._update_inplace(result)
7097 result = self
7098 else:
7099 for k, v in value.items():
7100 if k not in result:
7101 continue
7103 res_k = result[k].fillna(v, limit=limit)
7105 if not inplace:
7106 result[k] = res_k
7107 # We can write into our existing column(s) iff dtype
7108 # was preserved.
7109 elif isinstance(res_k, ABCSeries):
7110 # i.e. 'k' only shows up once in self.columns
7111 if res_k.dtype == result[k].dtype:
7112 result.loc[:, k] = res_k
7113 else:
7114 # Different dtype -> no way to do inplace.
7115 result[k] = res_k
7116 else:
7117 # see test_fillna_dict_inplace_nonunique_columns
7118 locs = result.columns.get_loc(k)
7119 if isinstance(locs, slice):
7120 locs = range(self.shape[1])[locs]
7121 elif isinstance(locs, np.ndarray) and locs.dtype.kind == "b":
7122 locs = locs.nonzero()[0]
7123 elif not (
7124 isinstance(locs, np.ndarray) and locs.dtype.kind == "i"
7125 ):
7126 # Should never be reached, but let's cover our bases
7127 raise NotImplementedError(
7128 "Unexpected get_loc result, please report a bug at "
7129 "https://github.com/pandas-dev/pandas"
7130 )
7132 for i, loc in enumerate(locs):
7133 res_loc = res_k.iloc[:, i]
7134 target = self.iloc[:, loc]
7136 if res_loc.dtype == target.dtype:
7137 result.iloc[:, loc] = res_loc
7138 else:
7139 result.isetitem(loc, res_loc)
7140 return result
7142 elif not is_list_like(value):
7143 if axis == 1:
7144 result = self.T.fillna(value=value, limit=limit).T
7145 new_data = result._mgr
7146 else:
7147 new_data = self._mgr.fillna(value=value, limit=limit, inplace=inplace)
7148 elif isinstance(value, ABCDataFrame) and self.ndim == 2:
7149 new_data = self.where(self.notna(), value)._mgr
7150 else:
7151 raise ValueError(f"invalid fill value with a {type(value)}")
7153 result = self._constructor_from_mgr(new_data, axes=new_data.axes)
7154 if inplace:
7155 self._update_inplace(result)
7156 return self
7157 else:
7158 return result.__finalize__(self, method="fillna")
7160 @final
7161 def ffill(
7162 self,
7163 *,
7164 axis: None | Axis = None,
7165 inplace: bool = False,
7166 limit: None | int = None,
7167 limit_area: Literal["inside", "outside"] | None = None,
7168 ) -> Self:
7169 """
7170 Fill NA/NaN values by propagating the last valid observation to next valid.
7172 Parameters
7173 ----------
7174 axis : {0 or 'index'} for Series, {0 or 'index', 1 or 'columns'} for DataFrame
7175 Axis along which to fill missing values. For `Series`
7176 this parameter is unused and defaults to 0.
7177 inplace : bool, default False
7178 If True, fill in-place. Note: this will modify any
7179 other views on this object (e.g., a no-copy slice for a column in a
7180 DataFrame).
7181 limit : int, default None
7182 If method is specified, this is the maximum number of consecutive
7183 NaN values to forward/backward fill. In other words, if there is
7184 a gap with more than this number of consecutive NaNs, it will only
7185 be partially filled. If method is not specified, this is the
7186 maximum number of entries along the entire axis where NaNs will be
7187 filled. Must be greater than 0 if not None.
7188 limit_area : {`None`, 'inside', 'outside'}, default None
7189 If limit is specified, consecutive NaNs will be filled with this
7190 restriction.
7192 * ``None``: No fill restriction.
7193 * 'inside': Only fill NaNs surrounded by valid values
7194 (interpolate).
7195 * 'outside': Only fill NaNs outside valid values (extrapolate).
7197 .. versionadded:: 2.2.0
7199 Returns
7200 -------
7201 Series/DataFrame
7202 Object with missing values filled.
7204 See Also
7205 --------
7206 DataFrame.bfill : Fill NA/NaN values by using the next valid observation
7207 to fill the gap.
7209 Examples
7210 --------
7211 >>> df = pd.DataFrame(
7212 ... [
7213 ... [np.nan, 2, np.nan, 0],
7214 ... [3, 4, np.nan, 1],
7215 ... [np.nan, np.nan, np.nan, np.nan],
7216 ... [np.nan, 3, np.nan, 4],
7217 ... ],
7218 ... columns=list("ABCD"),
7219 ... )
7220 >>> df
7221 A B C D
7222 0 NaN 2.0 NaN 0.0
7223 1 3.0 4.0 NaN 1.0
7224 2 NaN NaN NaN NaN
7225 3 NaN 3.0 NaN 4.0
7227 >>> df.ffill()
7228 A B C D
7229 0 NaN 2.0 NaN 0.0
7230 1 3.0 4.0 NaN 1.0
7231 2 3.0 4.0 NaN 1.0
7232 3 3.0 3.0 NaN 4.0
7234 >>> ser = pd.Series([1, np.nan, 2, 3])
7235 >>> ser.ffill()
7236 0 1.0
7237 1 1.0
7238 2 2.0
7239 3 3.0
7240 dtype: float64
7241 """
7242 inplace = validate_bool_kwarg(inplace, "inplace")
7243 if inplace:
7244 if not CHAINED_WARNING_DISABLED:
7245 if sys.getrefcount(
7246 self
7247 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
7248 warnings.warn(
7249 _chained_assignment_method_msg,
7250 ChainedAssignmentError,
7251 stacklevel=2,
7252 )
7254 return self._pad_or_backfill(
7255 "ffill",
7256 axis=axis,
7257 inplace=inplace,
7258 limit=limit,
7259 limit_area=limit_area,
7260 )
7262 @final
7263 def bfill(
7264 self,
7265 *,
7266 axis: None | Axis = None,
7267 inplace: bool = False,
7268 limit: None | int = None,
7269 limit_area: Literal["inside", "outside"] | None = None,
7270 ) -> Self:
7271 """
7272 Fill NA/NaN values by using the next valid observation to fill the gap.
7274 This method fills missing values in a backward direction along the
7275 specified axis, propagating non-null values from later positions to
7276 earlier positions containing NaN.
7278 Parameters
7279 ----------
7280 axis : {0 or 'index'} for Series, {0 or 'index', 1 or 'columns'} for DataFrame
7281 Axis along which to fill missing values. For `Series`
7282 this parameter is unused and defaults to 0.
7283 inplace : bool, default False
7284 If True, fill in-place. Note: this will modify any
7285 other views on this object (e.g., a no-copy slice for a column in a
7286 DataFrame).
7287 limit : int, default None
7288 If method is specified, this is the maximum number of consecutive
7289 NaN values to forward/backward fill. In other words, if there is
7290 a gap with more than this number of consecutive NaNs, it will only
7291 be partially filled. If method is not specified, this is the
7292 maximum number of entries along the entire axis where NaNs will be
7293 filled. Must be greater than 0 if not None.
7294 limit_area : {`None`, 'inside', 'outside'}, default None
7295 If limit is specified, consecutive NaNs will be filled with this
7296 restriction.
7298 * ``None``: No fill restriction.
7299 * 'inside': Only fill NaNs surrounded by valid values
7300 (interpolate).
7301 * 'outside': Only fill NaNs outside valid values (extrapolate).
7303 .. versionadded:: 2.2.0
7305 Returns
7306 -------
7307 Series/DataFrame
7308 Object with missing values filled.
7310 See Also
7311 --------
7312 DataFrame.ffill : Fill NA/NaN values by propagating the last valid
7313 observation to next valid.
7315 Examples
7316 --------
7317 For Series:
7319 >>> s = pd.Series([1, None, None, 2])
7320 >>> s.bfill()
7321 0 1.0
7322 1 2.0
7323 2 2.0
7324 3 2.0
7325 dtype: float64
7326 >>> s.bfill(limit=1)
7327 0 1.0
7328 1 NaN
7329 2 2.0
7330 3 2.0
7331 dtype: float64
7333 With DataFrame:
7335 >>> df = pd.DataFrame({"A": [1, None, None, 4], "B": [None, 5, None, 7]})
7336 >>> df
7337 A B
7338 0 1.0 NaN
7339 1 NaN 5.0
7340 2 NaN NaN
7341 3 4.0 7.0
7342 >>> df.bfill()
7343 A B
7344 0 1.0 5.0
7345 1 4.0 5.0
7346 2 4.0 7.0
7347 3 4.0 7.0
7348 >>> df.bfill(limit=1)
7349 A B
7350 0 1.0 5.0
7351 1 NaN 5.0
7352 2 4.0 7.0
7353 3 4.0 7.0
7354 """
7355 inplace = validate_bool_kwarg(inplace, "inplace")
7356 if inplace:
7357 if not CHAINED_WARNING_DISABLED:
7358 if sys.getrefcount(
7359 self
7360 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
7361 warnings.warn(
7362 _chained_assignment_method_msg,
7363 ChainedAssignmentError,
7364 stacklevel=2,
7365 )
7367 return self._pad_or_backfill(
7368 "bfill",
7369 axis=axis,
7370 inplace=inplace,
7371 limit=limit,
7372 limit_area=limit_area,
7373 )
7375 @final
7376 def replace(
7377 self,
7378 to_replace=None,
7379 value=lib.no_default,
7380 *,
7381 inplace: bool = False,
7382 regex: bool = False,
7383 ) -> Self:
7384 """
7385 Replace values given in `to_replace` with `value`.
7387 Values of the Series/DataFrame are replaced with other values dynamically.
7388 This differs from updating with ``.loc`` or ``.iloc``, which require
7389 you to specify a location to update with some value.
7391 Parameters
7392 ----------
7393 to_replace : str, regex, list, dict, Series, int, float, or None
7394 How to find the values that will be replaced.
7396 * numeric, str or regex:
7398 - numeric: numeric values equal to `to_replace` will be
7399 replaced with `value`
7400 - str: string exactly matching `to_replace` will be replaced
7401 with `value`
7402 - regex: regexes matching `to_replace` will be replaced with
7403 `value`
7405 * list of str, regex, or numeric:
7407 - First, if `to_replace` and `value` are both lists, they
7408 **must** be the same length.
7409 - Second, if ``regex=True`` then all of the strings in **both**
7410 lists will be interpreted as regexes otherwise they will match
7411 directly. This doesn't matter much for `value` since there
7412 are only a few possible substitution regexes you can use.
7413 - str, regex and numeric rules apply as above.
7415 * dict:
7417 - Dicts can be used to specify different replacement values
7418 for different existing values. For example,
7419 ``{'a': 'b', 'y': 'z'}`` replaces the value 'a' with 'b' and
7420 'y' with 'z'. To use a dict in this way, the optional `value`
7421 parameter should not be given.
7422 - For a DataFrame a dict can specify that different values
7423 should be replaced in different columns. For example,
7424 ``{'a': 1, 'b': 'z'}`` looks for the value 1 in column 'a'
7425 and the value 'z' in column 'b' and replaces these values
7426 with whatever is specified in `value`. The `value` parameter
7427 should not be ``None`` in this case. You can treat this as a
7428 special case of passing two lists except that you are
7429 specifying the column to search in.
7430 - For a DataFrame nested dictionaries, e.g.,
7431 ``{'a': {'b': np.nan}}``, are read as follows: look in column
7432 'a' for the value 'b' and replace it with NaN. The optional `value`
7433 parameter should not be specified to use a nested dict in this
7434 way. You can nest regular expressions as well. Note that
7435 column names (the top-level dictionary keys in a nested
7436 dictionary) **cannot** be regular expressions.
7438 * None:
7440 - This means that the `regex` argument must be a string,
7441 compiled regular expression, or list, dict, ndarray or
7442 Series of such elements. If `value` is also ``None`` then
7443 this **must** be a nested dictionary or Series.
7445 See the examples section for examples of each of these.
7446 value : scalar, dict, list, str, regex, default None
7447 Value to replace any values matching `to_replace` with.
7448 For a DataFrame a dict of values can be used to specify which
7449 value to use for each column (columns not in the dict will not be
7450 filled). Regular expressions, strings and lists or dicts of such
7451 objects are also allowed.
7453 inplace : bool, default False
7454 If True, performs operation inplace.
7455 regex : bool or same types as `to_replace`, default False
7456 Whether to interpret `to_replace` and/or `value` as regular
7457 expressions. Alternatively, this could be a regular expression or a
7458 list, dict, or array of regular expressions in which case
7459 `to_replace` must be ``None``.
7461 Returns
7462 -------
7463 Series/DataFrame
7464 Object after replacement.
7466 Raises
7467 ------
7468 AssertionError
7469 * If `regex` is not a ``bool`` and `to_replace` is not
7470 ``None``.
7472 TypeError
7473 * If `to_replace` is not a scalar, array-like, ``dict``, or ``None``
7474 * If `to_replace` is a ``dict`` and `value` is not a ``list``,
7475 ``dict``, ``ndarray``, or ``Series``
7476 * If `to_replace` is ``None`` and `regex` is not compilable
7477 into a regular expression or is a list, dict, ndarray, or
7478 Series.
7479 * When replacing multiple ``bool`` or ``datetime64`` objects and
7480 the arguments to `to_replace` does not match the type of the
7481 value being replaced
7483 ValueError
7484 * If a ``list`` or an ``ndarray`` is passed to `to_replace` and
7485 `value` but they are not the same length.
7487 See Also
7488 --------
7489 Series.fillna : Fill NA values.
7490 DataFrame.fillna : Fill NA values.
7491 Series.where : Replace values based on boolean condition.
7492 DataFrame.where : Replace values based on boolean condition.
7493 DataFrame.map: Apply a function to a Dataframe elementwise.
7494 Series.map: Map values of Series according to an input mapping or function.
7495 Series.str.replace : Simple string replacement.
7497 Notes
7498 -----
7499 * Regex substitution is performed under the hood with ``re.sub``. The
7500 rules for substitution for ``re.sub`` are the same.
7501 * Regular expressions will only substitute on strings, meaning you
7502 cannot provide, for example, a regular expression matching floating
7503 point numbers and expect the columns in your frame that have a
7504 numeric dtype to be matched. However, if those floating point
7505 numbers *are* strings, then you can do this.
7506 * This method has *a lot* of options. You are encouraged to experiment
7507 and play with this method to gain intuition about how it works.
7508 * When dict is used as the `to_replace` value, it is like
7509 key(s) in the dict are the to_replace part and
7510 value(s) in the dict are the value parameter.
7512 Examples
7513 --------
7515 **Scalar `to_replace` and `value`**
7517 >>> s = pd.Series([1, 2, 3, 4, 5])
7518 >>> s.replace(1, 5)
7519 0 5
7520 1 2
7521 2 3
7522 3 4
7523 4 5
7524 dtype: int64
7526 >>> df = pd.DataFrame(
7527 ... {
7528 ... "A": [0, 1, 2, 3, 4],
7529 ... "B": [5, 6, 7, 8, 9],
7530 ... "C": ["a", "b", "c", "d", "e"],
7531 ... }
7532 ... )
7533 >>> df.replace(0, 5)
7534 A B C
7535 0 5 5 a
7536 1 1 6 b
7537 2 2 7 c
7538 3 3 8 d
7539 4 4 9 e
7541 **List-like `to_replace`**
7543 >>> df.replace([0, 1, 2, 3], 4)
7544 A B C
7545 0 4 5 a
7546 1 4 6 b
7547 2 4 7 c
7548 3 4 8 d
7549 4 4 9 e
7551 >>> df.replace([0, 1, 2, 3], [4, 3, 2, 1])
7552 A B C
7553 0 4 5 a
7554 1 3 6 b
7555 2 2 7 c
7556 3 1 8 d
7557 4 4 9 e
7559 **dict-like `to_replace`**
7561 >>> df.replace({0: 10, 1: 100})
7562 A B C
7563 0 10 5 a
7564 1 100 6 b
7565 2 2 7 c
7566 3 3 8 d
7567 4 4 9 e
7569 >>> df.replace({"A": 0, "B": 5}, 100)
7570 A B C
7571 0 100 100 a
7572 1 1 6 b
7573 2 2 7 c
7574 3 3 8 d
7575 4 4 9 e
7577 >>> df.replace({"A": {0: 100, 4: 400}})
7578 A B C
7579 0 100 5 a
7580 1 1 6 b
7581 2 2 7 c
7582 3 3 8 d
7583 4 400 9 e
7585 **Regular expression `to_replace`**
7587 >>> df = pd.DataFrame({"A": ["bat", "foo", "bait"], "B": ["abc", "bar", "xyz"]})
7588 >>> df.replace(to_replace=r"^ba.$", value="new", regex=True)
7589 A B
7590 0 new abc
7591 1 foo new
7592 2 bait xyz
7594 >>> df.replace({"A": r"^ba.$"}, {"A": "new"}, regex=True)
7595 A B
7596 0 new abc
7597 1 foo bar
7598 2 bait xyz
7600 >>> df.replace(regex=r"^ba.$", value="new")
7601 A B
7602 0 new abc
7603 1 foo new
7604 2 bait xyz
7606 >>> df.replace(regex={r"^ba.$": "new", "foo": "xyz"})
7607 A B
7608 0 new abc
7609 1 xyz new
7610 2 bait xyz
7612 >>> df.replace(regex=[r"^ba.$", "foo"], value="new")
7613 A B
7614 0 new abc
7615 1 new new
7616 2 bait xyz
7618 Compare the behavior of ``s.replace({'a': None})`` and
7619 ``s.replace('a', None)`` to understand the peculiarities
7620 of the `to_replace` parameter:
7622 >>> s = pd.Series([10, "a", "a", "b", "a"])
7624 When one uses a dict as the `to_replace` value, it is like the
7625 value(s) in the dict are equal to the `value` parameter.
7626 ``s.replace({'a': None})`` is equivalent to
7627 ``s.replace(to_replace={'a': None}, value=None)``:
7629 >>> s.replace({"a": None})
7630 0 10
7631 1 None
7632 2 None
7633 3 b
7634 4 None
7635 dtype: object
7637 If ``None`` is explicitly passed for ``value``, it will be respected:
7639 >>> s.replace("a", None)
7640 0 10
7641 1 None
7642 2 None
7643 3 b
7644 4 None
7645 dtype: object
7647 When ``regex=True``, ``value`` is not ``None`` and `to_replace` is a string,
7648 the replacement will be applied in all columns of the DataFrame.
7650 >>> df = pd.DataFrame(
7651 ... {
7652 ... "A": [0, 1, 2, 3, 4],
7653 ... "B": ["a", "b", "c", "d", "e"],
7654 ... "C": ["f", "g", "h", "i", "j"],
7655 ... }
7656 ... )
7658 >>> df.replace(to_replace="^[a-g]", value="e", regex=True)
7659 A B C
7660 0 0 e e
7661 1 1 e e
7662 2 2 e h
7663 3 3 e i
7664 4 4 e j
7666 If ``value`` is not ``None`` and `to_replace` is a dictionary, the dictionary
7667 keys will be the DataFrame columns that the replacement will be applied.
7669 >>> df.replace(to_replace={"B": "^[a-c]", "C": "^[h-j]"}, value="e", regex=True)
7670 A B C
7671 0 0 e f
7672 1 1 e g
7673 2 2 e e
7674 3 3 d e
7675 4 4 e e
7676 """
7677 if not is_bool(regex) and to_replace is not None:
7678 raise ValueError("'to_replace' must be 'None' if 'regex' is not a bool")
7680 if not (
7681 is_scalar(to_replace)
7682 or is_re_compilable(to_replace)
7683 or is_list_like(to_replace)
7684 ):
7685 raise TypeError(
7686 "Expecting 'to_replace' to be either a scalar, array-like, "
7687 "dict or None, got invalid type "
7688 f"{type(to_replace).__name__!r}"
7689 )
7691 if value is lib.no_default and not (
7692 is_dict_like(to_replace) or is_dict_like(regex)
7693 ):
7694 raise ValueError(
7695 # GH#33302
7696 f"{type(self).__name__}.replace must specify either 'value', "
7697 "a dict-like 'to_replace', or dict-like 'regex'."
7698 )
7700 inplace = validate_bool_kwarg(inplace, "inplace")
7701 if inplace:
7702 if not CHAINED_WARNING_DISABLED:
7703 if sys.getrefcount(
7704 self
7705 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
7706 warnings.warn(
7707 _chained_assignment_method_msg,
7708 ChainedAssignmentError,
7709 stacklevel=2,
7710 )
7712 if value is lib.no_default:
7713 if not is_dict_like(to_replace):
7714 # In this case we have checked above that
7715 # 1) regex is dict-like and 2) to_replace is None
7716 to_replace = regex
7717 regex = True
7719 items = list(to_replace.items())
7720 if items:
7721 keys, values = zip(*items, strict=True)
7722 else:
7723 keys, values = ([], []) # type: ignore[assignment]
7725 are_mappings = [is_dict_like(v) for v in values]
7727 if any(are_mappings):
7728 if not all(are_mappings):
7729 raise TypeError(
7730 "If a nested mapping is passed, all values "
7731 "of the top level mapping must be mappings"
7732 )
7733 # passed a nested dict/Series
7734 to_rep_dict = {}
7735 value_dict = {}
7737 for k, v in items:
7738 # error: Incompatible types in assignment (expression has type
7739 # "list[Never]", variable has type "tuple[Any, ...]")
7740 keys, values = list(zip(*v.items(), strict=True)) or ( # type: ignore[assignment]
7741 [],
7742 [],
7743 )
7745 to_rep_dict[k] = list(keys)
7746 value_dict[k] = list(values)
7748 to_replace, value = to_rep_dict, value_dict
7749 else:
7750 to_replace, value = keys, values
7752 return self.replace(to_replace, value, inplace=inplace, regex=regex)
7753 else:
7754 # need a non-zero len on all axes
7755 if not self.size:
7756 return self if inplace else self.copy(deep=False)
7757 if is_dict_like(to_replace):
7758 if is_dict_like(value): # {'A' : NA} -> {'A' : 0}
7759 if isinstance(self, ABCSeries):
7760 raise ValueError(
7761 "to_replace and value cannot be dict-like for "
7762 "Series.replace"
7763 )
7764 # Note: Checking below for `in foo.keys()` instead of
7765 # `in foo` is needed for when we have a Series and not dict
7766 mapping = {
7767 col: (to_replace[col], value[col])
7768 for col in to_replace.keys()
7769 if col in value.keys() and col in self
7770 }
7771 return self._replace_columnwise(mapping, inplace, regex)
7773 # {'A': NA} -> 0
7774 elif not is_list_like(value):
7775 # Operate column-wise
7776 if self.ndim == 1:
7777 raise ValueError(
7778 "Series.replace cannot specify both a dict-like "
7779 "'to_replace' and a 'value'"
7780 )
7781 mapping = {
7782 col: (to_rep, value) for col, to_rep in to_replace.items()
7783 }
7784 return self._replace_columnwise(mapping, inplace, regex)
7785 else:
7786 raise TypeError("value argument must be scalar, dict, or Series")
7788 elif is_list_like(to_replace):
7789 if not is_list_like(value):
7790 # e.g. to_replace = [NA, ''] and value is 0,
7791 # so we replace NA with 0 and then replace '' with 0
7792 value = [value] * len(to_replace)
7794 # e.g. we have to_replace = [NA, ''] and value = [0, 'missing']
7795 if len(to_replace) != len(value):
7796 raise ValueError(
7797 f"Replacement lists must match in length. "
7798 f"Expecting {len(to_replace)} got {len(value)} "
7799 )
7800 new_data = self._mgr.replace_list(
7801 src_list=to_replace,
7802 dest_list=value,
7803 inplace=inplace,
7804 regex=regex,
7805 )
7807 elif to_replace is None:
7808 if not (
7809 is_re_compilable(regex)
7810 or is_list_like(regex)
7811 or is_dict_like(regex)
7812 ):
7813 raise TypeError(
7814 f"'regex' must be a string or a compiled regular expression "
7815 f"or a list or dict of strings or regular expressions, "
7816 f"you passed a {type(regex).__name__!r}"
7817 )
7818 return self.replace(regex, value, inplace=inplace, regex=True)
7819 # dest iterable dict-like
7820 elif is_dict_like(value): # NA -> {'A' : 0, 'B' : -1}
7821 # Operate column-wise
7822 if self.ndim == 1:
7823 raise ValueError(
7824 "Series.replace cannot use dict-value and non-None to_replace"
7825 )
7826 mapping = {col: (to_replace, val) for col, val in value.items()}
7827 return self._replace_columnwise(mapping, inplace, regex)
7829 elif not is_list_like(value): # NA -> 0
7830 regex = should_use_regex(regex, to_replace)
7831 if regex:
7832 new_data = self._mgr.replace_regex(
7833 to_replace=to_replace,
7834 value=value,
7835 inplace=inplace,
7836 )
7837 else:
7838 new_data = self._mgr.replace(
7839 to_replace=to_replace, value=value, inplace=inplace
7840 )
7841 else:
7842 raise TypeError(
7843 f'Invalid "to_replace" type: {type(to_replace).__name__!r}'
7844 )
7846 result = self._constructor_from_mgr(new_data, axes=new_data.axes)
7847 if inplace:
7848 self._update_inplace(result)
7849 return self
7850 else:
7851 return result.__finalize__(self, method="replace")
7853 @final
7854 def interpolate(
7855 self,
7856 method: InterpolateOptions = "linear",
7857 *,
7858 axis: Axis = 0,
7859 limit: int | None = None,
7860 inplace: bool = False,
7861 limit_direction: Literal["forward", "backward", "both"] | None = None,
7862 limit_area: Literal["inside", "outside"] | None = None,
7863 **kwargs,
7864 ) -> Self:
7865 """
7866 Fill NaN values using an interpolation method.
7868 Please note that only ``method='linear'`` is supported for
7869 DataFrame/Series with a MultiIndex.
7871 Parameters
7872 ----------
7873 method : str, default 'linear'
7874 Interpolation technique to use. One of:
7876 * 'linear': Ignore the index and treat the values as equally
7877 spaced. This is the only method supported on MultiIndexes.
7878 * 'time': Works on daily and higher resolution data to interpolate
7879 given length of interval. This interpolates values based on
7880 time interval between observations.
7881 * 'index': The interpolation uses the numerical values
7882 of the DataFrame's index to linearly calculate missing values.
7883 * 'values': Interpolation based on the numerical values
7884 in the DataFrame, treating them as equally spaced along the index.
7885 * 'nearest', 'zero', 'slinear', 'quadratic', 'cubic',
7886 'barycentric', 'polynomial': Passed to
7887 `scipy.interpolate.interp1d`, whereas 'spline' is passed to
7888 `scipy.interpolate.UnivariateSpline`. These methods use the numerical
7889 values of the index. Both 'polynomial' and 'spline' require that
7890 you also specify an `order` (int), e.g.
7891 ``df.interpolate(method='polynomial', order=5)``. Note that,
7892 `slinear` method in Pandas refers to the Scipy first order `spline`
7893 instead of Pandas first order `spline`.
7894 * 'krogh', 'piecewise_polynomial', 'spline', 'pchip', 'akima',
7895 'cubicspline': Wrappers around the SciPy interpolation methods of
7896 similar names. See `Notes`.
7897 * 'from_derivatives': Refers to
7898 `scipy.interpolate.BPoly.from_derivatives`.
7900 axis : {0 or 'index', 1 or 'columns', None}, default None
7901 Axis to interpolate along. For `Series` this parameter is unused
7902 and defaults to 0.
7903 limit : int, optional
7904 Maximum number of consecutive NaNs to fill. Must be greater than
7905 0.
7906 inplace : bool, default False
7907 Update the data in place if possible.
7908 limit_direction : {'forward', 'backward', 'both'}, optional, default 'forward'
7909 Consecutive NaNs will be filled in this direction.
7911 limit_area : {`None`, 'inside', 'outside'}, default None
7912 If limit is specified, consecutive NaNs will be filled with this
7913 restriction.
7915 * ``None``: No fill restriction.
7916 * 'inside': Only fill NaNs surrounded by valid values
7917 (interpolate).
7918 * 'outside': Only fill NaNs outside valid values (extrapolate).
7920 **kwargs : optional
7921 Keyword arguments to pass on to the interpolating function.
7923 Returns
7924 -------
7925 Series or DataFrame
7926 Returns the same object type as the caller, interpolated at
7927 some or all ``NaN`` values.
7929 See Also
7930 --------
7931 fillna : Fill missing values using different methods.
7932 scipy.interpolate.Akima1DInterpolator : Piecewise cubic polynomials
7933 (Akima interpolator).
7934 scipy.interpolate.BPoly.from_derivatives : Piecewise polynomial in the
7935 Bernstein basis.
7936 scipy.interpolate.interp1d : Interpolate a 1-D function.
7937 scipy.interpolate.KroghInterpolator : Interpolate polynomial (Krogh
7938 interpolator).
7939 scipy.interpolate.PchipInterpolator : PCHIP 1-d monotonic cubic
7940 interpolation.
7941 scipy.interpolate.CubicSpline : Cubic spline data interpolator.
7943 Notes
7944 -----
7945 The 'krogh', 'piecewise_polynomial', 'spline', 'pchip' and 'akima'
7946 methods are wrappers around the respective SciPy implementations of
7947 similar names. These use the actual numerical values of the index.
7948 For more information on their behavior, see the
7949 `SciPy documentation
7950 <https://docs.scipy.org/doc/scipy/reference/interpolate.html#univariate-interpolation>`__.
7952 Examples
7953 --------
7954 Filling in ``NaN`` in a :class:`~pandas.Series` via linear
7955 interpolation.
7957 >>> s = pd.Series([0, 1, np.nan, 3])
7958 >>> s
7959 0 0.0
7960 1 1.0
7961 2 NaN
7962 3 3.0
7963 dtype: float64
7964 >>> s.interpolate()
7965 0 0.0
7966 1 1.0
7967 2 2.0
7968 3 3.0
7969 dtype: float64
7971 Filling in ``NaN`` in a Series via polynomial interpolation or splines:
7972 Both 'polynomial' and 'spline' methods require that you also specify
7973 an ``order`` (int).
7975 >>> s = pd.Series([0, 2, np.nan, 8])
7976 >>> s.interpolate(method="polynomial", order=2)
7977 0 0.000000
7978 1 2.000000
7979 2 4.666667
7980 3 8.000000
7981 dtype: float64
7983 Fill the DataFrame forward (that is, going down) along each column
7984 using linear interpolation.
7986 Note how the last entry in column 'a' is interpolated differently,
7987 because there is no entry after it to use for interpolation.
7988 Note how the first entry in column 'b' remains ``NaN``, because there
7989 is no entry before it to use for interpolation.
7991 >>> df = pd.DataFrame(
7992 ... [
7993 ... (0.0, np.nan, -1.0, 1.0),
7994 ... (np.nan, 2.0, np.nan, np.nan),
7995 ... (2.0, 3.0, np.nan, 9.0),
7996 ... (np.nan, 4.0, -4.0, 16.0),
7997 ... ],
7998 ... columns=list("abcd"),
7999 ... )
8000 >>> df
8001 a b c d
8002 0 0.0 NaN -1.0 1.0
8003 1 NaN 2.0 NaN NaN
8004 2 2.0 3.0 NaN 9.0
8005 3 NaN 4.0 -4.0 16.0
8006 >>> df.interpolate(method="linear", limit_direction="forward", axis=0)
8007 a b c d
8008 0 0.0 NaN -1.0 1.0
8009 1 1.0 2.0 -2.0 5.0
8010 2 2.0 3.0 -3.0 9.0
8011 3 2.0 4.0 -4.0 16.0
8013 Using polynomial interpolation.
8015 >>> df["d"].interpolate(method="polynomial", order=2)
8016 0 1.0
8017 1 4.0
8018 2 9.0
8019 3 16.0
8020 Name: d, dtype: float64
8021 """
8022 inplace = validate_bool_kwarg(inplace, "inplace")
8024 if inplace:
8025 if not CHAINED_WARNING_DISABLED:
8026 if sys.getrefcount(
8027 self
8028 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
8029 warnings.warn(
8030 _chained_assignment_method_msg,
8031 ChainedAssignmentError,
8032 stacklevel=2,
8033 )
8035 axis = self._get_axis_number(axis)
8037 if self.empty:
8038 return self if inplace else self.copy()
8040 if not isinstance(method, str):
8041 raise ValueError("'method' should be a string, not None.")
8043 obj, should_transpose = (self.T, True) if axis == 1 else (self, False)
8045 if isinstance(obj.index, MultiIndex) and method != "linear":
8046 raise ValueError(
8047 "Only `method=linear` interpolation is supported on MultiIndexes."
8048 )
8050 limit_direction = missing.infer_limit_direction(limit_direction, method)
8052 index = missing.get_interp_index(method, obj.index)
8053 new_data = obj._mgr.interpolate(
8054 method=method,
8055 index=index,
8056 limit=limit,
8057 limit_direction=limit_direction,
8058 limit_area=limit_area,
8059 inplace=inplace,
8060 **kwargs,
8061 )
8063 result = self._constructor_from_mgr(new_data, axes=new_data.axes)
8064 if should_transpose:
8065 result = result.T
8066 if inplace:
8067 self._update_inplace(result)
8068 return self
8069 else:
8070 return result.__finalize__(self, method="interpolate")
8072 # ----------------------------------------------------------------------
8073 # Timeseries methods Methods
8075 @final
8076 def asof(self, where, subset=None):
8077 """
8078 Return the last row(s) without any NaNs before `where`.
8080 The last row (for each element in `where`, if list) without any
8081 NaN is taken.
8082 In case of a :class:`~pandas.DataFrame`, the last row without NaN
8083 considering only the subset of columns (if not `None`)
8085 If there is no good value, NaN is returned for a Series or
8086 a Series of NaN values for a DataFrame
8088 Parameters
8089 ----------
8090 where : date or array-like of dates
8091 Date(s) before which the last row(s) are returned.
8092 subset : str or array-like of str, default `None`
8093 For DataFrame, if not `None`, only use these columns to
8094 check for NaNs.
8096 Returns
8097 -------
8098 scalar, Series, or DataFrame
8100 The return can be:
8102 * scalar : when `self` is a Series and `where` is a scalar
8103 * Series: when `self` is a Series and `where` is an array-like,
8104 or when `self` is a DataFrame and `where` is a scalar
8105 * DataFrame : when `self` is a DataFrame and `where` is an
8106 array-like
8108 See Also
8109 --------
8110 merge_asof : Perform an asof merge. Similar to left join.
8112 Notes
8113 -----
8114 Dates are assumed to be sorted. Raises if this is not the case.
8116 Examples
8117 --------
8118 A Series and a scalar `where`.
8120 >>> s = pd.Series([1, 2, np.nan, 4], index=[10, 20, 30, 40])
8121 >>> s
8122 10 1.0
8123 20 2.0
8124 30 NaN
8125 40 4.0
8126 dtype: float64
8128 >>> s.asof(20)
8129 np.float64(2.0)
8131 For a sequence `where`, a Series is returned. The first value is
8132 NaN, because the first element of `where` is before the first
8133 index value.
8135 >>> s.asof([5, 20])
8136 5 NaN
8137 20 2.0
8138 dtype: float64
8140 Missing values are not considered. The following is ``2.0``, not
8141 NaN, even though NaN is at the index location for ``30``.
8143 >>> s.asof(30)
8144 np.float64(2.0)
8146 Take all columns into consideration
8148 >>> df = pd.DataFrame(
8149 ... {
8150 ... "a": [10.0, 20.0, 30.0, 40.0, 50.0],
8151 ... "b": [None, None, None, None, 500],
8152 ... },
8153 ... index=pd.DatetimeIndex(
8154 ... [
8155 ... "2018-02-27 09:01:00",
8156 ... "2018-02-27 09:02:00",
8157 ... "2018-02-27 09:03:00",
8158 ... "2018-02-27 09:04:00",
8159 ... "2018-02-27 09:05:00",
8160 ... ]
8161 ... ),
8162 ... )
8163 >>> df.asof(pd.DatetimeIndex(["2018-02-27 09:03:30", "2018-02-27 09:04:30"]))
8164 a b
8165 2018-02-27 09:03:30 NaN NaN
8166 2018-02-27 09:04:30 NaN NaN
8168 Take a single column into consideration
8170 >>> df.asof(
8171 ... pd.DatetimeIndex(["2018-02-27 09:03:30", "2018-02-27 09:04:30"]),
8172 ... subset=["a"],
8173 ... )
8174 a b
8175 2018-02-27 09:03:30 30.0 NaN
8176 2018-02-27 09:04:30 40.0 NaN
8177 """
8178 if isinstance(where, str):
8179 where = Timestamp(where)
8181 if not self.index.is_monotonic_increasing:
8182 raise ValueError("asof requires a sorted index")
8184 is_series = isinstance(self, ABCSeries)
8185 if is_series:
8186 if subset is not None:
8187 raise ValueError("subset is not valid for Series")
8188 else:
8189 if subset is None:
8190 subset = self.columns
8191 if not is_list_like(subset):
8192 subset = [subset]
8194 is_list = is_list_like(where)
8195 if not is_list:
8196 start = self.index[0]
8197 if isinstance(self.index, PeriodIndex):
8198 where = Period(where, freq=self.index.freq)
8200 if where < start:
8201 if not is_series:
8202 return self._constructor_sliced(
8203 index=self.columns, name=where, dtype=np.float64
8204 )
8205 return np.nan
8207 # It's always much faster to use a *while* loop here for
8208 # Series than pre-computing all the NAs. However a
8209 # *while* loop is extremely expensive for DataFrame
8210 # so we later pre-compute all the NAs and use the same
8211 # code path whether *where* is a scalar or list.
8212 # See PR: https://github.com/pandas-dev/pandas/pull/14476
8213 if is_series:
8214 loc = self.index.searchsorted(where, side="right")
8215 if loc > 0:
8216 loc -= 1
8218 values = self._values
8219 while loc > 0 and isna(values[loc]):
8220 loc -= 1
8221 return values[loc]
8223 if not isinstance(where, Index):
8224 where = Index(where) if is_list else Index([where])
8226 nulls = self.isna() if is_series else self[subset].isna().any(axis=1)
8227 if nulls.all():
8228 if is_series:
8229 self = cast("Series", self)
8230 return self._constructor(np.nan, index=where, name=self.name)
8231 elif is_list:
8232 self = cast("DataFrame", self)
8233 return self._constructor(np.nan, index=where, columns=self.columns)
8234 else:
8235 self = cast("DataFrame", self)
8236 return self._constructor_sliced(
8237 np.nan, index=self.columns, name=where[0]
8238 )
8240 # error: Unsupported operand type for
8241 # ~ ("ExtensionArray | ndarray[Any, Any] | Any")
8242 locs = self.index.asof_locs(where, ~nulls._values) # type: ignore[operator]
8244 # mask the missing
8245 mask = locs == -1
8246 data = self.take(locs)
8247 data.index = where
8248 if mask.any():
8249 # GH#16063 only do this setting when necessary, otherwise
8250 # we'd cast e.g. bools to floats
8251 data.loc[mask] = np.nan
8252 return data if is_list else data.iloc[-1]
8254 # ----------------------------------------------------------------------
8255 # Action Methods
8257 def isna(self) -> Self:
8258 """
8259 Detect missing values.
8261 Return a boolean same-sized object indicating if the values are NA.
8262 NA values, such as None or :attr:`numpy.NaN`, gets mapped to True
8263 values.
8264 Everything else gets mapped to False values. Characters such as empty
8265 strings ``''`` or :attr:`numpy.inf` are not considered NA values.
8267 Returns
8268 -------
8269 Series/DataFrame
8270 Mask of bool values for each element in Series/DataFrame
8271 that indicates whether an element is an NA value.
8273 See Also
8274 --------
8275 Series.isnull : Alias of isna.
8276 DataFrame.isnull : Alias of isna.
8277 Series.notna : Boolean inverse of isna.
8278 DataFrame.notna : Boolean inverse of isna.
8279 Series.dropna : Omit axes labels with missing values.
8280 DataFrame.dropna : Omit axes labels with missing values.
8281 isna : Top-level isna.
8283 Examples
8284 --------
8285 Show which entries in a DataFrame are NA.
8287 >>> df = pd.DataFrame(
8288 ... dict(
8289 ... age=[5, 6, np.nan],
8290 ... born=[
8291 ... pd.NaT,
8292 ... pd.Timestamp("1939-05-27"),
8293 ... pd.Timestamp("1940-04-25"),
8294 ... ],
8295 ... name=["Alfred", "Batman", ""],
8296 ... toy=[None, "Batmobile", "Joker"],
8297 ... )
8298 ... )
8299 >>> df
8300 age born name toy
8301 0 5.0 NaT Alfred NaN
8302 1 6.0 1939-05-27 Batman Batmobile
8303 2 NaN 1940-04-25 Joker
8305 >>> df.isna()
8306 age born name toy
8307 0 False True False True
8308 1 False False False False
8309 2 True False False False
8311 Show which entries in a Series are NA.
8313 >>> ser = pd.Series([5, 6, np.nan])
8314 >>> ser
8315 0 5.0
8316 1 6.0
8317 2 NaN
8318 dtype: float64
8320 >>> ser.isna()
8321 0 False
8322 1 False
8323 2 True
8324 dtype: bool
8325 """
8326 return isna(self).__finalize__(self, method="isna")
8328 def isnull(self) -> Self:
8329 """
8330 Detect missing values.
8332 Return a boolean same-sized object indicating if the values are NA.
8333 NA values, such as None or :attr:`numpy.NaN`, gets mapped to True
8334 values.
8335 Everything else gets mapped to False values. Characters such as empty
8336 strings ``''`` or :attr:`numpy.inf` are not considered NA values.
8338 Returns
8339 -------
8340 Series/DataFrame
8341 Mask of bool values for each element in Series/DataFrame
8342 that indicates whether an element is an NA value.
8344 See Also
8345 --------
8346 Series.isna : Alias of isnull.
8347 DataFrame.isna : Alias of isnull.
8348 Series.notna : Boolean inverse of isnull.
8349 DataFrame.notna : Boolean inverse of isnull.
8350 Series.dropna : Omit axes labels with missing values.
8351 DataFrame.dropna : Omit axes labels with missing values.
8352 isna : Top-level isna.
8354 Examples
8355 --------
8356 Show which entries in a DataFrame are NA.
8358 >>> df = pd.DataFrame(
8359 ... dict(
8360 ... age=[5, 6, np.nan],
8361 ... born=[
8362 ... pd.NaT,
8363 ... pd.Timestamp("1939-05-27"),
8364 ... pd.Timestamp("1940-04-25"),
8365 ... ],
8366 ... name=["Alfred", "Batman", ""],
8367 ... toy=[None, "Batmobile", "Joker"],
8368 ... )
8369 ... )
8370 >>> df
8371 age born name toy
8372 0 5.0 NaT Alfred NaN
8373 1 6.0 1939-05-27 Batman Batmobile
8374 2 NaN 1940-04-25 Joker
8376 >>> df.isna()
8377 age born name toy
8378 0 False True False True
8379 1 False False False False
8380 2 True False False False
8382 Show which entries in a Series are NA.
8384 >>> ser = pd.Series([5, 6, np.nan])
8385 >>> ser
8386 0 5.0
8387 1 6.0
8388 2 NaN
8389 dtype: float64
8391 >>> ser.isna()
8392 0 False
8393 1 False
8394 2 True
8395 dtype: bool
8396 """
8397 return isna(self).__finalize__(self, method="isnull")
8399 def notna(self) -> Self:
8400 """
8401 Detect existing (non-missing) values.
8403 Return a boolean same-sized object indicating if the values are not NA.
8404 Non-missing values get mapped to True. Characters such as empty
8405 strings ``''`` or :attr:`numpy.inf` are not considered NA values.
8406 NA values, such as None or :attr:`numpy.NaN`, get mapped to False
8407 values.
8409 Returns
8410 -------
8411 Series/DataFrame
8412 Mask of bool values for each element in Series/DataFrame
8413 that indicates whether an element is not an NA value.
8415 See Also
8416 --------
8417 Series.notnull : Alias of notna.
8418 DataFrame.notnull : Alias of notna.
8419 Series.isna : Boolean inverse of notna.
8420 DataFrame.isna : Boolean inverse of notna.
8421 Series.dropna : Omit axes labels with missing values.
8422 DataFrame.dropna : Omit axes labels with missing values.
8423 notna : Top-level notna.
8425 Examples
8426 --------
8427 Show which entries in a DataFrame are not NA.
8429 >>> df = pd.DataFrame(
8430 ... dict(
8431 ... age=[5, 6, np.nan],
8432 ... born=[
8433 ... pd.NaT,
8434 ... pd.Timestamp("1939-05-27"),
8435 ... pd.Timestamp("1940-04-25"),
8436 ... ],
8437 ... name=["Alfred", "Batman", ""],
8438 ... toy=[None, "Batmobile", "Joker"],
8439 ... )
8440 ... )
8441 >>> df
8442 age born name toy
8443 0 5.0 NaT Alfred NaN
8444 1 6.0 1939-05-27 Batman Batmobile
8445 2 NaN 1940-04-25 Joker
8447 >>> df.notna()
8448 age born name toy
8449 0 True False True False
8450 1 True True True True
8451 2 False True True True
8453 Show which entries in a Series are not NA.
8455 >>> ser = pd.Series([5, 6, np.nan])
8456 >>> ser
8457 0 5.0
8458 1 6.0
8459 2 NaN
8460 dtype: float64
8462 >>> ser.notna()
8463 0 True
8464 1 True
8465 2 False
8466 dtype: bool
8467 """
8468 return notna(self).__finalize__(self, method="notna")
8470 def notnull(self) -> Self:
8471 """
8472 Detect existing (non-missing) values.
8474 Return a boolean same-sized object indicating if the values are not NA.
8475 Non-missing values get mapped to True. Characters such as empty
8476 strings ``''`` or :attr:`numpy.inf` are not considered NA values.
8477 NA values, such as None or :attr:`numpy.NaN`, get mapped to False
8478 values.
8480 Returns
8481 -------
8482 Series/DataFrame
8483 Mask of bool values for each element in Series/DataFrame
8484 that indicates whether an element is not an NA value.
8486 See Also
8487 --------
8488 Series.notnull : Alias of notna.
8489 DataFrame.notnull : Alias of notna.
8490 Series.isna : Boolean inverse of notna.
8491 DataFrame.isna : Boolean inverse of notna.
8492 Series.dropna : Omit axes labels with missing values.
8493 DataFrame.dropna : Omit axes labels with missing values.
8494 notna : Top-level notna.
8496 Examples
8497 --------
8498 Show which entries in a DataFrame are not NA.
8500 >>> df = pd.DataFrame(
8501 ... dict(
8502 ... age=[5, 6, np.nan],
8503 ... born=[
8504 ... pd.NaT,
8505 ... pd.Timestamp("1939-05-27"),
8506 ... pd.Timestamp("1940-04-25"),
8507 ... ],
8508 ... name=["Alfred", "Batman", ""],
8509 ... toy=[None, "Batmobile", "Joker"],
8510 ... )
8511 ... )
8512 >>> df
8513 age born name toy
8514 0 5.0 NaT Alfred NaN
8515 1 6.0 1939-05-27 Batman Batmobile
8516 2 NaN 1940-04-25 Joker
8518 >>> df.notna()
8519 age born name toy
8520 0 True False True False
8521 1 True True True True
8522 2 False True True True
8524 Show which entries in a Series are not NA.
8526 >>> ser = pd.Series([5, 6, np.nan])
8527 >>> ser
8528 0 5.0
8529 1 6.0
8530 2 NaN
8531 dtype: float64
8533 >>> ser.notna()
8534 0 True
8535 1 True
8536 2 False
8537 dtype: bool
8538 """
8539 return notna(self).__finalize__(self, method="notnull")
8541 @final
8542 def _clip_with_scalar(self, lower, upper, inplace: bool = False):
8543 if (lower is not None and np.any(isna(lower))) or (
8544 upper is not None and np.any(isna(upper))
8545 ):
8546 raise ValueError("Cannot use an NA value as a clip threshold")
8548 result = self
8549 mask = self.isna()
8551 if lower is not None:
8552 cond = mask | (self >= lower)
8553 result = result.where(cond, lower, inplace=inplace)
8554 if upper is not None:
8555 cond = mask | (self <= upper)
8556 result = result.where(cond, upper, inplace=inplace)
8558 return result
8560 @final
8561 def _clip_with_one_bound(self, threshold, method, axis, inplace):
8562 if axis is not None:
8563 axis = self._get_axis_number(axis)
8565 # method is self.le for upper bound and self.ge for lower bound
8566 if is_scalar(threshold) and is_number(threshold):
8567 if method.__name__ == "le":
8568 return self._clip_with_scalar(None, threshold, inplace=inplace)
8569 return self._clip_with_scalar(threshold, None, inplace=inplace)
8571 # GH #15390
8572 # In order for where method to work, the threshold must
8573 # be transformed to NDFrame from other array like structure.
8574 if (not isinstance(threshold, ABCSeries)) and is_list_like(threshold):
8575 if isinstance(self, ABCSeries):
8576 threshold = self._constructor(threshold, index=self.index)
8577 else:
8578 threshold = self._align_for_op(threshold, axis, flex=None)[1]
8580 # GH 40420
8581 # Treat missing thresholds as no bounds, not clipping the values
8582 if is_list_like(threshold):
8583 fill_value = np.inf if method.__name__ == "le" else -np.inf
8584 threshold_inf = threshold.fillna(fill_value)
8585 else:
8586 threshold_inf = threshold
8588 subset = method(threshold_inf, axis=axis) | isna(self)
8590 # GH 40420
8591 return self.where(subset, threshold, axis=axis, inplace=inplace)
8593 @final
8594 def clip(
8595 self,
8596 lower=None,
8597 upper=None,
8598 *,
8599 axis: Axis | None = None,
8600 inplace: bool = False,
8601 **kwargs,
8602 ) -> Self:
8603 """
8604 Trim values at input threshold(s).
8606 Assigns values outside boundary to boundary values. Thresholds
8607 can be singular values or array like, and in the latter case
8608 the clipping is performed element-wise in the specified axis.
8610 Parameters
8611 ----------
8612 lower : float or array-like, default None
8613 Minimum threshold value. All values below this
8614 threshold will be set to it. A missing
8615 threshold (e.g `NA`) will not clip the value.
8616 upper : float or array-like, default None
8617 Maximum threshold value. All values above this
8618 threshold will be set to it. A missing
8619 threshold (e.g `NA`) will not clip the value.
8620 axis : {0 or 'index', 1 or 'columns', None}, default None
8621 Align object with lower and upper along the given axis.
8622 For `Series` this parameter is unused and defaults to `None`.
8623 inplace : bool, default False
8624 Whether to perform the operation in place on the data.
8625 **kwargs
8626 Additional keywords have no effect but might be accepted
8627 for compatibility with numpy.
8629 Returns
8630 -------
8631 Series or DataFrame
8632 Same type as calling object with the values outside the
8633 clip boundaries replaced.
8635 See Also
8636 --------
8637 Series.clip : Trim values at input threshold in series.
8638 DataFrame.clip : Trim values at input threshold in DataFrame.
8639 numpy.clip : Clip (limit) the values in an array.
8641 Examples
8642 --------
8643 >>> data = {"col_0": [9, -3, 0, -1, 5], "col_1": [-2, -7, 6, 8, -5]}
8644 >>> df = pd.DataFrame(data)
8645 >>> df
8646 col_0 col_1
8647 0 9 -2
8648 1 -3 -7
8649 2 0 6
8650 3 -1 8
8651 4 5 -5
8653 Clips per column using lower and upper thresholds:
8655 >>> df.clip(-4, 6)
8656 col_0 col_1
8657 0 6 -2
8658 1 -3 -4
8659 2 0 6
8660 3 -1 6
8661 4 5 -4
8663 Clips using specific lower and upper thresholds per column:
8665 >>> df.clip([-2, -1], [4, 5])
8666 col_0 col_1
8667 0 4 -1
8668 1 -2 -1
8669 2 0 5
8670 3 -1 5
8671 4 4 -1
8673 Clips using specific lower and upper thresholds per column element:
8675 >>> t = pd.Series([2, -4, -1, 6, 3])
8676 >>> t
8677 0 2
8678 1 -4
8679 2 -1
8680 3 6
8681 4 3
8682 dtype: int64
8684 >>> df.clip(t, t + 4, axis=0)
8685 col_0 col_1
8686 0 6 2
8687 1 -3 -4
8688 2 0 3
8689 3 6 8
8690 4 5 3
8692 Clips using specific lower threshold per column element, with missing values:
8694 >>> t = pd.Series([2, -4, np.nan, 6, 3])
8695 >>> t
8696 0 2.0
8697 1 -4.0
8698 2 NaN
8699 3 6.0
8700 4 3.0
8701 dtype: float64
8703 >>> df.clip(t, axis=0)
8704 col_0 col_1
8705 0 9.0 2.0
8706 1 -3.0 -4.0
8707 2 0.0 6.0
8708 3 6.0 8.0
8709 4 5.0 3.0
8710 """
8711 inplace = validate_bool_kwarg(inplace, "inplace")
8713 if inplace:
8714 if not CHAINED_WARNING_DISABLED:
8715 if sys.getrefcount(
8716 self
8717 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
8718 warnings.warn(
8719 _chained_assignment_method_msg,
8720 ChainedAssignmentError,
8721 stacklevel=2,
8722 )
8724 axis = nv.validate_clip_with_axis(axis, (), kwargs)
8725 if axis is not None:
8726 axis = self._get_axis_number(axis)
8728 # GH 17276
8729 # numpy doesn't like NaN as a clip value
8730 # so ignore
8731 # GH 19992
8732 # numpy doesn't drop a list-like bound containing NaN
8733 isna_lower = isna(lower)
8734 if not is_list_like(lower):
8735 if np.any(isna_lower):
8736 lower = None
8737 elif np.all(isna_lower):
8738 lower = None
8739 isna_upper = isna(upper)
8740 if not is_list_like(upper):
8741 if np.any(isna_upper):
8742 upper = None
8743 elif np.all(isna_upper):
8744 upper = None
8746 # GH 2747 (arguments were reversed)
8747 if (
8748 lower is not None
8749 and upper is not None
8750 and is_scalar(lower)
8751 and is_scalar(upper)
8752 ):
8753 lower, upper = min(lower, upper), max(lower, upper)
8755 # fast-path for scalars
8756 if (lower is None or is_number(lower)) and (upper is None or is_number(upper)):
8757 return self._clip_with_scalar(lower, upper, inplace=inplace)
8759 result = self
8760 if lower is not None:
8761 result = result._clip_with_one_bound(
8762 lower, method=self.ge, axis=axis, inplace=inplace
8763 )
8764 if upper is not None:
8765 if inplace:
8766 result = self
8767 result = result._clip_with_one_bound(
8768 upper, method=self.le, axis=axis, inplace=inplace
8769 )
8771 return result
8773 @final
8774 def asfreq(
8775 self,
8776 freq: Frequency,
8777 method: FillnaOptions | None = None,
8778 how: Literal["start", "end"] | None = None,
8779 normalize: bool = False,
8780 fill_value: Hashable | None = None,
8781 ) -> Self:
8782 """
8783 Convert time series to specified frequency.
8785 Returns the original data conformed to a new index with the specified
8786 frequency.
8788 If the index of this Series/DataFrame is a :class:`~pandas.PeriodIndex`, the
8789 new index is the result of transforming the original index with
8790 :meth:`PeriodIndex.asfreq <pandas.PeriodIndex.asfreq>` (so the original index
8791 will map one-to-one to the new index).
8793 Otherwise, the new index will be equivalent to ``pd.date_range(start, end,
8794 freq=freq)`` where ``start`` and ``end`` are, respectively, the min and
8795 max entries in the original index (see :func:`pandas.date_range`). The
8796 values corresponding to any timesteps in the new index which were not present
8797 in the original index will be null (``NaN``), unless a method for filling
8798 such unknowns is provided (see the ``method`` parameter below).
8800 The :meth:`resample` method is more appropriate if an operation on each group of
8801 timesteps (such as an aggregate) is necessary to represent the data at the new
8802 frequency.
8804 Parameters
8805 ----------
8806 freq : DateOffset or str
8807 Frequency DateOffset or string.
8808 method : {'backfill'/'bfill', 'pad'/'ffill'}, default None
8809 Method to use for filling holes in reindexed Series (note this
8810 does not fill NaNs that already were present):
8812 * 'pad' / 'ffill': propagate last valid observation forward to next
8813 valid based on the order of the index
8814 * 'backfill' / 'bfill': use NEXT valid observation to fill.
8815 how : {'start', 'end'}, default end
8816 For PeriodIndex only (see PeriodIndex.asfreq).
8817 normalize : bool, default False
8818 Whether to reset output index to midnight.
8819 fill_value : scalar, optional
8820 Value to use for missing values, applied during upsampling (note
8821 this does not fill NaNs that already were present).
8823 Returns
8824 -------
8825 Series/DataFrame
8826 Series/DataFrame object reindexed to the specified frequency.
8828 See Also
8829 --------
8830 reindex : Conform DataFrame to new index with optional filling logic.
8832 Notes
8833 -----
8834 To learn more about the frequency strings, please see
8835 :ref:`this link<timeseries.offset_aliases>`.
8837 Examples
8838 --------
8839 Start by creating a series with 4 one minute timestamps.
8841 >>> index = pd.date_range("1/1/2000", periods=4, freq="min")
8842 >>> series = pd.Series([0.0, None, 2.0, 3.0], index=index)
8843 >>> df = pd.DataFrame({"s": series})
8844 >>> df
8845 s
8846 2000-01-01 00:00:00 0.0
8847 2000-01-01 00:01:00 NaN
8848 2000-01-01 00:02:00 2.0
8849 2000-01-01 00:03:00 3.0
8851 Upsample the series into 30 second bins.
8853 >>> df.asfreq(freq="30s")
8854 s
8855 2000-01-01 00:00:00 0.0
8856 2000-01-01 00:00:30 NaN
8857 2000-01-01 00:01:00 NaN
8858 2000-01-01 00:01:30 NaN
8859 2000-01-01 00:02:00 2.0
8860 2000-01-01 00:02:30 NaN
8861 2000-01-01 00:03:00 3.0
8863 Upsample again, providing a ``fill value``.
8865 >>> df.asfreq(freq="30s", fill_value=9.0)
8866 s
8867 2000-01-01 00:00:00 0.0
8868 2000-01-01 00:00:30 9.0
8869 2000-01-01 00:01:00 NaN
8870 2000-01-01 00:01:30 9.0
8871 2000-01-01 00:02:00 2.0
8872 2000-01-01 00:02:30 9.0
8873 2000-01-01 00:03:00 3.0
8875 Upsample again, providing a ``method``.
8877 >>> df.asfreq(freq="30s", method="bfill")
8878 s
8879 2000-01-01 00:00:00 0.0
8880 2000-01-01 00:00:30 NaN
8881 2000-01-01 00:01:00 NaN
8882 2000-01-01 00:01:30 2.0
8883 2000-01-01 00:02:00 2.0
8884 2000-01-01 00:02:30 3.0
8885 2000-01-01 00:03:00 3.0
8886 """
8887 from pandas.core.resample import asfreq
8889 return asfreq(
8890 self,
8891 freq,
8892 method=method,
8893 how=how,
8894 normalize=normalize,
8895 fill_value=fill_value,
8896 )
8898 @final
8899 def at_time(self, time, asof: bool = False, axis: Axis | None = None) -> Self:
8900 """
8901 Select values at particular time of day (e.g., 9:30AM).
8903 Parameters
8904 ----------
8905 time : datetime.time or str
8906 The values to select.
8907 asof : bool, default False
8908 This parameter is currently not supported.
8909 axis : {0 or 'index', 1 or 'columns'}, default 0
8910 For `Series` this parameter is unused and defaults to 0.
8912 Returns
8913 -------
8914 Series or DataFrame
8915 The values with the specified time.
8917 Raises
8918 ------
8919 TypeError
8920 If the index is not a :class:`DatetimeIndex`
8922 See Also
8923 --------
8924 between_time : Select values between particular times of the day.
8925 first : Select initial periods of time series based on a date offset.
8926 last : Select final periods of time series based on a date offset.
8927 DatetimeIndex.indexer_at_time : Get just the index locations for
8928 values at particular time of the day.
8930 Examples
8931 --------
8932 >>> i = pd.date_range("2018-04-09", periods=4, freq="12h")
8933 >>> ts = pd.DataFrame({"A": [1, 2, 3, 4]}, index=i)
8934 >>> ts
8935 A
8936 2018-04-09 00:00:00 1
8937 2018-04-09 12:00:00 2
8938 2018-04-10 00:00:00 3
8939 2018-04-10 12:00:00 4
8941 >>> ts.at_time("12:00")
8942 A
8943 2018-04-09 12:00:00 2
8944 2018-04-10 12:00:00 4
8945 """
8946 if axis is None:
8947 axis = 0
8948 axis = self._get_axis_number(axis)
8950 index = self._get_axis(axis)
8952 if not isinstance(index, DatetimeIndex):
8953 raise TypeError("Index must be DatetimeIndex")
8955 indexer = index.indexer_at_time(time, asof=asof)
8956 return self.take(indexer, axis=axis)
8958 @final
8959 def between_time(
8960 self,
8961 start_time,
8962 end_time,
8963 inclusive: IntervalClosedType = "both",
8964 axis: Axis | None = None,
8965 ) -> Self:
8966 """
8967 Select values between particular times of the day (e.g., 9:00-9:30 AM).
8969 By setting ``start_time`` to be later than ``end_time``,
8970 you can get the times that are *not* between the two times.
8972 Parameters
8973 ----------
8974 start_time : datetime.time or str
8975 Initial time as a time filter limit.
8976 end_time : datetime.time or str
8977 End time as a time filter limit.
8978 inclusive : {"both", "neither", "left", "right"}, default "both"
8979 Include boundaries; whether to set each bound as closed or open.
8980 axis : {0 or 'index', 1 or 'columns'}, default 0
8981 Determine range time on index or columns value.
8982 For `Series` this parameter is unused and defaults to 0.
8984 Returns
8985 -------
8986 Series or DataFrame
8987 Data from the original object filtered to the specified dates range.
8989 Raises
8990 ------
8991 TypeError
8992 If the index is not a :class:`DatetimeIndex`
8994 See Also
8995 --------
8996 at_time : Select values at a particular time of the day.
8997 first : Select initial periods of time series based on a date offset.
8998 last : Select final periods of time series based on a date offset.
8999 DatetimeIndex.indexer_between_time : Get just the index locations for
9000 values between particular times of the day.
9002 Examples
9003 --------
9004 >>> i = pd.date_range("2018-04-09", periods=4, freq="1D20min")
9005 >>> ts = pd.DataFrame({"A": [1, 2, 3, 4]}, index=i)
9006 >>> ts
9007 A
9008 2018-04-09 00:00:00 1
9009 2018-04-10 00:20:00 2
9010 2018-04-11 00:40:00 3
9011 2018-04-12 01:00:00 4
9013 >>> ts.between_time("0:15", "0:45")
9014 A
9015 2018-04-10 00:20:00 2
9016 2018-04-11 00:40:00 3
9018 You get the times that are *not* between two times by setting
9019 ``start_time`` later than ``end_time``:
9021 >>> ts.between_time("0:45", "0:15")
9022 A
9023 2018-04-09 00:00:00 1
9024 2018-04-12 01:00:00 4
9025 """
9026 if axis is None:
9027 axis = 0
9028 axis = self._get_axis_number(axis)
9030 index = self._get_axis(axis)
9031 if not isinstance(index, DatetimeIndex):
9032 raise TypeError("Index must be DatetimeIndex")
9034 left_inclusive, right_inclusive = validate_inclusive(inclusive)
9035 indexer = index.indexer_between_time(
9036 start_time,
9037 end_time,
9038 include_start=left_inclusive,
9039 include_end=right_inclusive,
9040 )
9041 return self.take(indexer, axis=axis)
9043 @final
9044 def resample(
9045 self,
9046 rule,
9047 closed: Literal["right", "left"] | None = None,
9048 label: Literal["right", "left"] | None = None,
9049 convention: Literal["start", "end", "s", "e"] = "start",
9050 on: Level | None = None,
9051 level: Level | None = None,
9052 origin: str | TimestampConvertibleTypes = "start_day",
9053 offset: TimedeltaConvertibleTypes | None = None,
9054 group_keys: bool = False,
9055 ) -> Resampler:
9056 """
9057 Resample time-series data.
9059 Convenience method for frequency conversion and resampling of time series.
9060 The object must have a datetime-like index (`DatetimeIndex`, `PeriodIndex`,
9061 or `TimedeltaIndex`), or the caller must pass the label of a datetime-like
9062 series/index to the ``on``/``level`` keyword parameter.
9064 Parameters
9065 ----------
9066 rule : DateOffset, Timedelta or str
9067 The offset string or object representing target conversion.
9068 closed : {'right', 'left'}, default None
9069 Which side of bin interval is closed. The default is 'left'
9070 for all frequency offsets except for 'ME', 'YE', 'QE', 'BME',
9071 'BA', 'BQE', and 'W' which all have a default of 'right'.
9072 label : {'right', 'left'}, default None
9073 Which bin edge label to label bucket with. The default is 'left'
9074 for all frequency offsets except for 'ME', 'YE', 'QE', 'BME',
9075 'BA', 'BQE', and 'W' which all have a default of 'right'.
9076 convention : {'start', 'end', 's', 'e'}, default 'start'
9077 For `PeriodIndex` only, controls whether to use the start or
9078 end of `rule`.
9079 on : str, optional
9080 For a DataFrame, column to use instead of index for resampling.
9081 Column must be datetime-like.
9082 level : str or int, optional
9083 For a MultiIndex, level (name or number) to use for
9084 resampling. `level` must be datetime-like.
9085 origin : Timestamp or str, default 'start_day'
9086 The timestamp on which to adjust the grouping. The timezone of origin
9087 must match the timezone of the index.
9088 If string, must be Timestamp convertible or one of the following:
9090 - 'epoch': `origin` is 1970-01-01
9091 - 'start': `origin` is the first value of the timeseries
9092 - 'start_day': `origin` is the first day at midnight of the timeseries
9094 - 'end': `origin` is the last value of the timeseries
9095 - 'end_day': `origin` is the ceiling midnight of the last day
9097 .. note::
9099 Only takes effect for Tick-frequencies (i.e. fixed frequencies like
9100 days, hours, and minutes, rather than months or quarters).
9101 offset : Timedelta or str, default is None
9102 An offset timedelta added to the origin.
9104 group_keys : bool, default False
9105 Whether to include the group keys in the result index when using
9106 ``.apply()`` on the resampled object.
9108 .. versionchanged:: 2.0.0
9110 ``group_keys`` now defaults to ``False``.
9112 Returns
9113 -------
9114 pandas.api.typing.Resampler
9115 :class:`~pandas.core.Resampler` object.
9117 See Also
9118 --------
9119 Series.resample : Resample a Series.
9120 DataFrame.resample : Resample a DataFrame.
9121 groupby : Group Series/DataFrame by mapping, function, label, or list of labels.
9122 asfreq : Reindex a Series/DataFrame with the given frequency without grouping.
9124 Notes
9125 -----
9126 See the `user guide
9127 <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#resampling>`__
9128 for more.
9130 To learn more about the offset strings, please see `this link
9131 <https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects>`__.
9133 Examples
9134 --------
9135 Start by creating a series with 9 one minute timestamps.
9137 >>> index = pd.date_range("1/1/2000", periods=9, freq="min")
9138 >>> series = pd.Series(range(9), index=index)
9139 >>> series
9140 2000-01-01 00:00:00 0
9141 2000-01-01 00:01:00 1
9142 2000-01-01 00:02:00 2
9143 2000-01-01 00:03:00 3
9144 2000-01-01 00:04:00 4
9145 2000-01-01 00:05:00 5
9146 2000-01-01 00:06:00 6
9147 2000-01-01 00:07:00 7
9148 2000-01-01 00:08:00 8
9149 Freq: min, dtype: int64
9151 Downsample the series into 3 minute bins and sum the values
9152 of the timestamps falling into a bin.
9154 >>> series.resample("3min").sum()
9155 2000-01-01 00:00:00 3
9156 2000-01-01 00:03:00 12
9157 2000-01-01 00:06:00 21
9158 Freq: 3min, dtype: int64
9160 Downsample the series into 3 minute bins as above, but label each
9161 bin using the right edge instead of the left. Please note that the
9162 value in the bucket used as the label is not included in the bucket,
9163 which it labels. For example, in the original series the
9164 bucket ``2000-01-01 00:03:00`` contains the value 3, but the summed
9165 value in the resampled bucket with the label ``2000-01-01 00:03:00``
9166 does not include 3 (if it did, the summed value would be 6, not 3).
9168 >>> series.resample("3min", label="right").sum()
9169 2000-01-01 00:03:00 3
9170 2000-01-01 00:06:00 12
9171 2000-01-01 00:09:00 21
9172 Freq: 3min, dtype: int64
9174 To include this value close the right side of the bin interval,
9175 as shown below.
9177 >>> series.resample("3min", label="right", closed="right").sum()
9178 2000-01-01 00:00:00 0
9179 2000-01-01 00:03:00 6
9180 2000-01-01 00:06:00 15
9181 2000-01-01 00:09:00 15
9182 Freq: 3min, dtype: int64
9184 Upsample the series into 30 second bins.
9186 >>> series.resample("30s").asfreq()[0:5] # Select first 5 rows
9187 2000-01-01 00:00:00 0.0
9188 2000-01-01 00:00:30 NaN
9189 2000-01-01 00:01:00 1.0
9190 2000-01-01 00:01:30 NaN
9191 2000-01-01 00:02:00 2.0
9192 Freq: 30s, dtype: float64
9194 Upsample the series into 30 second bins and fill the ``NaN``
9195 values using the ``ffill`` method.
9197 >>> series.resample("30s").ffill()[0:5]
9198 2000-01-01 00:00:00 0
9199 2000-01-01 00:00:30 0
9200 2000-01-01 00:01:00 1
9201 2000-01-01 00:01:30 1
9202 2000-01-01 00:02:00 2
9203 Freq: 30s, dtype: int64
9205 Upsample the series into 30 second bins and fill the
9206 ``NaN`` values using the ``bfill`` method.
9208 >>> series.resample("30s").bfill()[0:5]
9209 2000-01-01 00:00:00 0
9210 2000-01-01 00:00:30 1
9211 2000-01-01 00:01:00 1
9212 2000-01-01 00:01:30 2
9213 2000-01-01 00:02:00 2
9214 Freq: 30s, dtype: int64
9216 Pass a custom function via ``apply``
9218 >>> def custom_resampler(arraylike):
9219 ... return np.sum(arraylike) + 5
9220 >>> series.resample("3min").apply(custom_resampler)
9221 2000-01-01 00:00:00 8
9222 2000-01-01 00:03:00 17
9223 2000-01-01 00:06:00 26
9224 Freq: 3min, dtype: int64
9226 For a Series with a PeriodIndex, the keyword `convention` can be
9227 used to control whether to use the start or end of `rule`.
9229 Resample a year by quarter using 'start' `convention`. Values are
9230 assigned to the first quarter of the period.
9232 >>> s = pd.Series(
9233 ... [1, 2], index=pd.period_range("2012-01-01", freq="Y", periods=2)
9234 ... )
9235 >>> s
9236 2012 1
9237 2013 2
9238 Freq: Y-DEC, dtype: int64
9239 >>> s.resample("Q", convention="start").asfreq()
9240 2012Q1 1.0
9241 2012Q2 NaN
9242 2012Q3 NaN
9243 2012Q4 NaN
9244 2013Q1 2.0
9245 2013Q2 NaN
9246 2013Q3 NaN
9247 2013Q4 NaN
9248 Freq: Q-DEC, dtype: float64
9250 Resample quarters by month using 'end' `convention`. Values are
9251 assigned to the last month of the period.
9253 >>> q = pd.Series(
9254 ... [1, 2, 3, 4], index=pd.period_range("2018-01-01", freq="Q", periods=4)
9255 ... )
9256 >>> q
9257 2018Q1 1
9258 2018Q2 2
9259 2018Q3 3
9260 2018Q4 4
9261 Freq: Q-DEC, dtype: int64
9262 >>> q.resample("M", convention="end").asfreq()
9263 2018-03 1.0
9264 2018-04 NaN
9265 2018-05 NaN
9266 2018-06 2.0
9267 2018-07 NaN
9268 2018-08 NaN
9269 2018-09 3.0
9270 2018-10 NaN
9271 2018-11 NaN
9272 2018-12 4.0
9273 Freq: M, dtype: float64
9275 For DataFrame objects, the keyword `on` can be used to specify the
9276 column instead of the index for resampling.
9278 >>> df = pd.DataFrame([10, 11, 9, 13, 14, 18, 17, 19], columns=["price"])
9279 >>> df["volume"] = [50, 60, 40, 100, 50, 100, 40, 50]
9280 >>> df["week_starting"] = pd.date_range("01/01/2018", periods=8, freq="W")
9281 >>> df
9282 price volume week_starting
9283 0 10 50 2018-01-07
9284 1 11 60 2018-01-14
9285 2 9 40 2018-01-21
9286 3 13 100 2018-01-28
9287 4 14 50 2018-02-04
9288 5 18 100 2018-02-11
9289 6 17 40 2018-02-18
9290 7 19 50 2018-02-25
9291 >>> df.resample("ME", on="week_starting").mean()
9292 price volume
9293 week_starting
9294 2018-01-31 10.75 62.5
9295 2018-02-28 17.00 60.0
9297 For a DataFrame with MultiIndex, the keyword `level` can be used to
9298 specify on which level the resampling needs to take place.
9300 >>> days = pd.date_range("1/1/2000", periods=4, freq="D")
9301 >>> df2 = pd.DataFrame(
9302 ... [
9303 ... [10, 50],
9304 ... [11, 60],
9305 ... [9, 40],
9306 ... [13, 100],
9307 ... [14, 50],
9308 ... [18, 100],
9309 ... [17, 40],
9310 ... [19, 50],
9311 ... ],
9312 ... columns=["price", "volume"],
9313 ... index=pd.MultiIndex.from_product([days, ["morning", "afternoon"]]),
9314 ... )
9315 >>> df2
9316 price volume
9317 2000-01-01 morning 10 50
9318 afternoon 11 60
9319 2000-01-02 morning 9 40
9320 afternoon 13 100
9321 2000-01-03 morning 14 50
9322 afternoon 18 100
9323 2000-01-04 morning 17 40
9324 afternoon 19 50
9325 >>> df2.resample("D", level=0).sum()
9326 price volume
9327 2000-01-01 21 110
9328 2000-01-02 22 140
9329 2000-01-03 32 150
9330 2000-01-04 36 90
9332 If you want to adjust the start of the bins based on a fixed timestamp:
9334 >>> start, end = "2000-10-01 23:30:00", "2000-10-02 00:30:00"
9335 >>> rng = pd.date_range(start, end, freq="7min")
9336 >>> ts = pd.Series(np.arange(len(rng)) * 3, index=rng)
9337 >>> ts
9338 2000-10-01 23:30:00 0
9339 2000-10-01 23:37:00 3
9340 2000-10-01 23:44:00 6
9341 2000-10-01 23:51:00 9
9342 2000-10-01 23:58:00 12
9343 2000-10-02 00:05:00 15
9344 2000-10-02 00:12:00 18
9345 2000-10-02 00:19:00 21
9346 2000-10-02 00:26:00 24
9347 Freq: 7min, dtype: int64
9349 >>> ts.resample("17min").sum()
9350 2000-10-01 23:14:00 0
9351 2000-10-01 23:31:00 9
9352 2000-10-01 23:48:00 21
9353 2000-10-02 00:05:00 54
9354 2000-10-02 00:22:00 24
9355 Freq: 17min, dtype: int64
9357 >>> ts.resample("17min", origin="epoch").sum()
9358 2000-10-01 23:18:00 0
9359 2000-10-01 23:35:00 18
9360 2000-10-01 23:52:00 27
9361 2000-10-02 00:09:00 39
9362 2000-10-02 00:26:00 24
9363 Freq: 17min, dtype: int64
9365 >>> ts.resample("17min", origin="2000-01-01").sum()
9366 2000-10-01 23:24:00 3
9367 2000-10-01 23:41:00 15
9368 2000-10-01 23:58:00 45
9369 2000-10-02 00:15:00 45
9370 Freq: 17min, dtype: int64
9372 If you want to adjust the start of the bins with an `offset` Timedelta, the two
9373 following lines are equivalent:
9375 >>> ts.resample("17min", origin="start").sum()
9376 2000-10-01 23:30:00 9
9377 2000-10-01 23:47:00 21
9378 2000-10-02 00:04:00 54
9379 2000-10-02 00:21:00 24
9380 Freq: 17min, dtype: int64
9382 >>> ts.resample("17min", offset="23h30min").sum()
9383 2000-10-01 23:30:00 9
9384 2000-10-01 23:47:00 21
9385 2000-10-02 00:04:00 54
9386 2000-10-02 00:21:00 24
9387 Freq: 17min, dtype: int64
9389 If you want to take the largest Timestamp as the end of the bins:
9391 >>> ts.resample("17min", origin="end").sum()
9392 2000-10-01 23:35:00 0
9393 2000-10-01 23:52:00 18
9394 2000-10-02 00:09:00 27
9395 2000-10-02 00:26:00 63
9396 Freq: 17min, dtype: int64
9398 In contrast with the `start_day`, you can use `end_day` to take the ceiling
9399 midnight of the largest Timestamp as the end of the bins and drop the bins
9400 not containing data:
9402 >>> ts.resample("17min", origin="end_day").sum()
9403 2000-10-01 23:38:00 3
9404 2000-10-01 23:55:00 15
9405 2000-10-02 00:12:00 45
9406 2000-10-02 00:29:00 45
9407 Freq: 17min, dtype: int64
9408 """
9409 from pandas.core.resample import get_resampler
9411 return get_resampler(
9412 cast("Series | DataFrame", self),
9413 freq=rule,
9414 label=label,
9415 closed=closed,
9416 convention=convention,
9417 key=on,
9418 level=level,
9419 origin=origin,
9420 offset=offset,
9421 group_keys=group_keys,
9422 )
9424 @final
9425 def rank(
9426 self,
9427 axis: Axis = 0,
9428 method: Literal["average", "min", "max", "first", "dense"] = "average",
9429 numeric_only: bool = False,
9430 na_option: Literal["keep", "top", "bottom"] = "keep",
9431 ascending: bool = True,
9432 pct: bool = False,
9433 ) -> Self:
9434 """
9435 Compute numerical data ranks (1 through n) along axis.
9437 By default, equal values are assigned a rank that is the average of the
9438 ranks of those values.
9440 Parameters
9441 ----------
9442 axis : {0 or 'index', 1 or 'columns'}, default 0
9443 Index to direct ranking.
9444 For `Series` this parameter is unused and defaults to 0.
9445 method : {'average', 'min', 'max', 'first', 'dense'}, default 'average'
9446 How to rank the group of records that have the same value (i.e. ties):
9448 * average: average rank of the group
9449 * min: lowest rank in the group
9450 * max: highest rank in the group
9451 * first: ranks assigned in order they appear in the array
9452 * dense: like 'min', but rank always increases by 1 between groups.
9454 numeric_only : bool, default False
9455 For DataFrame objects, rank only numeric columns if set to True.
9457 .. versionchanged:: 2.0.0
9458 The default value of ``numeric_only`` is now ``False``.
9460 na_option : {'keep', 'top', 'bottom'}, default 'keep'
9461 How to rank NaN values:
9463 * keep: assign NaN rank to NaN values
9464 * top: assign lowest rank to NaN values
9465 * bottom: assign highest rank to NaN values
9467 ascending : bool, default True
9468 Whether or not the elements should be ranked in ascending order.
9469 pct : bool, default False
9470 Whether or not to display the returned rankings in percentile
9471 form.
9473 Returns
9474 -------
9475 same type as caller
9476 Return a Series or DataFrame with data ranks as values.
9478 See Also
9479 --------
9480 core.groupby.DataFrameGroupBy.rank : Rank of values within each group.
9481 core.groupby.SeriesGroupBy.rank : Rank of values within each group.
9483 Examples
9484 --------
9485 >>> df = pd.DataFrame(
9486 ... data={
9487 ... "Animal": ["cat", "penguin", "dog", "spider", "snake"],
9488 ... "Number_legs": [4, 2, 4, 8, np.nan],
9489 ... }
9490 ... )
9491 >>> df
9492 Animal Number_legs
9493 0 cat 4.0
9494 1 penguin 2.0
9495 2 dog 4.0
9496 3 spider 8.0
9497 4 snake NaN
9499 Ties are assigned the mean of the ranks (by default) for the group.
9501 >>> s = pd.Series(range(5), index=list("abcde"))
9502 >>> s["d"] = s["b"]
9503 >>> s.rank()
9504 a 1.0
9505 b 2.5
9506 c 4.0
9507 d 2.5
9508 e 5.0
9509 dtype: float64
9511 The following example shows how the method behaves with the above
9512 parameters:
9514 * default_rank: this is the default behaviour obtained without using
9515 any parameter.
9516 * max_rank: setting ``method = 'max'`` the records that have the
9517 same values are ranked using the highest rank (e.g.: since 'cat'
9518 and 'dog' are both in the 2nd and 3rd position, rank 3 is assigned.)
9519 * NA_bottom: choosing ``na_option = 'bottom'``, if there are records
9520 with NaN values they are placed at the bottom of the ranking.
9521 * pct_rank: when setting ``pct = True``, the ranking is expressed as
9522 percentile rank.
9524 >>> df["default_rank"] = df["Number_legs"].rank()
9525 >>> df["max_rank"] = df["Number_legs"].rank(method="max")
9526 >>> df["NA_bottom"] = df["Number_legs"].rank(na_option="bottom")
9527 >>> df["pct_rank"] = df["Number_legs"].rank(pct=True)
9528 >>> df
9529 Animal Number_legs default_rank max_rank NA_bottom pct_rank
9530 0 cat 4.0 2.5 3.0 2.5 0.625
9531 1 penguin 2.0 1.0 1.0 1.0 0.250
9532 2 dog 4.0 2.5 3.0 2.5 0.625
9533 3 spider 8.0 4.0 4.0 4.0 1.000
9534 4 snake NaN NaN NaN 5.0 NaN
9535 """
9536 axis_int = self._get_axis_number(axis)
9538 if na_option not in {"keep", "top", "bottom"}:
9539 msg = "na_option must be one of 'keep', 'top', or 'bottom'"
9540 raise ValueError(msg)
9542 def ranker(data):
9543 if data.ndim == 2:
9544 # i.e. DataFrame, we cast to ndarray
9545 values = data.values
9546 else:
9547 # i.e. Series, can dispatch to EA
9548 values = data._values
9550 if isinstance(values, ExtensionArray):
9551 ranks = values._rank(
9552 axis=axis_int,
9553 method=method,
9554 ascending=ascending,
9555 na_option=na_option,
9556 pct=pct,
9557 )
9558 else:
9559 ranks = algos.rank(
9560 values,
9561 axis=axis_int,
9562 method=method,
9563 ascending=ascending,
9564 na_option=na_option,
9565 pct=pct,
9566 )
9568 ranks_obj = self._constructor(ranks, **data._construct_axes_dict())
9569 return ranks_obj.__finalize__(self, method="rank")
9571 if numeric_only:
9572 if self.ndim == 1 and not is_numeric_dtype(self.dtype):
9573 # GH#47500
9574 raise TypeError(
9575 "Series.rank does not allow numeric_only=True with "
9576 "non-numeric dtype."
9577 )
9578 data = self._get_numeric_data()
9579 else:
9580 data = self
9582 return ranker(data)
9584 def compare(
9585 self,
9586 other: Self,
9587 align_axis: Axis = 1,
9588 keep_shape: bool = False,
9589 keep_equal: bool = False,
9590 result_names: Suffixes = ("self", "other"),
9591 ):
9592 """
9593 Compare to another Series/DataFrame and show the differences.
9595 Parameters
9596 ----------
9597 other : Series/DataFrame
9598 Object to compare with.
9600 align_axis : {0 or 'index', 1 or 'columns'}, default 1
9601 Determine which axis to align the comparison on.
9603 * 0, or 'index' : Resulting differences are stacked vertically
9604 with rows drawn alternately from self and other.
9605 * 1, or 'columns' : Resulting differences are aligned horizontally
9606 with columns drawn alternately from self and other.
9608 keep_shape : bool, default False
9609 If true, all rows and columns are kept.
9610 Otherwise, only the ones with different values are kept.
9612 keep_equal : bool, default False
9613 If true, the result keeps values that are equal.
9614 Otherwise, equal values are shown as NaNs.
9616 result_names : tuple, default ('self', 'other')
9617 Set the dataframes names in the comparison.
9618 """
9619 if type(self) is not type(other):
9620 cls_self, cls_other = type(self).__name__, type(other).__name__
9621 raise TypeError(
9622 f"can only compare '{cls_self}' (not '{cls_other}') with '{cls_self}'"
9623 )
9625 # error: Unsupported left operand type for & ("Self")
9626 mask = ~((self == other) | (self.isna() & other.isna())) # type: ignore[operator]
9627 mask.fillna(True, inplace=True)
9629 if not keep_equal:
9630 self = self.where(mask)
9631 other = other.where(mask)
9633 if not keep_shape:
9634 if isinstance(self, ABCDataFrame):
9635 cmask = mask.any()
9636 rmask = mask.any(axis=1)
9637 self = self.loc[rmask, cmask]
9638 other = other.loc[rmask, cmask]
9639 else:
9640 self = self[mask]
9641 other = other[mask]
9642 if not isinstance(result_names, tuple):
9643 raise TypeError(
9644 f"Passing 'result_names' as a {type(result_names)} is not "
9645 "supported. Provide 'result_names' as a tuple instead."
9646 )
9648 if align_axis in (1, "columns"): # This is needed for Series
9649 axis = 1
9650 else:
9651 axis = self._get_axis_number(align_axis)
9653 # error: List item 0 has incompatible type "NDFrame"; expected
9654 # "Union[Series, DataFrame]"
9655 diff = concat(
9656 [self, other], # type: ignore[list-item]
9657 axis=axis,
9658 keys=result_names,
9659 )
9661 if axis >= self.ndim:
9662 # No need to reorganize data if stacking on new axis
9663 # This currently applies for stacking two Series on columns
9664 return diff
9666 ax = diff._get_axis(axis)
9667 ax_names = np.array(ax.names)
9669 # set index names to positions to avoid confusion
9670 ax.names = np.arange(len(ax_names))
9672 # bring self-other to inner level
9673 order = [*range(1, ax.nlevels), 0]
9674 if isinstance(diff, ABCDataFrame):
9675 diff = diff.reorder_levels(order, axis=axis)
9676 else:
9677 diff = diff.reorder_levels(order)
9679 # restore the index names in order
9680 diff._get_axis(axis=axis).names = ax_names[order]
9682 # reorder axis to keep things organized
9683 indices = (
9684 np.arange(diff.shape[axis])
9685 .reshape([2, diff.shape[axis] // 2])
9686 .T.reshape(-1)
9687 )
9688 diff = diff.take(indices, axis=axis)
9690 return diff
9692 @final
9693 def align(
9694 self,
9695 other: NDFrameT,
9696 join: AlignJoin = "outer",
9697 axis: Axis | None = None,
9698 level: Level | None = None,
9699 copy: bool | lib.NoDefault = lib.no_default,
9700 fill_value: Hashable | None = None,
9701 ) -> tuple[Self, NDFrameT]:
9702 """
9703 Align two objects on their axes with the specified join method.
9705 Join method is specified for each axis Index.
9707 Parameters
9708 ----------
9709 other : DataFrame or Series
9710 The object to align with.
9711 join : {'outer', 'inner', 'left', 'right'}, default 'outer'
9712 Type of alignment to be performed.
9714 * left: use only keys from left frame, preserve key order.
9715 * right: use only keys from right frame, preserve key order.
9716 * outer: use union of keys from both frames, sort keys lexicographically.
9717 * inner: use intersection of keys from both frames,
9718 preserve the order of the left keys.
9720 axis : allowed axis of the other object, default None
9721 Align on index (0), columns (1), or both (None).
9722 level : int or level name, default None
9723 Broadcast across a level, matching Index values on the
9724 passed MultiIndex level.
9725 copy : bool, default False
9726 This keyword is now ignored; changing its value will have no
9727 impact on the method.
9729 .. deprecated:: 3.0.0
9731 This keyword is ignored and will be removed in pandas 4.0. Since
9732 pandas 3.0, this method always returns a new object using a lazy
9733 copy mechanism that defers copies until necessary
9734 (Copy-on-Write). See the `user guide on Copy-on-Write
9735 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
9736 for more details.
9738 fill_value : scalar, default np.nan
9739 Value to use for missing values. Defaults to NaN, but can be any
9740 "compatible" value.
9742 Returns
9743 -------
9744 tuple of (Series/DataFrame, type of other)
9745 Aligned objects.
9747 See Also
9748 --------
9749 Series.align : Align two objects on their axes with specified join method.
9750 DataFrame.align : Align two objects on their axes with specified join method.
9752 Examples
9753 --------
9754 >>> df = pd.DataFrame(
9755 ... [[1, 2, 3, 4], [6, 7, 8, 9]], columns=["D", "B", "E", "A"], index=[1, 2]
9756 ... )
9757 >>> other = pd.DataFrame(
9758 ... [[10, 20, 30, 40], [60, 70, 80, 90], [600, 700, 800, 900]],
9759 ... columns=["A", "B", "C", "D"],
9760 ... index=[2, 3, 4],
9761 ... )
9762 >>> df
9763 D B E A
9764 1 1 2 3 4
9765 2 6 7 8 9
9766 >>> other
9767 A B C D
9768 2 10 20 30 40
9769 3 60 70 80 90
9770 4 600 700 800 900
9772 Align on columns:
9774 >>> left, right = df.align(other, join="outer", axis=1)
9775 >>> left
9776 A B C D E
9777 1 4 2 NaN 1 3
9778 2 9 7 NaN 6 8
9779 >>> right
9780 A B C D E
9781 2 10 20 30 40 NaN
9782 3 60 70 80 90 NaN
9783 4 600 700 800 900 NaN
9785 We can also align on the index:
9787 >>> left, right = df.align(other, join="outer", axis=0)
9788 >>> left
9789 D B E A
9790 1 1.0 2.0 3.0 4.0
9791 2 6.0 7.0 8.0 9.0
9792 3 NaN NaN NaN NaN
9793 4 NaN NaN NaN NaN
9794 >>> right
9795 A B C D
9796 1 NaN NaN NaN NaN
9797 2 10.0 20.0 30.0 40.0
9798 3 60.0 70.0 80.0 90.0
9799 4 600.0 700.0 800.0 900.0
9801 Finally, the default `axis=None` will align on both index and columns:
9803 >>> left, right = df.align(other, join="outer", axis=None)
9804 >>> left
9805 A B C D E
9806 1 4.0 2.0 NaN 1.0 3.0
9807 2 9.0 7.0 NaN 6.0 8.0
9808 3 NaN NaN NaN NaN NaN
9809 4 NaN NaN NaN NaN NaN
9810 >>> right
9811 A B C D E
9812 1 NaN NaN NaN NaN NaN
9813 2 10.0 20.0 30.0 40.0 NaN
9814 3 60.0 70.0 80.0 90.0 NaN
9815 4 600.0 700.0 800.0 900.0 NaN
9816 """
9817 self._check_copy_deprecation(copy)
9819 _right: DataFrame | Series
9820 if axis is not None:
9821 axis = self._get_axis_number(axis)
9822 if isinstance(other, ABCDataFrame):
9823 left, _right, join_index = self._align_frame(
9824 other,
9825 join=join,
9826 axis=axis,
9827 level=level,
9828 fill_value=fill_value,
9829 )
9831 elif isinstance(other, ABCSeries):
9832 left, _right, join_index = self._align_series(
9833 other,
9834 join=join,
9835 axis=axis,
9836 level=level,
9837 fill_value=fill_value,
9838 )
9839 else: # pragma: no cover
9840 raise TypeError(f"unsupported type: {type(other)}")
9842 right = cast(NDFrameT, _right)
9843 if self.ndim == 1 or axis == 0:
9844 # If we are aligning timezone-aware DatetimeIndexes and the timezones
9845 # do not match, convert both to UTC.
9846 if isinstance(left.index.dtype, DatetimeTZDtype):
9847 if left.index.tz != right.index.tz:
9848 if join_index is not None:
9849 # GH#33671 copy to ensure we don't change the index on
9850 # our original Series
9851 left = left.copy(deep=False)
9852 right = right.copy(deep=False)
9853 left.index = join_index
9854 right.index = join_index
9856 left = left.__finalize__(self)
9857 right = right.__finalize__(other)
9858 return left, right
9860 @final
9861 def _align_frame(
9862 self,
9863 other: DataFrame,
9864 join: AlignJoin = "outer",
9865 axis: Axis | None = None,
9866 level=None,
9867 fill_value=None,
9868 ) -> tuple[Self, DataFrame, Index | None]:
9869 # defaults
9870 join_index, join_columns = None, None
9871 ilidx, iridx = None, None
9872 clidx, cridx = None, None
9874 is_series = isinstance(self, ABCSeries)
9876 if (axis is None or axis == 0) and not self.index.equals(other.index):
9877 join_index, ilidx, iridx = self.index.join(
9878 other.index, how=join, level=level, return_indexers=True
9879 )
9881 if (
9882 (axis is None or axis == 1)
9883 and not is_series
9884 and not self.columns.equals(other.columns)
9885 ):
9886 join_columns, clidx, cridx = self.columns.join(
9887 other.columns, how=join, level=level, return_indexers=True
9888 )
9890 if is_series:
9891 reindexers = {0: [join_index, ilidx]}
9892 else:
9893 reindexers = {0: [join_index, ilidx], 1: [join_columns, clidx]}
9895 left = self._reindex_with_indexers(
9896 reindexers, fill_value=fill_value, allow_dups=True
9897 )
9898 # other must be always DataFrame
9899 right = other._reindex_with_indexers(
9900 {0: [join_index, iridx], 1: [join_columns, cridx]},
9901 fill_value=fill_value,
9902 allow_dups=True,
9903 )
9904 return left, right, join_index
9906 @final
9907 def _align_series(
9908 self,
9909 other: Series,
9910 join: AlignJoin = "outer",
9911 axis: Axis | None = None,
9912 level=None,
9913 fill_value=None,
9914 ) -> tuple[Self, Series, Index | None]:
9915 is_series = isinstance(self, ABCSeries)
9917 if (not is_series and axis is None) or axis not in [None, 0, 1]:
9918 raise ValueError("Must specify axis=0 or 1")
9920 if is_series and axis == 1:
9921 raise ValueError("cannot align series to a series other than axis 0")
9923 # series/series compat, other must always be a Series
9924 if not axis:
9925 # equal
9926 if self.index.equals(other.index):
9927 join_index, lidx, ridx = None, None, None
9928 else:
9929 join_index, lidx, ridx = self.index.join(
9930 other.index, how=join, level=level, return_indexers=True
9931 )
9933 if is_series:
9934 left = self._reindex_indexer(join_index, lidx)
9935 elif lidx is None or join_index is None:
9936 left = self.copy(deep=False)
9937 else:
9938 new_mgr = self._mgr.reindex_indexer(join_index, lidx, axis=1)
9939 left = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
9941 right = other._reindex_indexer(join_index, ridx)
9943 else:
9944 # one has > 1 ndim
9945 fdata = self._mgr
9946 join_index = self.axes[1]
9947 lidx, ridx = None, None
9948 if not join_index.equals(other.index):
9949 join_index, lidx, ridx = join_index.join(
9950 other.index, how=join, level=level, return_indexers=True
9951 )
9953 if lidx is not None:
9954 bm_axis = self._get_block_manager_axis(1)
9955 fdata = fdata.reindex_indexer(join_index, lidx, axis=bm_axis)
9957 left = self._constructor_from_mgr(fdata, axes=fdata.axes)
9959 right = other._reindex_indexer(join_index, ridx)
9961 # fill
9962 fill_na = notna(fill_value)
9963 if fill_na:
9964 left = left.fillna(fill_value)
9965 right = right.fillna(fill_value)
9967 return left, right, join_index
9969 @final
9970 def _where(
9971 self,
9972 cond,
9973 other=lib.no_default,
9974 *,
9975 inplace: bool = False,
9976 axis: Axis | None = None,
9977 level=None,
9978 ) -> Self:
9979 """
9980 Equivalent to public method `where`, except that `other` is not
9981 applied as a function even if callable. Used in __setitem__.
9982 """
9983 inplace = validate_bool_kwarg(inplace, "inplace")
9985 if axis is not None:
9986 axis = self._get_axis_number(axis)
9988 # align the cond to same shape as myself
9989 cond = common.apply_if_callable(cond, self)
9990 if isinstance(cond, NDFrame):
9991 # CoW: Make sure reference is not kept alive
9992 if cond.ndim == 1 and self.ndim == 2:
9993 cond = cond._constructor_expanddim(
9994 dict.fromkeys(range(len(self.columns)), cond),
9995 copy=False,
9996 )
9997 cond.columns = self.columns
9998 cond = cond.align(self, join="right")[0]
9999 else:
10000 if not hasattr(cond, "shape"):
10001 cond = np.asanyarray(cond)
10002 if cond.shape != self.shape:
10003 raise ValueError("Array conditional must be same shape as self")
10004 cond = self._constructor(cond, **self._construct_axes_dict(), copy=False)
10006 # make sure we are boolean
10007 fill_value = bool(inplace)
10008 cond = cond.fillna(fill_value)
10009 cond = cond.infer_objects()
10011 msg = "Boolean array expected for the condition, not {dtype}"
10013 if not cond.empty:
10014 if not isinstance(cond, ABCDataFrame):
10015 # This is a single-dimensional object.
10016 if not is_bool_dtype(cond):
10017 raise TypeError(msg.format(dtype=cond.dtype))
10018 else:
10019 for block in cond._mgr.blocks:
10020 if not is_bool_dtype(block.dtype):
10021 raise TypeError(msg.format(dtype=block.dtype))
10022 if cond._mgr.any_extension_types:
10023 # GH51574: avoid object ndarray conversion later on
10024 cond = cond._constructor(
10025 cond.to_numpy(dtype=bool, na_value=fill_value),
10026 **cond._construct_axes_dict(),
10027 )
10028 else:
10029 # GH#21947 we have an empty DataFrame/Series, could be object-dtype
10030 cond = cond.astype(bool)
10032 cond = -cond if inplace else cond
10033 cond = cond.reindex(self._info_axis, axis=self._info_axis_number)
10035 # try to align with other
10036 if isinstance(other, NDFrame):
10037 # align with me
10038 if other.ndim <= self.ndim:
10039 # CoW: Make sure reference is not kept alive
10040 other = self.align(
10041 other,
10042 join="left",
10043 axis=axis,
10044 level=level,
10045 fill_value=None,
10046 )[1]
10048 # if we are NOT aligned, raise as we cannot where index
10049 if axis is None and not other._indexed_same(self):
10050 raise InvalidIndexError
10052 if other.ndim < self.ndim:
10053 other = other._values
10054 if isinstance(other, np.ndarray):
10055 # TODO(EA2D): could also do this for NDArrayBackedEA cases?
10056 if axis == 0:
10057 other = np.reshape(other, (-1, 1))
10058 elif axis == 1:
10059 other = np.reshape(other, (1, -1))
10061 other = np.broadcast_to(other, self.shape)
10062 else:
10063 # GH#38729, GH#62038 avoid lossy casting or object-casting
10064 if axis == 0:
10065 res_cols = [
10066 self.iloc[:, i]._where(
10067 cond.iloc[:, i],
10068 other,
10069 )
10070 for i in range(self.shape[1])
10071 ]
10072 elif axis == 1:
10073 # TODO: can we use a zero-copy alternative to "repeat"?
10074 res_cols = [
10075 self.iloc[:, i]._where(
10076 cond.iloc[:, i],
10077 other[i : i + 1].repeat(len(self)),
10078 )
10079 for i in range(self.shape[1])
10080 ]
10081 res = self._constructor(dict(enumerate(res_cols)))
10082 res.index = self.index
10083 res.columns = self.columns
10084 if inplace:
10085 self._update_inplace(res)
10086 return self
10087 return res.__finalize__(self)
10089 # slice me out of the other
10090 else:
10091 raise NotImplementedError(
10092 "cannot align with a higher dimensional NDFrame"
10093 )
10095 elif not isinstance(other, (MultiIndex, NDFrame)):
10096 # mainly just catching Index here
10097 other = extract_array(other, extract_numpy=True)
10099 if isinstance(other, (np.ndarray, ExtensionArray)):
10100 if other.shape != self.shape:
10101 if self.ndim != 1:
10102 # In the ndim == 1 case we may have
10103 # other length 1, which we treat as scalar (GH#2745, GH#4192)
10104 # or len(other) == icond.sum(), which we treat like
10105 # __setitem__ (GH#3235)
10106 raise ValueError(
10107 "other must be the same shape as self when an ndarray"
10108 )
10110 # we are the same shape, so create an actual object for alignment
10111 else:
10112 other = self._constructor(
10113 other, **self._construct_axes_dict(), copy=False
10114 )
10116 if axis is None:
10117 axis = 0
10119 if self.ndim == getattr(other, "ndim", 0):
10120 align = True
10121 else:
10122 align = self._get_axis_number(axis) == 1
10124 if inplace:
10125 # we may have different type blocks come out of putmask, so
10126 # reconstruct the block manager
10128 new_data = self._mgr.putmask(mask=cond, new=other, align=align)
10129 result = self._constructor_from_mgr(new_data, axes=new_data.axes)
10130 self._update_inplace(result)
10131 return self
10133 else:
10134 new_data = self._mgr.where(
10135 other=other,
10136 cond=cond,
10137 align=align,
10138 )
10139 result = self._constructor_from_mgr(new_data, axes=new_data.axes)
10140 return result.__finalize__(self)
10142 @final
10143 def where(
10144 self,
10145 cond,
10146 other=lib.no_default,
10147 *,
10148 inplace: bool = False,
10149 axis: Axis | None = None,
10150 level: Level | None = None,
10151 ) -> Self:
10152 """
10153 Replace values where the condition is False.
10155 This method allows conditional replacement of values. Where the
10156 condition evaluates to True, the original values are retained; where
10157 it evaluates to False, values are replaced with corresponding entries
10158 from ``other``.
10160 Parameters
10161 ----------
10162 cond : bool Series/DataFrame, array-like, or callable
10163 Where `cond` is True, keep the original value. Where
10164 False, replace with corresponding value from `other`.
10165 If `cond` is callable, it is computed on the Series/DataFrame and
10166 should return boolean Series/DataFrame or array. The callable must
10167 not change input Series/DataFrame (though pandas doesn't check it).
10168 other : scalar, Series/DataFrame, or callable
10169 Entries where `cond` is False are replaced with
10170 corresponding value from `other`.
10171 If other is callable, it is computed on the Series/DataFrame and
10172 should return scalar or Series/DataFrame. The callable must not
10173 change input Series/DataFrame (though pandas doesn't check it).
10174 If not specified, entries will be filled with the corresponding
10175 NULL value (``np.nan`` for numpy dtypes, ``pd.NA`` for extension
10176 dtypes).
10177 inplace : bool, default False
10178 Whether to perform the operation in place on the data.
10179 axis : int, default None
10180 Alignment axis if needed. For `Series` this parameter is
10181 unused and defaults to 0.
10182 level : int, default None
10183 Alignment level if needed.
10185 Returns
10186 -------
10187 Series or DataFrame
10188 When applied to a Series, the function will return a Series,
10189 and when applied to a DataFrame, it will return a DataFrame.
10191 See Also
10192 --------
10193 :func:`DataFrame.mask` : Return an object of same shape as caller.
10194 :func:`Series.mask` : Return an object of same shape as caller.
10196 Notes
10197 -----
10198 The where method is an application of the if-then idiom. For each
10199 element in the caller, if ``cond`` is ``True`` the
10200 element is used; otherwise the corresponding element from
10201 ``other`` is used. If the axis of ``other`` does not align with axis of
10202 ``cond`` Series/DataFrame, the values of ``cond`` on misaligned index positions
10203 will be filled with False.
10205 The signature for :func:`Series.where` or
10206 :func:`DataFrame.where` differs from :func:`numpy.where`.
10207 Roughly ``df1.where(m, df2)`` is equivalent to ``np.where(m, df1, df2)``.
10209 For further details and examples see the ``where`` documentation in
10210 :ref:`indexing <indexing.where_mask>`.
10212 The dtype of the object takes precedence. The fill value is casted to
10213 the object's dtype, if this can be done losslessly.
10215 Examples
10216 --------
10217 >>> s = pd.Series(range(5))
10218 >>> s.where(s > 0)
10219 0 NaN
10220 1 1.0
10221 2 2.0
10222 3 3.0
10223 4 4.0
10224 dtype: float64
10225 >>> s.mask(s > 0)
10226 0 0.0
10227 1 NaN
10228 2 NaN
10229 3 NaN
10230 4 NaN
10231 dtype: float64
10233 >>> s = pd.Series(range(5))
10234 >>> t = pd.Series([True, False])
10235 >>> s.where(t, 99)
10236 0 0
10237 1 99
10238 2 99
10239 3 99
10240 4 99
10241 dtype: int64
10242 >>> s.mask(t, 99)
10243 0 99
10244 1 1
10245 2 99
10246 3 99
10247 4 99
10248 dtype: int64
10250 >>> s.where(s > 1, 10)
10251 0 10
10252 1 10
10253 2 2
10254 3 3
10255 4 4
10256 dtype: int64
10257 >>> s.mask(s > 1, 10)
10258 0 0
10259 1 1
10260 2 10
10261 3 10
10262 4 10
10263 dtype: int64
10265 >>> df = pd.DataFrame(np.arange(10).reshape(-1, 2), columns=["A", "B"])
10266 >>> df
10267 A B
10268 0 0 1
10269 1 2 3
10270 2 4 5
10271 3 6 7
10272 4 8 9
10273 >>> m = df % 3 == 0
10274 >>> df.where(m, -df)
10275 A B
10276 0 0 -1
10277 1 -2 3
10278 2 -4 -5
10279 3 6 -7
10280 4 -8 9
10281 >>> df.where(m, -df) == np.where(m, df, -df)
10282 A B
10283 0 True True
10284 1 True True
10285 2 True True
10286 3 True True
10287 4 True True
10288 >>> df.where(m, -df) == df.mask(~m, -df)
10289 A B
10290 0 True True
10291 1 True True
10292 2 True True
10293 3 True True
10294 4 True True
10295 """
10296 inplace = validate_bool_kwarg(inplace, "inplace")
10297 if inplace:
10298 if not CHAINED_WARNING_DISABLED:
10299 if sys.getrefcount(
10300 self
10301 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
10302 warnings.warn(
10303 _chained_assignment_method_msg,
10304 ChainedAssignmentError,
10305 stacklevel=2,
10306 )
10308 other = common.apply_if_callable(other, self)
10309 return self._where(cond, other, inplace=inplace, axis=axis, level=level)
10311 @final
10312 def mask(
10313 self,
10314 cond,
10315 other=lib.no_default,
10316 *,
10317 inplace: bool = False,
10318 axis: Axis | None = None,
10319 level: Level | None = None,
10320 ) -> Self:
10321 """
10322 Replace values where the condition is True.
10324 Parameters
10325 ----------
10326 cond : bool Series/DataFrame, array-like, or callable
10327 Where `cond` is False, keep the original value. Where
10328 True, replace with corresponding value from `other`.
10329 If `cond` is callable, it is computed on the Series/DataFrame and
10330 should return boolean Series/DataFrame or array. The callable must
10331 not change input Series/DataFrame (though pandas doesn't check it).
10332 other : scalar, Series/DataFrame, or callable
10333 Entries where `cond` is True are replaced with
10334 corresponding value from `other`.
10335 If other is callable, it is computed on the Series/DataFrame and
10336 should return scalar or Series/DataFrame. The callable must not
10337 change input Series/DataFrame (though pandas doesn't check it).
10338 If not specified, entries will be filled with the corresponding
10339 NULL value (``np.nan`` for numpy dtypes, ``pd.NA`` for extension
10340 dtypes).
10341 inplace : bool, default False
10342 Whether to perform the operation in place on the data.
10343 axis : int, default None
10344 Alignment axis if needed. For `Series` this parameter is
10345 unused and defaults to 0.
10346 level : int, default None
10347 Alignment level if needed.
10349 Returns
10350 -------
10351 Series or DataFrame
10352 When applied to a Series, the function will return a Series,
10353 and when applied to a DataFrame, it will return a DataFrame.
10355 See Also
10356 --------
10357 :func:`DataFrame.where` : Return an object of same shape as caller.
10358 :func:`Series.where` : Return an object of same shape as caller.
10360 Notes
10361 -----
10362 The mask method is an application of the if-then idiom. For each
10363 element in the caller, if ``cond`` is ``False`` the
10364 element is used; otherwise the corresponding element from
10365 ``other`` is used. If the axis of ``other`` does not align with axis of
10366 ``cond`` Series/DataFrame, the values of ``cond`` on misaligned index positions
10367 will be filled with True.
10369 The signature for :func:`Series.where` or
10370 :func:`DataFrame.where` differs from :func:`numpy.where`.
10371 Roughly ``df1.where(m, df2)`` is equivalent to ``np.where(m, df1, df2)``.
10373 For further details and examples see the ``mask`` documentation in
10374 :ref:`indexing <indexing.where_mask>`.
10376 The dtype of the object takes precedence. The fill value is casted to
10377 the object's dtype, if this can be done losslessly.
10379 Examples
10380 --------
10381 >>> s = pd.Series(range(5))
10382 >>> s.where(s > 0)
10383 0 NaN
10384 1 1.0
10385 2 2.0
10386 3 3.0
10387 4 4.0
10388 dtype: float64
10389 >>> s.mask(s > 0)
10390 0 0.0
10391 1 NaN
10392 2 NaN
10393 3 NaN
10394 4 NaN
10395 dtype: float64
10397 >>> s = pd.Series(range(5))
10398 >>> t = pd.Series([True, False])
10399 >>> s.where(t, 99)
10400 0 0
10401 1 99
10402 2 99
10403 3 99
10404 4 99
10405 dtype: int64
10406 >>> s.mask(t, 99)
10407 0 99
10408 1 1
10409 2 99
10410 3 99
10411 4 99
10412 dtype: int64
10414 >>> s.where(s > 1, 10)
10415 0 10
10416 1 10
10417 2 2
10418 3 3
10419 4 4
10420 dtype: int64
10421 >>> s.mask(s > 1, 10)
10422 0 0
10423 1 1
10424 2 10
10425 3 10
10426 4 10
10427 dtype: int64
10429 >>> df = pd.DataFrame(np.arange(10).reshape(-1, 2), columns=["A", "B"])
10430 >>> df
10431 A B
10432 0 0 1
10433 1 2 3
10434 2 4 5
10435 3 6 7
10436 4 8 9
10437 >>> m = df % 3 == 0
10438 >>> df.where(m, -df)
10439 A B
10440 0 0 -1
10441 1 -2 3
10442 2 -4 -5
10443 3 6 -7
10444 4 -8 9
10445 >>> df.where(m, -df) == np.where(m, df, -df)
10446 A B
10447 0 True True
10448 1 True True
10449 2 True True
10450 3 True True
10451 4 True True
10452 >>> df.where(m, -df) == df.mask(~m, -df)
10453 A B
10454 0 True True
10455 1 True True
10456 2 True True
10457 3 True True
10458 4 True True
10459 """
10460 inplace = validate_bool_kwarg(inplace, "inplace")
10461 if inplace:
10462 if not CHAINED_WARNING_DISABLED:
10463 if sys.getrefcount(
10464 self
10465 ) <= REF_COUNT_METHOD and not common.is_local_in_caller_frame(self):
10466 warnings.warn(
10467 _chained_assignment_method_msg,
10468 ChainedAssignmentError,
10469 stacklevel=2,
10470 )
10472 cond = common.apply_if_callable(cond, self)
10473 other = common.apply_if_callable(other, self)
10475 # see gh-21891
10476 if not hasattr(cond, "__invert__"):
10477 cond = np.array(cond)
10479 return self._where(
10480 ~cond,
10481 other=other,
10482 inplace=inplace,
10483 axis=axis,
10484 level=level,
10485 )
10487 def shift(
10488 self,
10489 periods: int | Sequence[int] = 1,
10490 freq=None,
10491 axis: Axis = 0,
10492 fill_value: Hashable = lib.no_default,
10493 suffix: str | None = None,
10494 ) -> Self | DataFrame:
10495 """
10496 Shift index by desired number of periods with an optional time `freq`.
10498 When `freq` is not passed, shift the index without realigning the data.
10499 If `freq` is passed (in this case, the index must be date or datetime,
10500 or it will raise a `NotImplementedError`), the index will be
10501 increased using the periods and the `freq`. `freq` can be inferred
10502 when specified as "infer" as long as either freq or inferred_freq
10503 attribute is set in the index.
10505 Parameters
10506 ----------
10507 periods : int or Sequence
10508 Number of periods to shift. Can be positive or negative.
10509 If an iterable of ints, the data will be shifted once by each int.
10510 This is equivalent to shifting by one value at a time and
10511 concatenating all resulting frames. The resulting columns will have
10512 the shift suffixed to their column names. For multiple periods,
10513 axis must not be 1.
10514 freq : DateOffset, tseries.offsets, timedelta, or str, optional
10515 Offset to use from the tseries module or time rule (e.g. 'EOM').
10516 If `freq` is specified then the index values are shifted but the
10517 data is not realigned. That is, use `freq` if you would like to
10518 extend the index when shifting and preserve the original data.
10519 If `freq` is specified as "infer" then it will be inferred from
10520 the freq or inferred_freq attributes of the index. If neither of
10521 those attributes exist, a ValueError is thrown.
10522 axis : {0 or 'index', 1 or 'columns', None}, default None
10523 Shift direction. For `Series` this parameter is unused and defaults to 0.
10524 fill_value : object, optional
10525 The scalar value to use for newly introduced missing values.
10526 the default depends on the dtype of `self`.
10527 For Boolean and numeric NumPy data types, ``np.nan`` is used.
10528 For datetime, timedelta, or period data, etc. :attr:`NaT` is used.
10529 For extension dtypes, ``self.dtype.na_value`` is used.
10530 suffix : str, optional
10531 If str and periods is an iterable, this is added after the column
10532 name and before the shift value for each shifted column name.
10533 For `Series` this parameter is unused and defaults to `None`.
10535 Returns
10536 -------
10537 Series/DataFrame
10538 Copy of input object, shifted.
10540 See Also
10541 --------
10542 Index.shift : Shift values of Index.
10543 DatetimeIndex.shift : Shift values of DatetimeIndex.
10544 PeriodIndex.shift : Shift values of PeriodIndex.
10546 Examples
10547 --------
10548 >>> df = pd.DataFrame(
10549 ... [[10, 13, 17], [20, 23, 27], [15, 18, 22], [30, 33, 37], [45, 48, 52]],
10550 ... columns=["Col1", "Col2", "Col3"],
10551 ... index=pd.date_range("2020-01-01", "2020-01-05"),
10552 ... )
10553 >>> df
10554 Col1 Col2 Col3
10555 2020-01-01 10 13 17
10556 2020-01-02 20 23 27
10557 2020-01-03 15 18 22
10558 2020-01-04 30 33 37
10559 2020-01-05 45 48 52
10561 >>> df.shift(periods=3)
10562 Col1 Col2 Col3
10563 2020-01-01 NaN NaN NaN
10564 2020-01-02 NaN NaN NaN
10565 2020-01-03 NaN NaN NaN
10566 2020-01-04 10.0 13.0 17.0
10567 2020-01-05 20.0 23.0 27.0
10569 >>> df.shift(periods=1, axis="columns")
10570 Col1 Col2 Col3
10571 2020-01-01 NaN 10 13
10572 2020-01-02 NaN 20 23
10573 2020-01-03 NaN 15 18
10574 2020-01-04 NaN 30 33
10575 2020-01-05 NaN 45 48
10577 >>> df.shift(periods=3, fill_value=0)
10578 Col1 Col2 Col3
10579 2020-01-01 0 0 0
10580 2020-01-02 0 0 0
10581 2020-01-03 0 0 0
10582 2020-01-04 10 13 17
10583 2020-01-05 20 23 27
10585 >>> df.shift(periods=3, freq="D")
10586 Col1 Col2 Col3
10587 2020-01-04 10 13 17
10588 2020-01-05 20 23 27
10589 2020-01-06 15 18 22
10590 2020-01-07 30 33 37
10591 2020-01-08 45 48 52
10593 >>> df.shift(periods=3, freq="infer")
10594 Col1 Col2 Col3
10595 2020-01-04 10 13 17
10596 2020-01-05 20 23 27
10597 2020-01-06 15 18 22
10598 2020-01-07 30 33 37
10599 2020-01-08 45 48 52
10601 >>> df["Col1"].shift(periods=[0, 1, 2])
10602 Col1_0 Col1_1 Col1_2
10603 2020-01-01 10 NaN NaN
10604 2020-01-02 20 10.0 NaN
10605 2020-01-03 15 20.0 10.0
10606 2020-01-04 30 15.0 20.0
10607 2020-01-05 45 30.0 15.0
10608 """
10609 axis = self._get_axis_number(axis)
10611 if freq is not None and fill_value is not lib.no_default:
10612 # GH#53832
10613 raise ValueError(
10614 "Passing a 'freq' together with a 'fill_value' is not allowed."
10615 )
10617 if periods == 0:
10618 return self.copy(deep=False)
10620 if is_list_like(periods) and isinstance(self, ABCSeries):
10621 return self.to_frame().shift(
10622 periods=periods, freq=freq, axis=axis, fill_value=fill_value
10623 )
10624 periods = cast(int, periods)
10626 if freq is None:
10627 # when freq is None, data is shifted, index is not
10628 axis = self._get_axis_number(axis)
10629 assert axis == 0 # axis == 1 cases handled in DataFrame.shift
10630 new_data = self._mgr.shift(periods=periods, fill_value=fill_value)
10631 return self._constructor_from_mgr(
10632 new_data, axes=new_data.axes
10633 ).__finalize__(self, method="shift")
10635 return self._shift_with_freq(periods, axis, freq)
10637 @final
10638 def _shift_with_freq(self, periods: int, axis: int, freq) -> Self:
10639 # see shift.__doc__
10640 # when freq is given, index is shifted, data is not
10641 index = self._get_axis(axis)
10643 if freq == "infer":
10644 freq = getattr(index, "freq", None)
10646 if freq is None:
10647 freq = getattr(index, "inferred_freq", None)
10649 if freq is None:
10650 msg = "Freq was not set in the index hence cannot be inferred"
10651 raise ValueError(msg)
10653 elif isinstance(freq, str):
10654 is_period = isinstance(index, PeriodIndex)
10655 freq = to_offset(freq, is_period=is_period)
10657 if isinstance(index, PeriodIndex):
10658 orig_freq = to_offset(index.freq)
10659 if freq != orig_freq:
10660 assert orig_freq is not None # for mypy
10661 raise ValueError(
10662 f"Given freq {PeriodDtype(freq)._freqstr} "
10663 f"does not match PeriodIndex freq "
10664 f"{PeriodDtype(orig_freq)._freqstr}"
10665 )
10666 new_ax: Index = index.shift(periods)
10667 else:
10668 new_ax = index.shift(periods, freq)
10670 result = self.set_axis(new_ax, axis=axis)
10671 return result.__finalize__(self, method="shift")
10673 @final
10674 def truncate(
10675 self,
10676 before=None,
10677 after=None,
10678 axis: Axis | None = None,
10679 copy: bool | lib.NoDefault = lib.no_default,
10680 ) -> Self:
10681 """
10682 Truncate a Series or DataFrame before and after some index value.
10684 This is a useful shorthand for boolean indexing based on index
10685 values above or below certain thresholds.
10687 Parameters
10688 ----------
10689 before : date, str, int
10690 Truncate all rows before this index value.
10691 after : date, str, int
10692 Truncate all rows after this index value.
10693 axis : {0 or 'index', 1 or 'columns'}, optional
10694 Axis to truncate. Truncates the index (rows) by default.
10695 For `Series` this parameter is unused and defaults to 0.
10696 copy : bool, default False
10697 This keyword is now ignored; changing its value will have no
10698 impact on the method.
10700 .. deprecated:: 3.0.0
10702 This keyword is ignored and will be removed in pandas 4.0. Since
10703 pandas 3.0, this method always returns a new object using a lazy
10704 copy mechanism that defers copies until necessary
10705 (Copy-on-Write). See the `user guide on Copy-on-Write
10706 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
10707 for more details.
10709 Returns
10710 -------
10711 type of caller
10712 The truncated Series or DataFrame.
10714 See Also
10715 --------
10716 DataFrame.loc : Select a subset of a DataFrame by label.
10717 DataFrame.iloc : Select a subset of a DataFrame by position.
10719 Notes
10720 -----
10721 If the index being truncated contains only datetime values,
10722 `before` and `after` may be specified as strings instead of
10723 Timestamps.
10725 Examples
10726 --------
10727 >>> df = pd.DataFrame(
10728 ... {
10729 ... "A": ["a", "b", "c", "d", "e"],
10730 ... "B": ["f", "g", "h", "i", "j"],
10731 ... "C": ["k", "l", "m", "n", "o"],
10732 ... },
10733 ... index=[1, 2, 3, 4, 5],
10734 ... )
10735 >>> df
10736 A B C
10737 1 a f k
10738 2 b g l
10739 3 c h m
10740 4 d i n
10741 5 e j o
10743 >>> df.truncate(before=2, after=4)
10744 A B C
10745 2 b g l
10746 3 c h m
10747 4 d i n
10749 The columns of a DataFrame can be truncated.
10751 >>> df.truncate(before="A", after="B", axis="columns")
10752 A B
10753 1 a f
10754 2 b g
10755 3 c h
10756 4 d i
10757 5 e j
10759 For Series, only rows can be truncated.
10761 >>> df["A"].truncate(before=2, after=4)
10762 2 b
10763 3 c
10764 4 d
10765 Name: A, dtype: str
10767 The index values in ``truncate`` can be datetimes or string
10768 dates.
10770 >>> dates = pd.date_range("2016-01-01", "2016-02-01", freq="s")
10771 >>> df = pd.DataFrame(index=dates, data={"A": 1})
10772 >>> df.tail()
10773 A
10774 2016-01-31 23:59:56 1
10775 2016-01-31 23:59:57 1
10776 2016-01-31 23:59:58 1
10777 2016-01-31 23:59:59 1
10778 2016-02-01 00:00:00 1
10780 >>> df.truncate(
10781 ... before=pd.Timestamp("2016-01-05"), after=pd.Timestamp("2016-01-10")
10782 ... ).tail()
10783 A
10784 2016-01-09 23:59:56 1
10785 2016-01-09 23:59:57 1
10786 2016-01-09 23:59:58 1
10787 2016-01-09 23:59:59 1
10788 2016-01-10 00:00:00 1
10790 Because the index is a DatetimeIndex containing only dates, we can
10791 specify `before` and `after` as strings. They will be coerced to
10792 Timestamps before truncation.
10794 >>> df.truncate("2016-01-05", "2016-01-10").tail()
10795 A
10796 2016-01-09 23:59:56 1
10797 2016-01-09 23:59:57 1
10798 2016-01-09 23:59:58 1
10799 2016-01-09 23:59:59 1
10800 2016-01-10 00:00:00 1
10802 Note that ``truncate`` assumes a 0 value for any unspecified time
10803 component (midnight). This differs from partial string slicing, which
10804 returns any partially matching dates.
10806 >>> df.loc["2016-01-05":"2016-01-10", :].tail()
10807 A
10808 2016-01-10 23:59:55 1
10809 2016-01-10 23:59:56 1
10810 2016-01-10 23:59:57 1
10811 2016-01-10 23:59:58 1
10812 2016-01-10 23:59:59 1
10813 """
10814 self._check_copy_deprecation(copy)
10816 if axis is None:
10817 axis = 0
10818 axis = self._get_axis_number(axis)
10819 ax = self._get_axis(axis)
10821 # GH 17935
10822 # Check that index is sorted
10823 if not ax.is_monotonic_increasing and not ax.is_monotonic_decreasing:
10824 raise ValueError("truncate requires a sorted index")
10826 # if we have a date index, convert to dates, otherwise
10827 # treat like a slice
10828 if ax._is_all_dates:
10829 from pandas.core.tools.datetimes import to_datetime
10831 if before is not None:
10832 # Avoid converting to NaT
10833 before = to_datetime(before)
10834 if after is not None:
10835 # Avoid converting to NaT
10836 after = to_datetime(after)
10838 if before is not None and after is not None and before > after:
10839 raise ValueError(f"Truncate: {after} must be after {before}")
10841 if len(ax) > 1 and ax.is_monotonic_decreasing and ax.nunique() > 1:
10842 before, after = after, before
10844 slicer = [slice(None, None)] * self._AXIS_LEN
10845 slicer[axis] = slice(before, after)
10846 result = self.loc[tuple(slicer)]
10848 if isinstance(ax, MultiIndex):
10849 setattr(result, self._get_axis_name(axis), ax.truncate(before, after))
10851 result = result.copy(deep=False)
10853 return result
10855 @final
10856 def tz_convert(
10857 self,
10858 tz,
10859 axis: Axis = 0,
10860 level=None,
10861 copy: bool | lib.NoDefault = lib.no_default,
10862 ) -> Self:
10863 """
10864 Convert tz-aware axis to target time zone.
10866 Parameters
10867 ----------
10868 tz : str or tzinfo object or None
10869 Target time zone. Passing ``None`` will convert to
10870 UTC and remove the timezone information.
10871 axis : {0 or 'index', 1 or 'columns'}, default 0
10872 The axis to convert
10873 level : int, str, default None
10874 If axis is a MultiIndex, convert a specific level. Otherwise
10875 must be None.
10876 copy : bool, default False
10877 This keyword is now ignored; changing its value will have no
10878 impact on the method.
10880 .. deprecated:: 3.0.0
10882 This keyword is ignored and will be removed in pandas 4.0. Since
10883 pandas 3.0, this method always returns a new object using a lazy
10884 copy mechanism that defers copies until necessary
10885 (Copy-on-Write). See the `user guide on Copy-on-Write
10886 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
10887 for more details.
10889 Returns
10890 -------
10891 Series/DataFrame
10892 Object with time zone converted axis.
10894 Raises
10895 ------
10896 TypeError
10897 If the axis is tz-naive.
10899 See Also
10900 --------
10901 DataFrame.tz_localize: Localize tz-naive index of DataFrame to target time zone.
10902 Series.tz_localize: Localize tz-naive index of Series to target time zone.
10904 Examples
10905 --------
10906 Change to another time zone:
10908 >>> s = pd.Series(
10909 ... [1],
10910 ... index=pd.DatetimeIndex(["2018-09-15 01:30:00+02:00"]),
10911 ... )
10912 >>> s.tz_convert("Asia/Shanghai")
10913 2018-09-15 07:30:00+08:00 1
10914 dtype: int64
10916 Pass None to convert to UTC and get a tz-naive index:
10918 >>> s = pd.Series([1], index=pd.DatetimeIndex(["2018-09-15 01:30:00+02:00"]))
10919 >>> s.tz_convert(None)
10920 2018-09-14 23:30:00 1
10921 dtype: int64
10922 """
10923 self._check_copy_deprecation(copy)
10924 axis = self._get_axis_number(axis)
10925 ax = self._get_axis(axis)
10927 def _tz_convert(ax, tz):
10928 if not hasattr(ax, "tz_convert"):
10929 if len(ax) > 0:
10930 ax_name = self._get_axis_name(axis)
10931 raise TypeError(
10932 f"{ax_name} is not a valid DatetimeIndex or PeriodIndex"
10933 )
10934 ax = DatetimeIndex([], tz=tz)
10935 else:
10936 ax = ax.tz_convert(tz)
10937 return ax
10939 # if a level is given it must be a MultiIndex level or
10940 # equivalent to the axis name
10941 if isinstance(ax, MultiIndex):
10942 level = ax._get_level_number(level)
10943 new_level = _tz_convert(ax.levels[level], tz)
10944 ax = ax.set_levels(new_level, level=level)
10945 else:
10946 if level not in (None, 0, ax.name):
10947 raise ValueError(f"The level {level} is not valid")
10948 ax = _tz_convert(ax, tz)
10950 result = self.copy(deep=False)
10951 result = result.set_axis(ax, axis=axis)
10952 return result.__finalize__(self, method="tz_convert")
10954 @final
10955 def tz_localize(
10956 self,
10957 tz,
10958 axis: Axis = 0,
10959 level=None,
10960 copy: bool | lib.NoDefault = lib.no_default,
10961 ambiguous: TimeAmbiguous = "raise",
10962 nonexistent: TimeNonexistent = "raise",
10963 ) -> Self:
10964 """
10965 Localize time zone naive index of a Series or DataFrame to target time zone.
10967 This operation localizes the Index. To localize the values in a
10968 time zone naive Series, use :meth:`Series.dt.tz_localize`.
10970 Parameters
10971 ----------
10972 tz : str or tzinfo or None
10973 Time zone to localize. Passing ``None`` will remove the
10974 time zone information and preserve local time.
10975 axis : {0 or 'index', 1 or 'columns'}, default 0
10976 The axis to localize
10977 level : int, str, default None
10978 If axis ia a MultiIndex, localize a specific level. Otherwise
10979 must be None.
10980 copy : bool, default False
10981 This keyword is now ignored; changing its value will have no
10982 impact on the method.
10984 .. deprecated:: 3.0.0
10986 This keyword is ignored and will be removed in pandas 4.0. Since
10987 pandas 3.0, this method always returns a new object using a lazy
10988 copy mechanism that defers copies until necessary
10989 (Copy-on-Write). See the `user guide on Copy-on-Write
10990 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
10991 for more details.
10993 ambiguous : 'infer', bool, bool-ndarray, 'NaT', default 'raise'
10994 When clocks moved backward due to DST, ambiguous times may arise.
10995 For example in Central European Time (UTC+01), when going from
10996 03:00 DST to 02:00 non-DST, 02:30:00 local time occurs both at
10997 00:30:00 UTC and at 01:30:00 UTC. In such a situation, the
10998 `ambiguous` parameter dictates how ambiguous times should be
10999 handled.
11001 - 'infer' will attempt to infer fall dst-transition hours based on
11002 order
11003 - bool (or bool-ndarray) where True signifies a DST time, False designates
11004 a non-DST time (note that this flag is only applicable for
11005 ambiguous times)
11006 - 'NaT' will return NaT where there are ambiguous times
11007 - 'raise' will raise a ValueError if there are ambiguous
11008 times.
11009 nonexistent : str, default 'raise'
11010 A nonexistent time does not exist in a particular timezone
11011 where clocks moved forward due to DST. Valid values are:
11013 - 'shift_forward' will shift the nonexistent time forward to the
11014 closest existing time
11015 - 'shift_backward' will shift the nonexistent time backward to the
11016 closest existing time
11017 - 'NaT' will return NaT where there are nonexistent times
11018 - timedelta objects will shift nonexistent times by the timedelta
11019 - 'raise' will raise a ValueError if there are
11020 nonexistent times.
11022 Returns
11023 -------
11024 Series/DataFrame
11025 Same type as the input, with time zone naive or aware index, depending on
11026 ``tz``.
11028 Raises
11029 ------
11030 TypeError
11031 If the TimeSeries is tz-aware and tz is not None.
11033 See Also
11034 --------
11035 Series.dt.tz_localize: Localize the values in a time zone naive Series.
11036 Timestamp.tz_localize: Localize the Timestamp to a timezone.
11038 Examples
11039 --------
11040 Localize local times:
11042 >>> s = pd.Series(
11043 ... [1],
11044 ... index=pd.DatetimeIndex(["2018-09-15 01:30:00"]),
11045 ... )
11046 >>> s.tz_localize("CET")
11047 2018-09-15 01:30:00+02:00 1
11048 dtype: int64
11050 Pass None to convert to tz-naive index and preserve local time:
11052 >>> s = pd.Series([1], index=pd.DatetimeIndex(["2018-09-15 01:30:00+02:00"]))
11053 >>> s.tz_localize(None)
11054 2018-09-15 01:30:00 1
11055 dtype: int64
11057 Be careful with DST changes. When there is sequential data, pandas
11058 can infer the DST time:
11060 >>> s = pd.Series(
11061 ... range(7),
11062 ... index=pd.DatetimeIndex(
11063 ... [
11064 ... "2018-10-28 01:30:00",
11065 ... "2018-10-28 02:00:00",
11066 ... "2018-10-28 02:30:00",
11067 ... "2018-10-28 02:00:00",
11068 ... "2018-10-28 02:30:00",
11069 ... "2018-10-28 03:00:00",
11070 ... "2018-10-28 03:30:00",
11071 ... ]
11072 ... ),
11073 ... )
11074 >>> s.tz_localize("CET", ambiguous="infer")
11075 2018-10-28 01:30:00+02:00 0
11076 2018-10-28 02:00:00+02:00 1
11077 2018-10-28 02:30:00+02:00 2
11078 2018-10-28 02:00:00+01:00 3
11079 2018-10-28 02:30:00+01:00 4
11080 2018-10-28 03:00:00+01:00 5
11081 2018-10-28 03:30:00+01:00 6
11082 dtype: int64
11084 In some cases, inferring the DST is impossible. In such cases, you can
11085 pass an ndarray to the ambiguous parameter to set the DST explicitly
11087 >>> s = pd.Series(
11088 ... range(3),
11089 ... index=pd.DatetimeIndex(
11090 ... [
11091 ... "2018-10-28 01:20:00",
11092 ... "2018-10-28 02:36:00",
11093 ... "2018-10-28 03:46:00",
11094 ... ]
11095 ... ),
11096 ... )
11097 >>> s.tz_localize("CET", ambiguous=np.array([True, True, False]))
11098 2018-10-28 01:20:00+02:00 0
11099 2018-10-28 02:36:00+02:00 1
11100 2018-10-28 03:46:00+01:00 2
11101 dtype: int64
11103 If the DST transition causes nonexistent times, you can shift these
11104 dates forward or backward with a timedelta object or `'shift_forward'`
11105 or `'shift_backward'`.
11107 >>> dti = pd.DatetimeIndex(
11108 ... ["2015-03-29 02:30:00", "2015-03-29 03:30:00"], dtype="M8[ns]"
11109 ... )
11110 >>> s = pd.Series(range(2), index=dti)
11111 >>> s.tz_localize("Europe/Warsaw", nonexistent="shift_forward")
11112 2015-03-29 03:00:00+02:00 0
11113 2015-03-29 03:30:00+02:00 1
11114 dtype: int64
11115 >>> s.tz_localize("Europe/Warsaw", nonexistent="shift_backward")
11116 2015-03-29 01:59:59.999999999+01:00 0
11117 2015-03-29 03:30:00+02:00 1
11118 dtype: int64
11119 >>> s.tz_localize("Europe/Warsaw", nonexistent=pd.Timedelta("1h"))
11120 2015-03-29 03:30:00+02:00 0
11121 2015-03-29 03:30:00+02:00 1
11122 dtype: int64
11123 """
11124 self._check_copy_deprecation(copy)
11125 nonexistent_options = ("raise", "NaT", "shift_forward", "shift_backward")
11126 if nonexistent not in nonexistent_options and not isinstance(
11127 nonexistent, dt.timedelta
11128 ):
11129 raise ValueError(
11130 "The nonexistent argument must be one of 'raise', "
11131 "'NaT', 'shift_forward', 'shift_backward' or "
11132 "a timedelta object"
11133 )
11135 axis = self._get_axis_number(axis)
11136 ax = self._get_axis(axis)
11138 def _tz_localize(ax, tz, ambiguous, nonexistent):
11139 if not hasattr(ax, "tz_localize"):
11140 if len(ax) > 0:
11141 ax_name = self._get_axis_name(axis)
11142 raise TypeError(
11143 f"{ax_name} is not a valid DatetimeIndex or PeriodIndex"
11144 )
11145 ax = DatetimeIndex([], tz=tz)
11146 else:
11147 ax = ax.tz_localize(tz, ambiguous=ambiguous, nonexistent=nonexistent)
11148 return ax
11150 # if a level is given it must be a MultiIndex level or
11151 # equivalent to the axis name
11152 if isinstance(ax, MultiIndex):
11153 level = ax._get_level_number(level)
11154 new_level = _tz_localize(ax.levels[level], tz, ambiguous, nonexistent)
11155 ax = ax.set_levels(new_level, level=level)
11156 else:
11157 if level not in (None, 0, ax.name):
11158 raise ValueError(f"The level {level} is not valid")
11159 ax = _tz_localize(ax, tz, ambiguous, nonexistent)
11161 result = self.copy(deep=False)
11162 result = result.set_axis(ax, axis=axis)
11163 return result.__finalize__(self, method="tz_localize")
11165 # ----------------------------------------------------------------------
11166 # Numeric Methods
11168 @final
11169 def describe(
11170 self,
11171 percentiles=None,
11172 include=None,
11173 exclude=None,
11174 ) -> Self:
11175 """
11176 Generate descriptive statistics.
11178 Descriptive statistics include those that summarize the central
11179 tendency, dispersion and shape of a
11180 dataset's distribution, excluding ``NaN`` values.
11182 Analyzes both numeric and object series, as well
11183 as ``DataFrame`` column sets of mixed data types. The output
11184 will vary depending on what is provided. Refer to the notes
11185 below for more detail.
11187 Parameters
11188 ----------
11189 percentiles : list-like of numbers, optional
11190 The percentiles to include in the output. All should
11191 fall between 0 and 1. The default, ``None``, will automatically
11192 return the 25th, 50th, and 75th percentiles.
11193 include : 'all', list-like of dtypes or None (default), optional
11194 A white list of data types to include in the result. Ignored
11195 for ``Series``. Here are the options:
11197 - 'all' : All columns of the input will be included in the output.
11198 - A list-like of dtypes : Limits the results to the
11199 provided data types.
11200 To limit the result to numeric types submit
11201 ``numpy.number``. To limit it instead to object columns submit
11202 the ``numpy.object`` data type. Strings
11203 can also be used in the style of
11204 ``select_dtypes`` (e.g. ``df.describe(include=['O'])``). To
11205 select pandas categorical columns, use ``'category'``
11206 - None (default) : The result will include all numeric columns.
11207 exclude : list-like of dtypes or None (default), optional,
11208 A black list of data types to omit from the result. Ignored
11209 for ``Series``. Here are the options:
11211 - A list-like of dtypes : Excludes the provided data types
11212 from the result. To exclude numeric types submit
11213 ``numpy.number``. To exclude object columns submit the data
11214 type ``numpy.object``. Strings can also be used in the style of
11215 ``select_dtypes`` (e.g. ``df.describe(exclude=['O'])``). To
11216 exclude pandas categorical columns, use ``'category'``
11217 - None (default) : The result will exclude nothing.
11219 Returns
11220 -------
11221 Series or DataFrame
11222 Summary statistics of the Series or Dataframe provided.
11224 See Also
11225 --------
11226 DataFrame.count: Count number of non-NA/null observations.
11227 DataFrame.max: Maximum of the values in the object.
11228 DataFrame.min: Minimum of the values in the object.
11229 DataFrame.mean: Mean of the values.
11230 DataFrame.std: Standard deviation of the observations.
11231 DataFrame.select_dtypes: Subset of a DataFrame including/excluding
11232 columns based on their dtype.
11234 Notes
11235 -----
11236 For numeric data, the result's index will include ``count``,
11237 ``mean``, ``std``, ``min``, ``max`` as well as lower, ``50`` and
11238 upper percentiles. By default the lower percentile is ``25`` and the
11239 upper percentile is ``75``. The ``50`` percentile is the
11240 same as the median.
11242 For object data (e.g. strings), the result's index
11243 will include ``count``, ``unique``, ``top``, and ``freq``. The ``top``
11244 is the most common value. The ``freq`` is the most common value's
11245 frequency.
11247 If multiple object values have the highest count, then the
11248 ``count`` and ``top`` results will be arbitrarily chosen from
11249 among those with the highest count.
11251 For mixed data types provided via a ``DataFrame``, the default is to
11252 return only an analysis of numeric columns. If the DataFrame consists
11253 only of object and categorical data without any numeric columns, the
11254 default is to return an analysis of both the object and categorical
11255 columns. If ``include='all'`` is provided as an option, the result
11256 will include a union of attributes of each type.
11258 The `include` and `exclude` parameters can be used to limit
11259 which columns in a ``DataFrame`` are analyzed for the output.
11260 The parameters are ignored when analyzing a ``Series``.
11262 Examples
11263 --------
11264 Describing a numeric ``Series``.
11266 >>> s = pd.Series([1, 2, 3])
11267 >>> s.describe()
11268 count 3.0
11269 mean 2.0
11270 std 1.0
11271 min 1.0
11272 25% 1.5
11273 50% 2.0
11274 75% 2.5
11275 max 3.0
11276 dtype: float64
11278 Describing a categorical ``Series``.
11280 >>> s = pd.Series(["a", "a", "b", "c"])
11281 >>> s.describe()
11282 count 4
11283 unique 3
11284 top a
11285 freq 2
11286 dtype: object
11288 Describing a timestamp ``Series``.
11290 >>> s = pd.Series(
11291 ... [
11292 ... np.datetime64("2000-01-01"),
11293 ... np.datetime64("2010-01-01"),
11294 ... np.datetime64("2010-01-01"),
11295 ... ]
11296 ... )
11297 >>> s.describe()
11298 count 3
11299 mean 2006-09-01 08:00:00
11300 min 2000-01-01 00:00:00
11301 25% 2004-12-31 12:00:00
11302 50% 2010-01-01 00:00:00
11303 75% 2010-01-01 00:00:00
11304 max 2010-01-01 00:00:00
11305 dtype: object
11307 Describing a ``DataFrame``. By default only numeric fields
11308 are returned.
11310 >>> df = pd.DataFrame(
11311 ... {
11312 ... "categorical": pd.Categorical(["d", "e", "f"]),
11313 ... "numeric": [1, 2, 3],
11314 ... "object": ["a", "b", "c"],
11315 ... }
11316 ... )
11317 >>> df.describe()
11318 numeric
11319 count 3.0
11320 mean 2.0
11321 std 1.0
11322 min 1.0
11323 25% 1.5
11324 50% 2.0
11325 75% 2.5
11326 max 3.0
11328 Describing all columns of a ``DataFrame`` regardless of data type.
11330 >>> df.describe(include="all") # doctest: +SKIP
11331 categorical numeric object
11332 count 3 3.0 3
11333 unique 3 NaN 3
11334 top f NaN a
11335 freq 1 NaN 1
11336 mean NaN 2.0 NaN
11337 std NaN 1.0 NaN
11338 min NaN 1.0 NaN
11339 25% NaN 1.5 NaN
11340 50% NaN 2.0 NaN
11341 75% NaN 2.5 NaN
11342 max NaN 3.0 NaN
11344 Describing a column from a ``DataFrame`` by accessing it as
11345 an attribute.
11347 >>> df.numeric.describe()
11348 count 3.0
11349 mean 2.0
11350 std 1.0
11351 min 1.0
11352 25% 1.5
11353 50% 2.0
11354 75% 2.5
11355 max 3.0
11356 Name: numeric, dtype: float64
11358 Including only numeric columns in a ``DataFrame`` description.
11360 >>> df.describe(include=[np.number])
11361 numeric
11362 count 3.0
11363 mean 2.0
11364 std 1.0
11365 min 1.0
11366 25% 1.5
11367 50% 2.0
11368 75% 2.5
11369 max 3.0
11371 Including only string columns in a ``DataFrame`` description.
11373 >>> df.describe(include=[object]) # doctest: +SKIP
11374 object
11375 count 3
11376 unique 3
11377 top a
11378 freq 1
11380 Including only categorical columns from a ``DataFrame`` description.
11382 >>> df.describe(include=["category"])
11383 categorical
11384 count 3
11385 unique 3
11386 top d
11387 freq 1
11389 Excluding numeric columns from a ``DataFrame`` description.
11391 >>> df.describe(exclude=[np.number]) # doctest: +SKIP
11392 categorical object
11393 count 3 3
11394 unique 3 3
11395 top f a
11396 freq 1 1
11398 Excluding object columns from a ``DataFrame`` description.
11400 >>> df.describe(exclude=[object]) # doctest: +SKIP
11401 categorical numeric
11402 count 3 3.0
11403 unique 3 NaN
11404 top f NaN
11405 freq 1 NaN
11406 mean NaN 2.0
11407 std NaN 1.0
11408 min NaN 1.0
11409 25% NaN 1.5
11410 50% NaN 2.0
11411 75% NaN 2.5
11412 max NaN 3.0
11413 """
11414 return describe_ndframe(
11415 obj=self,
11416 include=include,
11417 exclude=exclude,
11418 percentiles=percentiles,
11419 ).__finalize__(self, method="describe")
11421 @final
11422 def pct_change(
11423 self,
11424 periods: int = 1,
11425 fill_method: None = None,
11426 freq=None,
11427 **kwargs,
11428 ) -> Self:
11429 """
11430 Fractional change between the current and a prior element.
11432 Computes the fractional change from the immediately previous row by
11433 default. This is useful in comparing the fraction of change in a time
11434 series of elements.
11436 .. note::
11438 Despite the name of this method, it calculates fractional change
11439 (also known as per unit change or relative change) and not
11440 percentage change. If you need the percentage change, multiply
11441 these values by 100.
11443 Parameters
11444 ----------
11445 periods : int, default 1
11446 Periods to shift for forming percent change.
11447 fill_method : None
11448 Must be None. This argument will be removed in a future version of pandas.
11449 freq : DateOffset, timedelta, or str, optional
11450 Increment to use from time series API (e.g. 'ME' or BDay()).
11451 **kwargs
11452 Additional keyword arguments are passed into
11453 `DataFrame.shift` or `Series.shift`.
11455 Returns
11456 -------
11457 Series or DataFrame
11458 The same type as the calling object.
11460 See Also
11461 --------
11462 Series.diff : Compute the difference of two elements in a Series.
11463 DataFrame.diff : Compute the difference of two elements in a DataFrame.
11464 Series.shift : Shift the index by some number of periods.
11465 DataFrame.shift : Shift the index by some number of periods.
11467 Examples
11468 --------
11469 **Series**
11471 >>> s = pd.Series([90, 91, 85])
11472 >>> s
11473 0 90
11474 1 91
11475 2 85
11476 dtype: int64
11478 >>> s.pct_change()
11479 0 NaN
11480 1 0.011111
11481 2 -0.065934
11482 dtype: float64
11484 >>> s.pct_change(periods=2)
11485 0 NaN
11486 1 NaN
11487 2 -0.055556
11488 dtype: float64
11490 See the percentage change in a Series where filling NAs with last
11491 valid observation forward to next valid.
11493 >>> s = pd.Series([90, 91, None, 85])
11494 >>> s
11495 0 90.0
11496 1 91.0
11497 2 NaN
11498 3 85.0
11499 dtype: float64
11501 >>> s.ffill().pct_change()
11502 0 NaN
11503 1 0.011111
11504 2 0.000000
11505 3 -0.065934
11506 dtype: float64
11508 **DataFrame**
11510 Percentage change in French franc, Deutsche Mark, and Italian lira from
11511 1980-01-01 to 1980-03-01.
11513 >>> df = pd.DataFrame(
11514 ... {
11515 ... "FR": [4.0405, 4.0963, 4.3149],
11516 ... "GR": [1.7246, 1.7482, 1.8519],
11517 ... "IT": [804.74, 810.01, 860.13],
11518 ... },
11519 ... index=["1980-01-01", "1980-02-01", "1980-03-01"],
11520 ... )
11521 >>> df
11522 FR GR IT
11523 1980-01-01 4.0405 1.7246 804.74
11524 1980-02-01 4.0963 1.7482 810.01
11525 1980-03-01 4.3149 1.8519 860.13
11527 >>> df.pct_change()
11528 FR GR IT
11529 1980-01-01 NaN NaN NaN
11530 1980-02-01 0.013810 0.013684 0.006549
11531 1980-03-01 0.053365 0.059318 0.061876
11533 Percentage of change in GOOG and APPL stock volume. Shows computing
11534 the percentage change between columns.
11536 >>> df = pd.DataFrame(
11537 ... {
11538 ... "2016": [1769950, 30586265],
11539 ... "2015": [1500923, 40912316],
11540 ... "2014": [1371819, 41403351],
11541 ... },
11542 ... index=["GOOG", "APPL"],
11543 ... )
11544 >>> df
11545 2016 2015 2014
11546 GOOG 1769950 1500923 1371819
11547 APPL 30586265 40912316 41403351
11549 >>> df.pct_change(axis="columns", periods=-1)
11550 2016 2015 2014
11551 GOOG 0.179241 0.094112 NaN
11552 APPL -0.252395 -0.011860 NaN
11553 """
11554 # GH#53491
11555 if fill_method is not None:
11556 raise ValueError(f"fill_method must be None; got {fill_method=}.")
11558 axis = self._get_axis_number(kwargs.pop("axis", "index"))
11559 shifted = self.shift(periods=periods, freq=freq, axis=axis, **kwargs)
11560 # Unsupported left operand type for / ("Self")
11561 rs = self / shifted - 1 # type: ignore[operator]
11562 if freq is not None:
11563 # Shift method is implemented differently when freq is not None
11564 # We want to restore the original index
11565 rs = rs.loc[~rs.index.duplicated()]
11566 rs = rs.reindex_like(self)
11567 return rs.__finalize__(self, method="pct_change")
11569 @final
11570 def _logical_func(
11571 self,
11572 name: str,
11573 func,
11574 axis: Axis | None = 0,
11575 bool_only: bool = False,
11576 skipna: bool = True,
11577 **kwargs,
11578 ) -> Series | bool:
11579 nv.validate_logical_func((), kwargs, fname=name)
11580 validate_bool_kwarg(skipna, "skipna", none_allowed=False)
11582 if self.ndim > 1 and axis is None:
11583 # Reduce along one dimension then the other, to simplify DataFrame._reduce
11584 res = self._logical_func(
11585 name, func, axis=0, bool_only=bool_only, skipna=skipna, **kwargs
11586 )
11587 # error: Item "bool" of "Series | bool" has no attribute "_logical_func"
11588 return res._logical_func( # type: ignore[union-attr]
11589 name, func, skipna=skipna, **kwargs
11590 )
11591 elif axis is None:
11592 axis = 0
11594 if (
11595 self.ndim > 1
11596 and axis == 1
11597 and len(self._mgr.blocks) > 1
11598 # TODO(EA2D): special-case not needed
11599 and all(block.values.ndim == 2 for block in self._mgr.blocks)
11600 and not kwargs
11601 ):
11602 # Fastpath avoiding potentially expensive transpose
11603 obj = self
11604 if bool_only:
11605 obj = self._get_bool_data()
11606 return obj._reduce_axis1(name, func, skipna=skipna)
11608 return self._reduce(
11609 func,
11610 name=name,
11611 axis=axis,
11612 skipna=skipna,
11613 numeric_only=bool_only,
11614 filter_type="bool",
11615 )
11617 def any(
11618 self,
11619 *,
11620 axis: Axis | None = 0,
11621 bool_only: bool = False,
11622 skipna: bool = True,
11623 **kwargs,
11624 ) -> Series | bool:
11625 return self._logical_func(
11626 "any", nanops.nanany, axis, bool_only, skipna, **kwargs
11627 )
11629 def all(
11630 self,
11631 *,
11632 axis: Axis = 0,
11633 bool_only: bool = False,
11634 skipna: bool = True,
11635 **kwargs,
11636 ) -> Series | bool:
11637 return self._logical_func(
11638 "all", nanops.nanall, axis, bool_only, skipna, **kwargs
11639 )
11641 @final
11642 def _accum_func(
11643 self,
11644 name: str,
11645 func,
11646 axis: Axis | None = None,
11647 skipna: bool = True,
11648 *args,
11649 **kwargs,
11650 ):
11651 skipna = nv.validate_cum_func_with_skipna(skipna, args, kwargs, name)
11652 if axis is None:
11653 axis = 0
11654 else:
11655 axis = self._get_axis_number(axis)
11657 if axis == 1:
11658 return self.T._accum_func(
11659 name,
11660 func,
11661 axis=0,
11662 skipna=skipna,
11663 *args, # noqa: B026
11664 **kwargs,
11665 ).T
11667 def block_accum_func(blk_values):
11668 values = blk_values.T if hasattr(blk_values, "T") else blk_values
11670 result: np.ndarray | ExtensionArray
11671 if isinstance(values, ExtensionArray):
11672 result = values._accumulate(name, skipna=skipna, **kwargs)
11673 else:
11674 result = nanops.na_accum_func(values, func, skipna=skipna)
11676 result = result.T if hasattr(result, "T") else result
11677 return result
11679 result = self._mgr.apply(block_accum_func)
11681 return self._constructor_from_mgr(result, axes=result.axes).__finalize__(
11682 self, method=name
11683 )
11685 def cummax(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
11686 return self._accum_func(
11687 "cummax", np.maximum.accumulate, axis, skipna, *args, **kwargs
11688 )
11690 def cummin(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
11691 return self._accum_func(
11692 "cummin", np.minimum.accumulate, axis, skipna, *args, **kwargs
11693 )
11695 def cumsum(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
11696 return self._accum_func("cumsum", np.cumsum, axis, skipna, *args, **kwargs)
11698 def cumprod(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
11699 return self._accum_func("cumprod", np.cumprod, axis, skipna, *args, **kwargs)
11701 @final
11702 def _stat_function_ddof(
11703 self,
11704 name: str,
11705 func,
11706 axis: Axis | None = 0,
11707 skipna: bool = True,
11708 ddof: int = 1,
11709 numeric_only: bool = False,
11710 **kwargs,
11711 ) -> Series | float:
11712 nv.validate_stat_ddof_func((), kwargs, fname=name)
11713 validate_bool_kwarg(skipna, "skipna", none_allowed=False)
11715 return self._reduce(
11716 func, name, axis=axis, numeric_only=numeric_only, skipna=skipna, ddof=ddof
11717 )
11719 def sem(
11720 self,
11721 *,
11722 axis: Axis | None = 0,
11723 skipna: bool = True,
11724 ddof: int = 1,
11725 numeric_only: bool = False,
11726 **kwargs,
11727 ) -> Series | float:
11728 return self._stat_function_ddof(
11729 "sem", nanops.nansem, axis, skipna, ddof, numeric_only, **kwargs
11730 )
11732 def var(
11733 self,
11734 *,
11735 axis: Axis | None = 0,
11736 skipna: bool = True,
11737 ddof: int = 1,
11738 numeric_only: bool = False,
11739 **kwargs,
11740 ) -> Series | float:
11741 return self._stat_function_ddof(
11742 "var", nanops.nanvar, axis, skipna, ddof, numeric_only, **kwargs
11743 )
11745 def std(
11746 self,
11747 *,
11748 axis: Axis | None = 0,
11749 skipna: bool = True,
11750 ddof: int = 1,
11751 numeric_only: bool = False,
11752 **kwargs,
11753 ) -> Series | float:
11754 return self._stat_function_ddof(
11755 "std", nanops.nanstd, axis, skipna, ddof, numeric_only, **kwargs
11756 )
11758 @final
11759 def _stat_function(
11760 self,
11761 name: str,
11762 func,
11763 axis: Axis | None = 0,
11764 skipna: bool = True,
11765 numeric_only: bool = False,
11766 **kwargs,
11767 ):
11768 assert name in ["median", "mean", "min", "max", "kurt", "skew"], name
11769 nv.validate_func(name, (), kwargs)
11771 validate_bool_kwarg(skipna, "skipna", none_allowed=False)
11773 return self._reduce(
11774 func, name=name, axis=axis, skipna=skipna, numeric_only=numeric_only
11775 )
11777 def min(
11778 self,
11779 *,
11780 axis: Axis | None = 0,
11781 skipna: bool = True,
11782 numeric_only: bool = False,
11783 **kwargs,
11784 ):
11785 return self._stat_function(
11786 "min",
11787 nanops.nanmin,
11788 axis,
11789 skipna,
11790 numeric_only,
11791 **kwargs,
11792 )
11794 def max(
11795 self,
11796 *,
11797 axis: Axis | None = 0,
11798 skipna: bool = True,
11799 numeric_only: bool = False,
11800 **kwargs,
11801 ):
11802 return self._stat_function(
11803 "max",
11804 nanops.nanmax,
11805 axis,
11806 skipna,
11807 numeric_only,
11808 **kwargs,
11809 )
11811 def mean(
11812 self,
11813 *,
11814 axis: Axis | None = 0,
11815 skipna: bool = True,
11816 numeric_only: bool = False,
11817 **kwargs,
11818 ) -> Series | float:
11819 return self._stat_function(
11820 "mean", nanops.nanmean, axis, skipna, numeric_only, **kwargs
11821 )
11823 def median(
11824 self,
11825 *,
11826 axis: Axis | None = 0,
11827 skipna: bool = True,
11828 numeric_only: bool = False,
11829 **kwargs,
11830 ) -> Series | float:
11831 return self._stat_function(
11832 "median", nanops.nanmedian, axis, skipna, numeric_only, **kwargs
11833 )
11835 def skew(
11836 self,
11837 *,
11838 axis: Axis | None = 0,
11839 skipna: bool = True,
11840 numeric_only: bool = False,
11841 **kwargs,
11842 ) -> Series | float:
11843 return self._stat_function(
11844 "skew", nanops.nanskew, axis, skipna, numeric_only, **kwargs
11845 )
11847 def kurt(
11848 self,
11849 *,
11850 axis: Axis | None = 0,
11851 skipna: bool = True,
11852 numeric_only: bool = False,
11853 **kwargs,
11854 ) -> Series | float:
11855 return self._stat_function(
11856 "kurt", nanops.nankurt, axis, skipna, numeric_only, **kwargs
11857 )
11859 kurtosis = kurt
11861 @final
11862 def _min_count_stat_function(
11863 self,
11864 name: str,
11865 func,
11866 axis: Axis | None = 0,
11867 skipna: bool = True,
11868 numeric_only: bool = False,
11869 min_count: int = 0,
11870 **kwargs,
11871 ):
11872 assert name in ["sum", "prod"], name
11873 nv.validate_func(name, (), kwargs)
11875 validate_bool_kwarg(skipna, "skipna", none_allowed=False)
11877 return self._reduce(
11878 func,
11879 name=name,
11880 axis=axis,
11881 skipna=skipna,
11882 numeric_only=numeric_only,
11883 min_count=min_count,
11884 )
11886 def sum(
11887 self,
11888 *,
11889 axis: Axis | None = 0,
11890 skipna: bool = True,
11891 numeric_only: bool = False,
11892 min_count: int = 0,
11893 **kwargs,
11894 ):
11895 return self._min_count_stat_function(
11896 "sum", nanops.nansum, axis, skipna, numeric_only, min_count, **kwargs
11897 )
11899 def prod(
11900 self,
11901 *,
11902 axis: Axis | None = 0,
11903 skipna: bool = True,
11904 numeric_only: bool = False,
11905 min_count: int = 0,
11906 **kwargs,
11907 ):
11908 return self._min_count_stat_function(
11909 "prod",
11910 nanops.nanprod,
11911 axis,
11912 skipna,
11913 numeric_only,
11914 min_count,
11915 **kwargs,
11916 )
11918 product = prod
11920 @final
11921 def rolling(
11922 self,
11923 window: int | dt.timedelta | str | BaseOffset | BaseIndexer,
11924 min_periods: int | None = None,
11925 center: bool = False,
11926 win_type: str | None = None,
11927 on: str | None = None,
11928 closed: IntervalClosedType | None = None,
11929 step: int | None = None,
11930 method: str = "single",
11931 ) -> Window | Rolling:
11932 """
11933 Provide rolling window calculations.
11935 Parameters
11936 ----------
11937 window : int, timedelta, str, offset, or BaseIndexer subclass
11938 Interval of the moving window.
11940 If an integer, the delta between the start and end of each window.
11941 The number of points in the window depends on the ``closed`` argument.
11943 If a timedelta, str, or offset, the time period of each window. Each
11944 window will be a variable sized based on the observations included in
11945 the time-period. This is only valid for datetimelike indexes.
11946 To learn more about the offsets & frequency strings, please see
11947 :ref:`this link<timeseries.offset_aliases>`.
11949 If a BaseIndexer subclass, the window boundaries
11950 based on the defined ``get_window_bounds`` method. Additional rolling
11951 keyword arguments, namely ``min_periods``, ``center``, ``closed`` and
11952 ``step`` will be passed to ``get_window_bounds``.
11954 min_periods : int, default None
11955 Minimum number of observations in window required to have a value;
11956 otherwise, result is ``np.nan``.
11958 For a window that is specified by an offset, ``min_periods`` will default
11959 to 1.
11961 For a window that is specified by an integer, ``min_periods`` will default
11962 to the size of the window.
11964 center : bool, default False
11965 If False, set the window labels as the right edge of the window index.
11967 If True, set the window labels as the center of the window index.
11969 win_type : str, default None
11970 If ``None``, all points are evenly weighted.
11972 If a string, it must be a valid `scipy.signal window function
11973 <https://docs.scipy.org/doc/scipy/reference/signal.windows.html#module-scipy.signal.windows>`__.
11975 Certain Scipy window types require additional parameters to be passed
11976 in the aggregation function. The additional parameters must match
11977 the keywords specified in the Scipy window type method signature.
11979 on : str, optional
11980 For a DataFrame, a column label or Index level on which
11981 to calculate the rolling window, rather than the DataFrame's index.
11983 Provided integer column is ignored and excluded from result since
11984 an integer index is not used to calculate the rolling window.
11986 closed : str, default None
11987 Determines the inclusivity of points in the window
11989 If ``'right'``, uses the window (first, last] meaning the last point
11990 is included in the calculations.
11992 If ``'left'``, uses the window [first, last) meaning the first point
11993 is included in the calculations.
11995 If ``'both'``, uses the window [first, last] meaning all points in
11996 the window are included in the calculations.
11998 If ``'neither'``, uses the window (first, last) meaning the first
11999 and last points in the window are excluded from calculations.
12001 () and [] are referencing open and closed set
12002 notation respetively.
12004 Default ``None`` (``'right'``).
12006 step : int, default None
12007 Evaluate the window at every ``step`` result, equivalent to slicing as
12008 ``[::step]``. ``window`` must be an integer. Using a step argument other
12009 than None or 1 will produce a result with a different shape than the input.
12011 method : str {'single', 'table'}, default 'single'
12013 Execute the rolling operation per single column or row (``'single'``)
12014 or over the entire object (``'table'``).
12016 This argument is only implemented when specifying ``engine='numba'``
12017 in the method call.
12019 Returns
12020 -------
12021 pandas.api.typing.Window or pandas.api.typing.Rolling
12022 An instance of Window is returned if ``win_type`` is passed. Otherwise,
12023 an instance of Rolling is returned.
12025 See Also
12026 --------
12027 expanding : Provides expanding transformations.
12028 ewm : Provides exponential weighted functions.
12030 Notes
12031 -----
12032 See :ref:`Windowing Operations <window.generic>` for further usage details
12033 and examples.
12035 Examples
12036 --------
12037 >>> df = pd.DataFrame({"B": [0, 1, 2, np.nan, 4]})
12038 >>> df
12039 B
12040 0 0.0
12041 1 1.0
12042 2 2.0
12043 3 NaN
12044 4 4.0
12046 **window**
12048 Rolling sum with a window length of 2 observations.
12050 >>> df.rolling(2).sum()
12051 B
12052 0 NaN
12053 1 1.0
12054 2 3.0
12055 3 NaN
12056 4 NaN
12058 Rolling sum with a window span of 2 seconds.
12060 >>> df_time = pd.DataFrame(
12061 ... {"B": [0, 1, 2, np.nan, 4]},
12062 ... index=[
12063 ... pd.Timestamp("20130101 09:00:00"),
12064 ... pd.Timestamp("20130101 09:00:02"),
12065 ... pd.Timestamp("20130101 09:00:03"),
12066 ... pd.Timestamp("20130101 09:00:05"),
12067 ... pd.Timestamp("20130101 09:00:06"),
12068 ... ],
12069 ... )
12071 >>> df_time
12072 B
12073 2013-01-01 09:00:00 0.0
12074 2013-01-01 09:00:02 1.0
12075 2013-01-01 09:00:03 2.0
12076 2013-01-01 09:00:05 NaN
12077 2013-01-01 09:00:06 4.0
12079 >>> df_time.rolling("2s").sum()
12080 B
12081 2013-01-01 09:00:00 0.0
12082 2013-01-01 09:00:02 1.0
12083 2013-01-01 09:00:03 3.0
12084 2013-01-01 09:00:05 NaN
12085 2013-01-01 09:00:06 4.0
12087 Rolling sum with forward looking windows with 2 observations.
12089 >>> indexer = pd.api.indexers.FixedForwardWindowIndexer(window_size=2)
12090 >>> df.rolling(window=indexer, min_periods=1).sum()
12091 B
12092 0 1.0
12093 1 3.0
12094 2 2.0
12095 3 4.0
12096 4 4.0
12098 **min_periods**
12100 Rolling sum with a window length of 2 observations, but only needs a minimum
12101 of 1 observation to calculate a value.
12103 >>> df.rolling(2, min_periods=1).sum()
12104 B
12105 0 0.0
12106 1 1.0
12107 2 3.0
12108 3 2.0
12109 4 4.0
12111 **center**
12113 Rolling sum with the result assigned to the center of the window index.
12115 >>> df.rolling(3, min_periods=1, center=True).sum()
12116 B
12117 0 1.0
12118 1 3.0
12119 2 3.0
12120 3 6.0
12121 4 4.0
12123 >>> df.rolling(3, min_periods=1, center=False).sum()
12124 B
12125 0 0.0
12126 1 1.0
12127 2 3.0
12128 3 3.0
12129 4 6.0
12131 **step**
12133 Rolling sum with a window length of 2 observations, minimum of 1 observation to
12134 calculate a value, and a step of 2.
12136 >>> df.rolling(2, min_periods=1, step=2).sum()
12137 B
12138 0 0.0
12139 2 3.0
12140 4 4.0
12142 **win_type**
12144 Rolling sum with a window length of 2, using the Scipy ``'gaussian'``
12145 window type. ``std`` is required in the aggregation function.
12147 >>> df.rolling(2, win_type="gaussian").sum(std=3)
12148 B
12149 0 NaN
12150 1 0.986207
12151 2 2.958621
12152 3 NaN
12153 4 NaN
12155 **on**
12157 Rolling sum with a window length of 2 days.
12159 >>> df = pd.DataFrame(
12160 ... {
12161 ... "A": [
12162 ... pd.to_datetime("2020-01-01"),
12163 ... pd.to_datetime("2020-01-01"),
12164 ... pd.to_datetime("2020-01-02"),
12165 ... ],
12166 ... "B": [1, 2, 3],
12167 ... },
12168 ... index=pd.date_range("2020", periods=3),
12169 ... )
12171 >>> df
12172 A B
12173 2020-01-01 2020-01-01 1
12174 2020-01-02 2020-01-01 2
12175 2020-01-03 2020-01-02 3
12177 >>> df.rolling("2D", on="A").sum()
12178 A B
12179 2020-01-01 2020-01-01 1.0
12180 2020-01-02 2020-01-01 3.0
12181 2020-01-03 2020-01-02 6.0
12182 """
12183 if win_type is not None:
12184 return Window(
12185 self,
12186 window=window,
12187 min_periods=min_periods,
12188 center=center,
12189 win_type=win_type,
12190 on=on,
12191 closed=closed,
12192 step=step,
12193 method=method,
12194 )
12196 return Rolling(
12197 self,
12198 window=window,
12199 min_periods=min_periods,
12200 center=center,
12201 win_type=win_type,
12202 on=on,
12203 closed=closed,
12204 step=step,
12205 method=method,
12206 )
12208 @final
12209 def expanding(
12210 self,
12211 min_periods: int = 1,
12212 method: Literal["single", "table"] = "single",
12213 ) -> Expanding:
12214 """
12215 Provide expanding window calculations.
12217 An expanding window yields the value of an aggregation statistic with all
12218 the data available up to that point in time.
12220 Parameters
12221 ----------
12222 min_periods : int, default 1
12223 Minimum number of observations in window required to have a value;
12224 otherwise, result is ``np.nan``.
12226 method : str {'single', 'table'}, default 'single'
12227 Execute the rolling operation per single column or row (``'single'``)
12228 or over the entire object (``'table'``).
12230 This argument is only implemented when specifying ``engine='numba'``
12231 in the method call.
12233 Returns
12234 -------
12235 pandas.api.typing.Expanding
12236 An instance of Expanding for further expanding window calculations,
12237 e.g. using the ``sum`` method.
12239 See Also
12240 --------
12241 rolling : Provides rolling window calculations.
12242 ewm : Provides exponential weighted functions.
12244 Notes
12245 -----
12246 See :ref:`Windowing Operations <window.expanding>` for further usage details
12247 and examples.
12249 Examples
12250 --------
12251 >>> df = pd.DataFrame({"B": [0, 1, 2, np.nan, 4]})
12252 >>> df
12253 B
12254 0 0.0
12255 1 1.0
12256 2 2.0
12257 3 NaN
12258 4 4.0
12260 **min_periods**
12262 Expanding sum with 1 vs 3 observations needed to calculate a value.
12264 >>> df.expanding(1).sum()
12265 B
12266 0 0.0
12267 1 1.0
12268 2 3.0
12269 3 3.0
12270 4 7.0
12271 >>> df.expanding(3).sum()
12272 B
12273 0 NaN
12274 1 NaN
12275 2 3.0
12276 3 3.0
12277 4 7.0
12278 """
12279 return Expanding(self, min_periods=min_periods, method=method)
12281 @final
12282 @doc(ExponentialMovingWindow)
12283 def ewm(
12284 self,
12285 com: float | None = None,
12286 span: float | None = None,
12287 halflife: float | TimedeltaConvertibleTypes | None = None,
12288 alpha: float | None = None,
12289 min_periods: int | None = 0,
12290 adjust: bool = True,
12291 ignore_na: bool = False,
12292 times: np.ndarray | DataFrame | Series | None = None,
12293 method: Literal["single", "table"] = "single",
12294 ) -> ExponentialMovingWindow:
12295 return ExponentialMovingWindow(
12296 self,
12297 com=com,
12298 span=span,
12299 halflife=halflife,
12300 alpha=alpha,
12301 min_periods=min_periods,
12302 adjust=adjust,
12303 ignore_na=ignore_na,
12304 times=times,
12305 method=method,
12306 )
12308 # ----------------------------------------------------------------------
12309 # Arithmetic Methods
12311 @final
12312 def _inplace_method(self, other, op) -> Self:
12313 """
12314 Wrap arithmetic method to operate inplace.
12315 """
12316 result = op(self, other)
12318 # this makes sure that we are aligned like the input
12319 # we are updating inplace
12320 self._update_inplace(result.reindex_like(self))
12321 return self
12323 @final
12324 def __iadd__(self, other) -> Self:
12325 # error: Unsupported left operand type for + ("Type[NDFrame]")
12326 return self._inplace_method(other, type(self).__add__) # type: ignore[operator]
12328 @final
12329 def __isub__(self, other) -> Self:
12330 # error: Unsupported left operand type for - ("Type[NDFrame]")
12331 return self._inplace_method(other, type(self).__sub__) # type: ignore[operator]
12333 @final
12334 def __imul__(self, other) -> Self:
12335 # error: Unsupported left operand type for * ("Type[NDFrame]")
12336 return self._inplace_method(other, type(self).__mul__) # type: ignore[operator]
12338 @final
12339 def __itruediv__(self, other) -> Self:
12340 # error: Unsupported left operand type for / ("Type[NDFrame]")
12341 return self._inplace_method(
12342 other,
12343 type(self).__truediv__, # type: ignore[operator]
12344 )
12346 @final
12347 def __ifloordiv__(self, other) -> Self:
12348 # error: Unsupported left operand type for // ("Type[NDFrame]")
12349 return self._inplace_method(
12350 other,
12351 type(self).__floordiv__, # type: ignore[operator]
12352 )
12354 @final
12355 def __imod__(self, other) -> Self:
12356 # error: Unsupported left operand type for % ("Type[NDFrame]")
12357 return self._inplace_method(other, type(self).__mod__) # type: ignore[operator]
12359 @final
12360 def __ipow__(self, other) -> Self:
12361 # error: Unsupported left operand type for ** ("Type[NDFrame]")
12362 return self._inplace_method(other, type(self).__pow__) # type: ignore[operator]
12364 @final
12365 def __iand__(self, other) -> Self:
12366 # error: Unsupported left operand type for & ("Type[NDFrame]")
12367 return self._inplace_method(other, type(self).__and__) # type: ignore[operator]
12369 @final
12370 def __ior__(self, other) -> Self:
12371 return self._inplace_method(other, type(self).__or__)
12373 @final
12374 def __ixor__(self, other) -> Self:
12375 # error: Unsupported left operand type for ^ ("Type[NDFrame]")
12376 return self._inplace_method(other, type(self).__xor__) # type: ignore[operator]
12378 # ----------------------------------------------------------------------
12379 # Misc methods
12381 @final
12382 def _find_valid_index(self, *, how: str) -> Hashable:
12383 """
12384 Retrieves the index of the first valid value.
12386 Parameters
12387 ----------
12388 how : {'first', 'last'}
12389 Use this parameter to change between the first or last valid index.
12391 Returns
12392 -------
12393 idx_first_valid : type of index
12394 """
12395 is_valid = self.notna().values
12396 idxpos = find_valid_index(how=how, is_valid=is_valid)
12397 if idxpos is None:
12398 return None
12399 return self.index[idxpos]
12401 @final
12402 def first_valid_index(self) -> Hashable:
12403 """
12404 Return index for first non-missing value or None, if no value is found.
12406 See the :ref:`User Guide <missing_data>` for more information
12407 on which values are considered missing.
12409 Returns
12410 -------
12411 type of index
12412 Index of first non-missing value.
12414 See Also
12415 --------
12416 DataFrame.last_valid_index : Return index for last non-NA value or None, if
12417 no non-NA value is found.
12418 Series.last_valid_index : Return index for last non-NA value or None, if no
12419 non-NA value is found.
12420 DataFrame.isna : Detect missing values.
12422 Examples
12423 --------
12424 For Series:
12426 >>> s = pd.Series([None, 3, 4])
12427 >>> s.first_valid_index()
12428 1
12429 >>> s.last_valid_index()
12430 2
12432 >>> s = pd.Series([None, None])
12433 >>> print(s.first_valid_index())
12434 None
12435 >>> print(s.last_valid_index())
12436 None
12438 If all elements in Series are NA/null, returns None.
12440 >>> s = pd.Series()
12441 >>> print(s.first_valid_index())
12442 None
12443 >>> print(s.last_valid_index())
12444 None
12446 If Series is empty, returns None.
12448 For DataFrame:
12450 >>> df = pd.DataFrame({"A": [None, None, 2], "B": [None, 3, 4]})
12451 >>> df
12452 A B
12453 0 NaN NaN
12454 1 NaN 3.0
12455 2 2.0 4.0
12456 >>> df.first_valid_index()
12457 1
12458 >>> df.last_valid_index()
12459 2
12461 >>> df = pd.DataFrame({"A": [None, None, None], "B": [None, None, None]})
12462 >>> df
12463 A B
12464 0 None None
12465 1 None None
12466 2 None None
12467 >>> print(df.first_valid_index())
12468 None
12469 >>> print(df.last_valid_index())
12470 None
12472 If all elements in DataFrame are NA/null, returns None.
12474 >>> df = pd.DataFrame()
12475 >>> df
12476 Empty DataFrame
12477 Columns: []
12478 Index: []
12479 >>> print(df.first_valid_index())
12480 None
12481 >>> print(df.last_valid_index())
12482 None
12484 If DataFrame is empty, returns None.
12485 """
12486 return self._find_valid_index(how="first")
12488 @final
12489 def last_valid_index(self) -> Hashable:
12490 """
12491 Return index for last non-missing value or None, if no value is found.
12493 See the :ref:`User Guide <missing_data>` for more information
12494 on which values are considered missing.
12496 Returns
12497 -------
12498 type of index
12499 Index of last non-missing value.
12501 See Also
12502 --------
12503 DataFrame.first_valid_index : Return index for first non-NA value or None, if
12504 no non-NA value is found.
12505 Series.first_valid_index : Return index for first non-NA value or None, if no
12506 non-NA value is found.
12507 DataFrame.isna : Detect missing values.
12509 Examples
12510 --------
12511 For Series:
12513 >>> s = pd.Series([None, 3, 4])
12514 >>> s.first_valid_index()
12515 1
12516 >>> s.last_valid_index()
12517 2
12519 >>> s = pd.Series([None, None])
12520 >>> print(s.first_valid_index())
12521 None
12522 >>> print(s.last_valid_index())
12523 None
12525 If all elements in Series are NA/null, returns None.
12527 >>> s = pd.Series()
12528 >>> print(s.first_valid_index())
12529 None
12530 >>> print(s.last_valid_index())
12531 None
12533 If Series is empty, returns None.
12535 For DataFrame:
12537 >>> df = pd.DataFrame({"A": [None, None, 2], "B": [None, 3, 4]})
12538 >>> df
12539 A B
12540 0 NaN NaN
12541 1 NaN 3.0
12542 2 2.0 4.0
12543 >>> df.first_valid_index()
12544 1
12545 >>> df.last_valid_index()
12546 2
12548 >>> df = pd.DataFrame({"A": [None, None, None], "B": [None, None, None]})
12549 >>> df
12550 A B
12551 0 None None
12552 1 None None
12553 2 None None
12554 >>> print(df.first_valid_index())
12555 None
12556 >>> print(df.last_valid_index())
12557 None
12559 If all elements in DataFrame are NA/null, returns None.
12561 >>> df = pd.DataFrame()
12562 >>> df
12563 Empty DataFrame
12564 Columns: []
12565 Index: []
12566 >>> print(df.first_valid_index())
12567 None
12568 >>> print(df.last_valid_index())
12569 None
12571 If DataFrame is empty, returns None.
12572 """
12573 return self._find_valid_index(how="last")
12576_num_doc = """
12577{desc}
12579Parameters
12580----------
12581axis : {axis_descr}
12582 Axis for the function to be applied on.
12583 For `Series` this parameter is unused and defaults to 0.
12585 For DataFrames, specifying ``axis=None`` will apply the aggregation
12586 across both axes.
12588 .. versionadded:: 2.0.0
12590skipna : bool, default True
12591 Exclude NA/null values when computing the result.
12592numeric_only : bool, default False
12593 Include only float, int, boolean columns.
12595{min_count}\
12596**kwargs
12597 Additional keyword arguments to be passed to the function.
12599Returns
12600-------
12601{name1} or scalar\
12603 Value containing the calculation referenced in the description.\
12604{see_also}\
12605{examples}
12606"""
12608_sum_prod_doc = """
12609{desc}
12611Parameters
12612----------
12613axis : {axis_descr}
12614 Axis for the function to be applied on.
12615 For `Series` this parameter is unused and defaults to 0.
12617 .. warning::
12619 The behavior of DataFrame.{name} with ``axis=None`` is deprecated,
12620 in a future version this will reduce over both axes and return a scalar
12621 To retain the old behavior, pass axis=0 (or do not pass axis).
12623 .. versionadded:: 2.0.0
12625skipna : bool, default True
12626 Exclude NA/null values when computing the result.
12627numeric_only : bool, default False
12628 Include only float, int, boolean columns. Not implemented for Series.
12630{min_count}\
12631**kwargs
12632 Additional keyword arguments to be passed to the function.
12634Returns
12635-------
12636{name1} or scalar\
12638 Value containing the calculation referenced in the description.\
12639{see_also}\
12640{examples}
12641"""
12643_num_ddof_doc = """
12644{desc}
12646Parameters
12647----------
12648axis : {axis_descr}
12649 For `Series` this parameter is unused and defaults to 0.
12651 .. warning::
12653 The behavior of DataFrame.{name} with ``axis=None`` is deprecated,
12654 in a future version this will reduce over both axes and return a scalar
12655 To retain the old behavior, pass axis=0 (or do not pass axis).
12657skipna : bool, default True
12658 Exclude NA/null values. If an entire row/column is NA, the result
12659 will be NA.
12660ddof : int, default 1
12661 Delta Degrees of Freedom. The divisor used in calculations is N - ddof,
12662 where N represents the number of elements.
12663numeric_only : bool, default False
12664 Include only float, int, boolean columns. Not implemented for Series.
12665**kwargs :
12666 Additional keywords have no effect but might be accepted
12667 for compatibility with NumPy.
12669Returns
12670-------
12671{name1} or {name2} (if level specified)
12672 {return_desc}
12674See Also
12675--------
12676{see_also}\
12677{notes}\
12678{examples}
12679"""
12681_sem_see_also = """\
12682scipy.stats.sem : Compute standard error of the mean.
12683{name2}.std : Return sample standard deviation over requested axis.
12684{name2}.var : Return unbiased variance over requested axis.
12685{name2}.mean : Return the mean of the values over the requested axis.
12686{name2}.median : Return the median of the values over the requested axis.
12687{name2}.mode : Return the mode(s) of the Series."""
12689_sem_return_desc = """\
12690Unbiased standard error of the mean over requested axis."""
12692_std_see_also = """\
12693numpy.std : Compute the standard deviation along the specified axis.
12694{name2}.var : Return unbiased variance over requested axis.
12695{name2}.sem : Return unbiased standard error of the mean over requested axis.
12696{name2}.mean : Return the mean of the values over the requested axis.
12697{name2}.median : Return the median of the values over the requested axis.
12698{name2}.mode : Return the mode(s) of the Series."""
12700_std_return_desc = """\
12701Standard deviation over requested axis."""
12703_std_notes = """
12705Notes
12706-----
12707To have the same behaviour as `numpy.std`, use `ddof=0` (instead of the
12708default `ddof=1`)"""
12710_std_examples = """
12712Examples
12713--------
12714>>> df = pd.DataFrame({'person_id': [0, 1, 2, 3],
12715... 'age': [21, 25, 62, 43],
12716... 'height': [1.61, 1.87, 1.49, 2.01]}
12717... ).set_index('person_id')
12718>>> df
12719 age height
12720person_id
127210 21 1.61
127221 25 1.87
127232 62 1.49
127243 43 2.01
12726The standard deviation of the columns can be found as follows:
12728>>> df.std()
12729age 18.786076
12730height 0.237417
12731dtype: float64
12733Alternatively, `ddof=0` can be set to normalize by N instead of N-1:
12735>>> df.std(ddof=0)
12736age 16.269219
12737height 0.205609
12738dtype: float64"""
12740_var_examples = """
12742Examples
12743--------
12744>>> df = pd.DataFrame({'person_id': [0, 1, 2, 3],
12745... 'age': [21, 25, 62, 43],
12746... 'height': [1.61, 1.87, 1.49, 2.01]}
12747... ).set_index('person_id')
12748>>> df
12749 age height
12750person_id
127510 21 1.61
127521 25 1.87
127532 62 1.49
127543 43 2.01
12756>>> df.var()
12757age 352.916667
12758height 0.056367
12759dtype: float64
12761Alternatively, ``ddof=0`` can be set to normalize by N instead of N-1:
12763>>> df.var(ddof=0)
12764age 264.687500
12765height 0.042275
12766dtype: float64"""
12768_bool_doc = """
12769{desc}
12771Parameters
12772----------
12773axis : {{0 or 'index', 1 or 'columns', None}}, default 0
12774 Indicate which axis or axes should be reduced. For `Series` this parameter
12775 is unused and defaults to 0.
12777 * 0 / 'index' : reduce the index, return a Series whose index is the
12778 original column labels.
12779 * 1 / 'columns' : reduce the columns, return a Series whose index is the
12780 original index.
12781 * None : reduce all axes, return a scalar.
12783bool_only : bool, default False
12784 Include only boolean columns. Not implemented for Series.
12785skipna : bool, default True
12786 Exclude NA/null values. If the entire row/column is NA and skipna is
12787 True, then the result will be {empty_value}, as for an empty row/column.
12788 If skipna is False, then NA are treated as True, because these are not
12789 equal to zero.
12790**kwargs : any, default None
12791 Additional keywords have no effect but might be accepted for
12792 compatibility with NumPy.
12794Returns
12795-------
12796{name2} or {name1}
12797 If axis=None, then a scalar boolean is returned.
12798 Otherwise a Series is returned with index matching the index argument.
12800{see_also}
12801{examples}"""
12803_all_desc = """\
12804Return whether all elements are True, potentially over an axis.
12806Returns True unless there at least one element within a series or
12807along a Dataframe axis that is False or equivalent (e.g. zero or
12808empty)."""
12810_all_examples = """\
12811Examples
12812--------
12813**Series**
12815>>> pd.Series([True, True]).all()
12816True
12817>>> pd.Series([True, False]).all()
12818False
12819>>> pd.Series([], dtype="float64").all()
12820True
12821>>> pd.Series([np.nan]).all()
12822True
12823>>> pd.Series([np.nan]).all(skipna=False)
12824True
12826**DataFrames**
12828Create a DataFrame from a dictionary.
12830>>> df = pd.DataFrame({'col1': [True, True], 'col2': [True, False]})
12831>>> df
12832 col1 col2
128330 True True
128341 True False
12836Default behaviour checks if values in each column all return True.
12838>>> df.all()
12839col1 True
12840col2 False
12841dtype: bool
12843Specify ``axis='columns'`` to check if values in each row all return True.
12845>>> df.all(axis='columns')
128460 True
128471 False
12848dtype: bool
12850Or ``axis=None`` for whether every value is True.
12852>>> df.all(axis=None)
12853False
12854"""
12856_all_see_also = """\
12857See Also
12858--------
12859Series.all : Return True if all elements are True.
12860DataFrame.any : Return True if one (or more) elements are True.
12861"""
12863_cnum_pd_doc = """
12864Return cumulative {desc} over a DataFrame or Series axis.
12866Returns a DataFrame or Series of the same size containing the cumulative
12867{desc}.
12869Parameters
12870----------
12871axis : {{0 or 'index', 1 or 'columns'}}, default 0
12872 The index or the name of the axis. 0 is equivalent to None or 'index'.
12873 For `Series` this parameter is unused and defaults to 0.
12874skipna : bool, default True
12875 Exclude NA/null values. If an entire row/column is NA, the result
12876 will be NA.
12877numeric_only : bool, default False
12878 Include only float, int, boolean columns.
12879*args, **kwargs
12880 Additional keywords have no effect but might be accepted for
12881 compatibility with NumPy.
12883Returns
12884-------
12885{name1} or {name2}
12886 Return cumulative {desc} of {name1} or {name2}.
12888See Also
12889--------
12890core.window.expanding.Expanding.{accum_func_name} : Similar functionality
12891 but ignores ``NaN`` values.
12892{name2}.{accum_func_name} : Return the {desc} over
12893 {name2} axis.
12894{name2}.cummax : Return cumulative maximum over {name2} axis.
12895{name2}.cummin : Return cumulative minimum over {name2} axis.
12896{name2}.cumsum : Return cumulative sum over {name2} axis.
12897{name2}.cumprod : Return cumulative product over {name2} axis.
12899{examples}"""
12901_cnum_series_doc = """
12902Return cumulative {desc} over a DataFrame or Series axis.
12904Returns a DataFrame or Series of the same size containing the cumulative
12905{desc}.
12907Parameters
12908----------
12909axis : {{0 or 'index', 1 or 'columns'}}, default 0
12910 The index or the name of the axis. 0 is equivalent to None or 'index'.
12911 For `Series` this parameter is unused and defaults to 0.
12912skipna : bool, default True
12913 Exclude NA/null values. If an entire row/column is NA, the result
12914 will be NA.
12915*args, **kwargs
12916 Additional keywords have no effect but might be accepted for
12917 compatibility with NumPy.
12919Returns
12920-------
12921{name1} or {name2}
12922 Return cumulative {desc} of {name1} or {name2}.
12924See Also
12925--------
12926core.window.expanding.Expanding.{accum_func_name} : Similar functionality
12927 but ignores ``NaN`` values.
12928{name2}.{accum_func_name} : Return the {desc} over
12929 {name2} axis.
12930{name2}.cummax : Return cumulative maximum over {name2} axis.
12931{name2}.cummin : Return cumulative minimum over {name2} axis.
12932{name2}.cumsum : Return cumulative sum over {name2} axis.
12933{name2}.cumprod : Return cumulative product over {name2} axis.
12935{examples}"""
12937_cummin_examples = """\
12938Examples
12939--------
12940**Series**
12942>>> s = pd.Series([2, np.nan, 5, -1, 0])
12943>>> s
129440 2.0
129451 NaN
129462 5.0
129473 -1.0
129484 0.0
12949dtype: float64
12951By default, NA values are ignored.
12953>>> s.cummin()
129540 2.0
129551 NaN
129562 2.0
129573 -1.0
129584 -1.0
12959dtype: float64
12961To include NA values in the operation, use ``skipna=False``
12963>>> s.cummin(skipna=False)
129640 2.0
129651 NaN
129662 NaN
129673 NaN
129684 NaN
12969dtype: float64
12971**DataFrame**
12973>>> df = pd.DataFrame([[2.0, 1.0],
12974... [3.0, np.nan],
12975... [1.0, 0.0]],
12976... columns=list('AB'))
12977>>> df
12978 A B
129790 2.0 1.0
129801 3.0 NaN
129812 1.0 0.0
12983By default, iterates over rows and finds the minimum
12984in each column. This is equivalent to ``axis=None`` or ``axis='index'``.
12986>>> df.cummin()
12987 A B
129880 2.0 1.0
129891 2.0 NaN
129902 1.0 0.0
12992To iterate over columns and find the minimum in each row,
12993use ``axis=1``
12995>>> df.cummin(axis=1)
12996 A B
129970 2.0 1.0
129981 3.0 NaN
129992 1.0 0.0
13000"""
13002_cumsum_examples = """\
13003Examples
13004--------
13005**Series**
13007>>> s = pd.Series([2, np.nan, 5, -1, 0])
13008>>> s
130090 2.0
130101 NaN
130112 5.0
130123 -1.0
130134 0.0
13014dtype: float64
13016By default, NA values are ignored.
13018>>> s.cumsum()
130190 2.0
130201 NaN
130212 7.0
130223 6.0
130234 6.0
13024dtype: float64
13026To include NA values in the operation, use ``skipna=False``
13028>>> s.cumsum(skipna=False)
130290 2.0
130301 NaN
130312 NaN
130323 NaN
130334 NaN
13034dtype: float64
13036**DataFrame**
13038>>> df = pd.DataFrame([[2.0, 1.0],
13039... [3.0, np.nan],
13040... [1.0, 0.0]],
13041... columns=list('AB'))
13042>>> df
13043 A B
130440 2.0 1.0
130451 3.0 NaN
130462 1.0 0.0
13048By default, iterates over rows and finds the sum
13049in each column. This is equivalent to ``axis=None`` or ``axis='index'``.
13051>>> df.cumsum()
13052 A B
130530 2.0 1.0
130541 5.0 NaN
130552 6.0 1.0
13057To iterate over columns and find the sum in each row,
13058use ``axis=1``
13060>>> df.cumsum(axis=1)
13061 A B
130620 2.0 3.0
130631 3.0 NaN
130642 1.0 1.0
13065"""
13067_cumprod_examples = """\
13068Examples
13069--------
13070**Series**
13072>>> s = pd.Series([2, np.nan, 5, -1, 0])
13073>>> s
130740 2.0
130751 NaN
130762 5.0
130773 -1.0
130784 0.0
13079dtype: float64
13081By default, NA values are ignored.
13083>>> s.cumprod()
130840 2.0
130851 NaN
130862 10.0
130873 -10.0
130884 -0.0
13089dtype: float64
13091To include NA values in the operation, use ``skipna=False``
13093>>> s.cumprod(skipna=False)
130940 2.0
130951 NaN
130962 NaN
130973 NaN
130984 NaN
13099dtype: float64
13101**DataFrame**
13103>>> df = pd.DataFrame([[2.0, 1.0],
13104... [3.0, np.nan],
13105... [1.0, 0.0]],
13106... columns=list('AB'))
13107>>> df
13108 A B
131090 2.0 1.0
131101 3.0 NaN
131112 1.0 0.0
13113By default, iterates over rows and finds the product
13114in each column. This is equivalent to ``axis=None`` or ``axis='index'``.
13116>>> df.cumprod()
13117 A B
131180 2.0 1.0
131191 6.0 NaN
131202 6.0 0.0
13122To iterate over columns and find the product in each row,
13123use ``axis=1``
13125>>> df.cumprod(axis=1)
13126 A B
131270 2.0 2.0
131281 3.0 NaN
131292 1.0 0.0
13130"""
13132_cummax_examples = """\
13133Examples
13134--------
13135**Series**
13137>>> s = pd.Series([2, np.nan, 5, -1, 0])
13138>>> s
131390 2.0
131401 NaN
131412 5.0
131423 -1.0
131434 0.0
13144dtype: float64
13146By default, NA values are ignored.
13148>>> s.cummax()
131490 2.0
131501 NaN
131512 5.0
131523 5.0
131534 5.0
13154dtype: float64
13156To include NA values in the operation, use ``skipna=False``
13158>>> s.cummax(skipna=False)
131590 2.0
131601 NaN
131612 NaN
131623 NaN
131634 NaN
13164dtype: float64
13166**DataFrame**
13168>>> df = pd.DataFrame([[2.0, 1.0],
13169... [3.0, np.nan],
13170... [1.0, 0.0]],
13171... columns=list('AB'))
13172>>> df
13173 A B
131740 2.0 1.0
131751 3.0 NaN
131762 1.0 0.0
13178By default, iterates over rows and finds the maximum
13179in each column. This is equivalent to ``axis=None`` or ``axis='index'``.
13181>>> df.cummax()
13182 A B
131830 2.0 1.0
131841 3.0 NaN
131852 3.0 1.0
13187To iterate over columns and find the maximum in each row,
13188use ``axis=1``
13190>>> df.cummax(axis=1)
13191 A B
131920 2.0 2.0
131931 3.0 NaN
131942 1.0 1.0
13195"""
13197_any_see_also = """\
13198See Also
13199--------
13200numpy.any : Numpy version of this method.
13201Series.any : Return whether any element is True.
13202Series.all : Return whether all elements are True.
13203DataFrame.any : Return whether any element is True over requested axis.
13204DataFrame.all : Return whether all elements are True over requested axis.
13205"""
13207_any_desc = """\
13208Return whether any element is True, potentially over an axis.
13210Returns False unless there is at least one element within a series or
13211along a Dataframe axis that is True or equivalent (e.g. non-zero or
13212non-empty)."""
13214_any_examples = """\
13215Examples
13216--------
13217**Series**
13219For Series input, the output is a scalar indicating whether any element
13220is True.
13222>>> pd.Series([False, False]).any()
13223False
13224>>> pd.Series([True, False]).any()
13225True
13226>>> pd.Series([], dtype="float64").any()
13227False
13228>>> pd.Series([np.nan]).any()
13229False
13230>>> pd.Series([np.nan]).any(skipna=False)
13231True
13233**DataFrame**
13235Whether each column contains at least one True element (the default).
13237>>> df = pd.DataFrame({"A": [1, 2], "B": [0, 2], "C": [0, 0]})
13238>>> df
13239 A B C
132400 1 0 0
132411 2 2 0
13243>>> df.any()
13244A True
13245B True
13246C False
13247dtype: bool
13249Aggregating over the columns.
13251>>> df = pd.DataFrame({"A": [True, False], "B": [1, 2]})
13252>>> df
13253 A B
132540 True 1
132551 False 2
13257>>> df.any(axis='columns')
132580 True
132591 True
13260dtype: bool
13262>>> df = pd.DataFrame({"A": [True, False], "B": [1, 0]})
13263>>> df
13264 A B
132650 True 1
132661 False 0
13268>>> df.any(axis='columns')
132690 True
132701 False
13271dtype: bool
13273Aggregating over the entire DataFrame with ``axis=None``.
13275>>> df.any(axis=None)
13276True
13278`any` for an empty DataFrame is an empty Series.
13280>>> pd.DataFrame([]).any()
13281Series([], dtype: bool)
13282"""
13284_shared_docs["stat_func_example"] = """
13286Examples
13287--------
13288>>> idx = pd.MultiIndex.from_arrays([
13289... ['warm', 'warm', 'cold', 'cold'],
13290... ['dog', 'falcon', 'fish', 'spider']],
13291... names=['blooded', 'animal'])
13292>>> s = pd.Series([4, 2, 0, 8], name='legs', index=idx)
13293>>> s
13294blooded animal
13295warm dog 4
13296 falcon 2
13297cold fish 0
13298 spider 8
13299Name: legs, dtype: int64
13301>>> s.{stat_func}()
13302{default_output}"""
13304_sum_examples = _shared_docs["stat_func_example"].format(
13305 stat_func="sum", verb="Sum", default_output=14, level_output_0=6, level_output_1=8
13308_sum_examples += """
13310By default, the sum of an empty or all-NA Series is ``0``.
13312>>> pd.Series([], dtype="float64").sum() # min_count=0 is the default
133130.0
13315This can be controlled with the ``min_count`` parameter. For example, if
13316you'd like the sum of an empty series to be NaN, pass ``min_count=1``.
13318>>> pd.Series([], dtype="float64").sum(min_count=1)
13319nan
13321Thanks to the ``skipna`` parameter, ``min_count`` handles all-NA and
13322empty series identically.
13324>>> pd.Series([np.nan]).sum()
133250.0
13327>>> pd.Series([np.nan]).sum(min_count=1)
13328nan"""
13330_max_examples: str = _shared_docs["stat_func_example"].format(
13331 stat_func="max", verb="Max", default_output=8, level_output_0=4, level_output_1=8
13334_min_examples: str = _shared_docs["stat_func_example"].format(
13335 stat_func="min", verb="Min", default_output=0, level_output_0=2, level_output_1=0
13338_skew_see_also = """
13340See Also
13341--------
13342Series.skew : Return unbiased skew over requested axis.
13343Series.var : Return unbiased variance over requested axis.
13344Series.std : Return unbiased standard deviation over requested axis."""
13346_stat_func_see_also = """
13348See Also
13349--------
13350Series.sum : Return the sum.
13351Series.min : Return the minimum.
13352Series.max : Return the maximum.
13353Series.idxmin : Return the index of the minimum.
13354Series.idxmax : Return the index of the maximum.
13355DataFrame.sum : Return the sum over the requested axis.
13356DataFrame.min : Return the minimum over the requested axis.
13357DataFrame.max : Return the maximum over the requested axis.
13358DataFrame.idxmin : Return the index of the minimum over the requested axis.
13359DataFrame.idxmax : Return the index of the maximum over the requested axis."""
13361_prod_examples = """
13363Examples
13364--------
13365By default, the product of an empty or all-NA Series is ``1``
13367>>> pd.Series([], dtype="float64").prod()
133681.0
13370This can be controlled with the ``min_count`` parameter
13372>>> pd.Series([], dtype="float64").prod(min_count=1)
13373nan
13375Thanks to the ``skipna`` parameter, ``min_count`` handles all-NA and
13376empty series identically.
13378>>> pd.Series([np.nan]).prod()
133791.0
13381>>> pd.Series([np.nan]).prod(min_count=1)
13382nan"""
13384_min_count_stub = """\
13385min_count : int, default 0
13386 The required number of valid values to perform the operation. If fewer than
13387 ``min_count`` non-NA values are present the result will be NA.
13388"""
13391def make_doc(name: str, ndim: int) -> str:
13392 """
13393 Generate the docstring for a Series/DataFrame reduction.
13394 """
13395 if ndim == 1:
13396 name1 = "scalar"
13397 name2 = "Series"
13398 axis_descr = "{index (0)}"
13399 else:
13400 name1 = "Series"
13401 name2 = "DataFrame"
13402 axis_descr = "{index (0), columns (1)}"
13404 if name == "any":
13405 base_doc = _bool_doc
13406 desc = _any_desc
13407 see_also = _any_see_also
13408 examples = _any_examples
13409 kwargs = {"empty_value": "False"}
13410 elif name == "all":
13411 base_doc = _bool_doc
13412 desc = _all_desc
13413 see_also = _all_see_also
13414 examples = _all_examples
13415 kwargs = {"empty_value": "True"}
13416 elif name == "min":
13417 base_doc = _num_doc
13418 desc = (
13419 "Return the minimum of the values over the requested axis.\n\n"
13420 "If you want the *index* of the minimum, use ``idxmin``. This is "
13421 "the equivalent of the ``numpy.ndarray`` method ``argmin``."
13422 )
13423 see_also = _stat_func_see_also
13424 examples = _min_examples
13425 kwargs = {"min_count": ""}
13426 elif name == "max":
13427 base_doc = _num_doc
13428 desc = (
13429 "Return the maximum of the values over the requested axis.\n\n"
13430 "If you want the *index* of the maximum, use ``idxmax``. This is "
13431 "the equivalent of the ``numpy.ndarray`` method ``argmax``."
13432 )
13433 see_also = _stat_func_see_also
13434 examples = _max_examples
13435 kwargs = {"min_count": ""}
13437 elif name == "sum":
13438 base_doc = _sum_prod_doc
13439 desc = (
13440 "Return the sum of the values over the requested axis.\n\n"
13441 "This is equivalent to the method ``numpy.sum``."
13442 )
13443 see_also = _stat_func_see_also
13444 examples = _sum_examples
13445 kwargs = {"min_count": _min_count_stub}
13447 elif name == "prod":
13448 base_doc = _sum_prod_doc
13449 desc = "Return the product of the values over the requested axis."
13450 see_also = _stat_func_see_also
13451 examples = _prod_examples
13452 kwargs = {"min_count": _min_count_stub}
13454 elif name == "median":
13455 base_doc = _num_doc
13456 desc = "Return the median of the values over the requested axis."
13457 see_also = _stat_func_see_also
13458 examples = """
13460 Examples
13461 --------
13462 >>> s = pd.Series([1, 2, 3])
13463 >>> s.median()
13464 2.0
13466 With a DataFrame
13468 >>> df = pd.DataFrame({'a': [1, 2], 'b': [2, 3]}, index=['tiger', 'zebra'])
13469 >>> df
13470 a b
13471 tiger 1 2
13472 zebra 2 3
13473 >>> df.median()
13474 a 1.5
13475 b 2.5
13476 dtype: float64
13478 Using axis=1
13480 >>> df.median(axis=1)
13481 tiger 1.5
13482 zebra 2.5
13483 dtype: float64
13485 In this case, `numeric_only` should be set to `True`
13486 to avoid getting an error.
13488 >>> df = pd.DataFrame({'a': [1, 2], 'b': ['T', 'Z']},
13489 ... index=['tiger', 'zebra'])
13490 >>> df.median(numeric_only=True)
13491 a 1.5
13492 dtype: float64"""
13493 kwargs = {"min_count": ""}
13495 elif name == "mean":
13496 base_doc = _num_doc
13497 desc = "Return the mean of the values over the requested axis."
13498 see_also = _stat_func_see_also
13499 examples = """
13501 Examples
13502 --------
13503 >>> s = pd.Series([1, 2, 3])
13504 >>> s.mean()
13505 2.0
13507 With a DataFrame
13509 >>> df = pd.DataFrame({'a': [1, 2], 'b': [2, 3]}, index=['tiger', 'zebra'])
13510 >>> df
13511 a b
13512 tiger 1 2
13513 zebra 2 3
13514 >>> df.mean()
13515 a 1.5
13516 b 2.5
13517 dtype: float64
13519 Using axis=1
13521 >>> df.mean(axis=1)
13522 tiger 1.5
13523 zebra 2.5
13524 dtype: float64
13526 In this case, `numeric_only` should be set to `True` to avoid
13527 getting an error.
13529 >>> df = pd.DataFrame({'a': [1, 2], 'b': ['T', 'Z']},
13530 ... index=['tiger', 'zebra'])
13531 >>> df.mean(numeric_only=True)
13532 a 1.5
13533 dtype: float64"""
13534 kwargs = {"min_count": ""}
13536 elif name == "var":
13537 base_doc = _num_ddof_doc
13538 desc = (
13539 "Return unbiased variance over requested axis.\n\nNormalized by "
13540 "N-1 by default. This can be changed using the ddof argument."
13541 )
13542 examples = _var_examples
13543 see_also = ""
13544 kwargs = {"notes": ""}
13546 elif name == "std":
13547 base_doc = _num_ddof_doc
13548 desc = (
13549 "Return sample standard deviation over requested axis."
13550 "\n\nNormalized by N-1 by default. This can be changed using the "
13551 "ddof argument."
13552 )
13553 examples = _std_examples
13554 see_also = _std_see_also.format(name2=name2)
13555 kwargs = {"notes": "", "return_desc": _std_return_desc}
13557 elif name == "sem":
13558 base_doc = _num_ddof_doc
13559 desc = (
13560 "Return unbiased standard error of the mean over requested "
13561 "axis.\n\nNormalized by N-1 by default. This can be changed "
13562 "using the ddof argument"
13563 )
13564 examples = """
13566 Examples
13567 --------
13568 >>> s = pd.Series([1, 2, 3])
13569 >>> round(s.sem(), 6)
13570 0.57735
13572 With a DataFrame
13574 >>> df = pd.DataFrame({'a': [1, 2], 'b': [2, 3]}, index=['tiger', 'zebra'])
13575 >>> df
13576 a b
13577 tiger 1 2
13578 zebra 2 3
13579 >>> df.sem()
13580 a 0.5
13581 b 0.5
13582 dtype: float64
13584 Using axis=1
13586 >>> df.sem(axis=1)
13587 tiger 0.5
13588 zebra 0.5
13589 dtype: float64
13591 In this case, `numeric_only` should be set to `True`
13592 to avoid getting an error.
13594 >>> df = pd.DataFrame({'a': [1, 2], 'b': ['T', 'Z']},
13595 ... index=['tiger', 'zebra'])
13596 >>> df.sem(numeric_only=True)
13597 a 0.5
13598 dtype: float64"""
13599 see_also = _sem_see_also.format(name2=name2)
13600 kwargs = {"notes": "", "return_desc": _sem_return_desc}
13602 elif name == "skew":
13603 base_doc = _num_doc
13604 desc = "Return unbiased skew over requested axis.\n\nNormalized by N-1."
13605 see_also = _skew_see_also
13606 examples = """
13608 Examples
13609 --------
13610 >>> s = pd.Series([1, 2, 3])
13611 >>> s.skew()
13612 0.0
13614 With a DataFrame
13616 >>> df = pd.DataFrame({'a': [1, 2, 3], 'b': [2, 3, 4], 'c': [1, 3, 5]},
13617 ... index=['tiger', 'zebra', 'cow'])
13618 >>> df
13619 a b c
13620 tiger 1 2 1
13621 zebra 2 3 3
13622 cow 3 4 5
13623 >>> df.skew()
13624 a 0.0
13625 b 0.0
13626 c 0.0
13627 dtype: float64
13629 Using axis=1
13631 >>> df.skew(axis=1)
13632 tiger 1.732051
13633 zebra -1.732051
13634 cow 0.000000
13635 dtype: float64
13637 In this case, `numeric_only` should be set to `True` to avoid
13638 getting an error.
13640 >>> df = pd.DataFrame({'a': [1, 2, 3], 'b': ['T', 'Z', 'X']},
13641 ... index=['tiger', 'zebra', 'cow'])
13642 >>> df.skew(numeric_only=True)
13643 a 0.0
13644 dtype: float64"""
13645 kwargs = {"min_count": ""}
13647 elif name == "kurt":
13648 base_doc = _num_doc
13649 desc = (
13650 "Return unbiased kurtosis over requested axis.\n\n"
13651 "Kurtosis obtained using Fisher's definition of\n"
13652 "kurtosis (kurtosis of normal == 0.0). Normalized "
13653 "by N-1."
13654 )
13655 see_also = ""
13656 examples = """
13658 Examples
13659 --------
13660 >>> s = pd.Series([1, 2, 2, 3], index=['cat', 'dog', 'dog', 'mouse'])
13661 >>> s
13662 cat 1
13663 dog 2
13664 dog 2
13665 mouse 3
13666 dtype: int64
13667 >>> s.kurt()
13668 1.5
13670 With a DataFrame
13672 >>> df = pd.DataFrame({'a': [1, 2, 2, 3], 'b': [3, 4, 4, 4]},
13673 ... index=['cat', 'dog', 'dog', 'mouse'])
13674 >>> df
13675 a b
13676 cat 1 3
13677 dog 2 4
13678 dog 2 4
13679 mouse 3 4
13680 >>> df.kurt()
13681 a 1.5
13682 b 4.0
13683 dtype: float64
13685 With axis=None
13687 >>> df.kurt(axis=None)
13688 -0.9886927196984727
13690 Using axis=1
13692 >>> df = pd.DataFrame({'a': [1, 2], 'b': [3, 4], 'c': [3, 4], 'd': [1, 2]},
13693 ... index=['cat', 'dog'])
13694 >>> df.kurt(axis=1)
13695 cat -6.0
13696 dog -6.0
13697 dtype: float64"""
13698 kwargs = {"min_count": ""}
13700 elif name == "cumsum":
13701 if ndim == 1:
13702 base_doc = _cnum_series_doc
13703 else:
13704 base_doc = _cnum_pd_doc
13706 desc = "sum"
13707 see_also = ""
13708 examples = _cumsum_examples
13709 kwargs = {"accum_func_name": "sum"}
13711 elif name == "cumprod":
13712 if ndim == 1:
13713 base_doc = _cnum_series_doc
13714 else:
13715 base_doc = _cnum_pd_doc
13717 desc = "product"
13718 see_also = ""
13719 examples = _cumprod_examples
13720 kwargs = {"accum_func_name": "prod"}
13722 elif name == "cummin":
13723 if ndim == 1:
13724 base_doc = _cnum_series_doc
13725 else:
13726 base_doc = _cnum_pd_doc
13728 desc = "minimum"
13729 see_also = ""
13730 examples = _cummin_examples
13731 kwargs = {"accum_func_name": "min"}
13733 elif name == "cummax":
13734 if ndim == 1:
13735 base_doc = _cnum_series_doc
13736 else:
13737 base_doc = _cnum_pd_doc
13739 desc = "maximum"
13740 see_also = ""
13741 examples = _cummax_examples
13742 kwargs = {"accum_func_name": "max"}
13744 else:
13745 raise NotImplementedError
13747 docstr = base_doc.format(
13748 desc=desc,
13749 name=name,
13750 name1=name1,
13751 name2=name2,
13752 axis_descr=axis_descr,
13753 see_also=see_also,
13754 examples=examples,
13755 **kwargs,
13756 )
13757 return docstr