Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py: 14%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from collections import abc
4from contextlib import suppress
5import sys
6from typing import (
7 TYPE_CHECKING,
8 Any,
9 Self,
10 TypeAlias,
11 TypeVar,
12 cast,
13 final,
14)
15import warnings
17import numpy as np
19from pandas._libs.indexing import NDFrameIndexerBase
20from pandas._libs.lib import (
21 is_np_dtype,
22 item_from_zerodim,
23)
24from pandas.compat import CHAINED_WARNING_DISABLED
25from pandas.compat._constants import REF_COUNT_IDX
26from pandas.errors import (
27 AbstractMethodError,
28 ChainedAssignmentError,
29 IndexingError,
30 InvalidIndexError,
31 LossySetitemError,
32 Pandas4Warning,
33)
34from pandas.errors.cow import _chained_assignment_msg
35from pandas.util._exceptions import find_stack_level
37from pandas.core.dtypes.cast import (
38 can_hold_element,
39 floats_fit_integer_dtype,
40 maybe_downcast_to_dtype,
41)
42from pandas.core.dtypes.common import (
43 is_array_like_deprecate_non_pandas,
44 is_bool_dtype,
45 is_hashable,
46 is_integer,
47 is_iterator,
48 is_list_like,
49 is_numeric_dtype,
50 is_object_dtype,
51 is_scalar,
52 is_sequence,
53)
54from pandas.core.dtypes.concat import concat_compat
55from pandas.core.dtypes.dtypes import (
56 ArrowDtype,
57 BaseMaskedDtype,
58 DatetimeTZDtype,
59 ExtensionDtype,
60 IntervalDtype,
61 NumpyEADtype,
62 PeriodDtype,
63 SparseDtype,
64)
65from pandas.core.dtypes.generic import (
66 ABCDataFrame,
67 ABCExtensionArray,
68 ABCIndex,
69 ABCSeries,
70)
71from pandas.core.dtypes.missing import (
72 array_equivalent,
73 construct_1d_array_from_inferred_fill_value,
74 is_valid_na_for_dtype,
75 isna,
76 na_value_for_dtype,
77)
79from pandas.core import algorithms as algos
80import pandas.core.common as com
81from pandas.core.construction import (
82 array as pd_array,
83 ensure_wrapped_if_datetimelike,
84 extract_array,
85 sanitize_array,
86)
87from pandas.core.indexers import (
88 check_array_indexer,
89 is_list_like_indexer,
90 is_scalar_indexer,
91 length_of_indexer,
92)
93from pandas.core.indexes.api import (
94 Index,
95 MultiIndex,
96)
98if TYPE_CHECKING:
99 from collections.abc import (
100 Hashable,
101 Sequence,
102 )
104 from pandas._typing import (
105 ArrayLike,
106 Axis,
107 AxisInt,
108 DtypeObj,
109 Scalar,
110 T,
111 npt,
112 )
114 from pandas import (
115 DataFrame,
116 Series,
117 )
118 from pandas.core.arrays import ExtensionArray
119 from pandas.core.base import IndexOpsMixin
121 _IndexSliceTuple: TypeAlias = tuple[IndexOpsMixin | Scalar | Sequence | slice, ...]
123 _IndexSliceUnion: TypeAlias = (
124 Scalar | Sequence | slice | _IndexSliceTuple | tuple[_IndexSliceTuple, ...]
125 )
127 _IndexSliceUnionT = TypeVar("_IndexSliceUnionT", bound=_IndexSliceUnion)
130# "null slice"
131_NS = slice(None, None)
132_one_ellipsis_message = "indexer may only contain one '...' entry"
135# the public IndexSlicerMaker
136class _IndexSlice:
137 """
138 Create an object to more easily perform multi-index slicing.
140 ``IndexSlice`` is a convenience object that allows the use of natural
141 slice syntax (``start:stop``) when selecting from a :class:`MultiIndex`,
142 rather than requiring explicit ``slice()`` calls.
144 See Also
145 --------
146 MultiIndex.remove_unused_levels : New MultiIndex with no unused levels.
148 Notes
149 -----
150 See :ref:`Defined Levels <advanced.shown_levels>`
151 for further info on slicing a MultiIndex.
153 Examples
154 --------
155 >>> midx = pd.MultiIndex.from_product([["A0", "A1"], ["B0", "B1", "B2", "B3"]])
156 >>> columns = ["foo", "bar"]
157 >>> dfmi = pd.DataFrame(
158 ... np.arange(16).reshape((len(midx), len(columns))),
159 ... index=midx,
160 ... columns=columns,
161 ... )
163 Using the default slice command:
165 >>> dfmi.loc[(slice(None), slice("B0", "B1")), :]
166 foo bar
167 A0 B0 0 1
168 B1 2 3
169 A1 B0 8 9
170 B1 10 11
172 Using the IndexSlice class for a more intuitive command:
174 >>> idx = pd.IndexSlice
175 >>> dfmi.loc[idx[:, "B0":"B1"], :]
176 foo bar
177 A0 B0 0 1
178 B1 2 3
179 A1 B0 8 9
180 B1 10 11
181 """
183 def __getitem__(self, arg: _IndexSliceUnionT) -> _IndexSliceUnionT:
184 return arg
187IndexSlice = _IndexSlice()
188IndexSlice.__module__ = "pandas"
191class IndexingMixin:
192 """
193 Mixin for adding .loc/.iloc/.at/.iat to Dataframes and Series.
194 """
196 @property
197 def iloc(self) -> _iLocIndexer:
198 """
199 Purely integer-location based indexing for selection by position.
201 .. versionchanged:: 3.0
203 Callables which return a tuple are deprecated as input.
205 ``.iloc[]`` is primarily integer position based (from ``0`` to
206 ``length-1`` of the axis), but may also be used with a boolean
207 array.
209 Allowed inputs are:
211 - An integer, e.g. ``5``.
212 - A list or array of integers, e.g. ``[4, 3, 0]``.
213 - A slice object with ints, e.g. ``1:7``.
214 - A boolean array.
215 - A ``callable`` function with one argument (the calling Series or
216 DataFrame) and that returns valid output for indexing (one of the above).
217 This is useful in method chains, when you don't have a reference to the
218 calling object, but would like to base your selection on
219 some value.
220 - A tuple of row and column indexes. The tuple elements consist of one of the
221 above inputs, e.g. ``(0, 1)``.
223 ``.iloc`` will raise ``IndexError`` if a requested indexer is
224 out-of-bounds, except *slice* indexers which allow out-of-bounds
225 indexing (this conforms with python/numpy *slice* semantics).
227 See more at :ref:`Selection by Position <indexing.integer>`.
229 See Also
230 --------
231 DataFrame.iat : Fast integer location scalar accessor.
232 DataFrame.loc : Purely label-location based indexer for selection by label.
233 Series.iloc : Purely integer-location based indexing for
234 selection by position.
236 Examples
237 --------
238 >>> mydict = [
239 ... {"a": 1, "b": 2, "c": 3, "d": 4},
240 ... {"a": 100, "b": 200, "c": 300, "d": 400},
241 ... {"a": 1000, "b": 2000, "c": 3000, "d": 4000},
242 ... ]
243 >>> df = pd.DataFrame(mydict)
244 >>> df
245 a b c d
246 0 1 2 3 4
247 1 100 200 300 400
248 2 1000 2000 3000 4000
250 **Indexing just the rows**
252 With a scalar integer.
254 >>> type(df.iloc[0])
255 <class 'pandas.Series'>
256 >>> df.iloc[0]
257 a 1
258 b 2
259 c 3
260 d 4
261 Name: 0, dtype: int64
263 With a list of integers.
265 >>> df.iloc[[0]]
266 a b c d
267 0 1 2 3 4
268 >>> type(df.iloc[[0]])
269 <class 'pandas.DataFrame'>
271 >>> df.iloc[[0, 1]]
272 a b c d
273 0 1 2 3 4
274 1 100 200 300 400
276 With a `slice` object.
278 >>> df.iloc[:3]
279 a b c d
280 0 1 2 3 4
281 1 100 200 300 400
282 2 1000 2000 3000 4000
284 With a boolean mask the same length as the index.
286 >>> df.iloc[[True, False, True]]
287 a b c d
288 0 1 2 3 4
289 2 1000 2000 3000 4000
291 With a callable, useful in method chains. The `x` passed
292 to the ``lambda`` is the DataFrame being sliced. This selects
293 the rows whose index label even.
295 >>> df.iloc[lambda x: x.index % 2 == 0]
296 a b c d
297 0 1 2 3 4
298 2 1000 2000 3000 4000
300 **Indexing both axes**
302 You can mix the indexer types for the index and columns. Use ``:`` to
303 select the entire axis.
305 With scalar integers.
307 >>> df.iloc[0, 1]
308 2
310 With lists of integers.
312 >>> df.iloc[[0, 2], [1, 3]]
313 b d
314 0 2 4
315 2 2000 4000
317 With `slice` objects.
319 >>> df.iloc[1:3, 0:3]
320 a b c
321 1 100 200 300
322 2 1000 2000 3000
324 With a boolean array whose length matches the columns.
326 >>> df.iloc[:, [True, False, True, False]]
327 a c
328 0 1 3
329 1 100 300
330 2 1000 3000
332 With a callable function that expects the Series or DataFrame.
334 >>> df.iloc[:, lambda df: [0, 2]]
335 a c
336 0 1 3
337 1 100 300
338 2 1000 3000
340 **Selecting along an explicit axis**
342 ``.iloc`` can be called with an ``axis`` argument to apply a single
343 indexer to that axis.
345 >>> df.iloc(axis=0)[1]
346 a 100
347 b 200
348 c 300
349 d 400
350 Name: 1, dtype: int64
352 >>> df.iloc(axis=1)[1]
353 0 2
354 1 200
355 2 2000
356 Name: b, dtype: int64
357 """
358 return _iLocIndexer("iloc", self)
360 @property
361 def loc(self) -> _LocIndexer:
362 """
363 Access a group of rows and columns by label(s) or a boolean array.
365 ``.loc[]`` is primarily label based, but may also be used with a
366 boolean array.
368 Allowed inputs are:
370 - A single label, e.g. ``5`` or ``'a'``, (note that ``5`` is
371 interpreted as a *label* of the index, and **never** as an
372 integer position along the index).
373 - A list or array of labels, e.g. ``['a', 'b', 'c']``.
374 - A slice object with labels, e.g. ``'a':'f'``.
376 .. warning:: Note that contrary to usual python slices, **both** the
377 start and the stop are included
379 - A boolean array of the same length as the axis being sliced,
380 e.g. ``[True, False, True]``.
381 - An alignable boolean Series. The index of the key will be aligned before
382 masking.
383 - An alignable Index. The Index of the returned selection will be the input.
384 - A ``callable`` function with one argument (the calling Series or
385 DataFrame) and that returns valid output for indexing (one of the above)
387 See more at :ref:`Selection by Label <indexing.label>`.
389 Raises
390 ------
391 KeyError
392 If any items are not found.
393 IndexingError
394 If an indexed key is passed and its index is unalignable to the frame index.
396 See Also
397 --------
398 DataFrame.at : Access a single value for a row/column label pair.
399 DataFrame.iloc : Access group of rows and columns by integer position(s).
400 DataFrame.xs : Returns a cross-section (row(s) or column(s)) from the
401 Series/DataFrame.
402 Series.loc : Access group of values using labels.
404 Examples
405 --------
406 **Getting values**
408 >>> df = pd.DataFrame(
409 ... [[1, 2], [4, 5], [7, 8]],
410 ... index=["cobra", "viper", "sidewinder"],
411 ... columns=["max_speed", "shield"],
412 ... )
413 >>> df
414 max_speed shield
415 cobra 1 2
416 viper 4 5
417 sidewinder 7 8
419 Single label. Note this returns the row as a Series.
421 >>> df.loc["viper"]
422 max_speed 4
423 shield 5
424 Name: viper, dtype: int64
426 List of labels. Note using ``[[]]`` returns a DataFrame.
428 >>> df.loc[["viper", "sidewinder"]]
429 max_speed shield
430 viper 4 5
431 sidewinder 7 8
433 Single label for row and column
435 >>> df.loc["cobra", "shield"]
436 2
438 Slice with labels for row and single label for column. As mentioned
439 above, note that both the start and stop of the slice are included.
441 >>> df.loc["cobra":"viper", "max_speed"]
442 cobra 1
443 viper 4
444 Name: max_speed, dtype: int64
446 Boolean list with the same length as the row axis
448 >>> df.loc[[False, False, True]]
449 max_speed shield
450 sidewinder 7 8
452 Alignable boolean Series:
454 >>> df.loc[
455 ... pd.Series([False, True, False], index=["viper", "sidewinder", "cobra"])
456 ... ]
457 max_speed shield
458 sidewinder 7 8
460 Conditional that returns a boolean Series
462 >>> df.loc[df["shield"] > 6]
463 max_speed shield
464 sidewinder 7 8
466 Conditional that returns a boolean Series with column labels specified
468 >>> df.loc[df["shield"] > 6, ["max_speed"]]
469 max_speed
470 sidewinder 7
472 Multiple conditional using ``&`` that returns a boolean Series
474 >>> df.loc[(df["max_speed"] > 1) & (df["shield"] < 8)]
475 max_speed shield
476 viper 4 5
478 Multiple conditional using ``|`` that returns a boolean Series
480 >>> df.loc[(df["max_speed"] > 4) | (df["shield"] < 5)]
481 max_speed shield
482 cobra 1 2
483 sidewinder 7 8
485 Please ensure that each condition is wrapped in parentheses ``()``.
486 See the :ref:`user guide<indexing.boolean>`
487 for more details and explanations of Boolean indexing.
489 .. note::
490 If you find yourself using 3 or more conditionals in ``.loc[]``,
491 consider using :ref:`advanced indexing<advanced.advanced_hierarchical>`.
493 See below for using ``.loc[]`` on MultiIndex DataFrames.
495 Callable that returns a boolean Series
497 >>> df.loc[lambda df: df["shield"] == 8]
498 max_speed shield
499 sidewinder 7 8
501 **Selecting along an explicit axis**
503 ``.loc`` can be called with an ``axis`` argument to apply a single
504 indexer to that axis.
506 >>> df.loc(axis=0)["viper"]
507 max_speed 4
508 shield 5
509 Name: viper, dtype: int64
511 >>> df.loc(axis=1)["shield"]
512 cobra 2
513 viper 5
514 sidewinder 8
515 Name: shield, dtype: int64
517 **Setting values**
519 Set value for all items matching the list of labels
521 >>> df.loc[["viper", "sidewinder"], ["shield"]] = 50
522 >>> df
523 max_speed shield
524 cobra 1 2
525 viper 4 50
526 sidewinder 7 50
528 Set value for an entire row
530 >>> df.loc["cobra"] = 10
531 >>> df
532 max_speed shield
533 cobra 10 10
534 viper 4 50
535 sidewinder 7 50
537 Set value for an entire column
539 >>> df.loc[:, "max_speed"] = 30
540 >>> df
541 max_speed shield
542 cobra 30 10
543 viper 30 50
544 sidewinder 30 50
546 Set value for rows matching callable condition
548 >>> df.loc[df["shield"] > 35] = 0
549 >>> df
550 max_speed shield
551 cobra 30 10
552 viper 0 0
553 sidewinder 0 0
555 Add value matching location
557 >>> df.loc["viper", "shield"] += 5
558 >>> df
559 max_speed shield
560 cobra 30 10
561 viper 0 5
562 sidewinder 0 0
564 Setting using a ``Series`` or a ``DataFrame`` sets the values matching the
565 index labels, not the index positions.
567 >>> shuffled_df = df.loc[["viper", "cobra", "sidewinder"]]
568 >>> df.loc[:] += shuffled_df
569 >>> df
570 max_speed shield
571 cobra 60 20
572 viper 0 10
573 sidewinder 0 0
575 **Getting values on a DataFrame with an index that has integer labels**
577 Another example using integers for the index
579 >>> df = pd.DataFrame(
580 ... [[1, 2], [4, 5], [7, 8]],
581 ... index=[7, 8, 9],
582 ... columns=["max_speed", "shield"],
583 ... )
584 >>> df
585 max_speed shield
586 7 1 2
587 8 4 5
588 9 7 8
590 Slice with integer labels for rows. As mentioned above, note that both
591 the start and stop of the slice are included.
593 >>> df.loc[7:9]
594 max_speed shield
595 7 1 2
596 8 4 5
597 9 7 8
599 **Getting values with a MultiIndex**
601 A number of examples using a DataFrame with a MultiIndex
603 >>> tuples = [
604 ... ("cobra", "mark i"),
605 ... ("cobra", "mark ii"),
606 ... ("sidewinder", "mark i"),
607 ... ("sidewinder", "mark ii"),
608 ... ("viper", "mark ii"),
609 ... ("viper", "mark iii"),
610 ... ]
611 >>> index = pd.MultiIndex.from_tuples(tuples)
612 >>> values = [[12, 2], [0, 4], [10, 20], [1, 4], [7, 1], [16, 36]]
613 >>> df = pd.DataFrame(values, columns=["max_speed", "shield"], index=index)
614 >>> df
615 max_speed shield
616 cobra mark i 12 2
617 mark ii 0 4
618 sidewinder mark i 10 20
619 mark ii 1 4
620 viper mark ii 7 1
621 mark iii 16 36
623 Single label. Note this returns a DataFrame with a single index.
625 >>> df.loc["cobra"]
626 max_speed shield
627 mark i 12 2
628 mark ii 0 4
630 Single index tuple. Note this returns a Series.
632 >>> df.loc[("cobra", "mark ii")]
633 max_speed 0
634 shield 4
635 Name: (cobra, mark ii), dtype: int64
637 Single label for row and column. Similar to passing in a tuple, this
638 returns a Series.
640 >>> df.loc["cobra", "mark i"]
641 max_speed 12
642 shield 2
643 Name: (cobra, mark i), dtype: int64
645 Single tuple. Note using ``[[]]`` returns a DataFrame.
647 >>> df.loc[[("cobra", "mark ii")]]
648 max_speed shield
649 cobra mark ii 0 4
651 Single tuple for the index with a single label for the column
653 >>> df.loc[("cobra", "mark i"), "shield"]
654 2
656 Slice from index tuple to single label
658 >>> df.loc[("cobra", "mark i") : "viper"]
659 max_speed shield
660 cobra mark i 12 2
661 mark ii 0 4
662 sidewinder mark i 10 20
663 mark ii 1 4
664 viper mark ii 7 1
665 mark iii 16 36
667 Slice from index tuple to index tuple
669 >>> df.loc[("cobra", "mark i") : ("viper", "mark ii")]
670 max_speed shield
671 cobra mark i 12 2
672 mark ii 0 4
673 sidewinder mark i 10 20
674 mark ii 1 4
675 viper mark ii 7 1
677 Please see the :ref:`user guide<advanced.advanced_hierarchical>`
678 for more details and explanations of advanced indexing.
680 **Assignment with Series**
682 When assigning a Series to .loc[row_indexer, col_indexer], pandas aligns
683 the Series by index labels, not by order or position.
685 Series assignment with .loc and index alignment:
687 >>> df = pd.DataFrame({"A": [1, 2, 3]}, index=[0, 1, 2])
688 >>> s = pd.Series([10, 20], index=[1, 0]) # Note reversed order
689 >>> df.loc[:, "B"] = s # Aligns by index, not order
690 >>> df
691 A B
692 0 1 20.0
693 1 2 10.0
694 2 3 NaN
695 """
696 return _LocIndexer("loc", self)
698 @property
699 def at(self) -> _AtIndexer:
700 """
701 Access a single value for a row/column label pair.
703 Similar to ``loc``, in that both provide label-based lookups. Use
704 ``at`` if you only need to get or set a single value in a DataFrame
705 or Series.
707 Raises
708 ------
709 KeyError
710 If getting a value and 'label' does not exist in a DataFrame or Series.
712 ValueError
713 If row/column label pair is not a tuple or if any label
714 from the pair is not a scalar for DataFrame.
715 If label is list-like (*excluding* NamedTuple) for Series.
717 See Also
718 --------
719 DataFrame.at : Access a single value for a row/column pair by label.
720 DataFrame.iat : Access a single value for a row/column pair by integer
721 position.
722 DataFrame.loc : Access a group of rows and columns by label(s).
723 DataFrame.iloc : Access a group of rows and columns by integer
724 position(s).
725 Series.at : Access a single value by label.
726 Series.iat : Access a single value by integer position.
727 Series.loc : Access a group of rows by label(s).
728 Series.iloc : Access a group of rows by integer position(s).
730 Notes
731 -----
732 See :ref:`Fast scalar value getting and setting <indexing.basics.get_value>`
733 for more details.
735 Examples
736 --------
737 >>> df = pd.DataFrame(
738 ... [[0, 2, 3], [0, 4, 1], [10, 20, 30]],
739 ... index=[4, 5, 6],
740 ... columns=["A", "B", "C"],
741 ... )
742 >>> df
743 A B C
744 4 0 2 3
745 5 0 4 1
746 6 10 20 30
748 Get value at specified row/column pair
750 >>> df.at[4, "B"]
751 2
753 Set value at specified row/column pair
755 >>> df.at[4, "B"] = 10
756 >>> df.at[4, "B"]
757 10
759 Get value within a Series
761 >>> df.loc[5].at["B"]
762 4
763 """
764 return _AtIndexer("at", self)
766 @property
767 def iat(self) -> _iAtIndexer:
768 """
769 Access a single value for a row/column pair by integer position.
771 Similar to ``iloc``, in that both provide integer-based lookups. Use
772 ``iat`` if you only need to get or set a single value in a DataFrame
773 or Series.
775 Raises
776 ------
777 IndexError
778 When integer position is out of bounds.
780 See Also
781 --------
782 DataFrame.at : Access a single value for a row/column label pair.
783 DataFrame.loc : Access a group of rows and columns by label(s).
784 DataFrame.iloc : Access a group of rows and columns by integer position(s).
786 Examples
787 --------
788 >>> df = pd.DataFrame(
789 ... [[0, 2, 3], [0, 4, 1], [10, 20, 30]], columns=["A", "B", "C"]
790 ... )
791 >>> df
792 A B C
793 0 0 2 3
794 1 0 4 1
795 2 10 20 30
797 Get value at specified row/column pair
799 >>> df.iat[1, 2]
800 1
802 Set value at specified row/column pair
804 >>> df.iat[1, 2] = 10
805 >>> df.iat[1, 2]
806 10
808 Get value within a series
810 >>> df.loc[0].iat[1]
811 2
812 """
813 return _iAtIndexer("iat", self)
816class _LocationIndexer(NDFrameIndexerBase):
817 _valid_types: str
818 axis: AxisInt | None = None
820 # sub-classes need to set _takeable
821 _takeable: bool
823 @final
824 def __call__(self, axis: Axis | None = None) -> Self:
825 # we need to return a copy of ourselves
826 new_self = type(self)(self.name, self.obj)
828 if axis is not None:
829 axis_int_none = self.obj._get_axis_number(axis)
830 else:
831 axis_int_none = axis
832 new_self.axis = axis_int_none
833 return new_self
835 def _get_setitem_indexer(self, key):
836 """
837 Convert a potentially-label-based key into a positional indexer.
838 """
839 if self.name == "loc":
840 # always holds here bc iloc overrides _get_setitem_indexer
841 self._ensure_listlike_indexer(key, axis=self.axis)
843 if isinstance(key, tuple):
844 for x in key:
845 check_dict_or_set_indexers(x)
847 if self.axis is not None:
848 key = _tupleize_axis_indexer(self.ndim, self.axis, key)
850 ax = self.obj._get_axis(0)
852 if (
853 isinstance(ax, MultiIndex)
854 and self.name != "iloc"
855 and is_hashable(key, allow_slice=False)
856 ):
857 with suppress(KeyError, InvalidIndexError):
858 # TypeError e.g. passed a bool
859 return ax.get_loc(key)
861 if isinstance(key, tuple):
862 with suppress(IndexingError):
863 # suppress "Too many indexers"
864 return self._convert_tuple(key)
866 if isinstance(key, range):
867 # GH#45479 test_loc_setitem_range_key
868 key = list(key)
870 return self._convert_to_indexer(key, axis=0)
872 @final
873 def _maybe_mask_setitem_value(self, indexer, value):
874 """
875 If we have obj.iloc[mask] = series_or_frame and series_or_frame has the
876 same length as obj, we treat this as obj.iloc[mask] = series_or_frame[mask],
877 similar to Series.__setitem__.
879 Note this is only for loc, not iloc.
880 """
882 if (
883 isinstance(indexer, tuple)
884 and len(indexer) == 2
885 and isinstance(value, (ABCSeries, ABCDataFrame))
886 ):
887 pi, icols = indexer
888 ndim = value.ndim
889 if com.is_bool_indexer(pi) and len(value) == len(pi):
890 newkey = pi.nonzero()[0]
892 if is_scalar_indexer(icols, self.ndim - 1) and ndim == 1:
893 # e.g. test_loc_setitem_boolean_mask_allfalse
894 if len(newkey) == 0:
895 value = value.iloc[:0]
896 else:
897 # test_loc_setitem_ndframe_values_alignment
898 value = self.obj.iloc._align_series(indexer, value)
899 indexer = (newkey, icols)
901 elif (
902 isinstance(icols, np.ndarray)
903 and icols.dtype.kind == "i"
904 and len(icols) == 1
905 ):
906 if ndim == 1:
907 # We implicitly broadcast, though numpy does not, see
908 # github.com/pandas-dev/pandas/pull/45501#discussion_r789071825
909 # test_loc_setitem_ndframe_values_alignment
910 value = self.obj.iloc._align_series(indexer, value)
911 indexer = (newkey, icols)
913 elif ndim == 2 and value.shape[1] == 1:
914 if len(newkey) == 0:
915 value = value.iloc[:0]
916 else:
917 # test_loc_setitem_ndframe_values_alignment
918 value = self.obj.iloc._align_frame(indexer, value)
919 indexer = (newkey, icols)
920 elif com.is_bool_indexer(indexer):
921 indexer = indexer.nonzero()[0]
923 return indexer, value
925 @final
926 def _ensure_listlike_indexer(self, key, axis=None) -> None:
927 """
928 Ensure that a list-like of column labels are all present by adding them if
929 they do not already exist.
931 Parameters
932 ----------
933 key : list-like of column labels
934 Target labels.
935 axis : key axis if known
936 """
937 # column only exists in 2-dimensional DataFrame
938 if self.ndim != 2:
939 return
941 if isinstance(key, tuple) and len(key) > 1:
942 # key may be a tuple if we are .loc
943 # if length of key is > 1 set key to column part
944 # unless axis is already specified, then go with that
945 if axis is None:
946 axis = 1
947 key = key[axis]
949 if (
950 axis == 1
951 and not isinstance(self.obj.columns, MultiIndex)
952 and is_list_like_indexer(key)
953 and not com.is_bool_indexer(key)
954 and all(is_hashable(k) for k in key)
955 ):
956 # GH#38148
957 keys = self.obj.columns.union(key, sort=False)
958 diff = Index(key, copy=False).difference(self.obj.columns, sort=False)
960 if len(diff):
961 # e.g. if we are doing df.loc[:, ["A", "B"]] = 7 and "B"
962 # is a new column, add the new columns with dtype=np.void
963 # so that later when we go through setitem_single_column
964 # we will use isetitem. Without this, the reindex_axis
965 # below would create float64 columns in this example, which
966 # would successfully hold 7, so we would end up with the wrong
967 # dtype.
968 indexer = self.obj.columns.get_indexer(keys)
969 new_mgr = self.obj._mgr.reindex_indexer(
970 keys, indexer=indexer, axis=0, only_slice=True, use_na_proxy=True
971 )
972 self.obj._mgr = new_mgr
973 return
975 self.obj._mgr = self.obj._mgr.reindex_axis(keys, axis=0, only_slice=True)
977 @final
978 def __setitem__(self, key, value) -> None:
979 if not CHAINED_WARNING_DISABLED:
980 if sys.getrefcount(self.obj) <= REF_COUNT_IDX:
981 warnings.warn(
982 _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
983 )
985 check_dict_or_set_indexers(key)
986 if isinstance(key, tuple):
987 key = (list(x) if is_iterator(x) else x for x in key)
988 key = tuple(com.apply_if_callable(x, self.obj) for x in key)
989 else:
990 maybe_callable = com.apply_if_callable(key, self.obj)
991 key = self._raise_callable_usage(key, maybe_callable)
992 # Capture the row count before _get_setitem_indexer which may add
993 # columns (but not rows).
994 orig_nrows = self.obj.shape[0]
995 indexer = self._get_setitem_indexer(key) # may alter self.obj
996 self._has_valid_setitem_indexer(key)
998 # Capture dtype info cheaply between potential column expansion
999 # (above) and potential row expansion (below).
1000 if self.obj.ndim == 1:
1001 orig_dtype_info = self.obj.dtype
1002 orig_columns = None
1003 else:
1004 orig_dtype_info = self.obj._mgr.get_dtypes()
1005 orig_columns = self.obj.columns
1007 iloc: _iLocIndexer = (
1008 cast("_iLocIndexer", self) if self.name == "iloc" else self.obj.iloc
1009 )
1010 iloc._setitem_with_indexer(indexer, value, self.name)
1012 if self.obj.shape[0] != orig_nrows:
1013 self._post_expansion_casting(orig_dtype_info, orig_columns)
1015 def _post_expansion_casting(
1016 self,
1017 orig_dtype_info: DtypeObj | npt.NDArray[np.object_],
1018 orig_columns: Index | None,
1019 ) -> None:
1020 # setitem-with-expansion added new rows. Where a column's dtype
1021 # changed, warn if the original dtype could not hold the new values
1022 # (a future version will disallow the silent change).
1023 if self.obj.ndim == 1:
1024 assert not isinstance(orig_dtype_info, np.ndarray)
1025 if orig_dtype_info != self.obj.dtype:
1026 orig_arr = pd_array([], dtype=orig_dtype_info)
1027 new_arr = infer_and_maybe_downcast(orig_arr, self.obj._values)
1028 new_ser = self.obj._constructor(
1029 new_arr, index=self.obj.index, name=self.obj.name
1030 )
1031 self.obj._mgr = new_ser._mgr
1032 else:
1033 assert isinstance(orig_dtype_info, np.ndarray)
1034 assert orig_columns is not None
1035 orig_dtypes = orig_dtype_info
1036 new_dtypes = self.obj._mgr.get_dtypes()
1037 if len(orig_dtypes) == len(new_dtypes):
1038 # We added rows but not columns
1039 changed_dtypes = new_dtypes != orig_dtypes
1040 for idx in np.flatnonzero(changed_dtypes):
1041 if orig_dtypes[idx].kind == "V":
1042 # np.void NA-proxy placeholder from new-column
1043 # creation; not a meaningful dtype to retain
1044 continue
1045 orig_arr = pd_array([], dtype=orig_dtypes[idx])
1046 new_arr = infer_and_maybe_downcast(
1047 orig_arr, self.obj.iloc[:, idx]._values
1048 )
1049 self.obj.isetitem(idx, new_arr)
1051 elif orig_columns.is_unique and self.obj.columns.is_unique:
1052 # Added rows and columns; iterate by label since positional
1053 # correspondence between orig and new is not guaranteed.
1054 for col, orig_dtype in zip(orig_columns, orig_dtypes, strict=True):
1055 new_dtype = self.obj[col].dtype
1056 if new_dtype != orig_dtype:
1057 if orig_dtype.kind == "V":
1058 # np.void NA-proxy placeholder, as above
1059 continue
1060 orig_arr = pd_array([], dtype=orig_dtype)
1061 new_arr = infer_and_maybe_downcast(
1062 orig_arr, self.obj[col]._values
1063 )
1064 self.obj[col] = new_arr
1065 else:
1066 # In these cases there isn't a one-to-one correspondence between
1067 # old columns and new columns, which makes casting hairy.
1068 # Punt on these for now, as there are no tests that get here
1069 # as of 2025-09-29
1070 pass
1072 def _validate_key(self, key, axis: AxisInt) -> None:
1073 """
1074 Ensure that key is valid for current indexer.
1076 Parameters
1077 ----------
1078 key : scalar, slice or list-like
1079 Key requested.
1080 axis : int
1081 Dimension on which the indexing is being made.
1083 Raises
1084 ------
1085 TypeError
1086 If the key (or some element of it) has wrong type.
1087 IndexError
1088 If the key (or some element of it) is out of bounds.
1089 KeyError
1090 If the key was not found.
1091 """
1092 raise AbstractMethodError(self)
1094 @final
1095 def _expand_ellipsis(self, tup: tuple) -> tuple:
1096 """
1097 If a tuple key includes an Ellipsis, replace it with an appropriate
1098 number of null slices.
1099 """
1100 if any(x is Ellipsis for x in tup):
1101 if tup.count(Ellipsis) > 1:
1102 raise IndexingError(_one_ellipsis_message)
1104 if len(tup) == self.ndim:
1105 # It is unambiguous what axis this Ellipsis is indexing,
1106 # treat as a single null slice.
1107 i = tup.index(Ellipsis)
1108 new_key = (*tup[:i], _NS, *tup[i + 1 :])
1109 return new_key
1111 # TODO: other cases? only one test gets here, and that is covered
1112 # by _validate_key_length
1113 return tup
1115 @final
1116 def _validate_tuple_indexer(self, key: tuple) -> tuple:
1117 """
1118 Check the key for valid keys across my indexer.
1119 """
1120 key = self._validate_key_length(key)
1121 key = self._expand_ellipsis(key)
1122 for i, k in enumerate(key):
1123 try:
1124 self._validate_key(k, i)
1125 except ValueError as err:
1126 raise ValueError(
1127 f"Location based indexing can only have [{self._valid_types}] types"
1128 ) from err
1129 return key
1131 @final
1132 def _is_nested_tuple_indexer(self, tup: tuple) -> bool:
1133 """
1134 Returns
1135 -------
1136 bool
1137 """
1138 if any(isinstance(ax, MultiIndex) for ax in self.obj.axes):
1139 return any(is_nested_tuple(tup, ax) for ax in self.obj.axes)
1140 return False
1142 @final
1143 def _convert_tuple(self, key: tuple) -> tuple:
1144 # Note: we assume _tupleize_axis_indexer has been called, if necessary.
1145 self._validate_key_length(key)
1146 keyidx = [self._convert_to_indexer(k, axis=i) for i, k in enumerate(key)]
1147 return tuple(keyidx)
1149 @final
1150 def _validate_key_length(self, key: tuple) -> tuple:
1151 if len(key) > self.ndim:
1152 if key[0] is Ellipsis:
1153 # e.g. Series.iloc[..., 3] reduces to just Series.iloc[3]
1154 key = key[1:]
1155 if Ellipsis in key:
1156 raise IndexingError(_one_ellipsis_message)
1157 return self._validate_key_length(key)
1158 raise IndexingError("Too many indexers")
1159 return key
1161 @final
1162 def _getitem_tuple_same_dim(self, tup: tuple):
1163 """
1164 Index with indexers that should return an object of the same dimension
1165 as self.obj.
1167 This is only called after a failed call to _getitem_lowerdim.
1168 """
1169 retval = self.obj
1170 # Selecting columns before rows is significantly faster
1171 start_val = (self.ndim - len(tup)) + 1
1172 for i, key in enumerate(reversed(tup)):
1173 i = self.ndim - i - start_val
1174 if com.is_null_slice(key):
1175 continue
1177 retval = getattr(retval, self.name)._getitem_axis(key, axis=i)
1178 # We should never have retval.ndim < self.ndim, as that should
1179 # be handled by the _getitem_lowerdim call above.
1180 assert retval.ndim == self.ndim
1182 if retval is self.obj:
1183 # if all axes were a null slice (`df.loc[:, :]`), ensure we still
1184 # return a new object (https://github.com/pandas-dev/pandas/pull/49469)
1185 retval = retval.copy(deep=False)
1187 return retval
1189 @final
1190 def _getitem_lowerdim(self, tup: tuple):
1191 # we can directly get the axis result since the axis is specified
1192 if self.axis is not None:
1193 axis = self.obj._get_axis_number(self.axis)
1194 return self._getitem_axis(tup, axis=axis)
1196 # we may have a nested tuples indexer here
1197 if self._is_nested_tuple_indexer(tup):
1198 return self._getitem_nested_tuple(tup)
1200 # we may be using a tuple to represent multiple dimensions here
1201 ax0 = self.obj._get_axis(0)
1202 # ...but iloc should handle the tuple as simple integer-location
1203 # instead of checking it as multiindex representation (GH 13797)
1204 if (
1205 isinstance(ax0, MultiIndex)
1206 and self.name != "iloc"
1207 and not any(isinstance(x, slice) for x in tup)
1208 ):
1209 # Note: in all extant test cases, replacing the slice condition with
1210 # `all(is_hashable(x) or com.is_null_slice(x) for x in tup)`
1211 # is equivalent.
1212 # (see the other place where we call _handle_lowerdim_multi_index_axis0)
1213 with suppress(IndexingError):
1214 return cast("_LocIndexer", self)._handle_lowerdim_multi_index_axis0(tup)
1216 tup = self._validate_key_length(tup)
1218 # Reverse tuple so that we are indexing along columns before rows
1219 # and avoid unintended dtype inference. # GH60600
1220 for i, key in zip(range(len(tup) - 1, -1, -1), reversed(tup), strict=True):
1221 if (
1222 is_label_like(key)
1223 or is_list_like(key)
1224 or (isinstance(key, slice) and need_slice(key))
1225 ):
1226 # We don't need to check for tuples here because those are
1227 # caught by the _is_nested_tuple_indexer check above.
1228 section = self._getitem_axis(key, axis=i)
1230 # We should never have a scalar section here, because
1231 # _getitem_lowerdim is only called after a check for
1232 # is_scalar_access, which that would be.
1233 if section.ndim == self.ndim:
1234 # we're in the middle of slicing through a MultiIndex
1235 # revise the key wrt to `section` by inserting an _NS
1236 new_key = (*tup[:i], _NS, *tup[i + 1 :])
1238 else:
1239 # Note: the section.ndim == self.ndim check above
1240 # rules out having DataFrame here, so we dont need to worry
1241 # about transposing.
1242 new_key = tup[:i] + tup[i + 1 :]
1244 if len(new_key) == 1:
1245 new_key = new_key[0]
1247 # Slices should return views, but calling iloc/loc with a null
1248 # slice returns a new object.
1249 if com.is_null_slice(new_key):
1250 return section
1251 # This is an elided recursive call to iloc/loc
1252 return getattr(section, self.name)[new_key]
1254 raise IndexingError("not applicable")
1256 @final
1257 def _getitem_nested_tuple(self, tup: tuple):
1258 # we have a nested tuple so have at least 1 multi-index level
1259 # we should be able to match up the dimensionality here
1261 for key in tup:
1262 check_dict_or_set_indexers(key)
1264 # we have too many indexers for our dim, but have at least 1
1265 # multi-index dimension, try to see if we have something like
1266 # a tuple passed to a series with a multi-index
1267 if len(tup) > self.ndim:
1268 if self.name != "loc":
1269 # This should never be reached, but let's be explicit about it
1270 raise ValueError("Too many indices") # pragma: no cover
1271 if all(
1272 is_hashable(x, allow_slice=False) or com.is_null_slice(x) for x in tup
1273 ):
1274 # GH#10521 Series should reduce MultiIndex dimensions instead of
1275 # DataFrame, IndexingError is not raised when slice(None,None,None)
1276 # with one row.
1277 with suppress(IndexingError):
1278 return cast("_LocIndexer", self)._handle_lowerdim_multi_index_axis0(
1279 tup
1280 )
1281 elif isinstance(self.obj, ABCSeries) and any(
1282 isinstance(k, tuple) for k in tup
1283 ):
1284 # GH#35349 Raise if tuple in tuple for series
1285 # Do this after the all-hashable-or-null-slice check so that
1286 # we are only getting non-hashable tuples, in particular ones
1287 # that themselves contain a slice entry
1288 # See test_loc_series_getitem_too_many_dimensions
1289 raise IndexingError("Too many indexers")
1291 # this is a series with a multi-index specified a tuple of
1292 # selectors
1293 axis = self.axis or 0
1294 return self._getitem_axis(tup, axis=axis)
1296 # handle the multi-axis by taking sections and reducing
1297 # this is iterative
1298 obj = self.obj
1299 # GH#41369 Loop in reverse order ensures indexing along columns before rows
1300 # which selects only necessary blocks which avoids dtype conversion if possible
1301 axis = len(tup) - 1
1302 for key in reversed(tup):
1303 if com.is_null_slice(key):
1304 axis -= 1
1305 continue
1307 obj = getattr(obj, self.name)._getitem_axis(key, axis=axis)
1308 axis -= 1
1310 # if we have a scalar, we are done
1311 if is_scalar(obj) or not hasattr(obj, "ndim"):
1312 break
1314 return obj
1316 def _convert_to_indexer(self, key, axis: AxisInt):
1317 raise AbstractMethodError(self)
1319 def _raise_callable_usage(self, key: Any, maybe_callable: T) -> T:
1320 # GH53533
1321 if self.name == "iloc" and callable(key) and isinstance(maybe_callable, tuple):
1322 raise ValueError(
1323 "Returning a tuple from a callable with iloc is not allowed.",
1324 )
1325 return maybe_callable
1327 @final
1328 def __getitem__(self, key):
1329 check_dict_or_set_indexers(key)
1330 if type(key) is tuple:
1331 key = (list(x) if is_iterator(x) else x for x in key)
1332 key = tuple(com.apply_if_callable(x, self.obj) for x in key)
1333 if self._is_scalar_access(key):
1334 return self.obj._get_value(*key, takeable=self._takeable)
1335 return self._getitem_tuple(key)
1336 else:
1337 # we by definition only have the 0th axis
1338 axis = self.axis or 0
1340 maybe_callable = com.apply_if_callable(key, self.obj)
1341 maybe_callable = self._raise_callable_usage(key, maybe_callable)
1342 return self._getitem_axis(maybe_callable, axis=axis)
1344 def _is_scalar_access(self, key: tuple):
1345 raise NotImplementedError
1347 def _getitem_tuple(self, tup: tuple):
1348 raise AbstractMethodError(self)
1350 def _getitem_axis(self, key, axis: AxisInt):
1351 raise NotImplementedError
1353 def _has_valid_setitem_indexer(self, indexer) -> bool:
1354 raise AbstractMethodError(self)
1356 @final
1357 def _getbool_axis(self, key, axis: AxisInt):
1358 # caller is responsible for ensuring non-None axis
1359 labels = self.obj._get_axis(axis)
1360 key = check_bool_indexer(labels, key)
1361 inds = key.nonzero()[0]
1362 return self.obj.take(inds, axis=axis)
1365class _LocIndexer(_LocationIndexer):
1366 """
1367 Access a group of rows and columns by label(s) or a boolean array.
1369 ``.loc[]`` is primarily label based, but may also be used with a
1370 boolean array.
1372 Allowed inputs are:
1374 - A single label, e.g. ``5`` or ``'a'``, (note that ``5`` is
1375 interpreted as a *label* of the index, and **never** as an
1376 integer position along the index).
1377 - A list or array of labels, e.g. ``['a', 'b', 'c']``.
1378 - A slice object with labels, e.g. ``'a':'f'``.
1380 .. warning:: Note that contrary to usual python slices, **both** the
1381 start and the stop are included
1383 - A boolean array of the same length as the axis being sliced,
1384 e.g. ``[True, False, True]``.
1385 - An alignable boolean Series. The index of the key will be aligned before
1386 masking.
1387 - An alignable Index. The Index of the returned selection will be the input.
1388 - A ``callable`` function with one argument (the calling Series or
1389 DataFrame) and that returns valid output for indexing (one of the above)
1391 See more at :ref:`Selection by Label <indexing.label>`.
1393 Raises
1394 ------
1395 KeyError
1396 If any items are not found.
1397 IndexingError
1398 If an indexed key is passed and its index is unalignable to the frame index.
1400 See Also
1401 --------
1402 DataFrame.at : Access a single value for a row/column label pair.
1403 DataFrame.iloc : Access group of rows and columns by integer position(s).
1404 DataFrame.xs : Returns a cross-section (row(s) or column(s)) from the
1405 Series/DataFrame.
1406 Series.loc : Access group of values using labels.
1408 Examples
1409 --------
1410 **Getting values**
1412 >>> df = pd.DataFrame(
1413 ... [[1, 2], [4, 5], [7, 8]],
1414 ... index=["cobra", "viper", "sidewinder"],
1415 ... columns=["max_speed", "shield"],
1416 ... )
1417 >>> df
1418 max_speed shield
1419 cobra 1 2
1420 viper 4 5
1421 sidewinder 7 8
1423 Single label. Note this returns the row as a Series.
1425 >>> df.loc["viper"]
1426 max_speed 4
1427 shield 5
1428 Name: viper, dtype: int64
1430 List of labels. Note using ``[[]]`` returns a DataFrame.
1432 >>> df.loc[["viper", "sidewinder"]]
1433 max_speed shield
1434 viper 4 5
1435 sidewinder 7 8
1437 Single column label. Note this returns the column as a Series.
1439 >>> df.loc[:, "max_speed"]
1440 cobra 1
1441 viper 4
1442 sidewinder 7
1443 Name: max_speed, dtype: int64
1445 List with a single column label. Note this returns a DataFrame.
1447 >>> df.loc[:, ["max_speed"]]
1448 max_speed
1449 cobra 1
1450 viper 4
1451 sidewinder 7
1453 Single label for row and column
1455 >>> df.loc["cobra", "shield"]
1456 2
1458 Slice with labels for row and single label for column. As mentioned
1459 above, note that both the start and stop of the slice are included.
1461 >>> df.loc["cobra":"viper", "max_speed"]
1462 cobra 1
1463 viper 4
1464 Name: max_speed, dtype: int64
1466 Boolean list with the same length as the row axis
1468 >>> df.loc[[False, False, True]]
1469 max_speed shield
1470 sidewinder 7 8
1472 Alignable boolean Series:
1474 >>> df.loc[pd.Series([False, True, False], index=["viper", "sidewinder", "cobra"])]
1475 max_speed shield
1476 sidewinder 7 8
1478 Conditional that returns a boolean Series
1480 >>> df.loc[df["shield"] > 6]
1481 max_speed shield
1482 sidewinder 7 8
1484 Conditional that returns a boolean Series with column labels specified
1486 >>> df.loc[df["shield"] > 6, ["max_speed"]]
1487 max_speed
1488 sidewinder 7
1490 Multiple conditional using ``&`` that returns a boolean Series
1492 >>> df.loc[(df["max_speed"] > 1) & (df["shield"] < 8)]
1493 max_speed shield
1494 viper 4 5
1496 Multiple conditional using ``|`` that returns a boolean Series
1498 >>> df.loc[(df["max_speed"] > 4) | (df["shield"] < 5)]
1499 max_speed shield
1500 cobra 1 2
1501 sidewinder 7 8
1503 Please ensure that each condition is wrapped in parentheses ``()``.
1504 See the :ref:`user guide<indexing.boolean>`
1505 for more details and explanations of Boolean indexing.
1507 .. note::
1508 If you find yourself using 3 or more conditionals in ``.loc[]``,
1509 consider using :ref:`advanced indexing<advanced.advanced_hierarchical>`.
1511 See below for using ``.loc[]`` on MultiIndex DataFrames.
1513 Callable that returns a boolean Series
1515 >>> df.loc[lambda df: df["shield"] == 8]
1516 max_speed shield
1517 sidewinder 7 8
1519 **Setting values**
1521 Set value for all items matching the list of labels
1523 >>> df.loc[["viper", "sidewinder"], ["shield"]] = 50
1524 >>> df
1525 max_speed shield
1526 cobra 1 2
1527 viper 4 50
1528 sidewinder 7 50
1530 Set value for an entire row
1532 >>> df.loc["cobra"] = 10
1533 >>> df
1534 max_speed shield
1535 cobra 10 10
1536 viper 4 50
1537 sidewinder 7 50
1539 Set value for an entire column
1541 >>> df.loc[:, "max_speed"] = 30
1542 >>> df
1543 max_speed shield
1544 cobra 30 10
1545 viper 30 50
1546 sidewinder 30 50
1548 Set value for rows matching callable condition
1550 >>> df.loc[df["shield"] > 35] = 0
1551 >>> df
1552 max_speed shield
1553 cobra 30 10
1554 viper 0 0
1555 sidewinder 0 0
1557 Add value matching location
1559 >>> df.loc["viper", "shield"] += 5
1560 >>> df
1561 max_speed shield
1562 cobra 30 10
1563 viper 0 5
1564 sidewinder 0 0
1566 Setting using a ``Series`` or a ``DataFrame`` sets the values matching the
1567 index labels, not the index positions.
1569 >>> shuffled_df = df.loc[["viper", "cobra", "sidewinder"]]
1570 >>> df.loc[:] += shuffled_df
1571 >>> df
1572 max_speed shield
1573 cobra 60 20
1574 viper 0 10
1575 sidewinder 0 0
1577 **Getting values on a DataFrame with an index that has integer labels**
1579 Another example using integers for the index
1581 >>> df = pd.DataFrame(
1582 ... [[1, 2], [4, 5], [7, 8]],
1583 ... index=[7, 8, 9],
1584 ... columns=["max_speed", "shield"],
1585 ... )
1586 >>> df
1587 max_speed shield
1588 7 1 2
1589 8 4 5
1590 9 7 8
1592 Slice with integer labels for rows. As mentioned above, note that both
1593 the start and stop of the slice are included.
1595 >>> df.loc[7:9]
1596 max_speed shield
1597 7 1 2
1598 8 4 5
1599 9 7 8
1601 **Getting values with a MultiIndex**
1603 A number of examples using a DataFrame with a MultiIndex
1605 >>> tuples = [
1606 ... ("cobra", "mark i"),
1607 ... ("cobra", "mark ii"),
1608 ... ("sidewinder", "mark i"),
1609 ... ("sidewinder", "mark ii"),
1610 ... ("viper", "mark ii"),
1611 ... ("viper", "mark iii"),
1612 ... ]
1613 >>> index = pd.MultiIndex.from_tuples(tuples)
1614 >>> values = [[12, 2], [0, 4], [10, 20], [1, 4], [7, 1], [16, 36]]
1615 >>> df = pd.DataFrame(values, columns=["max_speed", "shield"], index=index)
1616 >>> df
1617 max_speed shield
1618 cobra mark i 12 2
1619 mark ii 0 4
1620 sidewinder mark i 10 20
1621 mark ii 1 4
1622 viper mark ii 7 1
1623 mark iii 16 36
1625 Single label. Note this returns a DataFrame with a single index.
1627 >>> df.loc["cobra"]
1628 max_speed shield
1629 mark i 12 2
1630 mark ii 0 4
1632 Single index tuple. Note this returns a Series.
1634 >>> df.loc[("cobra", "mark ii")]
1635 max_speed 0
1636 shield 4
1637 Name: (cobra, mark ii), dtype: int64
1639 Single label for row and column. Similar to passing in a tuple, this
1640 returns a Series.
1642 >>> df.loc["cobra", "mark i"]
1643 max_speed 12
1644 shield 2
1645 Name: (cobra, mark i), dtype: int64
1647 Single tuple. Note using ``[[]]`` returns a DataFrame.
1649 >>> df.loc[[("cobra", "mark ii")]]
1650 max_speed shield
1651 cobra mark ii 0 4
1653 Single tuple for the index with a single label for the column
1655 >>> df.loc[("cobra", "mark i"), "shield"]
1656 2
1658 Slice from index tuple to single label
1660 >>> df.loc[("cobra", "mark i") : "viper"]
1661 max_speed shield
1662 cobra mark i 12 2
1663 mark ii 0 4
1664 sidewinder mark i 10 20
1665 mark ii 1 4
1666 viper mark ii 7 1
1667 mark iii 16 36
1669 Slice from index tuple to index tuple
1671 >>> df.loc[("cobra", "mark i") : ("viper", "mark ii")]
1672 max_speed shield
1673 cobra mark i 12 2
1674 mark ii 0 4
1675 sidewinder mark i 10 20
1676 mark ii 1 4
1677 viper mark ii 7 1
1679 Please see the :ref:`user guide<advanced.advanced_hierarchical>`
1680 for more details and explanations of advanced indexing.
1682 **Assignment with Series**
1684 When assigning a Series to .loc[row_indexer, col_indexer], pandas aligns
1685 the Series by index labels, not by order or position.
1687 Series assignment with .loc and index alignment:
1689 >>> df = pd.DataFrame({"A": [1, 2, 3]}, index=[0, 1, 2])
1690 >>> s = pd.Series([10, 20], index=[1, 0]) # Note reversed order
1691 >>> df.loc[:, "B"] = s # Aligns by index, not order
1692 >>> df
1693 A B
1694 0 1 20.0
1695 1 2 10.0
1696 2 3 NaN
1697 """
1699 _takeable: bool = False
1700 _valid_types = (
1701 "labels (MUST BE IN THE INDEX), slices of labels (BOTH "
1702 "endpoints included! Can be slices of integers if the "
1703 "index is integers), listlike of labels, boolean"
1704 )
1706 # -------------------------------------------------------------------
1707 # Key Checks
1709 def _validate_key(self, key, axis: Axis) -> None:
1710 """
1711 Ensure that key is valid for current indexer.
1713 Parameters
1714 ----------
1715 key : scalar, slice or list-like
1716 Key requested.
1717 axis : int
1718 Dimension on which the indexing is being made.
1720 Raises
1721 ------
1722 TypeError
1723 If the key (or some element of it) has wrong type.
1724 IndexError
1725 If the key (or some element of it) is out of bounds.
1726 KeyError
1727 If the key was not found.
1728 """
1729 # valid for a collection of labels (we check their presence later)
1730 # slice of labels (where start-end in labels)
1731 # slice of integers (only if in the labels)
1732 # boolean not in slice and with boolean index
1733 ax = self.obj._get_axis(axis)
1734 if isinstance(key, bool) and not (
1735 is_bool_dtype(ax.dtype)
1736 or ax.dtype.name == "boolean"
1737 or (
1738 isinstance(ax, MultiIndex)
1739 and is_bool_dtype(ax.get_level_values(0).dtype)
1740 )
1741 ):
1742 raise KeyError(
1743 f"{key}: boolean label can not be used without a boolean index"
1744 )
1746 if isinstance(key, slice) and (
1747 isinstance(key.start, bool) or isinstance(key.stop, bool)
1748 ):
1749 raise TypeError(f"{key}: boolean values can not be used in a slice")
1751 def _has_valid_setitem_indexer(self, indexer) -> bool:
1752 return True
1754 def _is_scalar_access(self, key: tuple) -> bool:
1755 """
1756 Returns
1757 -------
1758 bool
1759 """
1760 # this is a shortcut accessor to both .loc and .iloc
1761 # that provide the equivalent access of .at and .iat
1762 # a) avoid getting things via sections and (to minimize dtype changes)
1763 # b) provide a performant path
1764 if len(key) != self.ndim:
1765 return False
1767 for i, k in enumerate(key):
1768 if not is_scalar(k):
1769 return False
1771 ax = self.obj.axes[i]
1772 if isinstance(ax, MultiIndex):
1773 return False
1775 if isinstance(k, str) and ax._supports_partial_string_indexing:
1776 # partial string indexing, df.loc['2000', 'A']
1777 # should not be considered scalar
1778 return False
1780 if not ax._index_as_unique:
1781 return False
1783 return True
1785 # -------------------------------------------------------------------
1786 # MultiIndex Handling
1788 def _multi_take_opportunity(self, tup: tuple) -> bool:
1789 """
1790 Check whether there is the possibility to use ``_multi_take``.
1792 Currently the limit is that all axes being indexed, must be indexed with
1793 list-likes.
1795 Parameters
1796 ----------
1797 tup : tuple
1798 Tuple of indexers, one per axis.
1800 Returns
1801 -------
1802 bool
1803 Whether the current indexing,
1804 can be passed through `_multi_take`.
1805 """
1806 if not all(is_list_like_indexer(x) for x in tup):
1807 return False
1809 # just too complicated
1810 return not any(com.is_bool_indexer(x) for x in tup)
1812 def _multi_take(self, tup: tuple):
1813 """
1814 Create the indexers for the passed tuple of keys, and
1815 executes the take operation. This allows the take operation to be
1816 executed all at once, rather than once for each dimension.
1817 Improving efficiency.
1819 Parameters
1820 ----------
1821 tup : tuple
1822 Tuple of indexers, one per axis.
1824 Returns
1825 -------
1826 values: same type as the object being indexed
1827 """
1828 # GH 836
1829 d = {
1830 axis: self._get_listlike_indexer(key, axis)
1831 for (key, axis) in zip(tup, self.obj._AXIS_ORDERS, strict=True)
1832 }
1833 return self.obj._reindex_with_indexers(d, allow_dups=True)
1835 # -------------------------------------------------------------------
1837 def _getitem_iterable(self, key, axis: AxisInt):
1838 """
1839 Index current object with an iterable collection of keys.
1841 Parameters
1842 ----------
1843 key : iterable
1844 Targeted labels.
1845 axis : int
1846 Dimension on which the indexing is being made.
1848 Raises
1849 ------
1850 KeyError
1851 If no key was found. Will change in the future to raise if not all
1852 keys were found.
1854 Returns
1855 -------
1856 scalar, DataFrame, or Series: indexed value(s).
1857 """
1858 # we assume that not com.is_bool_indexer(key), as that is
1859 # handled before we get here.
1860 self._validate_key(key, axis)
1862 # A collection of keys
1863 keyarr, indexer = self._get_listlike_indexer(key, axis)
1864 return self.obj._reindex_with_indexers(
1865 {axis: [keyarr, indexer]}, allow_dups=True
1866 )
1868 def _getitem_tuple(self, tup: tuple):
1869 with suppress(IndexingError):
1870 tup = self._expand_ellipsis(tup)
1871 return self._getitem_lowerdim(tup)
1873 # no multi-index, so validate all of the indexers
1874 tup = self._validate_tuple_indexer(tup)
1876 # ugly hack for GH #836
1877 if self._multi_take_opportunity(tup):
1878 return self._multi_take(tup)
1880 return self._getitem_tuple_same_dim(tup)
1882 def _get_label(self, label, axis: AxisInt):
1883 # GH#5567 this will fail if the label is not present in the axis.
1884 return self.obj.xs(label, axis=axis)
1886 def _handle_lowerdim_multi_index_axis0(self, tup: tuple):
1887 # we have an axis0 multi-index, handle or raise
1888 axis = self.axis or 0
1889 try:
1890 # fast path for series or for tup devoid of slices
1891 return self._get_label(tup, axis=axis)
1893 except KeyError as ek:
1894 # raise KeyError if number of indexers match
1895 # else IndexingError will be raised
1896 if self.ndim < len(tup) <= self.obj.index.nlevels:
1897 raise ek
1898 raise IndexingError("No label returned") from ek
1900 def _getitem_axis(self, key, axis: AxisInt):
1901 key = item_from_zerodim(key)
1902 if is_iterator(key):
1903 key = list(key)
1904 if key is Ellipsis:
1905 key = slice(None)
1907 labels = self.obj._get_axis(axis)
1909 if isinstance(key, tuple) and isinstance(labels, MultiIndex):
1910 key = tuple(key)
1912 if isinstance(key, slice):
1913 self._validate_key(key, axis)
1914 return self._get_slice_axis(key, axis=axis)
1915 elif com.is_bool_indexer(key):
1916 return self._getbool_axis(key, axis=axis)
1917 elif is_list_like_indexer(key):
1918 # an iterable multi-selection
1919 if not (isinstance(key, tuple) and isinstance(labels, MultiIndex)):
1920 if hasattr(key, "ndim") and key.ndim > 1:
1921 raise ValueError("Cannot index with multidimensional key")
1923 return self._getitem_iterable(key, axis=axis)
1925 # nested tuple slicing
1926 if is_nested_tuple(key, labels):
1927 locs = labels.get_locs(key)
1928 indexer: list[slice | npt.NDArray[np.intp]] = [slice(None)] * self.ndim
1929 indexer[axis] = locs
1930 result = self.obj.iloc[tuple(indexer)]
1932 # GH#18631 Drop levels that were indexed with scalars,
1933 # but only when the key has no slices or bool indexers.
1934 # Dropping scalar levels in the presence of slices would
1935 # be correct in principle, but many internal operations
1936 # (e.g. stack/unstack) rely on the current behavior.
1937 has_slice_or_mask = any(
1938 isinstance(k, slice) or com.is_bool_indexer(k) for k in key
1939 )
1940 if not has_slice_or_mask:
1941 levels_to_drop = []
1942 for idx, k in enumerate(key):
1943 if not is_list_like(k):
1944 levels_to_drop.append(idx)
1945 elif isinstance(k, tuple):
1946 # GH#27591 A tuple might be a single label
1947 # in this level rather than a sequence of labels
1948 try:
1949 labels.levels[idx].get_loc(k)
1950 levels_to_drop.append(idx)
1951 except KeyError:
1952 pass
1953 if levels_to_drop:
1954 if len(levels_to_drop) >= labels.nlevels:
1955 # All levels are scalar-indexed; reduce
1956 # dimensionality instead of droplevel (which
1957 # cannot remove every level).
1958 result = result._ixs(0, axis=axis)
1959 else:
1960 result = result.droplevel(levels_to_drop, axis=axis)
1962 return result
1964 # fall thru to straight lookup
1965 self._validate_key(key, axis)
1966 return self._get_label(key, axis=axis)
1968 def _get_slice_axis(self, slice_obj: slice, axis: AxisInt):
1969 """
1970 This is pretty simple as we just have to deal with labels.
1971 """
1972 # caller is responsible for ensuring non-None axis
1973 obj = self.obj
1974 if not need_slice(slice_obj):
1975 return obj.copy(deep=False)
1977 labels = obj._get_axis(axis)
1978 indexer = labels.slice_indexer(slice_obj.start, slice_obj.stop, slice_obj.step)
1980 if isinstance(indexer, slice):
1981 return self.obj._slice(indexer, axis=axis)
1982 else:
1983 # DatetimeIndex overrides Index.slice_indexer and may
1984 # return a DatetimeIndex instead of a slice object.
1985 return self.obj.take(indexer, axis=axis)
1987 def _convert_to_indexer(self, key, axis: AxisInt):
1988 """
1989 Convert indexing key into something we can use to do actual fancy
1990 indexing on an ndarray.
1992 Examples
1993 ix[:5] -> slice(0, 5)
1994 ix[[1,2,3]] -> [1,2,3]
1995 ix[['foo', 'bar', 'baz']] -> [i, j, k] (indices of foo, bar, baz)
1997 Going by Zen of Python?
1998 'In the face of ambiguity, refuse the temptation to guess.'
1999 raise AmbiguousIndexError with integer labels?
2000 - No, prefer label-based indexing
2001 """
2002 labels = self.obj._get_axis(axis)
2004 if isinstance(key, slice):
2005 return labels._convert_slice_indexer(key, kind="loc")
2007 if (
2008 isinstance(key, tuple)
2009 and not isinstance(labels, MultiIndex)
2010 and self.ndim < 2
2011 and len(key) > 1
2012 ):
2013 raise IndexingError("Too many indexers")
2015 # Slices are not valid keys passed in by the user,
2016 # even though they are hashable in Python 3.12
2017 contains_slice = False
2018 if isinstance(key, tuple):
2019 contains_slice = any(isinstance(v, slice) for v in key)
2021 if is_scalar(key) or (
2022 isinstance(labels, MultiIndex) and is_hashable(key) and not contains_slice
2023 ):
2024 # Otherwise get_loc will raise InvalidIndexError
2026 # if we are a label return me
2027 try:
2028 return labels.get_loc(key)
2029 except LookupError:
2030 if isinstance(key, tuple) and isinstance(labels, MultiIndex):
2031 if len(key) == labels.nlevels:
2032 return {"key": key}
2033 raise
2034 except InvalidIndexError:
2035 # GH35015, using datetime as column indices raises exception
2036 if not isinstance(labels, MultiIndex):
2037 raise
2038 except ValueError:
2039 if not is_integer(key):
2040 raise
2041 return {"key": key}
2043 if is_nested_tuple(key, labels):
2044 if self.ndim == 1 and any(isinstance(k, tuple) for k in key):
2045 # GH#35349 Raise if tuple in tuple for series
2046 raise IndexingError("Too many indexers")
2047 return labels.get_locs(key)
2049 elif is_list_like_indexer(key):
2050 if is_iterator(key):
2051 key = list(key)
2053 if com.is_bool_indexer(key):
2054 key = check_bool_indexer(labels, key)
2055 return key
2056 else:
2057 return self._get_listlike_indexer(key, axis)[1]
2058 else:
2059 try:
2060 return labels.get_loc(key)
2061 except LookupError:
2062 # allow a not found key only if we are a setter
2063 if not is_list_like_indexer(key):
2064 return {"key": key}
2065 raise
2067 def _get_listlike_indexer(self, key, axis: AxisInt):
2068 """
2069 Transform a list-like of keys into a new index and an indexer.
2071 Parameters
2072 ----------
2073 key : list-like
2074 Targeted labels.
2075 axis: int
2076 Dimension on which the indexing is being made.
2078 Raises
2079 ------
2080 KeyError
2081 If at least one key was requested but none was found.
2083 Returns
2084 -------
2085 keyarr: Index
2086 New index (coinciding with 'key' if the axis is unique).
2087 values : array-like
2088 Indexer for the return object, -1 denotes keys not found.
2089 """
2090 ax = self.obj._get_axis(axis)
2091 axis_name = self.obj._get_axis_name(axis)
2093 keyarr, indexer = ax._get_indexer_strict(key, axis_name)
2095 return keyarr, indexer
2098class _iLocIndexer(_LocationIndexer):
2099 """
2100 Purely integer-location based indexing for selection by position.
2102 .. versionchanged:: 3.0
2104 Callables which return a tuple are deprecated as input.
2106 ``.iloc[]`` is primarily integer position based (from ``0`` to
2107 ``length-1`` of the axis), but may also be used with a boolean
2108 array.
2110 Allowed inputs are:
2112 - An integer, e.g. ``5``.
2113 - A list or array of integers, e.g. ``[4, 3, 0]``.
2114 - A slice object with ints, e.g. ``1:7``.
2115 - A boolean array.
2116 - A ``callable`` function with one argument (the calling Series or
2117 DataFrame) and that returns valid output for indexing (one of the above).
2118 This is useful in method chains, when you don't have a reference to the
2119 calling object, but would like to base your selection on
2120 some value.
2121 - A tuple of row and column indexes. The tuple elements consist of one of the
2122 above inputs, e.g. ``(0, 1)``.
2124 ``.iloc`` will raise ``IndexError`` if a requested indexer is
2125 out-of-bounds, except *slice* indexers which allow out-of-bounds
2126 indexing (this conforms with python/numpy *slice* semantics).
2128 See more at :ref:`Selection by Position <indexing.integer>`.
2130 See Also
2131 --------
2132 DataFrame.iat : Fast integer location scalar accessor.
2133 DataFrame.loc : Purely label-location based indexer for selection by label.
2134 Series.iloc : Purely integer-location based indexing for
2135 selection by position.
2137 Examples
2138 --------
2139 >>> mydict = [
2140 ... {"a": 1, "b": 2, "c": 3, "d": 4},
2141 ... {"a": 100, "b": 200, "c": 300, "d": 400},
2142 ... {"a": 1000, "b": 2000, "c": 3000, "d": 4000},
2143 ... ]
2144 >>> df = pd.DataFrame(mydict)
2145 >>> df
2146 a b c d
2147 0 1 2 3 4
2148 1 100 200 300 400
2149 2 1000 2000 3000 4000
2151 **Indexing just the rows**
2153 With a scalar integer.
2155 >>> type(df.iloc[0])
2156 <class 'pandas.Series'>
2157 >>> df.iloc[0]
2158 a 1
2159 b 2
2160 c 3
2161 d 4
2162 Name: 0, dtype: int64
2164 With a list of integers.
2166 >>> df.iloc[[0]]
2167 a b c d
2168 0 1 2 3 4
2169 >>> type(df.iloc[[0]])
2170 <class 'pandas.DataFrame'>
2172 >>> df.iloc[[0, 1]]
2173 a b c d
2174 0 1 2 3 4
2175 1 100 200 300 400
2177 With a `slice` object.
2179 >>> df.iloc[:3]
2180 a b c d
2181 0 1 2 3 4
2182 1 100 200 300 400
2183 2 1000 2000 3000 4000
2185 With a boolean mask the same length as the index.
2187 >>> df.iloc[[True, False, True]]
2188 a b c d
2189 0 1 2 3 4
2190 2 1000 2000 3000 4000
2192 With a callable, useful in method chains. The `x` passed
2193 to the ``lambda`` is the DataFrame being sliced. This selects
2194 the rows whose index label even.
2196 >>> df.iloc[lambda x: x.index % 2 == 0]
2197 a b c d
2198 0 1 2 3 4
2199 2 1000 2000 3000 4000
2201 **Indexing both axes**
2203 You can mix the indexer types for the index and columns. Use ``:`` to
2204 select the entire axis.
2206 With scalar integers.
2208 >>> df.iloc[0, 1]
2209 2
2211 With lists of integers.
2213 >>> df.iloc[[0, 2], [1, 3]]
2214 b d
2215 0 2 4
2216 2 2000 4000
2218 With `slice` objects.
2220 >>> df.iloc[1:3, 0:3]
2221 a b c
2222 1 100 200 300
2223 2 1000 2000 3000
2225 With a boolean array whose length matches the columns.
2227 >>> df.iloc[:, [True, False, True, False]]
2228 a c
2229 0 1 3
2230 1 100 300
2231 2 1000 3000
2233 With a callable function that expects the Series or DataFrame.
2235 >>> df.iloc[:, lambda df: [0, 2]]
2236 a c
2237 0 1 3
2238 1 100 300
2239 2 1000 3000
2240 """
2242 _valid_types = (
2243 "integer, integer slice (START point is INCLUDED, END "
2244 "point is EXCLUDED), listlike of integers, boolean array"
2245 )
2246 _takeable = True
2248 # -------------------------------------------------------------------
2249 # Key Checks
2251 def _validate_key(self, key, axis: AxisInt) -> None:
2252 if com.is_bool_indexer(key):
2253 if hasattr(key, "index") and isinstance(key.index, Index):
2254 if key.index.inferred_type == "integer":
2255 return
2256 raise ValueError(
2257 "iLocation based boolean indexing cannot use an indexable as a mask"
2258 )
2259 return
2261 if isinstance(key, slice):
2262 return
2263 elif is_integer(key):
2264 self._validate_integer(key, axis)
2265 elif isinstance(key, tuple):
2266 # a tuple should already have been caught by this point
2267 # so don't treat a tuple as a valid indexer
2268 raise IndexingError("Too many indexers")
2269 elif is_list_like_indexer(key):
2270 if isinstance(key, ABCSeries):
2271 arr = key._values
2272 elif is_array_like_deprecate_non_pandas(key):
2273 arr = key
2274 else:
2275 arr = np.array(key)
2276 len_axis = len(self.obj._get_axis(axis))
2278 # check that the key has a numeric dtype
2279 if not is_numeric_dtype(arr.dtype):
2280 raise IndexError(f".iloc requires numeric indexers, got {arr}")
2282 if len(arr):
2283 if isinstance(arr.dtype, ExtensionDtype):
2284 arr_max = arr._reduce("max")
2285 arr_min = arr._reduce("min")
2286 else:
2287 arr_max = np.max(arr)
2288 arr_min = np.min(arr)
2290 # check that the key does not exceed the maximum size
2291 if arr_max >= len_axis or arr_min < -len_axis:
2292 raise IndexError("positional indexers are out-of-bounds")
2293 else:
2294 raise ValueError(f"Can only index by location with a [{self._valid_types}]")
2296 def _has_valid_setitem_indexer(self, indexer) -> bool:
2297 """
2298 Validate that a positional indexer cannot enlarge its target
2299 will raise if needed, does not modify the indexer externally.
2301 Returns
2302 -------
2303 bool
2304 """
2305 if isinstance(indexer, dict):
2306 raise IndexError("iloc cannot enlarge its target object")
2308 if isinstance(indexer, ABCDataFrame):
2309 raise TypeError(
2310 "DataFrame indexer for .iloc is not supported. "
2311 "Consider using .loc with a DataFrame indexer for automatic alignment.",
2312 )
2314 if not isinstance(indexer, tuple):
2315 indexer = _tuplify(self.ndim, indexer)
2317 for ax, i in zip(self.obj.axes, indexer, strict=False):
2318 if isinstance(i, slice):
2319 # should check the stop slice?
2320 pass
2321 elif is_list_like_indexer(i):
2322 # should check the elements?
2323 pass
2324 elif is_integer(i):
2325 if i >= len(ax):
2326 raise IndexError("iloc cannot enlarge its target object")
2327 elif isinstance(i, dict):
2328 raise IndexError("iloc cannot enlarge its target object")
2330 return True
2332 def _is_scalar_access(self, key: tuple) -> bool:
2333 """
2334 Returns
2335 -------
2336 bool
2337 """
2338 # this is a shortcut accessor to both .loc and .iloc
2339 # that provide the equivalent access of .at and .iat
2340 # a) avoid getting things via sections and (to minimize dtype changes)
2341 # b) provide a performant path
2342 if len(key) != self.ndim:
2343 return False
2345 return all(is_integer(k) for k in key)
2347 def _validate_integer(self, key: int | np.integer, axis: AxisInt) -> None:
2348 """
2349 Check that 'key' is a valid position in the desired axis.
2351 Parameters
2352 ----------
2353 key : int
2354 Requested position.
2355 axis : int
2356 Desired axis.
2358 Raises
2359 ------
2360 IndexError
2361 If 'key' is not a valid position in axis 'axis'.
2362 """
2363 len_axis = len(self.obj._get_axis(axis))
2364 if key >= len_axis or key < -len_axis:
2365 raise IndexError("single positional indexer is out-of-bounds")
2367 # -------------------------------------------------------------------
2369 def _getitem_tuple(self, tup: tuple):
2370 tup = self._validate_tuple_indexer(tup)
2371 with suppress(IndexingError):
2372 return self._getitem_lowerdim(tup)
2374 return self._getitem_tuple_same_dim(tup)
2376 def _get_list_axis(self, key, axis: AxisInt):
2377 """
2378 Return Series values by list or array of integers.
2380 Parameters
2381 ----------
2382 key : list-like positional indexer
2383 axis : int
2385 Returns
2386 -------
2387 Series object
2389 Notes
2390 -----
2391 `axis` can only be zero.
2392 """
2393 try:
2394 return self.obj.take(key, axis=axis)
2395 except IndexError as err:
2396 # re-raise with different error message, e.g. test_getitem_ndarray_3d
2397 raise IndexError("positional indexers are out-of-bounds") from err
2399 def _getitem_axis(self, key, axis: AxisInt):
2400 if key is Ellipsis:
2401 key = slice(None)
2402 elif isinstance(key, ABCDataFrame):
2403 raise IndexError(
2404 "DataFrame indexer is not allowed for .iloc\n"
2405 "Consider using .loc for automatic alignment."
2406 )
2408 if isinstance(key, slice):
2409 return self._get_slice_axis(key, axis=axis)
2411 if is_iterator(key):
2412 key = list(key)
2414 if isinstance(key, list):
2415 key = np.asarray(key)
2417 if com.is_bool_indexer(key):
2418 self._validate_key(key, axis)
2419 return self._getbool_axis(key, axis=axis)
2421 # a list of integers
2422 elif is_list_like_indexer(key):
2423 return self._get_list_axis(key, axis=axis)
2425 # a single integer
2426 else:
2427 key = item_from_zerodim(key)
2428 if not is_integer(key):
2429 raise TypeError("Cannot index by location index with a non-integer key")
2431 # validate the location
2432 self._validate_integer(key, axis)
2434 return self.obj._ixs(key, axis=axis)
2436 def _get_slice_axis(self, slice_obj: slice, axis: AxisInt):
2437 # caller is responsible for ensuring non-None axis
2438 obj = self.obj
2440 if not need_slice(slice_obj):
2441 return obj.copy(deep=False)
2443 labels = obj._get_axis(axis)
2444 labels._validate_positional_slice(slice_obj)
2445 return self.obj._slice(slice_obj, axis=axis)
2447 def _convert_to_indexer(self, key: T, axis: AxisInt) -> T:
2448 """
2449 Much simpler as we only have to deal with our valid types.
2450 """
2451 return key
2453 def _get_setitem_indexer(self, key):
2454 # GH#32257 Fall through to let numpy do validation
2455 if is_iterator(key):
2456 key = list(key)
2458 if self.axis is not None:
2459 key = _tupleize_axis_indexer(self.ndim, self.axis, key)
2461 return key
2463 # -------------------------------------------------------------------
2465 def _decide_split_path(self, indexer, value) -> bool:
2466 """
2467 Decide whether we will take a block-by-block path.
2468 """
2469 take_split_path = not self.obj._mgr.is_single_block
2471 if not take_split_path and isinstance(value, ABCDataFrame):
2472 # Avoid cast of values
2473 take_split_path = not value._mgr.is_single_block
2475 # if there is only one block/type, still have to take split path
2476 # unless the block is one-dimensional or it can hold the value
2477 if not take_split_path and len(self.obj._mgr.blocks) and self.ndim > 1:
2478 # in case of dict, keys are indices
2479 val = list(value.values()) if isinstance(value, dict) else value
2480 arr = self.obj._mgr.blocks[0].values
2481 take_split_path = not can_hold_element(
2482 arr, extract_array(val, extract_numpy=True)
2483 )
2485 # if we have any multi-indexes that have non-trivial slices
2486 # (not null slices) then we must take the split path, xref
2487 # GH 10360, GH 27841
2488 if isinstance(indexer, tuple) and len(indexer) == len(self.obj.axes):
2489 for i, ax in zip(indexer, self.obj.axes, strict=True):
2490 if isinstance(ax, MultiIndex) and not (
2491 is_integer(i) or com.is_null_slice(i)
2492 ):
2493 take_split_path = True
2494 break
2496 # GH#44103 - setting a scalar row across columns with a list-like
2497 # value must go through the split path so each column gets its
2498 # corresponding scalar value. Restricted to ExtensionArray blocks:
2499 # the non-split path already handles the others correctly, and forcing
2500 # the split path there transposes rectangular nested values (GH#65241).
2501 if (
2502 not take_split_path
2503 and isinstance(indexer, tuple)
2504 and len(indexer) == 2
2505 and is_integer(indexer[0])
2506 and not is_integer(indexer[1])
2507 and is_list_like(value)
2508 and not isinstance(value, (ABCSeries, ABCDataFrame))
2509 # not take_split_path guarantees exactly one block
2510 and self.obj._mgr.blocks[0].is_extension
2511 ):
2512 take_split_path = True
2514 return take_split_path
2516 def _setitem_new_column(self, indexer, key, value, name: str) -> None:
2517 """
2518 _setitem_with_indexer cases that can go through DataFrame.__setitem__.
2519 """
2520 # GH#65418 a dict is a label->value mapping; treat it as a Series so it
2521 # is aligned by key like the non-expansion paths, rather than being
2522 # sanitized positionally into its keys.
2523 if isinstance(value, dict):
2524 from pandas import Series
2526 value = Series(value)
2528 # add the new item, and set the value
2529 # must have all defined axes if we have a scalar
2530 # or a list-like on the non-info axes if we have a
2531 # list-like
2532 if not len(self.obj):
2533 if not is_list_like_indexer(value):
2534 raise ValueError(
2535 "cannot set a frame with no defined index and a scalar"
2536 )
2537 self.obj[key] = value
2538 return
2540 # add a new item with the dtype setup
2541 if com.is_null_slice(indexer[0]):
2542 # We are setting an entire column
2543 self.obj[key] = value
2544 return
2545 elif is_array_like_deprecate_non_pandas(value):
2546 # GH#42099
2547 arr = extract_array(value, extract_numpy=True)
2548 taker = -1 * np.ones(len(self.obj), dtype=np.intp)
2549 empty_value = algos.take_nd(arr, taker)
2550 if not isinstance(value, ABCSeries):
2551 # if not Series (in which case we need to align),
2552 # we can short-circuit
2553 if isinstance(arr, np.ndarray) and arr.ndim == 1 and len(arr) == 1:
2554 arr = arr[0]
2555 empty_value[indexer[0]] = arr
2556 self.obj[key] = empty_value
2557 return
2559 self.obj[key] = empty_value
2560 elif not is_list_like(value):
2561 self.obj[key] = construct_1d_array_from_inferred_fill_value(
2562 value, len(self.obj)
2563 )
2564 else:
2565 # GH#42099 list-like (non-array-like): convert to an array first
2566 # so we can preserve dtype the same way as the is_array_like path
2567 arr = sanitize_array(value, Index(range(len(value))), copy=False)
2568 taker = -1 * np.ones(len(self.obj), dtype=np.intp)
2569 empty_value = algos.take_nd(arr, taker)
2570 if isinstance(arr, np.ndarray) and arr.ndim == 1 and len(arr) == 1:
2571 arr = arr[0]
2572 empty_value[indexer[0]] = arr
2573 self.obj[key] = empty_value
2574 return
2576 new_indexer = convert_from_missing_indexer_tuple(indexer, self.obj.axes)
2577 self._setitem_with_indexer(new_indexer, value, name)
2579 return
2581 def _setitem_with_indexer(self, indexer, value, name: str = "iloc") -> None:
2582 """
2583 _setitem_with_indexer is for setting values on a Series/DataFrame
2584 using positional indexers.
2586 If the relevant keys are not present, the Series/DataFrame may be
2587 expanded.
2588 """
2589 info_axis = self.obj._info_axis_number
2590 take_split_path = self._decide_split_path(indexer, value)
2592 if isinstance(indexer, tuple):
2593 nindexer = []
2594 for i, idx in enumerate(indexer):
2595 idx, missing = convert_missing_indexer(idx)
2596 if missing:
2597 # reindex the axis to the new value
2598 # and set inplace
2599 key = idx
2601 # if this is the items axes, then take the main missing
2602 # path first
2603 # this correctly sets the dtype
2604 # essentially this separates out the block that is needed
2605 # to possibly be modified
2606 if self.ndim > 1 and i == info_axis:
2607 self._setitem_new_column(indexer, key, value, name=name)
2608 return
2610 # reindex the axis
2611 index = self.obj._get_axis(i)
2612 maybe_warn_multiindex_expansion(
2613 index,
2614 key,
2615 target="row on a DataFrame",
2616 hint="Use a full-length tuple key instead.",
2617 )
2618 labels = index.insert(len(index), key)
2620 # We are expanding the Series/DataFrame values to match
2621 # the length of the new index `labels`. GH#40096 ensure
2622 # this is valid even if the index has duplicates.
2623 taker = np.arange(len(index) + 1, dtype=np.intp)
2624 taker[-1] = -1
2625 reindexers = {i: (labels, taker)}
2626 new_obj = self.obj._reindex_with_indexers(
2627 reindexers, allow_dups=True
2628 )
2629 self.obj._mgr = new_obj._mgr
2631 nindexer.append(labels.get_loc(key))
2633 else:
2634 nindexer.append(idx)
2636 indexer = tuple(nindexer)
2637 else:
2638 indexer, missing = convert_missing_indexer(indexer)
2640 if missing:
2641 self._setitem_with_indexer_missing(indexer, value)
2642 return
2644 if name == "loc":
2645 # must come after setting of missing
2646 indexer, value = self._maybe_mask_setitem_value(indexer, value)
2648 # align and set the values
2649 if take_split_path:
2650 # We have to operate column-wise
2651 self._setitem_with_indexer_split_path(indexer, value, name)
2652 else:
2653 self._setitem_single_block(indexer, value, name)
2655 def _setitem_with_indexer_split_path(self, indexer, value, name: str):
2656 """
2657 Setitem column-wise.
2658 """
2659 # Above we only set take_split_path to True for 2D cases
2660 assert self.ndim == 2
2662 if not isinstance(indexer, tuple):
2663 indexer = _tuplify(self.ndim, indexer)
2664 if len(indexer) > self.ndim:
2665 raise IndexError("too many indices for array")
2666 if isinstance(indexer[0], np.ndarray) and indexer[0].ndim > 2:
2667 raise ValueError(r"Cannot set values with ndim > 2")
2669 if (isinstance(value, ABCSeries) and name != "iloc") or isinstance(value, dict):
2670 from pandas import Series
2672 value = self._align_series(indexer, Series(value))
2674 # Ensure we have something we can iterate over
2675 info_axis = indexer[1]
2676 ilocs = self._ensure_iterable_column_indexer(info_axis)
2678 pi = indexer[0]
2679 lplane_indexer = length_of_indexer(pi, self.obj.index)
2680 # lplane_indexer gives the expected length of obj[indexer[0]]
2682 # we need an iterable, with an ndim of at least 1
2683 # eg. don't pass through np.array(0)
2684 if is_list_like_indexer(value) and getattr(value, "ndim", 1) > 0:
2685 if isinstance(value, ABCDataFrame):
2686 self._setitem_with_indexer_frame_value(indexer, value, name)
2688 elif _is_2d_value_for_columns(value, len(ilocs)):
2689 self._setitem_with_indexer_2d_value(indexer, value)
2691 elif len(ilocs) == 1 and lplane_indexer == len(value) and not is_scalar(pi):
2692 # We are setting multiple rows in a single column.
2693 self._setitem_single_column(ilocs[0], value, pi)
2695 elif self._is_scalar_access(indexer) and is_object_dtype(
2696 self.obj.dtypes._values[ilocs[0]]
2697 ):
2698 # We are setting nested data into a single cell,
2699 # only possible for object dtype. Bypasses length checks.
2700 self._setitem_single_column(ilocs[0], value, pi)
2702 elif len(ilocs) == 1 and 0 != lplane_indexer != len(value):
2703 # We are trying to set N values into M entries of a single
2704 # column, which is invalid for N != M
2705 # Exclude zero-len for e.g. boolean masking that is all-false
2707 if len(value) == 1 and not is_integer(info_axis):
2708 # This is a case like df.iloc[:3, [1]] = [0]
2709 # where we treat as df.iloc[:3, 1] = 0
2710 return self._setitem_with_indexer((pi, info_axis[0]), value[0])
2712 raise ValueError(
2713 "Must have equal len keys and value when setting with an iterable"
2714 )
2716 elif lplane_indexer == 0 and len(value) == len(self.obj.index):
2717 # We get here in one case via .loc with an all-False mask
2718 pass
2720 elif len(ilocs) == len(value):
2721 # We are setting multiple columns in a single row.
2722 for loc, v in zip(ilocs, value, strict=True):
2723 self._setitem_single_column(loc, v, pi)
2725 elif len(ilocs) == 1 and com.is_null_slice(pi) and len(self.obj) == 0:
2726 # This is a setitem-with-expansion, see
2727 # test_loc_setitem_empty_append_expands_rows_mixed_dtype
2728 # e.g. df = DataFrame(columns=["x", "y"])
2729 # df["x"] = df["x"].astype(np.int64)
2730 # df.loc[:, "x"] = [1, 2, 3]
2731 self._setitem_single_column(ilocs[0], value, pi)
2733 else:
2734 raise ValueError(
2735 "Must have equal len keys and value when setting with an iterable"
2736 )
2738 else:
2739 # scalar value
2740 for loc in ilocs:
2741 self._setitem_single_column(loc, value, pi)
2743 def _setitem_with_indexer_2d_value(self, indexer, value) -> None:
2744 # We get here with a 2D value (array-like or list-of-lists),
2745 # excluding DataFrame which goes through _setitem_with_indexer_frame_value
2746 pi = indexer[0]
2747 ilocs = self._ensure_iterable_column_indexer(indexer[1])
2749 if not isinstance(value, list) and not is_array_like_deprecate_non_pandas(
2750 value
2751 ):
2752 value = np.asarray(value)
2754 msg = "Must have equal len keys and value when setting with an ndarray"
2755 if isinstance(value, list):
2756 nkeys = len(ilocs)
2757 rows = []
2758 for row in value:
2759 if len(row) != nkeys:
2760 raise ValueError(msg)
2761 if type(row) is not list:
2762 row = _positional_row(row)
2763 rows.append(row)
2764 value = rows
2765 elif value.shape[1] != len(ilocs):
2766 raise ValueError(msg)
2768 for i, loc in enumerate(ilocs):
2769 if isinstance(value, list):
2770 value_col = [row[i] for row in value]
2771 else:
2772 value_col = value[:, i]
2773 if is_object_dtype(value_col.dtype):
2774 value_col = value_col.tolist()
2775 self._setitem_single_column(loc, value_col, pi)
2777 def _setitem_with_indexer_frame_value(
2778 self, indexer, value: DataFrame, name: str
2779 ) -> None:
2780 ilocs = self._ensure_iterable_column_indexer(indexer[1])
2782 sub_indexer = list(indexer)
2783 pi = indexer[0]
2785 multiindex_indexer = isinstance(self.obj.columns, MultiIndex)
2787 unique_cols = value.columns.is_unique
2789 # We do not want to align the value in case of iloc GH#37728
2790 if name == "iloc":
2791 for i, loc in enumerate(ilocs):
2792 val = value.iloc[:, i]
2793 self._setitem_single_column(loc, val, pi)
2795 elif not unique_cols and value.columns.equals(self.obj.columns):
2796 # We assume we are already aligned, see
2797 # test_iloc_setitem_frame_duplicate_columns_multiple_blocks
2798 for loc in ilocs:
2799 item = self.obj.columns[loc]
2800 if item in value:
2801 sub_indexer[1] = item
2802 val = self._align_series(
2803 tuple(sub_indexer),
2804 value.iloc[:, loc],
2805 multiindex_indexer,
2806 )
2807 else:
2808 val = np.nan
2810 self._setitem_single_column(loc, val, pi)
2812 elif not unique_cols:
2813 raise ValueError("Setting with non-unique columns is not allowed.")
2815 else:
2816 for loc in ilocs:
2817 item = self.obj.columns[loc]
2818 if item in value:
2819 sub_indexer[1] = item
2820 ser = value[item]
2821 val = self._align_series(
2822 tuple(sub_indexer),
2823 ser,
2824 multiindex_indexer,
2825 )
2826 # If _align_series did not need to reindex, pass the
2827 # original Series so that _setitem_single_column can
2828 # preserve CoW ref tracking through isetitem.
2829 if val is ser._values:
2830 val = ser
2831 else:
2832 val = np.nan
2834 self._setitem_single_column(loc, val, pi)
2836 def _setitem_single_column(self, loc: int, value, plane_indexer) -> None:
2837 """
2839 Parameters
2840 ----------
2841 loc : int
2842 Indexer for column position
2843 plane_indexer : int, slice, listlike[int]
2844 The indexer we use for setitem along axis=0.
2845 """
2846 pi = plane_indexer
2848 is_full_setter = com.is_null_slice(pi) or com.is_full_slice(pi, len(self.obj))
2850 is_null_setter = com.is_empty_slice(pi) or (
2851 is_array_like_deprecate_non_pandas(pi) and len(pi) == 0
2852 )
2854 if is_null_setter:
2855 # no-op, don't cast dtype later
2856 return
2858 elif is_full_setter:
2859 try:
2860 self.obj._mgr.column_setitem(
2861 loc, plane_indexer, value, inplace_only=True
2862 )
2863 except (ValueError, TypeError, LossySetitemError) as exc:
2864 # If we're setting an entire column and we can't do it inplace,
2865 # then we can use value's dtype (or inferred dtype)
2866 # instead of object
2867 dtype = self.obj.dtypes.iloc[loc]
2868 if dtype not in (np.void, object) and not self.obj.empty:
2869 # - Exclude np.void, as that is a special case for expansion.
2870 # We want to raise for
2871 # df = pd.DataFrame({'a': [1, 2]})
2872 # df.loc[:, 'a'] = .3
2873 # but not for
2874 # df = pd.DataFrame({'a': [1, 2]})
2875 # df.loc[:, 'b'] = .3
2876 # - Exclude `object`, as then no upcasting happens.
2877 # - Exclude empty initial object with enlargement,
2878 # as then there's nothing to be inconsistent with.
2879 raise TypeError(
2880 f"Invalid value '{value}' for dtype '{dtype}'"
2881 ) from exc
2882 self.obj.isetitem(loc, value)
2883 else:
2884 # set value into the column (first attempting to operate inplace, then
2885 # falling back to casting if necessary)
2886 dtype = self.obj.dtypes.iloc[loc]
2887 if dtype == np.void:
2888 # This means we're expanding, with multiple columns, e.g.
2889 # df = pd.DataFrame({'A': [1,2,3], 'B': [4,5,6]})
2890 # df.loc[df.index <= 2, ['F', 'G']] = (1, 'abc')
2891 # Columns F and G will initially be set to np.void.
2892 # Here, we replace those temporary `np.void` columns with
2893 # columns of the appropriate dtype, based on `value`.
2894 new_arr = construct_1d_array_from_inferred_fill_value(
2895 value, len(self.obj)
2896 )
2897 self.obj.isetitem(loc, new_arr)
2898 self.obj._mgr.column_setitem(loc, plane_indexer, value)
2900 def _setitem_single_block(self, indexer, value, name: str) -> None:
2901 """
2902 _setitem_with_indexer for the case when we have a single Block.
2903 """
2904 from pandas import Series
2906 if (isinstance(value, ABCSeries) and name != "iloc") or isinstance(value, dict):
2907 # TODO(EA): ExtensionBlock.setitem this causes issues with
2908 # setting for extensionarrays that store dicts. Need to decide
2909 # if it's worth supporting that.
2910 value = self._align_series(indexer, Series(value))
2912 info_axis = self.obj._info_axis_number
2913 item_labels = self.obj._get_axis(info_axis)
2914 if isinstance(indexer, tuple):
2915 # if we are setting on the info axis ONLY
2916 # set using those methods to avoid block-splitting
2917 # logic here
2918 if (
2919 self.ndim == len(indexer) == 2
2920 and is_integer(indexer[1])
2921 and com.is_null_slice(indexer[0])
2922 ):
2923 col = item_labels[indexer[info_axis]]
2924 if len(item_labels.get_indexer_for([col])) == 1:
2925 # e.g. test_loc_setitem_empty_append_expands_rows
2926 loc = item_labels.get_loc(col)
2927 self._setitem_single_column(loc, value, indexer[0])
2928 return
2930 if (
2931 self.ndim == 2
2932 and len(indexer) == 2
2933 and self.obj.shape[1] > 1
2934 and not com.is_null_slice(indexer[1])
2935 and not isinstance(value, ABCDataFrame)
2936 and not can_hold_element(
2937 self.obj._mgr.blocks[0].values,
2938 extract_array(value, extract_numpy=True),
2939 )
2940 ):
2941 self._setitem_with_indexer_split_path(indexer, value, name)
2942 return
2944 if (
2945 self.ndim == 2
2946 and len(indexer) == 2
2947 # an integer column key drops the column axis, so the selection
2948 # stays 1-D and numpy broadcasts the value into it just fine
2949 and not is_integer(indexer[1])
2950 and not is_scalar(indexer[0])
2951 and not isinstance(value, ABCDataFrame)
2952 and is_list_like_indexer(value)
2953 and getattr(value, "ndim", 1) == 1
2954 # length_of_indexer below measures only these, and only in
2955 # one dimension; every other row key (masked/Categorical/tuple,
2956 # 0-d or 2-D ndarray) has to keep taking the whole-block path,
2957 # which reports its own, clearer errors. GH#68021
2958 and isinstance(
2959 indexer[0], (slice, range, list, np.ndarray, ABCSeries, ABCIndex)
2960 )
2961 and getattr(indexer[0], "ndim", 1) == 1
2962 ):
2963 ilocs = self._ensure_iterable_column_indexer(indexer[1])
2964 if len(ilocs) == 1 and not _is_2d_value_for_columns(value, 1):
2965 # _ensure_iterable_column_indexer leaves a non-ndarray
2966 # boolean key alone, so ilocs[0] can be True rather than a
2967 # position; that is not ours to set column-wise
2968 loc = ilocs[0]
2969 if is_integer(loc):
2970 nrows = length_of_indexer(indexer[0], self.obj.index)
2971 if nrows == len(value):
2972 # GH#68021 the cross-product selection is (N, 1) but
2973 # the value is 1-D of length N, which numpy cannot
2974 # broadcast into it. Set the single column row-wise,
2975 # as the split path does.
2976 self._setitem_single_column(int(loc), value, indexer[0])
2977 return
2979 indexer = maybe_convert_ix(*indexer) # e.g. test_setitem_frame_align
2981 if isinstance(value, ABCDataFrame) and name != "iloc":
2982 value = self._align_frame(indexer, value)._values
2984 # actually do the set
2985 self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
2987 def _setitem_with_indexer_missing(self, indexer, value):
2988 """
2989 Insert new row(s) or column(s) into the Series or DataFrame.
2990 """
2991 from pandas import Series
2993 # reindex the axis to the new value
2994 # and set inplace
2995 if self.ndim == 1:
2996 index = self.obj.index
2997 maybe_warn_multiindex_expansion(
2998 index,
2999 indexer,
3000 target="value on a Series",
3001 hint="Use a full-length tuple key instead.",
3002 )
3003 new_index = index.insert(len(index), indexer)
3005 # we have a coerced indexer, e.g. a float
3006 # that matches in an int64 Index, so
3007 # we will not create a duplicate index, rather
3008 # index to that element
3009 # e.g. 0.0 -> 0
3010 # GH#12246
3011 if index.is_unique:
3012 # pass new_index[-1:] instead if [new_index[-1]]
3013 # so that we retain dtype
3014 new_indexer = index.get_indexer(new_index[-1:])
3015 if (new_indexer != -1).any():
3016 # We get only here with loc, so can hard code
3017 return self._setitem_with_indexer(new_indexer, value, "loc")
3019 # this preserves dtype of the value and of the object
3020 if not is_scalar(value):
3021 pass
3023 elif is_valid_na_for_dtype(value, self.obj.dtype):
3024 if not is_object_dtype(self.obj.dtype):
3025 # Every NA value is suitable for object, no conversion needed
3026 value = na_value_for_dtype(self.obj.dtype, compat=False)
3028 new_values = infer_and_maybe_downcast(
3029 self.obj.array, [value], warn_if_cast=False
3030 )
3032 if len(self.obj._values):
3033 # GH#22717 handle casting compatibility that np.concatenate
3034 # does incorrectly
3035 # GH#62523 infer_and_maybe_downcast may return a raw M8/m8
3036 # ndarray for datetimelike; wrap it so concat_compat does not
3037 # mix a bare ndarray with the DatetimeArray/TimedeltaArray
3038 # _values
3039 new_values = ensure_wrapped_if_datetimelike(new_values)
3040 new_values = concat_compat([self.obj._values, new_values])
3041 self.obj._mgr = self.obj._constructor(
3042 new_values, index=new_index, name=self.obj.name
3043 )._mgr
3045 elif self.ndim == 2:
3046 maybe_warn_multiindex_expansion(
3047 self.obj.index,
3048 indexer,
3049 target="row on a DataFrame",
3050 hint="Use a full-length tuple key and an explicit column "
3051 "indexer instead, e.g. df.loc[key, :] = values.",
3052 )
3054 if not len(self.obj.columns):
3055 # GH#17895 no columns, just expand the index
3056 new_index = self.obj.index.insert(len(self.obj.index), indexer)
3057 self.obj._mgr = self.obj._constructor(
3058 index=new_index, columns=self.obj.columns
3059 )._mgr
3060 return
3062 has_dtype = hasattr(value, "dtype")
3063 if isinstance(value, ABCSeries):
3064 # append a Series
3065 value = value.reindex(index=self.obj.columns)
3066 value.name = indexer
3067 elif isinstance(value, dict):
3068 value = Series(
3069 value, index=self.obj.columns, name=indexer, dtype=object
3070 )
3071 else:
3072 # a list-list
3073 if is_list_like_indexer(value):
3074 # must have conforming columns
3075 if len(value) != len(self.obj.columns):
3076 raise ValueError("cannot set a row with mismatched columns")
3078 value = Series(value, index=self.obj.columns, name=indexer)
3080 if not len(self.obj):
3081 # We will ignore the existing dtypes instead of using
3082 # internals.concat logic
3083 df = value.to_frame().T
3085 idx = self.obj.index
3086 if isinstance(idx, MultiIndex):
3087 name = idx.names
3088 else:
3089 name = idx.name
3091 df.index = Index([indexer], name=name)
3092 if not has_dtype:
3093 # i.e. if we already had a Series or ndarray, keep that
3094 # dtype. But if we had a list or dict, then do inference
3095 df = df.infer_objects()
3096 self.obj._mgr = df._mgr
3097 else:
3098 self.obj._mgr = self.obj._append_internal(value)._mgr
3100 def _ensure_iterable_column_indexer(self, column_indexer):
3101 """
3102 Ensure that our column indexer is something that can be iterated over.
3103 """
3104 ilocs: Sequence[int | np.integer] | np.ndarray | range
3105 if is_integer(column_indexer):
3106 ilocs = [column_indexer]
3107 elif isinstance(column_indexer, slice):
3108 ilocs = range(len(self.obj.columns))[column_indexer]
3109 elif com.is_bool_indexer(column_indexer):
3110 ilocs = np.arange(len(column_indexer))[
3111 np.asarray(column_indexer, dtype=bool)
3112 ]
3113 else:
3114 ilocs = column_indexer
3115 return ilocs
3117 def _align_series(
3118 self,
3119 indexer,
3120 ser: Series,
3121 multiindex_indexer: bool = False,
3122 ) -> ArrayLike | Series:
3123 """
3124 Parameters
3125 ----------
3126 indexer : tuple, slice, scalar
3127 Indexer used to get the locations that will be set to `ser`.
3128 ser : pd.Series
3129 Values to assign to the locations specified by `indexer`.
3130 multiindex_indexer : bool, optional
3131 Defaults to False. Should be set to True if `indexer` was from
3132 a `pd.MultiIndex`, to avoid unnecessary broadcasting.
3134 Returns
3135 -------
3136 `np.array` of `ser` broadcast to the appropriate shape for assignment
3137 to the locations selected by `indexer`
3138 """
3139 if isinstance(indexer, (slice, np.ndarray, list, Index)):
3140 indexer = (indexer,)
3142 if isinstance(indexer, tuple):
3143 # flatten np.ndarray indexers
3144 if (
3145 len(indexer) == 2
3146 and isinstance(indexer[1], np.ndarray)
3147 and indexer[1].dtype == np.bool_
3148 ):
3149 indexer = (indexer[0], np.where(indexer[1])[0])
3151 def ravel(i):
3152 return i.ravel() if isinstance(i, np.ndarray) else i
3154 indexer = tuple(map(ravel, indexer))
3155 aligners = [not com.is_null_slice(idx) for idx in indexer]
3156 sum_aligners = sum(aligners)
3157 single_aligner = sum_aligners == 1
3158 is_frame = self.ndim == 2
3159 obj = self.obj
3161 # are we a single alignable value on a non-primary
3162 # dim (e.g. panel: 1,2, or frame: 0) ?
3163 # hence need to align to a single axis dimension
3164 # rather that find all valid dims
3166 # frame
3167 if is_frame:
3168 single_aligner = single_aligner and aligners[0]
3170 # we have a frame, with multiple indexers on both axes; and a
3171 # series, so need to broadcast (see GH5206)
3172 if all(is_sequence(_) or isinstance(_, slice) for _ in indexer):
3173 ser_values = ser.reindex(obj.axes[0][indexer[0]])._values
3175 # single indexer
3176 if len(indexer) > 1 and not multiindex_indexer:
3177 if isinstance(indexer[1], slice):
3178 len_indexer = len(obj.axes[1][indexer[1]])
3179 else:
3180 len_indexer = len(indexer[1])
3181 ser_values = (
3182 np.tile(ser_values, len_indexer).reshape(len_indexer, -1).T
3183 )
3185 return ser_values
3187 for i, idx in enumerate(indexer):
3188 ax = obj.axes[i]
3190 # multiple aligners (or null slices)
3191 if is_sequence(idx) or isinstance(idx, slice):
3192 if single_aligner and com.is_null_slice(idx):
3193 continue
3194 new_ix = ax[idx]
3195 if not is_list_like_indexer(new_ix):
3196 new_ix = Index([new_ix])
3197 else:
3198 new_ix = Index(new_ix)
3199 if not len(new_ix) or ser.index.equals(new_ix):
3200 return ser._values
3202 return ser.reindex(new_ix)._values
3204 # 2 dims
3205 elif single_aligner:
3206 # reindex along index
3207 ax = self.obj.axes[1]
3208 if ser.index.equals(ax) or not len(ax):
3209 return ser._values.copy()
3210 return ser.reindex(ax)._values
3212 elif is_integer(indexer) and self.ndim == 1:
3213 if is_object_dtype(self.obj.dtype):
3214 # Store the Series as a scalar element in object-dtype Series
3215 return ser
3216 ax = self.obj._get_axis(0)
3218 if ser.index.equals(ax):
3219 return ser._values.copy()
3221 return ser.reindex(ax)._values[indexer]
3223 elif is_integer(indexer):
3224 ax = self.obj._get_axis(1)
3226 if ser.index.equals(ax):
3227 return ser._values.copy()
3229 return ser.reindex(ax)._values
3231 raise ValueError("Incompatible indexer with Series")
3233 def _align_frame(self, indexer, df: DataFrame) -> DataFrame:
3234 is_frame = self.ndim == 2
3236 if isinstance(indexer, tuple):
3237 idx, cols = None, None
3238 sindexers = []
3239 for i, ix in enumerate(indexer):
3240 ax = self.obj.axes[i]
3241 if is_sequence(ix) or isinstance(ix, slice):
3242 if isinstance(ix, np.ndarray):
3243 ix = ix.reshape(-1)
3244 if idx is None:
3245 idx = ax[ix]
3246 elif cols is None:
3247 cols = ax[ix]
3248 else:
3249 break
3250 else:
3251 sindexers.append(i)
3253 if idx is not None and cols is not None:
3254 if df.index.equals(idx) and df.columns.equals(cols):
3255 val = df.copy()
3256 else:
3257 val = df.reindex(idx, columns=cols)
3258 return val
3260 elif (isinstance(indexer, slice) or is_list_like_indexer(indexer)) and is_frame:
3261 ax = self.obj.index[indexer]
3262 if df.index.equals(ax):
3263 val = df.copy()
3264 else:
3265 # we have a multi-index and are trying to align
3266 # with a particular, level GH3738
3267 if (
3268 isinstance(ax, MultiIndex)
3269 and isinstance(df.index, MultiIndex)
3270 and ax.nlevels != df.index.nlevels
3271 ):
3272 raise TypeError(
3273 "cannot align on a multi-index with out "
3274 "specifying the join levels"
3275 )
3277 val = df.reindex(index=ax)
3278 return val
3280 raise ValueError("Incompatible indexer with DataFrame")
3283class _ScalarAccessIndexer(NDFrameIndexerBase):
3284 """
3285 Access scalars quickly.
3286 """
3288 # sub-classes need to set _takeable
3289 _takeable: bool
3291 def _convert_key(self, key):
3292 raise AbstractMethodError(self)
3294 def __getitem__(self, key):
3295 if not isinstance(key, tuple):
3296 # we could have a convertible item here (e.g. Timestamp)
3297 if not is_list_like_indexer(key):
3298 key = (key,)
3299 else:
3300 raise ValueError("Invalid call for scalar access (getting)!")
3302 key = self._convert_key(key)
3303 return self.obj._get_value(*key, takeable=self._takeable)
3305 def __setitem__(self, key, value) -> None:
3306 if isinstance(key, tuple):
3307 key = tuple(com.apply_if_callable(x, self.obj) for x in key)
3308 else:
3309 # scalar callable may return tuple
3310 key = com.apply_if_callable(key, self.obj)
3312 if not isinstance(key, tuple):
3313 key = _tuplify(self.ndim, key)
3314 key = list(self._convert_key(key))
3315 if len(key) != self.ndim:
3316 raise ValueError("Not enough indexers for scalar access (setting)!")
3318 self.obj._set_value(*key, value=value, takeable=self._takeable)
3321class _AtIndexer(_ScalarAccessIndexer):
3322 """
3323 Access a single value for a row/column label pair.
3325 Similar to ``loc``, in that both provide label-based lookups. Use
3326 ``at`` if you only need to get or set a single value in a DataFrame
3327 or Series.
3329 Raises
3330 ------
3331 KeyError
3332 If getting a value and 'label' does not exist in a DataFrame or Series.
3334 ValueError
3335 If row/column label pair is not a tuple or if any label
3336 from the pair is not a scalar for DataFrame.
3337 If label is list-like (*excluding* NamedTuple) for Series.
3339 See Also
3340 --------
3341 DataFrame.at : Access a single value for a row/column pair by label.
3342 DataFrame.iat : Access a single value for a row/column pair by integer
3343 position.
3344 DataFrame.loc : Access a group of rows and columns by label(s).
3345 DataFrame.iloc : Access a group of rows and columns by integer
3346 position(s).
3347 Series.at : Access a single value by label.
3348 Series.iat : Access a single value by integer position.
3349 Series.loc : Access a group of rows by label(s).
3350 Series.iloc : Access a group of rows by integer position(s).
3352 Notes
3353 -----
3354 See :ref:`Fast scalar value getting and setting <indexing.basics.get_value>`
3355 for more details.
3357 Examples
3358 --------
3359 >>> df = pd.DataFrame(
3360 ... [[0, 2, 3], [0, 4, 1], [10, 20, 30]],
3361 ... index=[4, 5, 6],
3362 ... columns=["A", "B", "C"],
3363 ... )
3364 >>> df
3365 A B C
3366 4 0 2 3
3367 5 0 4 1
3368 6 10 20 30
3370 Get value at specified row/column pair
3372 >>> df.at[4, "B"]
3373 2
3375 Set value at specified row/column pair
3377 >>> df.at[4, "B"] = 10
3378 >>> df.at[4, "B"]
3379 10
3381 Get value within a Series
3383 >>> df.loc[5].at["B"]
3384 4
3385 """
3387 _takeable = False
3389 def _convert_key(self, key):
3390 """
3391 Require they keys to be the same type as the index. (so we don't
3392 fallback)
3393 """
3394 # GH 26989
3395 # For series, unpacking key needs to result in the label.
3396 # This is already the case for len(key) == 1; e.g. (1,)
3397 if self.ndim == 1 and len(key) > 1:
3398 key = (key,)
3400 return key
3402 def _warn_if_expanding(self, key) -> None:
3403 """
3404 GH#48323 - Warn if .at setitem would expand the object.
3405 """
3406 if self.ndim == 2:
3407 if not isinstance(key, tuple) or len(key) != 2:
3408 return
3409 checks = [(key[0], self.obj.index), (key[1], self.obj.columns)]
3410 else:
3411 check_key = key
3412 if isinstance(key, tuple) and len(key) == 1:
3413 check_key = key[0]
3414 checks = [(check_key, self.obj.index)]
3416 for check_key, axis in checks:
3417 # Only check for scalar-like keys (including tuples for MultiIndex).
3418 # Slices and list-likes are invalid for .at and will raise elsewhere.
3419 if isinstance(check_key, slice) or is_list_like_indexer(check_key):
3420 if not isinstance(check_key, tuple):
3421 continue
3423 try:
3424 is_expanding = check_key not in axis
3425 except (TypeError, InvalidIndexError):
3426 continue
3428 if is_expanding:
3429 obj_type = "DataFrame" if self.ndim == 2 else "Series"
3430 warnings.warn(
3431 f"Setting a value on a {obj_type} via .at with a key "
3432 "that does not exist in the index is deprecated "
3433 "and will raise a KeyError in a future version. "
3434 "Use .loc instead.",
3435 Pandas4Warning,
3436 stacklevel=find_stack_level(),
3437 )
3438 return
3440 @property
3441 def _axes_are_unique(self) -> bool:
3442 # Only relevant for self.ndim == 2
3443 assert self.ndim == 2
3444 return self.obj.index.is_unique and self.obj.columns.is_unique
3446 def __getitem__(self, key):
3447 if self.ndim == 2 and not self._axes_are_unique:
3448 # GH#33041 fall back to .loc
3449 if not isinstance(key, tuple) or not all(is_scalar(x) for x in key):
3450 raise ValueError("Invalid call for scalar access (getting)!")
3451 return self.obj.loc[key]
3453 return super().__getitem__(key)
3455 def __setitem__(self, key, value) -> None:
3456 if not CHAINED_WARNING_DISABLED:
3457 if sys.getrefcount(self.obj) <= REF_COUNT_IDX:
3458 warnings.warn(
3459 _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
3460 )
3462 # GH#48323 - deprecate .at setitem with expansion
3463 self._warn_if_expanding(key)
3465 if self.ndim == 2 and not self._axes_are_unique:
3466 # GH#33041 fall back to .loc
3467 if not isinstance(key, tuple) or not all(is_scalar(x) for x in key):
3468 raise ValueError("Invalid call for scalar access (setting)!")
3470 self.obj.loc[key] = value
3471 return
3473 return super().__setitem__(key, value)
3476class _iAtIndexer(_ScalarAccessIndexer):
3477 """
3478 Access a single value for a row/column pair by integer position.
3480 Similar to ``iloc``, in that both provide integer-based lookups. Use
3481 ``iat`` if you only need to get or set a single value in a DataFrame
3482 or Series.
3484 Raises
3485 ------
3486 IndexError
3487 When integer position is out of bounds.
3489 See Also
3490 --------
3491 DataFrame.at : Access a single value for a row/column label pair.
3492 DataFrame.loc : Access a group of rows and columns by label(s).
3493 DataFrame.iloc : Access a group of rows and columns by integer position(s).
3495 Examples
3496 --------
3497 >>> df = pd.DataFrame([[0, 2, 3], [0, 4, 1], [10, 20, 30]], columns=["A", "B", "C"])
3498 >>> df
3499 A B C
3500 0 0 2 3
3501 1 0 4 1
3502 2 10 20 30
3504 Get value at specified row/column pair
3506 >>> df.iat[1, 2]
3507 1
3509 Set value at specified row/column pair
3511 >>> df.iat[1, 2] = 10
3512 >>> df.iat[1, 2]
3513 10
3515 Get value within a series
3517 >>> df.loc[0].iat[1]
3518 2
3519 """
3521 _takeable = True
3523 def _convert_key(self, key):
3524 """
3525 Require integer args. (and convert to label arguments)
3526 """
3527 for i in key:
3528 if not is_integer(i):
3529 raise ValueError("iAt based indexing can only have integer indexers")
3530 return key
3532 def __setitem__(self, key, value) -> None:
3533 if not CHAINED_WARNING_DISABLED:
3534 if sys.getrefcount(self.obj) <= REF_COUNT_IDX:
3535 warnings.warn(
3536 _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
3537 )
3539 return super().__setitem__(key, value)
3542def _positional_row(row):
3543 """
3544 Return a row of a 2D setitem value that can be indexed by position.
3546 ``_setitem_with_indexer_2d_value`` extracts each column with ``row[i]``,
3547 which is wrong for a Series (indexed by *label*) and for a foreign array
3548 such as ``pyarrow.Array`` (``__getitem__`` hands back a boxed scalar).
3549 Rows that already index positionally are returned as-is, to skip a copy.
3550 Anything else goes through ``np.asarray``, as every row did before GH#64230
3551 replaced it with ``row[i]``; ``dtype=object`` is what keeps a heterogeneous
3552 row from being collapsed onto one numpy dtype, and ``pd.NA`` from becoming
3553 ``nan``.
3554 """
3555 if isinstance(row, ABCSeries):
3556 return row._values
3557 if isinstance(row, (tuple, np.ndarray, ABCExtensionArray, ABCIndex)):
3558 return row
3559 return np.asarray(row, dtype=object)
3562def _is_2d_value_for_columns(value, ncols: int) -> bool:
3563 """
3564 Whether ``value`` should be treated as a 2-D value for ``ncols`` columns.
3566 A list of tuples aimed at a single column is per-cell values unless the
3567 tuples are length-1 (GH#37629). With more columns it takes the 2-D path
3568 whatever the tuple width, which rejects a mismatched one. GH#65264
3569 """
3570 if not _is_2d_value(value):
3571 return False
3572 return not (
3573 isinstance(value, list)
3574 and ncols == 1
3575 and isinstance(value[0], tuple)
3576 and len(value[0]) != 1
3577 )
3580def _is_2d_value(value) -> bool:
3581 """Check if value is 2-dimensional, avoiding np.asarray for plain lists."""
3582 if isinstance(value, list):
3583 if len(value) == 0:
3584 return False
3585 first = value[0]
3586 if isinstance(first, (list, tuple)):
3587 return True
3588 # A row is anything np.asarray would have stacked into a 2-D array: a
3589 # sized, subscriptable, non-mapping sequence, i.e. ndarray/Series/Index/
3590 # ExtensionArray but also e.g. deque/array.array/range. A 0-d element is
3591 # a scalar and a 2-D element makes value 3D.
3592 # Note: deliberately not is_list_like, which also requires __iter__ and
3593 # so would exclude sequences that only implement __len__/__getitem__.
3594 return (
3595 not isinstance(first, (str, bytes, type, abc.Mapping))
3596 and hasattr(first, "__len__")
3597 and hasattr(first, "__getitem__")
3598 and getattr(first, "ndim", 1) == 1
3599 )
3600 return np.ndim(value) == 2
3603def _tuplify(ndim: int, loc: Hashable) -> tuple[Hashable | slice, ...]:
3604 """
3605 Given an indexer for the first dimension, create an equivalent tuple
3606 for indexing over all dimensions.
3608 Parameters
3609 ----------
3610 ndim : int
3611 loc : object
3613 Returns
3614 -------
3615 tuple
3616 """
3617 _tup: list[Hashable | slice]
3618 _tup = [slice(None, None) for _ in range(ndim)]
3619 _tup[0] = loc
3620 return tuple(_tup)
3623def _tupleize_axis_indexer(ndim: int, axis: AxisInt, key) -> tuple:
3624 """
3625 If we have an axis, adapt the given key to be axis-independent.
3626 """
3627 new_key = [slice(None)] * ndim
3628 new_key[axis] = key
3629 return tuple(new_key)
3632def check_bool_indexer(index: Index, key) -> np.ndarray:
3633 """
3634 Check if key is a valid boolean indexer for an object with such index and
3635 perform reindexing or conversion if needed.
3637 This function assumes that is_bool_indexer(key) == True.
3639 Parameters
3640 ----------
3641 index : Index
3642 Index of the object on which the indexing is done.
3643 key : list-like
3644 Boolean indexer to check.
3646 Returns
3647 -------
3648 np.array
3649 Resulting key.
3651 Raises
3652 ------
3653 IndexError
3654 If the key does not have the same length as index.
3655 IndexingError
3656 If the index of the key is unalignable to index.
3657 """
3658 result = key
3659 if isinstance(key, ABCSeries) and not key.index.equals(index):
3660 indexer = result.index.get_indexer_for(index)
3661 if -1 in indexer:
3662 raise IndexingError(
3663 "Unalignable boolean Series provided as "
3664 "indexer (index of the boolean Series and of "
3665 "the indexed object do not match)."
3666 )
3668 result = result.take(indexer)
3670 # fall through for boolean
3671 if not isinstance(result.dtype, ExtensionDtype):
3672 return result.astype(bool)._values
3674 if is_object_dtype(key):
3675 # key might be object-dtype bool, check_array_indexer needs bool array
3676 result = np.asarray(result, dtype=bool)
3677 elif not is_array_like_deprecate_non_pandas(result):
3678 # GH 33924
3679 # key may contain nan elements, check_array_indexer needs bool array
3680 result = pd_array(result, dtype=bool)
3681 return check_array_indexer(index, result)
3684def convert_missing_indexer(indexer):
3685 """
3686 Reverse convert a missing indexer, which is a dict
3687 return the scalar indexer and a boolean indicating if we converted
3688 """
3689 if isinstance(indexer, dict):
3690 # a missing key (but not a tuple indexer)
3691 indexer = indexer["key"]
3693 if isinstance(indexer, bool):
3694 raise KeyError("cannot use a single bool to index into setitem")
3695 return indexer, True
3697 return indexer, False
3700def convert_from_missing_indexer_tuple(indexer: tuple, axes: list[Index]) -> tuple:
3701 """
3702 Create a filtered indexer that doesn't have any missing indexers.
3703 """
3705 def get_indexer(_i, _idx):
3706 return axes[_i].get_loc(_idx["key"]) if isinstance(_idx, dict) else _idx
3708 return tuple(get_indexer(_i, _idx) for _i, _idx in enumerate(indexer))
3711def maybe_convert_ix(*args):
3712 """
3713 We likely want to take the cross-product.
3714 """
3715 for arg in args:
3716 if not isinstance(arg, (np.ndarray, list, ABCSeries, Index)):
3717 return args
3718 return np.ix_(*args)
3721def is_nested_tuple(tup, labels) -> bool:
3722 """
3723 Returns
3724 -------
3725 bool
3726 """
3727 # check for a compatible nested tuple and multiindexes among the axes
3728 if not isinstance(tup, tuple):
3729 return False
3731 for k in tup:
3732 if is_list_like(k) or isinstance(k, slice):
3733 return isinstance(labels, MultiIndex)
3735 return False
3738def is_label_like(key) -> bool:
3739 """
3740 Returns
3741 -------
3742 bool
3743 """
3744 # select a label or row
3745 return (
3746 not isinstance(key, slice)
3747 and not is_list_like_indexer(key)
3748 and key is not Ellipsis
3749 )
3752def need_slice(obj: slice) -> bool:
3753 """
3754 Returns
3755 -------
3756 bool
3757 """
3758 return (
3759 obj.start is not None
3760 or obj.stop is not None
3761 or (obj.step is not None and obj.step != 1)
3762 )
3765def check_dict_or_set_indexers(key) -> None:
3766 """
3767 Check if the indexer is or contains a dict or set, which is no longer allowed.
3768 """
3769 if isinstance(key, set) or (
3770 isinstance(key, tuple) and any(isinstance(x, set) for x in key)
3771 ):
3772 raise TypeError(
3773 "Passing a set as an indexer is not supported. Use a list instead."
3774 )
3776 if isinstance(key, dict) or (
3777 isinstance(key, tuple) and any(isinstance(x, dict) for x in key)
3778 ):
3779 raise TypeError(
3780 "Passing a dict as an indexer is not supported. Use a list instead."
3781 )
3784def _expansion_can_hold(orig: ExtensionArray, new_arr) -> bool:
3785 """
3786 Can new_arr's values be set losslessly into an array of orig's dtype?
3788 Decides whether a dtype change during setitem-with-expansion gets the
3789 GH#62369 deprecation warning.
3790 """
3791 dtype = orig.dtype
3792 np_dtype = dtype.numpy_dtype if isinstance(dtype, NumpyEADtype) else dtype
3793 if (
3794 isinstance(np_dtype, np.dtype)
3795 and np_dtype.kind in "iu"
3796 and isinstance(new_arr, np.ndarray)
3797 and new_arr.dtype.kind == "f"
3798 and not floats_fit_integer_dtype(new_arr, np_dtype)
3799 ):
3800 # GH#66394 can_hold_element's round-trip check cannot see an
3801 # out-of-range float->int cast, since e.g. 2**63 and 2**63 - 1 are
3802 # the same float64, so it would report the value as holdable.
3803 return False
3805 if isinstance(dtype, NumpyEADtype):
3806 # can_hold_element treats NumpyExtensionArray as a generic
3807 # ExtensionArray; unwrap so the underlying numpy dtype is checked.
3808 return can_hold_element(np.asarray(orig), new_arr)
3809 if isinstance(dtype, (PeriodDtype, IntervalDtype, DatetimeTZDtype, np.dtype)) or (
3810 dtype == "string"
3811 ):
3812 # the dtypes for which can_hold_element does a real setitem-validation
3813 # check, as opposed to unconditionally returning True
3814 return can_hold_element(orig, new_arr)
3816 # Generic ExtensionArray (masked, Categorical, ArrowDtype, ...): check
3817 # whether the values round-trip losslessly at the original dtype.
3818 try:
3819 with warnings.catch_warnings():
3820 # this is an internal probe whose result we compare below, so
3821 # constructor warnings (e.g. Categorical's on out-of-category
3822 # values) are spurious
3823 warnings.simplefilter("ignore")
3824 converted = pd_array(new_arr, dtype=dtype)
3825 except Exception:
3826 # EA constructors raise assorted exception types for un-holdable
3827 # values (e.g. pyarrow.lib.ArrowInvalid, decimal.InvalidOperation);
3828 # any failure means the original dtype cannot hold the new values.
3829 return False
3830 return array_equivalent(
3831 np.asarray(converted, dtype=object), np.asarray(new_arr, dtype=object)
3832 )
3835def maybe_warn_multiindex_expansion(index: Index, key, target: str, hint: str) -> None:
3836 """
3837 GH#17024 warn when expanding a MultiIndex axis with a key that is not a
3838 full-length tuple. Depending on the path, such keys currently either
3839 flatten the MultiIndex to an Index of tuples or get padded with "".
3840 """
3841 if isinstance(index, MultiIndex) and not (
3842 isinstance(key, tuple) and len(key) == index.nlevels
3843 ):
3844 warnings.warn(
3845 f"Setting a new {target} with a MultiIndex using a key that is "
3846 "not a full-length tuple is deprecated and will raise in a "
3847 f"future version. {hint}",
3848 Pandas4Warning,
3849 stacklevel=find_stack_level(),
3850 )
3853def _as_float64(arr: ArrayLike) -> np.ndarray:
3854 """
3855 Numpy float64 representation of a numeric array, with NA as NaN.
3856 """
3857 if isinstance(arr, np.ndarray):
3858 return arr.astype(np.float64, copy=False)
3859 return arr.to_numpy(dtype=np.float64, na_value=np.nan)
3862def infer_and_maybe_downcast(
3863 orig: ExtensionArray,
3864 new_arr,
3865 *,
3866 warn_if_cast: bool = True,
3867) -> ArrayLike:
3868 new_arr = orig._cast_pointwise_result(new_arr)
3870 dtype = orig.dtype
3871 if isinstance(dtype, NumpyEADtype):
3872 # error: Incompatible types in assignment (expression has
3873 # type "dtype[Any]", variable has type "ExtensionDtype")
3874 # [assignment]
3875 dtype = dtype.numpy_dtype # type: ignore[assignment]
3877 # _cast_pointwise_result may return NumpyExtensionArray; unwrap so
3878 # maybe_downcast_to_dtype sees a plain np.ndarray it can downcast.
3879 arr_type = None
3880 if isinstance(new_arr.dtype, NumpyEADtype):
3881 arr_type = type(new_arr)
3882 new_arr = new_arr._ndarray
3884 if is_np_dtype(new_arr.dtype, "f") and is_np_dtype(dtype, "iu"):
3885 new_arr = maybe_downcast_to_dtype(new_arr, dtype)
3886 elif (
3887 isinstance(dtype, ExtensionDtype)
3888 and dtype.kind in "iu"
3889 and new_arr.dtype.kind == "f"
3890 ):
3891 try:
3892 floats = _as_float64(new_arr)
3893 # GH#66394 an out-of-range float->int cast saturates or wraps
3894 # instead of raising, which the roundtrip check below cannot
3895 # detect, since e.g. 2**63 and 2**63 - 1 are the same float64.
3896 if isinstance(dtype, (ArrowDtype, BaseMaskedDtype)):
3897 np_dtype = dtype.numpy_dtype
3898 elif isinstance(dtype, SparseDtype):
3899 np_dtype = dtype.subtype
3900 else:
3901 # a third-party ExtensionDtype we have no bounds for
3902 np_dtype = None
3903 if np_dtype is None or floats_fit_integer_dtype(
3904 floats[~np.isnan(floats)], np_dtype
3905 ):
3906 converted = new_arr.astype(orig.dtype)
3907 # Only accept the conversion if no values were truncated. The
3908 # comparison is done in numpy float space because pyarrow
3909 # refuses its own int64->double cast above 2**53, even for
3910 # values that survive it intact.
3911 if array_equivalent(_as_float64(converted), floats):
3912 new_arr = converted
3913 except (ValueError, TypeError):
3914 pass
3915 elif dtype.kind in "mM" and new_arr.dtype != dtype:
3916 # GH#66402 inference re-derives the unit from the scalars, which for a
3917 # freshly-constructed Timestamp/Timedelta is us. Restore the original
3918 # resolution when every value still fits in it.
3919 wrapped = ensure_wrapped_if_datetimelike(new_arr)
3920 try:
3921 converted = wrapped.as_unit(orig.unit, round_ok=False) # type: ignore[attr-defined]
3922 except (AttributeError, ValueError, TypeError):
3923 pass
3924 else:
3925 # Guards against e.g. a tz-naive/tz-aware mismatch, where the units
3926 # line up but the dtypes are still not interchangeable.
3927 if converted.dtype == dtype:
3928 new_arr = converted
3930 # The inferred result is returned unchanged, matching the historical
3931 # behavior. We only warn when setitem-with-expansion lands on a dtype the
3932 # original cannot hold, which a future version will disallow. warn_if_cast
3933 # is False for the intermediate inference inside _setitem_with_indexer_missing
3934 # and the frame.py concat path; those warn later from the post-expansion pass.
3935 if new_arr.dtype != dtype and warn_if_cast:
3936 # PDEP6 exception: int/uint -> float when result contains NaN
3937 pdep6_allowed = (
3938 is_np_dtype(dtype, "iu")
3939 and is_np_dtype(new_arr.dtype, "f")
3940 and isna(new_arr).any()
3941 )
3942 if not pdep6_allowed and not _expansion_can_hold(orig, new_arr):
3943 warnings.warn(
3944 f"Setting an item of incompatible dtype is deprecated "
3945 f"and will raise an error in a future version of pandas. "
3946 f"The existing dtype {dtype} cannot hold the value being "
3947 f"set, so the result has dtype {new_arr.dtype}. Cast the "
3948 f"object to {new_arr.dtype} before this operation to "
3949 f"retain the current behavior.",
3950 Pandas4Warning,
3951 stacklevel=find_stack_level(),
3952 )
3954 if arr_type is not None:
3955 new_arr = arr_type(new_arr)
3956 return new_arr