Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py: 16%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from contextlib import suppress
4import sys
5from typing import (
6 TYPE_CHECKING,
7 Any,
8 Self,
9 cast,
10 final,
11)
12import warnings
14import numpy as np
16from pandas._libs.indexing import NDFrameIndexerBase
17from pandas._libs.lib import item_from_zerodim
18from pandas.compat import CHAINED_WARNING_DISABLED
19from pandas.compat._constants import REF_COUNT_IDX
20from pandas.errors import (
21 AbstractMethodError,
22 ChainedAssignmentError,
23 IndexingError,
24 InvalidIndexError,
25 LossySetitemError,
26)
27from pandas.errors.cow import _chained_assignment_msg
28from pandas.util._decorators import (
29 doc,
30)
32from pandas.core.dtypes.cast import (
33 can_hold_element,
34 maybe_promote,
35)
36from pandas.core.dtypes.common import (
37 is_array_like,
38 is_bool_dtype,
39 is_hashable,
40 is_integer,
41 is_iterator,
42 is_list_like,
43 is_numeric_dtype,
44 is_object_dtype,
45 is_scalar,
46 is_sequence,
47)
48from pandas.core.dtypes.concat import concat_compat
49from pandas.core.dtypes.dtypes import ExtensionDtype
50from pandas.core.dtypes.generic import (
51 ABCDataFrame,
52 ABCSeries,
53)
54from pandas.core.dtypes.missing import (
55 construct_1d_array_from_inferred_fill_value,
56 infer_fill_value,
57 is_valid_na_for_dtype,
58 isna,
59 na_value_for_dtype,
60)
62from pandas.core import algorithms as algos
63import pandas.core.common as com
64from pandas.core.construction import (
65 array as pd_array,
66 extract_array,
67)
68from pandas.core.indexers import (
69 check_array_indexer,
70 is_list_like_indexer,
71 is_scalar_indexer,
72 length_of_indexer,
73)
74from pandas.core.indexes.api import (
75 Index,
76 MultiIndex,
77)
79if TYPE_CHECKING:
80 from collections.abc import (
81 Hashable,
82 Sequence,
83 )
85 from pandas._typing import (
86 Axis,
87 AxisInt,
88 T,
89 npt,
90 )
92 from pandas import (
93 DataFrame,
94 Series,
95 )
97# "null slice"
98_NS = slice(None, None)
99_one_ellipsis_message = "indexer may only contain one '...' entry"
102# the public IndexSlicerMaker
103class _IndexSlice:
104 """
105 Create an object to more easily perform multi-index slicing.
107 See Also
108 --------
109 MultiIndex.remove_unused_levels : New MultiIndex with no unused levels.
111 Notes
112 -----
113 See :ref:`Defined Levels <advanced.shown_levels>`
114 for further info on slicing a MultiIndex.
116 Examples
117 --------
118 >>> midx = pd.MultiIndex.from_product([["A0", "A1"], ["B0", "B1", "B2", "B3"]])
119 >>> columns = ["foo", "bar"]
120 >>> dfmi = pd.DataFrame(
121 ... np.arange(16).reshape((len(midx), len(columns))),
122 ... index=midx,
123 ... columns=columns,
124 ... )
126 Using the default slice command:
128 >>> dfmi.loc[(slice(None), slice("B0", "B1")), :]
129 foo bar
130 A0 B0 0 1
131 B1 2 3
132 A1 B0 8 9
133 B1 10 11
135 Using the IndexSlice class for a more intuitive command:
137 >>> idx = pd.IndexSlice
138 >>> dfmi.loc[idx[:, "B0":"B1"], :]
139 foo bar
140 A0 B0 0 1
141 B1 2 3
142 A1 B0 8 9
143 B1 10 11
144 """
146 def __getitem__(self, arg):
147 return arg
150IndexSlice = _IndexSlice()
151IndexSlice.__module__ = "pandas"
154class IndexingMixin:
155 """
156 Mixin for adding .loc/.iloc/.at/.iat to Dataframes and Series.
157 """
159 @property
160 def iloc(self) -> _iLocIndexer:
161 """
162 Purely integer-location based indexing for selection by position.
164 .. versionchanged:: 3.0
166 Callables which return a tuple are deprecated as input.
168 ``.iloc[]`` is primarily integer position based (from ``0`` to
169 ``length-1`` of the axis), but may also be used with a boolean
170 array.
172 Allowed inputs are:
174 - An integer, e.g. ``5``.
175 - A list or array of integers, e.g. ``[4, 3, 0]``.
176 - A slice object with ints, e.g. ``1:7``.
177 - A boolean array.
178 - A ``callable`` function with one argument (the calling Series or
179 DataFrame) and that returns valid output for indexing (one of the above).
180 This is useful in method chains, when you don't have a reference to the
181 calling object, but would like to base your selection on
182 some value.
183 - A tuple of row and column indexes. The tuple elements consist of one of the
184 above inputs, e.g. ``(0, 1)``.
186 ``.iloc`` will raise ``IndexError`` if a requested indexer is
187 out-of-bounds, except *slice* indexers which allow out-of-bounds
188 indexing (this conforms with python/numpy *slice* semantics).
190 See more at :ref:`Selection by Position <indexing.integer>`.
192 See Also
193 --------
194 DataFrame.iat : Fast integer location scalar accessor.
195 DataFrame.loc : Purely label-location based indexer for selection by label.
196 Series.iloc : Purely integer-location based indexing for
197 selection by position.
199 Examples
200 --------
201 >>> mydict = [
202 ... {"a": 1, "b": 2, "c": 3, "d": 4},
203 ... {"a": 100, "b": 200, "c": 300, "d": 400},
204 ... {"a": 1000, "b": 2000, "c": 3000, "d": 4000},
205 ... ]
206 >>> df = pd.DataFrame(mydict)
207 >>> df
208 a b c d
209 0 1 2 3 4
210 1 100 200 300 400
211 2 1000 2000 3000 4000
213 **Indexing just the rows**
215 With a scalar integer.
217 >>> type(df.iloc[0])
218 <class 'pandas.Series'>
219 >>> df.iloc[0]
220 a 1
221 b 2
222 c 3
223 d 4
224 Name: 0, dtype: int64
226 With a list of integers.
228 >>> df.iloc[[0]]
229 a b c d
230 0 1 2 3 4
231 >>> type(df.iloc[[0]])
232 <class 'pandas.DataFrame'>
234 >>> df.iloc[[0, 1]]
235 a b c d
236 0 1 2 3 4
237 1 100 200 300 400
239 With a `slice` object.
241 >>> df.iloc[:3]
242 a b c d
243 0 1 2 3 4
244 1 100 200 300 400
245 2 1000 2000 3000 4000
247 With a boolean mask the same length as the index.
249 >>> df.iloc[[True, False, True]]
250 a b c d
251 0 1 2 3 4
252 2 1000 2000 3000 4000
254 With a callable, useful in method chains. The `x` passed
255 to the ``lambda`` is the DataFrame being sliced. This selects
256 the rows whose index label even.
258 >>> df.iloc[lambda x: x.index % 2 == 0]
259 a b c d
260 0 1 2 3 4
261 2 1000 2000 3000 4000
263 **Indexing both axes**
265 You can mix the indexer types for the index and columns. Use ``:`` to
266 select the entire axis.
268 With scalar integers.
270 >>> df.iloc[0, 1]
271 np.int64(2)
273 With lists of integers.
275 >>> df.iloc[[0, 2], [1, 3]]
276 b d
277 0 2 4
278 2 2000 4000
280 With `slice` objects.
282 >>> df.iloc[1:3, 0:3]
283 a b c
284 1 100 200 300
285 2 1000 2000 3000
287 With a boolean array whose length matches the columns.
289 >>> df.iloc[:, [True, False, True, False]]
290 a c
291 0 1 3
292 1 100 300
293 2 1000 3000
295 With a callable function that expects the Series or DataFrame.
297 >>> df.iloc[:, lambda df: [0, 2]]
298 a c
299 0 1 3
300 1 100 300
301 2 1000 3000
302 """
303 return _iLocIndexer("iloc", self)
305 @property
306 def loc(self) -> _LocIndexer:
307 """
308 Access a group of rows and columns by label(s) or a boolean array.
310 ``.loc[]`` is primarily label based, but may also be used with a
311 boolean array.
313 Allowed inputs are:
315 - A single label, e.g. ``5`` or ``'a'``, (note that ``5`` is
316 interpreted as a *label* of the index, and **never** as an
317 integer position along the index).
318 - A list or array of labels, e.g. ``['a', 'b', 'c']``.
319 - A slice object with labels, e.g. ``'a':'f'``.
321 .. warning:: Note that contrary to usual python slices, **both** the
322 start and the stop are included
324 - A boolean array of the same length as the axis being sliced,
325 e.g. ``[True, False, True]``.
326 - An alignable boolean Series. The index of the key will be aligned before
327 masking.
328 - An alignable Index. The Index of the returned selection will be the input.
329 - A ``callable`` function with one argument (the calling Series or
330 DataFrame) and that returns valid output for indexing (one of the above)
332 See more at :ref:`Selection by Label <indexing.label>`.
334 Raises
335 ------
336 KeyError
337 If any items are not found.
338 IndexingError
339 If an indexed key is passed and its index is unalignable to the frame index.
341 See Also
342 --------
343 DataFrame.at : Access a single value for a row/column label pair.
344 DataFrame.iloc : Access group of rows and columns by integer position(s).
345 DataFrame.xs : Returns a cross-section (row(s) or column(s)) from the
346 Series/DataFrame.
347 Series.loc : Access group of values using labels.
349 Examples
350 --------
351 **Getting values**
353 >>> df = pd.DataFrame(
354 ... [[1, 2], [4, 5], [7, 8]],
355 ... index=["cobra", "viper", "sidewinder"],
356 ... columns=["max_speed", "shield"],
357 ... )
358 >>> df
359 max_speed shield
360 cobra 1 2
361 viper 4 5
362 sidewinder 7 8
364 Single label. Note this returns the row as a Series.
366 >>> df.loc["viper"]
367 max_speed 4
368 shield 5
369 Name: viper, dtype: int64
371 List of labels. Note using ``[[]]`` returns a DataFrame.
373 >>> df.loc[["viper", "sidewinder"]]
374 max_speed shield
375 viper 4 5
376 sidewinder 7 8
378 Single label for row and column
380 >>> df.loc["cobra", "shield"]
381 np.int64(2)
383 Slice with labels for row and single label for column. As mentioned
384 above, note that both the start and stop of the slice are included.
386 >>> df.loc["cobra":"viper", "max_speed"]
387 cobra 1
388 viper 4
389 Name: max_speed, dtype: int64
391 Boolean list with the same length as the row axis
393 >>> df.loc[[False, False, True]]
394 max_speed shield
395 sidewinder 7 8
397 Alignable boolean Series:
399 >>> df.loc[
400 ... pd.Series([False, True, False], index=["viper", "sidewinder", "cobra"])
401 ... ]
402 max_speed shield
403 sidewinder 7 8
405 Index (same behavior as ``df.reindex``)
407 >>> df.loc[pd.Index(["cobra", "viper"], name="foo")]
408 max_speed shield
409 foo
410 cobra 1 2
411 viper 4 5
413 Conditional that returns a boolean Series
415 >>> df.loc[df["shield"] > 6]
416 max_speed shield
417 sidewinder 7 8
419 Conditional that returns a boolean Series with column labels specified
421 >>> df.loc[df["shield"] > 6, ["max_speed"]]
422 max_speed
423 sidewinder 7
425 Multiple conditional using ``&`` that returns a boolean Series
427 >>> df.loc[(df["max_speed"] > 1) & (df["shield"] < 8)]
428 max_speed shield
429 viper 4 5
431 Multiple conditional using ``|`` that returns a boolean Series
433 >>> df.loc[(df["max_speed"] > 4) | (df["shield"] < 5)]
434 max_speed shield
435 cobra 1 2
436 sidewinder 7 8
438 Please ensure that each condition is wrapped in parentheses ``()``.
439 See the :ref:`user guide<indexing.boolean>`
440 for more details and explanations of Boolean indexing.
442 .. note::
443 If you find yourself using 3 or more conditionals in ``.loc[]``,
444 consider using :ref:`advanced indexing<advanced.advanced_hierarchical>`.
446 See below for using ``.loc[]`` on MultiIndex DataFrames.
448 Callable that returns a boolean Series
450 >>> df.loc[lambda df: df["shield"] == 8]
451 max_speed shield
452 sidewinder 7 8
454 **Setting values**
456 Set value for all items matching the list of labels
458 >>> df.loc[["viper", "sidewinder"], ["shield"]] = 50
459 >>> df
460 max_speed shield
461 cobra 1 2
462 viper 4 50
463 sidewinder 7 50
465 Set value for an entire row
467 >>> df.loc["cobra"] = 10
468 >>> df
469 max_speed shield
470 cobra 10 10
471 viper 4 50
472 sidewinder 7 50
474 Set value for an entire column
476 >>> df.loc[:, "max_speed"] = 30
477 >>> df
478 max_speed shield
479 cobra 30 10
480 viper 30 50
481 sidewinder 30 50
483 Set value for rows matching callable condition
485 >>> df.loc[df["shield"] > 35] = 0
486 >>> df
487 max_speed shield
488 cobra 30 10
489 viper 0 0
490 sidewinder 0 0
492 Add value matching location
494 >>> df.loc["viper", "shield"] += 5
495 >>> df
496 max_speed shield
497 cobra 30 10
498 viper 0 5
499 sidewinder 0 0
501 Setting using a ``Series`` or a ``DataFrame`` sets the values matching the
502 index labels, not the index positions.
504 >>> shuffled_df = df.loc[["viper", "cobra", "sidewinder"]]
505 >>> df.loc[:] += shuffled_df
506 >>> df
507 max_speed shield
508 cobra 60 20
509 viper 0 10
510 sidewinder 0 0
512 **Getting values on a DataFrame with an index that has integer labels**
514 Another example using integers for the index
516 >>> df = pd.DataFrame(
517 ... [[1, 2], [4, 5], [7, 8]],
518 ... index=[7, 8, 9],
519 ... columns=["max_speed", "shield"],
520 ... )
521 >>> df
522 max_speed shield
523 7 1 2
524 8 4 5
525 9 7 8
527 Slice with integer labels for rows. As mentioned above, note that both
528 the start and stop of the slice are included.
530 >>> df.loc[7:9]
531 max_speed shield
532 7 1 2
533 8 4 5
534 9 7 8
536 **Getting values with a MultiIndex**
538 A number of examples using a DataFrame with a MultiIndex
540 >>> tuples = [
541 ... ("cobra", "mark i"),
542 ... ("cobra", "mark ii"),
543 ... ("sidewinder", "mark i"),
544 ... ("sidewinder", "mark ii"),
545 ... ("viper", "mark ii"),
546 ... ("viper", "mark iii"),
547 ... ]
548 >>> index = pd.MultiIndex.from_tuples(tuples)
549 >>> values = [[12, 2], [0, 4], [10, 20], [1, 4], [7, 1], [16, 36]]
550 >>> df = pd.DataFrame(values, columns=["max_speed", "shield"], index=index)
551 >>> df
552 max_speed shield
553 cobra mark i 12 2
554 mark ii 0 4
555 sidewinder mark i 10 20
556 mark ii 1 4
557 viper mark ii 7 1
558 mark iii 16 36
560 Single label. Note this returns a DataFrame with a single index.
562 >>> df.loc["cobra"]
563 max_speed shield
564 mark i 12 2
565 mark ii 0 4
567 Single index tuple. Note this returns a Series.
569 >>> df.loc[("cobra", "mark ii")]
570 max_speed 0
571 shield 4
572 Name: (cobra, mark ii), dtype: int64
574 Single label for row and column. Similar to passing in a tuple, this
575 returns a Series.
577 >>> df.loc["cobra", "mark i"]
578 max_speed 12
579 shield 2
580 Name: (cobra, mark i), dtype: int64
582 Single tuple. Note using ``[[]]`` returns a DataFrame.
584 >>> df.loc[[("cobra", "mark ii")]]
585 max_speed shield
586 cobra mark ii 0 4
588 Single tuple for the index with a single label for the column
590 >>> df.loc[("cobra", "mark i"), "shield"]
591 np.int64(2)
593 Slice from index tuple to single label
595 >>> df.loc[("cobra", "mark i") : "viper"]
596 max_speed shield
597 cobra mark i 12 2
598 mark ii 0 4
599 sidewinder mark i 10 20
600 mark ii 1 4
601 viper mark ii 7 1
602 mark iii 16 36
604 Slice from index tuple to index tuple
606 >>> df.loc[("cobra", "mark i") : ("viper", "mark ii")]
607 max_speed shield
608 cobra mark i 12 2
609 mark ii 0 4
610 sidewinder mark i 10 20
611 mark ii 1 4
612 viper mark ii 7 1
614 Please see the :ref:`user guide<advanced.advanced_hierarchical>`
615 for more details and explanations of advanced indexing.
617 **Assignment with Series**
619 When assigning a Series to .loc[row_indexer, col_indexer], pandas aligns
620 the Series by index labels, not by order or position.
622 Series assignment with .loc and index alignment:
624 >>> df = pd.DataFrame({"A": [1, 2, 3]}, index=[0, 1, 2])
625 >>> s = pd.Series([10, 20], index=[1, 0]) # Note reversed order
626 >>> df.loc[:, "B"] = s # Aligns by index, not order
627 >>> df
628 A B
629 0 1 20.0
630 1 2 10.0
631 2 3 NaN
632 """
633 return _LocIndexer("loc", self)
635 @property
636 def at(self) -> _AtIndexer:
637 """
638 Access a single value for a row/column label pair.
640 Similar to ``loc``, in that both provide label-based lookups. Use
641 ``at`` if you only need to get or set a single value in a DataFrame
642 or Series.
644 Raises
645 ------
646 KeyError
647 If getting a value and 'label' does not exist in a DataFrame or Series.
649 ValueError
650 If row/column label pair is not a tuple or if any label
651 from the pair is not a scalar for DataFrame.
652 If label is list-like (*excluding* NamedTuple) for Series.
654 See Also
655 --------
656 DataFrame.at : Access a single value for a row/column pair by label.
657 DataFrame.iat : Access a single value for a row/column pair by integer
658 position.
659 DataFrame.loc : Access a group of rows and columns by label(s).
660 DataFrame.iloc : Access a group of rows and columns by integer
661 position(s).
662 Series.at : Access a single value by label.
663 Series.iat : Access a single value by integer position.
664 Series.loc : Access a group of rows by label(s).
665 Series.iloc : Access a group of rows by integer position(s).
667 Notes
668 -----
669 See :ref:`Fast scalar value getting and setting <indexing.basics.get_value>`
670 for more details.
672 Examples
673 --------
674 >>> df = pd.DataFrame(
675 ... [[0, 2, 3], [0, 4, 1], [10, 20, 30]],
676 ... index=[4, 5, 6],
677 ... columns=["A", "B", "C"],
678 ... )
679 >>> df
680 A B C
681 4 0 2 3
682 5 0 4 1
683 6 10 20 30
685 Get value at specified row/column pair
687 >>> df.at[4, "B"]
688 np.int64(2)
690 Set value at specified row/column pair
692 >>> df.at[4, "B"] = 10
693 >>> df.at[4, "B"]
694 np.int64(10)
696 Get value within a Series
698 >>> df.loc[5].at["B"]
699 np.int64(4)
700 """
701 return _AtIndexer("at", self)
703 @property
704 def iat(self) -> _iAtIndexer:
705 """
706 Access a single value for a row/column pair by integer position.
708 Similar to ``iloc``, in that both provide integer-based lookups. Use
709 ``iat`` if you only need to get or set a single value in a DataFrame
710 or Series.
712 Raises
713 ------
714 IndexError
715 When integer position is out of bounds.
717 See Also
718 --------
719 DataFrame.at : Access a single value for a row/column label pair.
720 DataFrame.loc : Access a group of rows and columns by label(s).
721 DataFrame.iloc : Access a group of rows and columns by integer position(s).
723 Examples
724 --------
725 >>> df = pd.DataFrame(
726 ... [[0, 2, 3], [0, 4, 1], [10, 20, 30]], columns=["A", "B", "C"]
727 ... )
728 >>> df
729 A B C
730 0 0 2 3
731 1 0 4 1
732 2 10 20 30
734 Get value at specified row/column pair
736 >>> df.iat[1, 2]
737 np.int64(1)
739 Set value at specified row/column pair
741 >>> df.iat[1, 2] = 10
742 >>> df.iat[1, 2]
743 np.int64(10)
745 Get value within a series
747 >>> df.loc[0].iat[1]
748 np.int64(2)
749 """
750 return _iAtIndexer("iat", self)
753class _LocationIndexer(NDFrameIndexerBase):
754 _valid_types: str
755 axis: AxisInt | None = None
757 # sub-classes need to set _takeable
758 _takeable: bool
760 @final
761 def __call__(self, axis: Axis | None = None) -> Self:
762 # we need to return a copy of ourselves
763 new_self = type(self)(self.name, self.obj)
765 if axis is not None:
766 axis_int_none = self.obj._get_axis_number(axis)
767 else:
768 axis_int_none = axis
769 new_self.axis = axis_int_none
770 return new_self
772 def _get_setitem_indexer(self, key):
773 """
774 Convert a potentially-label-based key into a positional indexer.
775 """
776 if self.name == "loc":
777 # always holds here bc iloc overrides _get_setitem_indexer
778 self._ensure_listlike_indexer(key, axis=self.axis)
780 if isinstance(key, tuple):
781 for x in key:
782 check_dict_or_set_indexers(x)
784 if self.axis is not None:
785 key = _tupleize_axis_indexer(self.ndim, self.axis, key)
787 ax = self.obj._get_axis(0)
789 if (
790 isinstance(ax, MultiIndex)
791 and self.name != "iloc"
792 and is_hashable(key, allow_slice=False)
793 ):
794 with suppress(KeyError, InvalidIndexError):
795 # TypeError e.g. passed a bool
796 return ax.get_loc(key)
798 if isinstance(key, tuple):
799 with suppress(IndexingError):
800 # suppress "Too many indexers"
801 return self._convert_tuple(key)
803 if isinstance(key, range):
804 # GH#45479 test_loc_setitem_range_key
805 key = list(key)
807 return self._convert_to_indexer(key, axis=0)
809 @final
810 def _maybe_mask_setitem_value(self, indexer, value):
811 """
812 If we have obj.iloc[mask] = series_or_frame and series_or_frame has the
813 same length as obj, we treat this as obj.iloc[mask] = series_or_frame[mask],
814 similar to Series.__setitem__.
816 Note this is only for loc, not iloc.
817 """
819 if (
820 isinstance(indexer, tuple)
821 and len(indexer) == 2
822 and isinstance(value, (ABCSeries, ABCDataFrame))
823 ):
824 pi, icols = indexer
825 ndim = value.ndim
826 if com.is_bool_indexer(pi) and len(value) == len(pi):
827 newkey = pi.nonzero()[0]
829 if is_scalar_indexer(icols, self.ndim - 1) and ndim == 1:
830 # e.g. test_loc_setitem_boolean_mask_allfalse
831 if len(newkey) == 0:
832 value = value.iloc[:0]
833 else:
834 # test_loc_setitem_ndframe_values_alignment
835 value = self.obj.iloc._align_series(indexer, value)
836 indexer = (newkey, icols)
838 elif (
839 isinstance(icols, np.ndarray)
840 and icols.dtype.kind == "i"
841 and len(icols) == 1
842 ):
843 if ndim == 1:
844 # We implicitly broadcast, though numpy does not, see
845 # github.com/pandas-dev/pandas/pull/45501#discussion_r789071825
846 # test_loc_setitem_ndframe_values_alignment
847 value = self.obj.iloc._align_series(indexer, value)
848 indexer = (newkey, icols)
850 elif ndim == 2 and value.shape[1] == 1:
851 if len(newkey) == 0:
852 value = value.iloc[:0]
853 else:
854 # test_loc_setitem_ndframe_values_alignment
855 value = self.obj.iloc._align_frame(indexer, value)
856 indexer = (newkey, icols)
857 elif com.is_bool_indexer(indexer):
858 indexer = indexer.nonzero()[0]
860 return indexer, value
862 @final
863 def _ensure_listlike_indexer(self, key, axis=None, value=None) -> None:
864 """
865 Ensure that a list-like of column labels are all present by adding them if
866 they do not already exist.
868 Parameters
869 ----------
870 key : list-like of column labels
871 Target labels.
872 axis : key axis if known
873 """
874 column_axis = 1
876 # column only exists in 2-dimensional DataFrame
877 if self.ndim != 2:
878 return
880 if isinstance(key, tuple) and len(key) > 1:
881 # key may be a tuple if we are .loc
882 # if length of key is > 1 set key to column part
883 # unless axis is already specified, then go with that
884 if axis is None:
885 axis = column_axis
886 key = key[axis]
888 if (
889 axis == column_axis
890 and not isinstance(self.obj.columns, MultiIndex)
891 and is_list_like_indexer(key)
892 and not com.is_bool_indexer(key)
893 and all(is_hashable(k) for k in key)
894 ):
895 # GH#38148
896 keys = self.obj.columns.union(key, sort=False)
897 diff = Index(key, copy=False).difference(self.obj.columns, sort=False)
899 if len(diff):
900 # e.g. if we are doing df.loc[:, ["A", "B"]] = 7 and "B"
901 # is a new column, add the new columns with dtype=np.void
902 # so that later when we go through setitem_single_column
903 # we will use isetitem. Without this, the reindex_axis
904 # below would create float64 columns in this example, which
905 # would successfully hold 7, so we would end up with the wrong
906 # dtype.
907 indexer = np.arange(len(keys), dtype=np.intp)
908 indexer[len(self.obj.columns) :] = -1
909 new_mgr = self.obj._mgr.reindex_indexer(
910 keys, indexer=indexer, axis=0, only_slice=True, use_na_proxy=True
911 )
912 self.obj._mgr = new_mgr
913 return
915 self.obj._mgr = self.obj._mgr.reindex_axis(keys, axis=0, only_slice=True)
917 @final
918 def __setitem__(self, key, value) -> None:
919 if not CHAINED_WARNING_DISABLED:
920 if sys.getrefcount(self.obj) <= REF_COUNT_IDX:
921 warnings.warn(
922 _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
923 )
925 check_dict_or_set_indexers(key)
926 if isinstance(key, tuple):
927 key = (list(x) if is_iterator(x) else x for x in key)
928 key = tuple(com.apply_if_callable(x, self.obj) for x in key)
929 else:
930 maybe_callable = com.apply_if_callable(key, self.obj)
931 key = self._raise_callable_usage(key, maybe_callable)
932 indexer = self._get_setitem_indexer(key)
933 self._has_valid_setitem_indexer(key)
935 iloc: _iLocIndexer = (
936 cast("_iLocIndexer", self) if self.name == "iloc" else self.obj.iloc
937 )
938 iloc._setitem_with_indexer(indexer, value, self.name)
940 def _validate_key(self, key, axis: AxisInt) -> None:
941 """
942 Ensure that key is valid for current indexer.
944 Parameters
945 ----------
946 key : scalar, slice or list-like
947 Key requested.
948 axis : int
949 Dimension on which the indexing is being made.
951 Raises
952 ------
953 TypeError
954 If the key (or some element of it) has wrong type.
955 IndexError
956 If the key (or some element of it) is out of bounds.
957 KeyError
958 If the key was not found.
959 """
960 raise AbstractMethodError(self)
962 @final
963 def _expand_ellipsis(self, tup: tuple) -> tuple:
964 """
965 If a tuple key includes an Ellipsis, replace it with an appropriate
966 number of null slices.
967 """
968 if any(x is Ellipsis for x in tup):
969 if tup.count(Ellipsis) > 1:
970 raise IndexingError(_one_ellipsis_message)
972 if len(tup) == self.ndim:
973 # It is unambiguous what axis this Ellipsis is indexing,
974 # treat as a single null slice.
975 i = tup.index(Ellipsis)
976 # FIXME: this assumes only one Ellipsis
977 new_key = (*tup[:i], _NS, *tup[i + 1 :])
978 return new_key
980 # TODO: other cases? only one test gets here, and that is covered
981 # by _validate_key_length
982 return tup
984 @final
985 def _validate_tuple_indexer(self, key: tuple) -> tuple:
986 """
987 Check the key for valid keys across my indexer.
988 """
989 key = self._validate_key_length(key)
990 key = self._expand_ellipsis(key)
991 for i, k in enumerate(key):
992 try:
993 self._validate_key(k, i)
994 except ValueError as err:
995 raise ValueError(
996 f"Location based indexing can only have [{self._valid_types}] types"
997 ) from err
998 return key
1000 @final
1001 def _is_nested_tuple_indexer(self, tup: tuple) -> bool:
1002 """
1003 Returns
1004 -------
1005 bool
1006 """
1007 if any(isinstance(ax, MultiIndex) for ax in self.obj.axes):
1008 return any(is_nested_tuple(tup, ax) for ax in self.obj.axes)
1009 return False
1011 @final
1012 def _convert_tuple(self, key: tuple) -> tuple:
1013 # Note: we assume _tupleize_axis_indexer has been called, if necessary.
1014 self._validate_key_length(key)
1015 keyidx = [self._convert_to_indexer(k, axis=i) for i, k in enumerate(key)]
1016 return tuple(keyidx)
1018 @final
1019 def _validate_key_length(self, key: tuple) -> tuple:
1020 if len(key) > self.ndim:
1021 if key[0] is Ellipsis:
1022 # e.g. Series.iloc[..., 3] reduces to just Series.iloc[3]
1023 key = key[1:]
1024 if Ellipsis in key:
1025 raise IndexingError(_one_ellipsis_message)
1026 return self._validate_key_length(key)
1027 raise IndexingError("Too many indexers")
1028 return key
1030 @final
1031 def _getitem_tuple_same_dim(self, tup: tuple):
1032 """
1033 Index with indexers that should return an object of the same dimension
1034 as self.obj.
1036 This is only called after a failed call to _getitem_lowerdim.
1037 """
1038 retval = self.obj
1039 # Selecting columns before rows is significantly faster
1040 start_val = (self.ndim - len(tup)) + 1
1041 for i, key in enumerate(reversed(tup)):
1042 i = self.ndim - i - start_val
1043 if com.is_null_slice(key):
1044 continue
1046 retval = getattr(retval, self.name)._getitem_axis(key, axis=i)
1047 # We should never have retval.ndim < self.ndim, as that should
1048 # be handled by the _getitem_lowerdim call above.
1049 assert retval.ndim == self.ndim
1051 if retval is self.obj:
1052 # if all axes were a null slice (`df.loc[:, :]`), ensure we still
1053 # return a new object (https://github.com/pandas-dev/pandas/pull/49469)
1054 retval = retval.copy(deep=False)
1056 return retval
1058 @final
1059 def _getitem_lowerdim(self, tup: tuple):
1060 # we can directly get the axis result since the axis is specified
1061 if self.axis is not None:
1062 axis = self.obj._get_axis_number(self.axis)
1063 return self._getitem_axis(tup, axis=axis)
1065 # we may have a nested tuples indexer here
1066 if self._is_nested_tuple_indexer(tup):
1067 return self._getitem_nested_tuple(tup)
1069 # we maybe be using a tuple to represent multiple dimensions here
1070 ax0 = self.obj._get_axis(0)
1071 # ...but iloc should handle the tuple as simple integer-location
1072 # instead of checking it as multiindex representation (GH 13797)
1073 if (
1074 isinstance(ax0, MultiIndex)
1075 and self.name != "iloc"
1076 and not any(isinstance(x, slice) for x in tup)
1077 ):
1078 # Note: in all extant test cases, replacing the slice condition with
1079 # `all(is_hashable(x) or com.is_null_slice(x) for x in tup)`
1080 # is equivalent.
1081 # (see the other place where we call _handle_lowerdim_multi_index_axis0)
1082 with suppress(IndexingError):
1083 return cast(_LocIndexer, self)._handle_lowerdim_multi_index_axis0(tup)
1085 tup = self._validate_key_length(tup)
1087 # Reverse tuple so that we are indexing along columns before rows
1088 # and avoid unintended dtype inference. # GH60600
1089 for i, key in zip(range(len(tup) - 1, -1, -1), reversed(tup), strict=True):
1090 if is_label_like(key) or is_list_like(key):
1091 # We don't need to check for tuples here because those are
1092 # caught by the _is_nested_tuple_indexer check above.
1093 section = self._getitem_axis(key, axis=i)
1095 # We should never have a scalar section here, because
1096 # _getitem_lowerdim is only called after a check for
1097 # is_scalar_access, which that would be.
1098 if section.ndim == self.ndim:
1099 # we're in the middle of slicing through a MultiIndex
1100 # revise the key wrt to `section` by inserting an _NS
1101 new_key = (*tup[:i], _NS, *tup[i + 1 :])
1103 else:
1104 # Note: the section.ndim == self.ndim check above
1105 # rules out having DataFrame here, so we dont need to worry
1106 # about transposing.
1107 new_key = tup[:i] + tup[i + 1 :]
1109 if len(new_key) == 1:
1110 new_key = new_key[0]
1112 # Slices should return views, but calling iloc/loc with a null
1113 # slice returns a new object.
1114 if com.is_null_slice(new_key):
1115 return section
1116 # This is an elided recursive call to iloc/loc
1117 return getattr(section, self.name)[new_key]
1119 raise IndexingError("not applicable")
1121 @final
1122 def _getitem_nested_tuple(self, tup: tuple):
1123 # we have a nested tuple so have at least 1 multi-index level
1124 # we should be able to match up the dimensionality here
1126 for key in tup:
1127 check_dict_or_set_indexers(key)
1129 # we have too many indexers for our dim, but have at least 1
1130 # multi-index dimension, try to see if we have something like
1131 # a tuple passed to a series with a multi-index
1132 if len(tup) > self.ndim:
1133 if self.name != "loc":
1134 # This should never be reached, but let's be explicit about it
1135 raise ValueError("Too many indices") # pragma: no cover
1136 if all(
1137 is_hashable(x, allow_slice=False) or com.is_null_slice(x) for x in tup
1138 ):
1139 # GH#10521 Series should reduce MultiIndex dimensions instead of
1140 # DataFrame, IndexingError is not raised when slice(None,None,None)
1141 # with one row.
1142 with suppress(IndexingError):
1143 return cast(_LocIndexer, self)._handle_lowerdim_multi_index_axis0(
1144 tup
1145 )
1146 elif isinstance(self.obj, ABCSeries) and any(
1147 isinstance(k, tuple) for k in tup
1148 ):
1149 # GH#35349 Raise if tuple in tuple for series
1150 # Do this after the all-hashable-or-null-slice check so that
1151 # we are only getting non-hashable tuples, in particular ones
1152 # that themselves contain a slice entry
1153 # See test_loc_series_getitem_too_many_dimensions
1154 raise IndexingError("Too many indexers")
1156 # this is a series with a multi-index specified a tuple of
1157 # selectors
1158 axis = self.axis or 0
1159 return self._getitem_axis(tup, axis=axis)
1161 # handle the multi-axis by taking sections and reducing
1162 # this is iterative
1163 obj = self.obj
1164 # GH#41369 Loop in reverse order ensures indexing along columns before rows
1165 # which selects only necessary blocks which avoids dtype conversion if possible
1166 axis = len(tup) - 1
1167 for key in reversed(tup):
1168 if com.is_null_slice(key):
1169 axis -= 1
1170 continue
1172 obj = getattr(obj, self.name)._getitem_axis(key, axis=axis)
1173 axis -= 1
1175 # if we have a scalar, we are done
1176 if is_scalar(obj) or not hasattr(obj, "ndim"):
1177 break
1179 return obj
1181 def _convert_to_indexer(self, key, axis: AxisInt):
1182 raise AbstractMethodError(self)
1184 def _raise_callable_usage(self, key: Any, maybe_callable: T) -> T:
1185 # GH53533
1186 if self.name == "iloc" and callable(key) and isinstance(maybe_callable, tuple):
1187 raise ValueError(
1188 "Returning a tuple from a callable with iloc is not allowed.",
1189 )
1190 return maybe_callable
1192 @final
1193 def __getitem__(self, key):
1194 check_dict_or_set_indexers(key)
1195 if type(key) is tuple:
1196 key = (list(x) if is_iterator(x) else x for x in key)
1197 key = tuple(com.apply_if_callable(x, self.obj) for x in key)
1198 if self._is_scalar_access(key):
1199 return self.obj._get_value(*key, takeable=self._takeable)
1200 return self._getitem_tuple(key)
1201 else:
1202 # we by definition only have the 0th axis
1203 axis = self.axis or 0
1205 maybe_callable = com.apply_if_callable(key, self.obj)
1206 maybe_callable = self._raise_callable_usage(key, maybe_callable)
1207 return self._getitem_axis(maybe_callable, axis=axis)
1209 def _is_scalar_access(self, key: tuple):
1210 raise NotImplementedError
1212 def _getitem_tuple(self, tup: tuple):
1213 raise AbstractMethodError(self)
1215 def _getitem_axis(self, key, axis: AxisInt):
1216 raise NotImplementedError
1218 def _has_valid_setitem_indexer(self, indexer) -> bool:
1219 raise AbstractMethodError(self)
1221 @final
1222 def _getbool_axis(self, key, axis: AxisInt):
1223 # caller is responsible for ensuring non-None axis
1224 labels = self.obj._get_axis(axis)
1225 key = check_bool_indexer(labels, key)
1226 inds = key.nonzero()[0]
1227 return self.obj.take(inds, axis=axis)
1230@doc(IndexingMixin.loc)
1231class _LocIndexer(_LocationIndexer):
1232 _takeable: bool = False
1233 _valid_types = (
1234 "labels (MUST BE IN THE INDEX), slices of labels (BOTH "
1235 "endpoints included! Can be slices of integers if the "
1236 "index is integers), listlike of labels, boolean"
1237 )
1239 # -------------------------------------------------------------------
1240 # Key Checks
1242 @doc(_LocationIndexer._validate_key)
1243 def _validate_key(self, key, axis: Axis) -> None:
1244 # valid for a collection of labels (we check their presence later)
1245 # slice of labels (where start-end in labels)
1246 # slice of integers (only if in the labels)
1247 # boolean not in slice and with boolean index
1248 ax = self.obj._get_axis(axis)
1249 if isinstance(key, bool) and not (
1250 is_bool_dtype(ax.dtype)
1251 or ax.dtype.name == "boolean"
1252 or (
1253 isinstance(ax, MultiIndex)
1254 and is_bool_dtype(ax.get_level_values(0).dtype)
1255 )
1256 ):
1257 raise KeyError(
1258 f"{key}: boolean label can not be used without a boolean index"
1259 )
1261 if isinstance(key, slice) and (
1262 isinstance(key.start, bool) or isinstance(key.stop, bool)
1263 ):
1264 raise TypeError(f"{key}: boolean values can not be used in a slice")
1266 def _has_valid_setitem_indexer(self, indexer) -> bool:
1267 return True
1269 def _is_scalar_access(self, key: tuple) -> bool:
1270 """
1271 Returns
1272 -------
1273 bool
1274 """
1275 # this is a shortcut accessor to both .loc and .iloc
1276 # that provide the equivalent access of .at and .iat
1277 # a) avoid getting things via sections and (to minimize dtype changes)
1278 # b) provide a performant path
1279 if len(key) != self.ndim:
1280 return False
1282 for i, k in enumerate(key):
1283 if not is_scalar(k):
1284 return False
1286 ax = self.obj.axes[i]
1287 if isinstance(ax, MultiIndex):
1288 return False
1290 if isinstance(k, str) and ax._supports_partial_string_indexing:
1291 # partial string indexing, df.loc['2000', 'A']
1292 # should not be considered scalar
1293 return False
1295 if not ax._index_as_unique:
1296 return False
1298 return True
1300 # -------------------------------------------------------------------
1301 # MultiIndex Handling
1303 def _multi_take_opportunity(self, tup: tuple) -> bool:
1304 """
1305 Check whether there is the possibility to use ``_multi_take``.
1307 Currently the limit is that all axes being indexed, must be indexed with
1308 list-likes.
1310 Parameters
1311 ----------
1312 tup : tuple
1313 Tuple of indexers, one per axis.
1315 Returns
1316 -------
1317 bool
1318 Whether the current indexing,
1319 can be passed through `_multi_take`.
1320 """
1321 if not all(is_list_like_indexer(x) for x in tup):
1322 return False
1324 # just too complicated
1325 return not any(com.is_bool_indexer(x) for x in tup)
1327 def _multi_take(self, tup: tuple):
1328 """
1329 Create the indexers for the passed tuple of keys, and
1330 executes the take operation. This allows the take operation to be
1331 executed all at once, rather than once for each dimension.
1332 Improving efficiency.
1334 Parameters
1335 ----------
1336 tup : tuple
1337 Tuple of indexers, one per axis.
1339 Returns
1340 -------
1341 values: same type as the object being indexed
1342 """
1343 # GH 836
1344 d = {
1345 axis: self._get_listlike_indexer(key, axis)
1346 for (key, axis) in zip(tup, self.obj._AXIS_ORDERS, strict=True)
1347 }
1348 return self.obj._reindex_with_indexers(d, allow_dups=True)
1350 # -------------------------------------------------------------------
1352 def _getitem_iterable(self, key, axis: AxisInt):
1353 """
1354 Index current object with an iterable collection of keys.
1356 Parameters
1357 ----------
1358 key : iterable
1359 Targeted labels.
1360 axis : int
1361 Dimension on which the indexing is being made.
1363 Raises
1364 ------
1365 KeyError
1366 If no key was found. Will change in the future to raise if not all
1367 keys were found.
1369 Returns
1370 -------
1371 scalar, DataFrame, or Series: indexed value(s).
1372 """
1373 # we assume that not com.is_bool_indexer(key), as that is
1374 # handled before we get here.
1375 self._validate_key(key, axis)
1377 # A collection of keys
1378 keyarr, indexer = self._get_listlike_indexer(key, axis)
1379 return self.obj._reindex_with_indexers(
1380 {axis: [keyarr, indexer]}, allow_dups=True
1381 )
1383 def _getitem_tuple(self, tup: tuple):
1384 with suppress(IndexingError):
1385 tup = self._expand_ellipsis(tup)
1386 return self._getitem_lowerdim(tup)
1388 # no multi-index, so validate all of the indexers
1389 tup = self._validate_tuple_indexer(tup)
1391 # ugly hack for GH #836
1392 if self._multi_take_opportunity(tup):
1393 return self._multi_take(tup)
1395 return self._getitem_tuple_same_dim(tup)
1397 def _get_label(self, label, axis: AxisInt):
1398 # GH#5567 this will fail if the label is not present in the axis.
1399 return self.obj.xs(label, axis=axis)
1401 def _handle_lowerdim_multi_index_axis0(self, tup: tuple):
1402 # we have an axis0 multi-index, handle or raise
1403 axis = self.axis or 0
1404 try:
1405 # fast path for series or for tup devoid of slices
1406 return self._get_label(tup, axis=axis)
1408 except KeyError as ek:
1409 # raise KeyError if number of indexers match
1410 # else IndexingError will be raised
1411 if self.ndim < len(tup) <= self.obj.index.nlevels:
1412 raise ek
1413 raise IndexingError("No label returned") from ek
1415 def _getitem_axis(self, key, axis: AxisInt):
1416 key = item_from_zerodim(key)
1417 if is_iterator(key):
1418 key = list(key)
1419 if key is Ellipsis:
1420 key = slice(None)
1422 labels = self.obj._get_axis(axis)
1424 if isinstance(key, tuple) and isinstance(labels, MultiIndex):
1425 key = tuple(key)
1427 if isinstance(key, slice):
1428 self._validate_key(key, axis)
1429 return self._get_slice_axis(key, axis=axis)
1430 elif com.is_bool_indexer(key):
1431 return self._getbool_axis(key, axis=axis)
1432 elif is_list_like_indexer(key):
1433 # an iterable multi-selection
1434 if not (isinstance(key, tuple) and isinstance(labels, MultiIndex)):
1435 if hasattr(key, "ndim") and key.ndim > 1:
1436 raise ValueError("Cannot index with multidimensional key")
1438 return self._getitem_iterable(key, axis=axis)
1440 # nested tuple slicing
1441 if is_nested_tuple(key, labels):
1442 locs = labels.get_locs(key)
1443 indexer: list[slice | npt.NDArray[np.intp]] = [slice(None)] * self.ndim
1444 indexer[axis] = locs
1445 return self.obj.iloc[tuple(indexer)]
1447 # fall thru to straight lookup
1448 self._validate_key(key, axis)
1449 return self._get_label(key, axis=axis)
1451 def _get_slice_axis(self, slice_obj: slice, axis: AxisInt):
1452 """
1453 This is pretty simple as we just have to deal with labels.
1454 """
1455 # caller is responsible for ensuring non-None axis
1456 obj = self.obj
1457 if not need_slice(slice_obj):
1458 return obj.copy(deep=False)
1460 labels = obj._get_axis(axis)
1461 indexer = labels.slice_indexer(slice_obj.start, slice_obj.stop, slice_obj.step)
1463 if isinstance(indexer, slice):
1464 return self.obj._slice(indexer, axis=axis)
1465 else:
1466 # DatetimeIndex overrides Index.slice_indexer and may
1467 # return a DatetimeIndex instead of a slice object.
1468 return self.obj.take(indexer, axis=axis)
1470 def _convert_to_indexer(self, key, axis: AxisInt):
1471 """
1472 Convert indexing key into something we can use to do actual fancy
1473 indexing on an ndarray.
1475 Examples
1476 ix[:5] -> slice(0, 5)
1477 ix[[1,2,3]] -> [1,2,3]
1478 ix[['foo', 'bar', 'baz']] -> [i, j, k] (indices of foo, bar, baz)
1480 Going by Zen of Python?
1481 'In the face of ambiguity, refuse the temptation to guess.'
1482 raise AmbiguousIndexError with integer labels?
1483 - No, prefer label-based indexing
1484 """
1485 labels = self.obj._get_axis(axis)
1487 if isinstance(key, slice):
1488 return labels._convert_slice_indexer(key, kind="loc")
1490 if (
1491 isinstance(key, tuple)
1492 and not isinstance(labels, MultiIndex)
1493 and self.ndim < 2
1494 and len(key) > 1
1495 ):
1496 raise IndexingError("Too many indexers")
1498 # Slices are not valid keys passed in by the user,
1499 # even though they are hashable in Python 3.12
1500 contains_slice = False
1501 if isinstance(key, tuple):
1502 contains_slice = any(isinstance(v, slice) for v in key)
1504 if is_scalar(key) or (
1505 isinstance(labels, MultiIndex) and is_hashable(key) and not contains_slice
1506 ):
1507 # Otherwise get_loc will raise InvalidIndexError
1509 # if we are a label return me
1510 try:
1511 return labels.get_loc(key)
1512 except LookupError:
1513 if isinstance(key, tuple) and isinstance(labels, MultiIndex):
1514 if len(key) == labels.nlevels:
1515 return {"key": key}
1516 raise
1517 except InvalidIndexError:
1518 # GH35015, using datetime as column indices raises exception
1519 if not isinstance(labels, MultiIndex):
1520 raise
1521 except ValueError:
1522 if not is_integer(key):
1523 raise
1524 return {"key": key}
1526 if is_nested_tuple(key, labels):
1527 if self.ndim == 1 and any(isinstance(k, tuple) for k in key):
1528 # GH#35349 Raise if tuple in tuple for series
1529 raise IndexingError("Too many indexers")
1530 return labels.get_locs(key)
1532 elif is_list_like_indexer(key):
1533 if is_iterator(key):
1534 key = list(key)
1536 if com.is_bool_indexer(key):
1537 key = check_bool_indexer(labels, key)
1538 return key
1539 else:
1540 return self._get_listlike_indexer(key, axis)[1]
1541 else:
1542 try:
1543 return labels.get_loc(key)
1544 except LookupError:
1545 # allow a not found key only if we are a setter
1546 if not is_list_like_indexer(key):
1547 return {"key": key}
1548 raise
1550 def _get_listlike_indexer(self, key, axis: AxisInt):
1551 """
1552 Transform a list-like of keys into a new index and an indexer.
1554 Parameters
1555 ----------
1556 key : list-like
1557 Targeted labels.
1558 axis: int
1559 Dimension on which the indexing is being made.
1561 Raises
1562 ------
1563 KeyError
1564 If at least one key was requested but none was found.
1566 Returns
1567 -------
1568 keyarr: Index
1569 New index (coinciding with 'key' if the axis is unique).
1570 values : array-like
1571 Indexer for the return object, -1 denotes keys not found.
1572 """
1573 ax = self.obj._get_axis(axis)
1574 axis_name = self.obj._get_axis_name(axis)
1576 keyarr, indexer = ax._get_indexer_strict(key, axis_name)
1578 return keyarr, indexer
1581@doc(IndexingMixin.iloc)
1582class _iLocIndexer(_LocationIndexer):
1583 _valid_types = (
1584 "integer, integer slice (START point is INCLUDED, END "
1585 "point is EXCLUDED), listlike of integers, boolean array"
1586 )
1587 _takeable = True
1589 # -------------------------------------------------------------------
1590 # Key Checks
1592 def _validate_key(self, key, axis: AxisInt) -> None:
1593 if com.is_bool_indexer(key):
1594 if hasattr(key, "index") and isinstance(key.index, Index):
1595 if key.index.inferred_type == "integer":
1596 return
1597 raise ValueError(
1598 "iLocation based boolean indexing cannot use an indexable as a mask"
1599 )
1600 return
1602 if isinstance(key, slice):
1603 return
1604 elif is_integer(key):
1605 self._validate_integer(key, axis)
1606 elif isinstance(key, tuple):
1607 # a tuple should already have been caught by this point
1608 # so don't treat a tuple as a valid indexer
1609 raise IndexingError("Too many indexers")
1610 elif is_list_like_indexer(key):
1611 if isinstance(key, ABCSeries):
1612 arr = key._values
1613 elif is_array_like(key):
1614 arr = key
1615 else:
1616 arr = np.array(key)
1617 len_axis = len(self.obj._get_axis(axis))
1619 # check that the key has a numeric dtype
1620 if not is_numeric_dtype(arr.dtype):
1621 raise IndexError(f".iloc requires numeric indexers, got {arr}")
1623 if len(arr):
1624 if isinstance(arr.dtype, ExtensionDtype):
1625 arr_max = arr._reduce("max")
1626 arr_min = arr._reduce("min")
1627 else:
1628 arr_max = np.max(arr)
1629 arr_min = np.min(arr)
1631 # check that the key does not exceed the maximum size
1632 if arr_max >= len_axis or arr_min < -len_axis:
1633 raise IndexError("positional indexers are out-of-bounds")
1634 else:
1635 raise ValueError(f"Can only index by location with a [{self._valid_types}]")
1637 def _has_valid_setitem_indexer(self, indexer) -> bool:
1638 """
1639 Validate that a positional indexer cannot enlarge its target
1640 will raise if needed, does not modify the indexer externally.
1642 Returns
1643 -------
1644 bool
1645 """
1646 if isinstance(indexer, dict):
1647 raise IndexError("iloc cannot enlarge its target object")
1649 if isinstance(indexer, ABCDataFrame):
1650 raise TypeError(
1651 "DataFrame indexer for .iloc is not supported. "
1652 "Consider using .loc with a DataFrame indexer for automatic alignment.",
1653 )
1655 if not isinstance(indexer, tuple):
1656 indexer = _tuplify(self.ndim, indexer)
1658 for ax, i in zip(self.obj.axes, indexer, strict=False):
1659 if isinstance(i, slice):
1660 # should check the stop slice?
1661 pass
1662 elif is_list_like_indexer(i):
1663 # should check the elements?
1664 pass
1665 elif is_integer(i):
1666 if i >= len(ax):
1667 raise IndexError("iloc cannot enlarge its target object")
1668 elif isinstance(i, dict):
1669 raise IndexError("iloc cannot enlarge its target object")
1671 return True
1673 def _is_scalar_access(self, key: tuple) -> bool:
1674 """
1675 Returns
1676 -------
1677 bool
1678 """
1679 # this is a shortcut accessor to both .loc and .iloc
1680 # that provide the equivalent access of .at and .iat
1681 # a) avoid getting things via sections and (to minimize dtype changes)
1682 # b) provide a performant path
1683 if len(key) != self.ndim:
1684 return False
1686 return all(is_integer(k) for k in key)
1688 def _validate_integer(self, key: int | np.integer, axis: AxisInt) -> None:
1689 """
1690 Check that 'key' is a valid position in the desired axis.
1692 Parameters
1693 ----------
1694 key : int
1695 Requested position.
1696 axis : int
1697 Desired axis.
1699 Raises
1700 ------
1701 IndexError
1702 If 'key' is not a valid position in axis 'axis'.
1703 """
1704 len_axis = len(self.obj._get_axis(axis))
1705 if key >= len_axis or key < -len_axis:
1706 raise IndexError("single positional indexer is out-of-bounds")
1708 # -------------------------------------------------------------------
1710 def _getitem_tuple(self, tup: tuple):
1711 tup = self._validate_tuple_indexer(tup)
1712 with suppress(IndexingError):
1713 return self._getitem_lowerdim(tup)
1715 return self._getitem_tuple_same_dim(tup)
1717 def _get_list_axis(self, key, axis: AxisInt):
1718 """
1719 Return Series values by list or array of integers.
1721 Parameters
1722 ----------
1723 key : list-like positional indexer
1724 axis : int
1726 Returns
1727 -------
1728 Series object
1730 Notes
1731 -----
1732 `axis` can only be zero.
1733 """
1734 try:
1735 return self.obj.take(key, axis=axis)
1736 except IndexError as err:
1737 # re-raise with different error message, e.g. test_getitem_ndarray_3d
1738 raise IndexError("positional indexers are out-of-bounds") from err
1740 def _getitem_axis(self, key, axis: AxisInt):
1741 if key is Ellipsis:
1742 key = slice(None)
1743 elif isinstance(key, ABCDataFrame):
1744 raise IndexError(
1745 "DataFrame indexer is not allowed for .iloc\n"
1746 "Consider using .loc for automatic alignment."
1747 )
1749 if isinstance(key, slice):
1750 return self._get_slice_axis(key, axis=axis)
1752 if is_iterator(key):
1753 key = list(key)
1755 if isinstance(key, list):
1756 key = np.asarray(key)
1758 if com.is_bool_indexer(key):
1759 self._validate_key(key, axis)
1760 return self._getbool_axis(key, axis=axis)
1762 # a list of integers
1763 elif is_list_like_indexer(key):
1764 return self._get_list_axis(key, axis=axis)
1766 # a single integer
1767 else:
1768 key = item_from_zerodim(key)
1769 if not is_integer(key):
1770 raise TypeError("Cannot index by location index with a non-integer key")
1772 # validate the location
1773 self._validate_integer(key, axis)
1775 return self.obj._ixs(key, axis=axis)
1777 def _get_slice_axis(self, slice_obj: slice, axis: AxisInt):
1778 # caller is responsible for ensuring non-None axis
1779 obj = self.obj
1781 if not need_slice(slice_obj):
1782 return obj.copy(deep=False)
1784 labels = obj._get_axis(axis)
1785 labels._validate_positional_slice(slice_obj)
1786 return self.obj._slice(slice_obj, axis=axis)
1788 def _convert_to_indexer(self, key: T, axis: AxisInt) -> T:
1789 """
1790 Much simpler as we only have to deal with our valid types.
1791 """
1792 return key
1794 def _get_setitem_indexer(self, key):
1795 # GH#32257 Fall through to let numpy do validation
1796 if is_iterator(key):
1797 key = list(key)
1799 if self.axis is not None:
1800 key = _tupleize_axis_indexer(self.ndim, self.axis, key)
1802 return key
1804 # -------------------------------------------------------------------
1806 def _decide_split_path(self, indexer, value) -> bool:
1807 """
1808 Decide whether we will take a block-by-block path.
1809 """
1810 take_split_path = not self.obj._mgr.is_single_block
1812 if not take_split_path and isinstance(value, ABCDataFrame):
1813 # Avoid cast of values
1814 take_split_path = not value._mgr.is_single_block
1816 # if there is only one block/type, still have to take split path
1817 # unless the block is one-dimensional or it can hold the value
1818 if not take_split_path and len(self.obj._mgr.blocks) and self.ndim > 1:
1819 # in case of dict, keys are indices
1820 val = list(value.values()) if isinstance(value, dict) else value
1821 arr = self.obj._mgr.blocks[0].values
1822 take_split_path = not can_hold_element(
1823 arr, extract_array(val, extract_numpy=True)
1824 )
1826 # if we have any multi-indexes that have non-trivial slices
1827 # (not null slices) then we must take the split path, xref
1828 # GH 10360, GH 27841
1829 if isinstance(indexer, tuple) and len(indexer) == len(self.obj.axes):
1830 for i, ax in zip(indexer, self.obj.axes, strict=True):
1831 if isinstance(ax, MultiIndex) and not (
1832 is_integer(i) or com.is_null_slice(i)
1833 ):
1834 take_split_path = True
1835 break
1837 return take_split_path
1839 def _setitem_new_column(self, indexer, key, value, name: str) -> None:
1840 """
1841 _setitem_with_indexer cases that can go through DataFrame.__setitem__.
1842 """
1843 # add the new item, and set the value
1844 # must have all defined axes if we have a scalar
1845 # or a list-like on the non-info axes if we have a
1846 # list-like
1847 if not len(self.obj):
1848 if not is_list_like_indexer(value):
1849 raise ValueError(
1850 "cannot set a frame with no defined index and a scalar"
1851 )
1852 self.obj[key] = value
1853 return
1855 # add a new item with the dtype setup
1856 if com.is_null_slice(indexer[0]):
1857 # We are setting an entire column
1858 self.obj[key] = value
1859 return
1860 elif is_array_like(value):
1861 # GH#42099
1862 arr = extract_array(value, extract_numpy=True)
1863 taker = -1 * np.ones(len(self.obj), dtype=np.intp)
1864 empty_value = algos.take_nd(arr, taker)
1865 if not isinstance(value, ABCSeries):
1866 # if not Series (in which case we need to align),
1867 # we can short-circuit
1868 if isinstance(arr, np.ndarray) and arr.ndim == 1 and len(arr) == 1:
1869 # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615
1870 arr = arr[0, ...]
1871 empty_value[indexer[0]] = arr
1872 self.obj[key] = empty_value
1873 return
1875 self.obj[key] = empty_value
1876 elif not is_list_like(value):
1877 self.obj[key] = construct_1d_array_from_inferred_fill_value(
1878 value, len(self.obj)
1879 )
1880 else:
1881 # FIXME: GH#42099#issuecomment-864326014
1882 self.obj[key] = infer_fill_value(value)
1884 new_indexer = convert_from_missing_indexer_tuple(indexer, self.obj.axes)
1885 self._setitem_with_indexer(new_indexer, value, name)
1887 return
1889 def _setitem_with_indexer(self, indexer, value, name: str = "iloc") -> None:
1890 """
1891 _setitem_with_indexer is for setting values on a Series/DataFrame
1892 using positional indexers.
1894 If the relevant keys are not present, the Series/DataFrame may be
1895 expanded.
1896 """
1897 info_axis = self.obj._info_axis_number
1898 take_split_path = self._decide_split_path(indexer, value)
1900 if isinstance(indexer, tuple):
1901 nindexer = []
1902 for i, idx in enumerate(indexer):
1903 idx, missing = convert_missing_indexer(idx)
1904 if missing:
1905 # reindex the axis to the new value
1906 # and set inplace
1907 key = idx
1909 # if this is the items axes, then take the main missing
1910 # path first
1911 # this correctly sets the dtype
1912 # essentially this separates out the block that is needed
1913 # to possibly be modified
1914 if self.ndim > 1 and i == info_axis:
1915 self._setitem_new_column(indexer, key, value, name=name)
1916 return
1918 # reindex the axis
1919 index = self.obj._get_axis(i)
1920 labels = index.insert(len(index), key)
1922 # We are expanding the Series/DataFrame values to match
1923 # the length of the new index `labels`. GH#40096 ensure
1924 # this is valid even if the index has duplicates.
1925 taker = np.arange(len(index) + 1, dtype=np.intp)
1926 taker[-1] = -1
1927 reindexers = {i: (labels, taker)}
1928 new_obj = self.obj._reindex_with_indexers(
1929 reindexers, allow_dups=True
1930 )
1931 self.obj._mgr = new_obj._mgr
1933 nindexer.append(labels.get_loc(key))
1935 else:
1936 nindexer.append(idx)
1938 indexer = tuple(nindexer)
1939 else:
1940 indexer, missing = convert_missing_indexer(indexer)
1942 if missing:
1943 self._setitem_with_indexer_missing(indexer, value)
1944 return
1946 if name == "loc":
1947 # must come after setting of missing
1948 indexer, value = self._maybe_mask_setitem_value(indexer, value)
1950 # align and set the values
1951 if take_split_path:
1952 # We have to operate column-wise
1953 self._setitem_with_indexer_split_path(indexer, value, name)
1954 else:
1955 self._setitem_single_block(indexer, value, name)
1957 def _setitem_with_indexer_split_path(self, indexer, value, name: str):
1958 """
1959 Setitem column-wise.
1960 """
1961 # Above we only set take_split_path to True for 2D cases
1962 assert self.ndim == 2
1964 if not isinstance(indexer, tuple):
1965 indexer = _tuplify(self.ndim, indexer)
1966 if len(indexer) > self.ndim:
1967 raise IndexError("too many indices for array")
1968 if isinstance(indexer[0], np.ndarray) and indexer[0].ndim > 2:
1969 raise ValueError(r"Cannot set values with ndim > 2")
1971 if (isinstance(value, ABCSeries) and name != "iloc") or isinstance(value, dict):
1972 from pandas import Series
1974 value = self._align_series(indexer, Series(value))
1976 # Ensure we have something we can iterate over
1977 info_axis = indexer[1]
1978 ilocs = self._ensure_iterable_column_indexer(info_axis)
1980 pi = indexer[0]
1981 lplane_indexer = length_of_indexer(pi, self.obj.index)
1982 # lplane_indexer gives the expected length of obj[indexer[0]]
1984 # we need an iterable, with an ndim of at least 1
1985 # eg. don't pass through np.array(0)
1986 if is_list_like_indexer(value) and getattr(value, "ndim", 1) > 0:
1987 if isinstance(value, ABCDataFrame):
1988 self._setitem_with_indexer_frame_value(indexer, value, name)
1990 elif np.ndim(value) == 2:
1991 # TODO: avoid np.ndim call in case it isn't an ndarray, since
1992 # that will construct an ndarray, which will be wasteful
1993 self._setitem_with_indexer_2d_value(indexer, value)
1995 elif len(ilocs) == 1 and lplane_indexer == len(value) and not is_scalar(pi):
1996 # We are setting multiple rows in a single column.
1997 self._setitem_single_column(ilocs[0], value, pi)
1999 elif len(ilocs) == 1 and 0 != lplane_indexer != len(value):
2000 # We are trying to set N values into M entries of a single
2001 # column, which is invalid for N != M
2002 # Exclude zero-len for e.g. boolean masking that is all-false
2004 if len(value) == 1 and not is_integer(info_axis):
2005 # This is a case like df.iloc[:3, [1]] = [0]
2006 # where we treat as df.iloc[:3, 1] = 0
2007 return self._setitem_with_indexer((pi, info_axis[0]), value[0])
2009 raise ValueError(
2010 "Must have equal len keys and value when setting with an iterable"
2011 )
2013 elif lplane_indexer == 0 and len(value) == len(self.obj.index):
2014 # We get here in one case via .loc with an all-False mask
2015 pass
2017 elif self._is_scalar_access(indexer) and is_object_dtype(
2018 self.obj.dtypes._values[ilocs[0]]
2019 ):
2020 # We are setting nested data, only possible for object dtype data
2021 self._setitem_single_column(indexer[1], value, pi)
2023 elif len(ilocs) == len(value):
2024 # We are setting multiple columns in a single row.
2025 for loc, v in zip(ilocs, value, strict=True):
2026 self._setitem_single_column(loc, v, pi)
2028 elif len(ilocs) == 1 and com.is_null_slice(pi) and len(self.obj) == 0:
2029 # This is a setitem-with-expansion, see
2030 # test_loc_setitem_empty_append_expands_rows_mixed_dtype
2031 # e.g. df = DataFrame(columns=["x", "y"])
2032 # df["x"] = df["x"].astype(np.int64)
2033 # df.loc[:, "x"] = [1, 2, 3]
2034 self._setitem_single_column(ilocs[0], value, pi)
2036 else:
2037 raise ValueError(
2038 "Must have equal len keys and value when setting with an iterable"
2039 )
2041 else:
2042 # scalar value
2043 for loc in ilocs:
2044 self._setitem_single_column(loc, value, pi)
2046 def _setitem_with_indexer_2d_value(self, indexer, value) -> None:
2047 # We get here with np.ndim(value) == 2, excluding DataFrame,
2048 # which goes through _setitem_with_indexer_frame_value
2049 pi = indexer[0]
2051 ilocs = self._ensure_iterable_column_indexer(indexer[1])
2053 if not is_array_like(value):
2054 # cast lists to array
2055 value = np.array(value, dtype=object)
2056 if len(ilocs) != value.shape[1]:
2057 raise ValueError(
2058 "Must have equal len keys and value when setting with an ndarray"
2059 )
2061 for i, loc in enumerate(ilocs):
2062 value_col = value[:, i]
2063 if is_object_dtype(value_col.dtype):
2064 # casting to list so that we do type inference in setitem_single_column
2065 value_col = value_col.tolist()
2066 self._setitem_single_column(loc, value_col, pi)
2068 def _setitem_with_indexer_frame_value(
2069 self, indexer, value: DataFrame, name: str
2070 ) -> None:
2071 ilocs = self._ensure_iterable_column_indexer(indexer[1])
2073 sub_indexer = list(indexer)
2074 pi = indexer[0]
2076 multiindex_indexer = isinstance(self.obj.columns, MultiIndex)
2078 unique_cols = value.columns.is_unique
2080 # We do not want to align the value in case of iloc GH#37728
2081 if name == "iloc":
2082 for i, loc in enumerate(ilocs):
2083 val = value.iloc[:, i]
2084 self._setitem_single_column(loc, val, pi)
2086 elif not unique_cols and value.columns.equals(self.obj.columns):
2087 # We assume we are already aligned, see
2088 # test_iloc_setitem_frame_duplicate_columns_multiple_blocks
2089 for loc in ilocs:
2090 item = self.obj.columns[loc]
2091 if item in value:
2092 sub_indexer[1] = item
2093 val = self._align_series(
2094 tuple(sub_indexer),
2095 value.iloc[:, loc],
2096 multiindex_indexer,
2097 )
2098 else:
2099 val = np.nan
2101 self._setitem_single_column(loc, val, pi)
2103 elif not unique_cols:
2104 raise ValueError("Setting with non-unique columns is not allowed.")
2106 else:
2107 for loc in ilocs:
2108 item = self.obj.columns[loc]
2109 if item in value:
2110 sub_indexer[1] = item
2111 val = self._align_series(
2112 tuple(sub_indexer),
2113 value[item],
2114 multiindex_indexer,
2115 using_cow=True,
2116 )
2117 else:
2118 val = np.nan
2120 self._setitem_single_column(loc, val, pi)
2122 def _setitem_single_column(self, loc: int, value, plane_indexer) -> None:
2123 """
2125 Parameters
2126 ----------
2127 loc : int
2128 Indexer for column position
2129 plane_indexer : int, slice, listlike[int]
2130 The indexer we use for setitem along axis=0.
2131 """
2132 pi = plane_indexer
2134 is_full_setter = com.is_null_slice(pi) or com.is_full_slice(pi, len(self.obj))
2136 is_null_setter = com.is_empty_slice(pi) or (is_array_like(pi) and len(pi) == 0)
2138 if is_null_setter:
2139 # no-op, don't cast dtype later
2140 return
2142 elif is_full_setter:
2143 try:
2144 self.obj._mgr.column_setitem(
2145 loc, plane_indexer, value, inplace_only=True
2146 )
2147 except (ValueError, TypeError, LossySetitemError) as exc:
2148 # If we're setting an entire column and we can't do it inplace,
2149 # then we can use value's dtype (or inferred dtype)
2150 # instead of object
2151 dtype = self.obj.dtypes.iloc[loc]
2152 if dtype not in (np.void, object) and not self.obj.empty:
2153 # - Exclude np.void, as that is a special case for expansion.
2154 # We want to raise for
2155 # df = pd.DataFrame({'a': [1, 2]})
2156 # df.loc[:, 'a'] = .3
2157 # but not for
2158 # df = pd.DataFrame({'a': [1, 2]})
2159 # df.loc[:, 'b'] = .3
2160 # - Exclude `object`, as then no upcasting happens.
2161 # - Exclude empty initial object with enlargement,
2162 # as then there's nothing to be inconsistent with.
2163 raise TypeError(
2164 f"Invalid value '{value}' for dtype '{dtype}'"
2165 ) from exc
2166 self.obj.isetitem(loc, value)
2167 else:
2168 # set value into the column (first attempting to operate inplace, then
2169 # falling back to casting if necessary)
2170 dtype = self.obj.dtypes.iloc[loc]
2171 if dtype == np.void:
2172 # This means we're expanding, with multiple columns, e.g.
2173 # df = pd.DataFrame({'A': [1,2,3], 'B': [4,5,6]})
2174 # df.loc[df.index <= 2, ['F', 'G']] = (1, 'abc')
2175 # Columns F and G will initially be set to np.void.
2176 # Here, we replace those temporary `np.void` columns with
2177 # columns of the appropriate dtype, based on `value`.
2178 self.obj.iloc[:, loc] = construct_1d_array_from_inferred_fill_value(
2179 value, len(self.obj)
2180 )
2181 self.obj._mgr.column_setitem(loc, plane_indexer, value)
2183 def _setitem_single_block(self, indexer, value, name: str) -> None:
2184 """
2185 _setitem_with_indexer for the case when we have a single Block.
2186 """
2187 from pandas import Series
2189 if (isinstance(value, ABCSeries) and name != "iloc") or isinstance(value, dict):
2190 # TODO(EA): ExtensionBlock.setitem this causes issues with
2191 # setting for extensionarrays that store dicts. Need to decide
2192 # if it's worth supporting that.
2193 value = self._align_series(indexer, Series(value))
2195 info_axis = self.obj._info_axis_number
2196 item_labels = self.obj._get_axis(info_axis)
2197 if isinstance(indexer, tuple):
2198 # if we are setting on the info axis ONLY
2199 # set using those methods to avoid block-splitting
2200 # logic here
2201 if (
2202 self.ndim == len(indexer) == 2
2203 and is_integer(indexer[1])
2204 and com.is_null_slice(indexer[0])
2205 ):
2206 col = item_labels[indexer[info_axis]]
2207 if len(item_labels.get_indexer_for([col])) == 1:
2208 # e.g. test_loc_setitem_empty_append_expands_rows
2209 loc = item_labels.get_loc(col)
2210 self._setitem_single_column(loc, value, indexer[0])
2211 return
2213 indexer = maybe_convert_ix(*indexer) # e.g. test_setitem_frame_align
2215 if isinstance(value, ABCDataFrame) and name != "iloc":
2216 value = self._align_frame(indexer, value)._values
2218 # actually do the set
2219 self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
2221 def _setitem_with_indexer_missing(self, indexer, value):
2222 """
2223 Insert new row(s) or column(s) into the Series or DataFrame.
2224 """
2225 from pandas import Series
2227 # reindex the axis to the new value
2228 # and set inplace
2229 if self.ndim == 1:
2230 index = self.obj.index
2231 new_index = index.insert(len(index), indexer)
2233 # we have a coerced indexer, e.g. a float
2234 # that matches in an int64 Index, so
2235 # we will not create a duplicate index, rather
2236 # index to that element
2237 # e.g. 0.0 -> 0
2238 # GH#12246
2239 if index.is_unique:
2240 # pass new_index[-1:] instead if [new_index[-1]]
2241 # so that we retain dtype
2242 new_indexer = index.get_indexer(new_index[-1:])
2243 if (new_indexer != -1).any():
2244 # We get only here with loc, so can hard code
2245 return self._setitem_with_indexer(new_indexer, value, "loc")
2247 # this preserves dtype of the value and of the object
2248 if not is_scalar(value):
2249 new_dtype = None
2251 elif is_valid_na_for_dtype(value, self.obj.dtype):
2252 if not is_object_dtype(self.obj.dtype):
2253 # Every NA value is suitable for object, no conversion needed
2254 value = na_value_for_dtype(self.obj.dtype, compat=False)
2256 new_dtype = maybe_promote(self.obj.dtype, value)[0]
2258 elif isna(value):
2259 new_dtype = None
2260 elif not self.obj.empty and not is_object_dtype(self.obj.dtype):
2261 # We should not cast, if we have object dtype because we can
2262 # set timedeltas into object series
2263 curr_dtype = self.obj.dtype
2264 curr_dtype = getattr(curr_dtype, "numpy_dtype", curr_dtype)
2265 new_dtype = maybe_promote(curr_dtype, value)[0]
2266 else:
2267 new_dtype = None
2269 new_values = Series([value], dtype=new_dtype)._values
2271 if len(self.obj._values):
2272 # GH#22717 handle casting compatibility that np.concatenate
2273 # does incorrectly
2274 new_values = concat_compat([self.obj._values, new_values])
2275 self.obj._mgr = self.obj._constructor(
2276 new_values, index=new_index, name=self.obj.name
2277 )._mgr
2279 elif self.ndim == 2:
2280 if not len(self.obj.columns):
2281 # no columns and scalar
2282 raise ValueError("cannot set a frame with no defined columns")
2284 has_dtype = hasattr(value, "dtype")
2285 if isinstance(value, ABCSeries):
2286 # append a Series
2287 value = value.reindex(index=self.obj.columns)
2288 value.name = indexer
2289 elif isinstance(value, dict):
2290 value = Series(
2291 value, index=self.obj.columns, name=indexer, dtype=object
2292 )
2293 else:
2294 # a list-list
2295 if is_list_like_indexer(value):
2296 # must have conforming columns
2297 if len(value) != len(self.obj.columns):
2298 raise ValueError("cannot set a row with mismatched columns")
2300 value = Series(value, index=self.obj.columns, name=indexer)
2302 if not len(self.obj):
2303 # We will ignore the existing dtypes instead of using
2304 # internals.concat logic
2305 df = value.to_frame().T
2307 idx = self.obj.index
2308 if isinstance(idx, MultiIndex):
2309 name = idx.names
2310 else:
2311 name = idx.name
2313 df.index = Index([indexer], name=name)
2314 if not has_dtype:
2315 # i.e. if we already had a Series or ndarray, keep that
2316 # dtype. But if we had a list or dict, then do inference
2317 df = df.infer_objects()
2318 self.obj._mgr = df._mgr
2319 else:
2320 self.obj._mgr = self.obj._append_internal(value)._mgr
2322 def _ensure_iterable_column_indexer(self, column_indexer):
2323 """
2324 Ensure that our column indexer is something that can be iterated over.
2325 """
2326 ilocs: Sequence[int | np.integer] | np.ndarray | range
2327 if is_integer(column_indexer):
2328 ilocs = [column_indexer]
2329 elif isinstance(column_indexer, slice):
2330 ilocs = range(len(self.obj.columns))[column_indexer]
2331 elif (
2332 isinstance(column_indexer, np.ndarray) and column_indexer.dtype.kind == "b"
2333 ):
2334 ilocs = np.arange(len(column_indexer))[column_indexer]
2335 else:
2336 ilocs = column_indexer
2337 return ilocs
2339 def _align_series(
2340 self,
2341 indexer,
2342 ser: Series,
2343 multiindex_indexer: bool = False,
2344 using_cow: bool = False,
2345 ):
2346 """
2347 Parameters
2348 ----------
2349 indexer : tuple, slice, scalar
2350 Indexer used to get the locations that will be set to `ser`.
2351 ser : pd.Series
2352 Values to assign to the locations specified by `indexer`.
2353 multiindex_indexer : bool, optional
2354 Defaults to False. Should be set to True if `indexer` was from
2355 a `pd.MultiIndex`, to avoid unnecessary broadcasting.
2357 Returns
2358 -------
2359 `np.array` of `ser` broadcast to the appropriate shape for assignment
2360 to the locations selected by `indexer`
2361 """
2362 if isinstance(indexer, (slice, np.ndarray, list, Index)):
2363 indexer = (indexer,)
2365 if isinstance(indexer, tuple):
2366 # flatten np.ndarray indexers
2367 if (
2368 len(indexer) == 2
2369 and isinstance(indexer[1], np.ndarray)
2370 and indexer[1].dtype == np.bool_
2371 ):
2372 indexer = (indexer[0], np.where(indexer[1])[0])
2374 def ravel(i):
2375 return i.ravel() if isinstance(i, np.ndarray) else i
2377 indexer = tuple(map(ravel, indexer))
2378 aligners = [not com.is_null_slice(idx) for idx in indexer]
2379 sum_aligners = sum(aligners)
2380 single_aligner = sum_aligners == 1
2381 is_frame = self.ndim == 2
2382 obj = self.obj
2384 # are we a single alignable value on a non-primary
2385 # dim (e.g. panel: 1,2, or frame: 0) ?
2386 # hence need to align to a single axis dimension
2387 # rather that find all valid dims
2389 # frame
2390 if is_frame:
2391 single_aligner = single_aligner and aligners[0]
2393 # we have a frame, with multiple indexers on both axes; and a
2394 # series, so need to broadcast (see GH5206)
2395 if all(is_sequence(_) or isinstance(_, slice) for _ in indexer):
2396 ser_values = ser.reindex(obj.axes[0][indexer[0]])._values
2398 # single indexer
2399 if len(indexer) > 1 and not multiindex_indexer:
2400 if isinstance(indexer[1], slice):
2401 len_indexer = len(obj.axes[1][indexer[1]])
2402 else:
2403 len_indexer = len(indexer[1])
2404 ser_values = (
2405 np.tile(ser_values, len_indexer).reshape(len_indexer, -1).T
2406 )
2408 return ser_values
2410 for i, idx in enumerate(indexer):
2411 ax = obj.axes[i]
2413 # multiple aligners (or null slices)
2414 if is_sequence(idx) or isinstance(idx, slice):
2415 if single_aligner and com.is_null_slice(idx):
2416 continue
2417 new_ix = ax[idx]
2418 if not is_list_like_indexer(new_ix):
2419 new_ix = Index([new_ix])
2420 else:
2421 new_ix = Index(new_ix)
2422 if not len(new_ix) or ser.index.equals(new_ix):
2423 if using_cow:
2424 return ser
2425 return ser._values.copy()
2427 return ser.reindex(new_ix)._values
2429 # 2 dims
2430 elif single_aligner:
2431 # reindex along index
2432 ax = self.obj.axes[1]
2433 if ser.index.equals(ax) or not len(ax):
2434 return ser._values.copy()
2435 return ser.reindex(ax)._values
2437 elif is_integer(indexer) and self.ndim == 1:
2438 if is_object_dtype(self.obj.dtype):
2439 return ser
2440 ax = self.obj._get_axis(0)
2442 if ser.index.equals(ax):
2443 return ser._values.copy()
2445 return ser.reindex(ax)._values[indexer]
2447 elif is_integer(indexer):
2448 ax = self.obj._get_axis(1)
2450 if ser.index.equals(ax):
2451 return ser._values.copy()
2453 return ser.reindex(ax)._values
2455 raise ValueError("Incompatible indexer with Series")
2457 def _align_frame(self, indexer, df: DataFrame) -> DataFrame:
2458 is_frame = self.ndim == 2
2460 if isinstance(indexer, tuple):
2461 idx, cols = None, None
2462 sindexers = []
2463 for i, ix in enumerate(indexer):
2464 ax = self.obj.axes[i]
2465 if is_sequence(ix) or isinstance(ix, slice):
2466 if isinstance(ix, np.ndarray):
2467 ix = ix.reshape(-1)
2468 if idx is None:
2469 idx = ax[ix]
2470 elif cols is None:
2471 cols = ax[ix]
2472 else:
2473 break
2474 else:
2475 sindexers.append(i)
2477 if idx is not None and cols is not None:
2478 if df.index.equals(idx) and df.columns.equals(cols):
2479 val = df.copy()
2480 else:
2481 val = df.reindex(idx, columns=cols)
2482 return val
2484 elif (isinstance(indexer, slice) or is_list_like_indexer(indexer)) and is_frame:
2485 ax = self.obj.index[indexer]
2486 if df.index.equals(ax):
2487 val = df.copy()
2488 else:
2489 # we have a multi-index and are trying to align
2490 # with a particular, level GH3738
2491 if (
2492 isinstance(ax, MultiIndex)
2493 and isinstance(df.index, MultiIndex)
2494 and ax.nlevels != df.index.nlevels
2495 ):
2496 raise TypeError(
2497 "cannot align on a multi-index with out "
2498 "specifying the join levels"
2499 )
2501 val = df.reindex(index=ax)
2502 return val
2504 raise ValueError("Incompatible indexer with DataFrame")
2507class _ScalarAccessIndexer(NDFrameIndexerBase):
2508 """
2509 Access scalars quickly.
2510 """
2512 # sub-classes need to set _takeable
2513 _takeable: bool
2515 def _convert_key(self, key):
2516 raise AbstractMethodError(self)
2518 def __getitem__(self, key):
2519 if not isinstance(key, tuple):
2520 # we could have a convertible item here (e.g. Timestamp)
2521 if not is_list_like_indexer(key):
2522 key = (key,)
2523 else:
2524 raise ValueError("Invalid call for scalar access (getting)!")
2526 key = self._convert_key(key)
2527 return self.obj._get_value(*key, takeable=self._takeable)
2529 def __setitem__(self, key, value) -> None:
2530 if isinstance(key, tuple):
2531 key = tuple(com.apply_if_callable(x, self.obj) for x in key)
2532 else:
2533 # scalar callable may return tuple
2534 key = com.apply_if_callable(key, self.obj)
2536 if not isinstance(key, tuple):
2537 key = _tuplify(self.ndim, key)
2538 key = list(self._convert_key(key))
2539 if len(key) != self.ndim:
2540 raise ValueError("Not enough indexers for scalar access (setting)!")
2542 self.obj._set_value(*key, value=value, takeable=self._takeable)
2545@doc(IndexingMixin.at)
2546class _AtIndexer(_ScalarAccessIndexer):
2547 _takeable = False
2549 def _convert_key(self, key):
2550 """
2551 Require they keys to be the same type as the index. (so we don't
2552 fallback)
2553 """
2554 # GH 26989
2555 # For series, unpacking key needs to result in the label.
2556 # This is already the case for len(key) == 1; e.g. (1,)
2557 if self.ndim == 1 and len(key) > 1:
2558 key = (key,)
2560 return key
2562 @property
2563 def _axes_are_unique(self) -> bool:
2564 # Only relevant for self.ndim == 2
2565 assert self.ndim == 2
2566 return self.obj.index.is_unique and self.obj.columns.is_unique
2568 def __getitem__(self, key):
2569 if self.ndim == 2 and not self._axes_are_unique:
2570 # GH#33041 fall back to .loc
2571 if not isinstance(key, tuple) or not all(is_scalar(x) for x in key):
2572 raise ValueError("Invalid call for scalar access (getting)!")
2573 return self.obj.loc[key]
2575 return super().__getitem__(key)
2577 def __setitem__(self, key, value) -> None:
2578 if not CHAINED_WARNING_DISABLED:
2579 if sys.getrefcount(self.obj) <= REF_COUNT_IDX:
2580 warnings.warn(
2581 _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
2582 )
2584 if self.ndim == 2 and not self._axes_are_unique:
2585 # GH#33041 fall back to .loc
2586 if not isinstance(key, tuple) or not all(is_scalar(x) for x in key):
2587 raise ValueError("Invalid call for scalar access (setting)!")
2589 self.obj.loc[key] = value
2590 return
2592 return super().__setitem__(key, value)
2595@doc(IndexingMixin.iat)
2596class _iAtIndexer(_ScalarAccessIndexer):
2597 _takeable = True
2599 def _convert_key(self, key):
2600 """
2601 Require integer args. (and convert to label arguments)
2602 """
2603 for i in key:
2604 if not is_integer(i):
2605 raise ValueError("iAt based indexing can only have integer indexers")
2606 return key
2608 def __setitem__(self, key, value) -> None:
2609 if not CHAINED_WARNING_DISABLED:
2610 if sys.getrefcount(self.obj) <= REF_COUNT_IDX:
2611 warnings.warn(
2612 _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
2613 )
2615 return super().__setitem__(key, value)
2618def _tuplify(ndim: int, loc: Hashable) -> tuple[Hashable | slice, ...]:
2619 """
2620 Given an indexer for the first dimension, create an equivalent tuple
2621 for indexing over all dimensions.
2623 Parameters
2624 ----------
2625 ndim : int
2626 loc : object
2628 Returns
2629 -------
2630 tuple
2631 """
2632 _tup: list[Hashable | slice]
2633 _tup = [slice(None, None) for _ in range(ndim)]
2634 _tup[0] = loc
2635 return tuple(_tup)
2638def _tupleize_axis_indexer(ndim: int, axis: AxisInt, key) -> tuple:
2639 """
2640 If we have an axis, adapt the given key to be axis-independent.
2641 """
2642 new_key = [slice(None)] * ndim
2643 new_key[axis] = key
2644 return tuple(new_key)
2647def check_bool_indexer(index: Index, key) -> np.ndarray:
2648 """
2649 Check if key is a valid boolean indexer for an object with such index and
2650 perform reindexing or conversion if needed.
2652 This function assumes that is_bool_indexer(key) == True.
2654 Parameters
2655 ----------
2656 index : Index
2657 Index of the object on which the indexing is done.
2658 key : list-like
2659 Boolean indexer to check.
2661 Returns
2662 -------
2663 np.array
2664 Resulting key.
2666 Raises
2667 ------
2668 IndexError
2669 If the key does not have the same length as index.
2670 IndexingError
2671 If the index of the key is unalignable to index.
2672 """
2673 result = key
2674 if isinstance(key, ABCSeries) and not key.index.equals(index):
2675 indexer = result.index.get_indexer_for(index)
2676 if -1 in indexer:
2677 raise IndexingError(
2678 "Unalignable boolean Series provided as "
2679 "indexer (index of the boolean Series and of "
2680 "the indexed object do not match)."
2681 )
2683 result = result.take(indexer)
2685 # fall through for boolean
2686 if not isinstance(result.dtype, ExtensionDtype):
2687 return result.astype(bool)._values
2689 if is_object_dtype(key):
2690 # key might be object-dtype bool, check_array_indexer needs bool array
2691 result = np.asarray(result, dtype=bool)
2692 elif not is_array_like(result):
2693 # GH 33924
2694 # key may contain nan elements, check_array_indexer needs bool array
2695 result = pd_array(result, dtype=bool)
2696 return check_array_indexer(index, result)
2699def convert_missing_indexer(indexer):
2700 """
2701 Reverse convert a missing indexer, which is a dict
2702 return the scalar indexer and a boolean indicating if we converted
2703 """
2704 if isinstance(indexer, dict):
2705 # a missing key (but not a tuple indexer)
2706 indexer = indexer["key"]
2708 if isinstance(indexer, bool):
2709 raise KeyError("cannot use a single bool to index into setitem")
2710 return indexer, True
2712 return indexer, False
2715def convert_from_missing_indexer_tuple(indexer: tuple, axes: list[Index]) -> tuple:
2716 """
2717 Create a filtered indexer that doesn't have any missing indexers.
2718 """
2720 def get_indexer(_i, _idx):
2721 return axes[_i].get_loc(_idx["key"]) if isinstance(_idx, dict) else _idx
2723 return tuple(get_indexer(_i, _idx) for _i, _idx in enumerate(indexer))
2726def maybe_convert_ix(*args):
2727 """
2728 We likely want to take the cross-product.
2729 """
2730 for arg in args:
2731 if not isinstance(arg, (np.ndarray, list, ABCSeries, Index)):
2732 return args
2733 return np.ix_(*args)
2736def is_nested_tuple(tup, labels) -> bool:
2737 """
2738 Returns
2739 -------
2740 bool
2741 """
2742 # check for a compatible nested tuple and multiindexes among the axes
2743 if not isinstance(tup, tuple):
2744 return False
2746 for k in tup:
2747 if is_list_like(k) or isinstance(k, slice):
2748 return isinstance(labels, MultiIndex)
2750 return False
2753def is_label_like(key) -> bool:
2754 """
2755 Returns
2756 -------
2757 bool
2758 """
2759 # select a label or row
2760 return (
2761 not isinstance(key, slice)
2762 and not is_list_like_indexer(key)
2763 and key is not Ellipsis
2764 )
2767def need_slice(obj: slice) -> bool:
2768 """
2769 Returns
2770 -------
2771 bool
2772 """
2773 return (
2774 obj.start is not None
2775 or obj.stop is not None
2776 or (obj.step is not None and obj.step != 1)
2777 )
2780def check_dict_or_set_indexers(key) -> None:
2781 """
2782 Check if the indexer is or contains a dict or set, which is no longer allowed.
2783 """
2784 if isinstance(key, set) or (
2785 isinstance(key, tuple) and any(isinstance(x, set) for x in key)
2786 ):
2787 raise TypeError(
2788 "Passing a set as an indexer is not supported. Use a list instead."
2789 )
2791 if isinstance(key, dict) or (
2792 isinstance(key, tuple) and any(isinstance(x, dict) for x in key)
2793 ):
2794 raise TypeError(
2795 "Passing a dict as an indexer is not supported. Use a list instead."
2796 )