Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Low-dependency indexing utilities.
3"""
5from __future__ import annotations
7from typing import (
8 TYPE_CHECKING,
9 Any,
10)
12import numpy as np
14from pandas._libs import lib
15from pandas.util._decorators import set_module
17from pandas.core.dtypes.common import (
18 is_array_like,
19 is_bool_dtype,
20 is_integer,
21 is_integer_dtype,
22 is_list_like,
23)
24from pandas.core.dtypes.dtypes import ExtensionDtype
25from pandas.core.dtypes.generic import (
26 ABCIndex,
27 ABCSeries,
28)
30if TYPE_CHECKING:
31 from pandas._typing import AnyArrayLike
33 from pandas.core.frame import DataFrame
34 from pandas.core.indexes.base import Index
36# -----------------------------------------------------------
37# Indexer Identification
40def is_valid_positional_slice(slc: slice) -> bool:
41 """
42 Check if a slice object can be interpreted as a positional indexer.
44 Parameters
45 ----------
46 slc : slice
48 Returns
49 -------
50 bool
52 Notes
53 -----
54 A valid positional slice may also be interpreted as a label-based slice
55 depending on the index being sliced.
56 """
57 return (
58 lib.is_int_or_none(slc.start)
59 and lib.is_int_or_none(slc.stop)
60 and lib.is_int_or_none(slc.step)
61 )
64def is_list_like_indexer(key) -> bool:
65 """
66 Check if we have a list-like indexer that is *not* a NamedTuple.
68 Parameters
69 ----------
70 key : object
72 Returns
73 -------
74 bool
75 """
76 # allow a list_like, but exclude NamedTuples which can be indexers
77 return is_list_like(key) and not (isinstance(key, tuple) and type(key) is not tuple)
80def is_scalar_indexer(indexer, ndim: int) -> bool:
81 """
82 Return True if we are all scalar indexers.
84 Parameters
85 ----------
86 indexer : object
87 ndim : int
88 Number of dimensions in the object being indexed.
90 Returns
91 -------
92 bool
93 """
94 if ndim == 1 and is_integer(indexer):
95 # GH37748: allow indexer to be an integer for Series
96 return True
97 if isinstance(indexer, tuple) and len(indexer) == ndim:
98 return all(is_integer(x) for x in indexer)
99 return False
102def is_empty_indexer(indexer) -> bool:
103 """
104 Check if we have an empty indexer.
106 Parameters
107 ----------
108 indexer : object
110 Returns
111 -------
112 bool
113 """
114 if is_list_like(indexer) and not len(indexer):
115 return True
116 if not isinstance(indexer, tuple):
117 indexer = (indexer,)
118 return any(isinstance(idx, np.ndarray) and len(idx) == 0 for idx in indexer)
121# -----------------------------------------------------------
122# Indexer Validation
125def check_setitem_lengths(indexer, value, values) -> bool:
126 """
127 Validate that value and indexer are the same length.
129 A special-case is allowed for when the indexer is a boolean array
130 and the number of true values equals the length of ``value``. In
131 this case, no exception is raised.
133 Parameters
134 ----------
135 indexer : sequence
136 Key for the setitem.
137 value : array-like
138 Value for the setitem.
139 values : array-like
140 Values being set into.
142 Returns
143 -------
144 bool
145 Whether this is an empty listlike setting which is a no-op.
147 Raises
148 ------
149 ValueError
150 When the indexer is an ndarray or list and the lengths don't match.
151 """
152 no_op = False
154 if isinstance(indexer, (np.ndarray, list)):
155 # We can ignore other listlikes because they are either
156 # a) not necessarily 1-D indexers, e.g. tuple
157 # b) boolean indexers e.g. BoolArray
158 if is_list_like(value):
159 if len(indexer) != len(value) and values.ndim == 1:
160 # boolean with truth values == len of the value is ok too
161 if isinstance(indexer, list):
162 indexer = np.array(indexer)
163 if not (
164 isinstance(indexer, np.ndarray)
165 and indexer.dtype == np.bool_
166 and indexer.sum() == len(value)
167 ):
168 raise ValueError(
169 "cannot set using a list-like indexer "
170 "with a different length than the value"
171 )
172 if not len(indexer):
173 no_op = True
175 elif isinstance(indexer, slice):
176 if is_list_like(value):
177 if len(value) != length_of_indexer(indexer, values) and values.ndim == 1:
178 # In case of two dimensional value is used row-wise and broadcasted
179 raise ValueError(
180 "cannot set using a slice indexer with a "
181 "different length than the value"
182 )
183 if not len(value):
184 no_op = True
186 return no_op
189def validate_indices(indices: np.ndarray, n: int) -> None:
190 """
191 Perform bounds-checking for an indexer.
193 -1 is allowed for indicating missing values.
195 Parameters
196 ----------
197 indices : ndarray
198 n : int
199 Length of the array being indexed.
201 Raises
202 ------
203 ValueError
205 Examples
206 --------
207 >>> validate_indices(np.array([1, 2]), 3) # OK
209 >>> validate_indices(np.array([1, -2]), 3)
210 Traceback (most recent call last):
211 ...
212 ValueError: negative dimensions are not allowed
214 >>> validate_indices(np.array([1, 2, 3]), 3)
215 Traceback (most recent call last):
216 ...
217 IndexError: indices are out-of-bounds
219 >>> validate_indices(np.array([-1, -1]), 0) # OK
221 >>> validate_indices(np.array([0, 1]), 0)
222 Traceback (most recent call last):
223 ...
224 IndexError: indices are out-of-bounds
225 """
226 if len(indices):
227 min_idx = indices.min()
228 if min_idx < -1:
229 msg = f"'indices' contains values less than allowed ({min_idx} < -1)"
230 raise ValueError(msg)
232 max_idx = indices.max()
233 if max_idx >= n:
234 raise IndexError("indices are out-of-bounds")
237# -----------------------------------------------------------
238# Indexer Conversion
241def maybe_convert_indices(indices, n: int, verify: bool = True) -> np.ndarray:
242 """
243 Attempt to convert indices into valid, positive indices.
245 If we have negative indices, translate to positive here.
246 If we have indices that are out-of-bounds, raise an IndexError.
248 Parameters
249 ----------
250 indices : array-like
251 Array of indices that we are to convert.
252 n : int
253 Number of elements in the array that we are indexing.
254 verify : bool, default True
255 Check that all entries are between 0 and n - 1, inclusive.
257 Returns
258 -------
259 array-like
260 An array-like of positive indices that correspond to the ones
261 that were passed in initially to this function.
263 Raises
264 ------
265 IndexError
266 One of the converted indices either exceeded the number of,
267 elements (specified by `n`), or was still negative.
268 """
269 if isinstance(indices, list):
270 indices = np.array(indices)
271 if len(indices) == 0:
272 # If `indices` is empty, np.array will return a float,
273 # and will cause indexing errors.
274 return np.empty(0, dtype=np.intp)
276 mask = indices < 0
277 if mask.any():
278 indices = indices.copy()
279 indices[mask] += n
281 if verify:
282 mask = (indices >= n) | (indices < 0)
283 if mask.any():
284 raise IndexError("indices are out-of-bounds")
285 return indices
288# -----------------------------------------------------------
289# Unsorted
292def length_of_indexer(indexer, target=None) -> int:
293 """
294 Return the expected length of target[indexer]
296 Returns
297 -------
298 int
299 """
300 if target is not None and isinstance(indexer, slice):
301 target_len = len(target)
302 start = indexer.start
303 stop = indexer.stop
304 step = indexer.step
305 if start is None:
306 start = 0
307 elif start < 0:
308 start += target_len
309 if stop is None or stop > target_len:
310 stop = target_len
311 elif stop < 0:
312 stop += target_len
313 if step is None:
314 step = 1
315 elif step < 0:
316 start, stop = stop + 1, start + 1
317 step = -step
318 return (stop - start + step - 1) // step
319 elif isinstance(indexer, (ABCSeries, ABCIndex, np.ndarray, list)):
320 if isinstance(indexer, list):
321 indexer = np.array(indexer)
323 if indexer.dtype == bool:
324 # GH#25774
325 return indexer.sum()
326 return len(indexer)
327 elif isinstance(indexer, range):
328 return (indexer.stop - indexer.start) // indexer.step
329 elif not is_list_like_indexer(indexer):
330 return 1
331 raise AssertionError("cannot find the length of the indexer")
334def disallow_ndim_indexing(result) -> None:
335 """
336 Helper function to disallow multi-dimensional indexing on 1D Series/Index.
338 GH#27125 indexer like idx[:, None] expands dim, but we cannot do that
339 and keep an index, so we used to return ndarray, which was deprecated
340 in GH#30588.
341 """
342 if np.ndim(result) > 1:
343 raise ValueError(
344 "Multi-dimensional indexing (e.g. `obj[:, None]`) is no longer "
345 "supported. Convert to a numpy array before indexing instead."
346 )
349def unpack_1tuple(tup):
350 """
351 If we have a length-1 tuple/list that contains a slice, unpack to just
352 the slice.
354 Notes
355 -----
356 The list case is deprecated.
357 """
358 if len(tup) == 1 and isinstance(tup[0], slice):
359 # if we don't have a MultiIndex, we may still be able to handle
360 # a 1-tuple. see test_1tuple_without_multiindex
362 if isinstance(tup, list):
363 # GH#31299
364 raise ValueError(
365 "Indexing with a single-item list containing a "
366 "slice is not allowed. Pass a tuple instead.",
367 )
369 return tup[0]
370 return tup
373def check_key_length(columns: Index, key, value: DataFrame) -> None:
374 """
375 Checks if a key used as indexer has the same length as the columns it is
376 associated with.
378 Parameters
379 ----------
380 columns : Index The columns of the DataFrame to index.
381 key : A list-like of keys to index with.
382 value : DataFrame The value to set for the keys.
384 Raises
385 ------
386 ValueError: If the length of key is not equal to the number of columns in value
387 or if the number of columns referenced by key is not equal to number
388 of columns.
389 """
390 if columns.is_unique:
391 if len(value.columns) != len(key):
392 raise ValueError("Columns must be same length as key")
393 # Missing keys in columns are represented as -1
394 elif len(columns.get_indexer_non_unique(key)[0]) != len(value.columns):
395 raise ValueError("Columns must be same length as key")
398def unpack_tuple_and_ellipses(item: tuple):
399 """
400 Possibly unpack arr[..., n] to arr[n]
401 """
402 if len(item) > 1:
403 # Note: we are assuming this indexing is being done on a 1D arraylike
404 if item[0] is Ellipsis:
405 item = item[1:]
406 elif item[-1] is Ellipsis:
407 item = item[:-1]
409 if len(item) > 1:
410 raise IndexError("too many indices for array.")
412 item = item[0]
413 return item
416def getitem_returns_view(arr, key) -> bool:
417 """
418 Check if an ``arr.__getitem__`` call with given ``key`` would return a view
419 or not.
420 """
421 if not isinstance(key, tuple):
422 key = (key,)
424 # filter out Ellipsis and np.newaxis
425 key = tuple(k for k in key if k is not Ellipsis and k is not np.newaxis)
426 if not key:
427 return True
428 # single integer gives view if selecting subset of 2D array
429 if arr.ndim == 2 and lib.is_integer(key[0]):
430 return True
431 # slices always give views
432 if all(isinstance(k, slice) for k in key):
433 return True
434 return False
437# -----------------------------------------------------------
438# Public indexer validation
441@set_module("pandas.api.indexers")
442def check_array_indexer(array: AnyArrayLike, indexer: Any) -> Any:
443 """
444 Check if `indexer` is a valid array indexer for `array`.
446 For a boolean mask, `array` and `indexer` are checked to have the same
447 length. The dtype is validated, and if it is an integer or boolean
448 ExtensionArray, it is checked if there are missing values present, and
449 it is converted to the appropriate numpy array. Other dtypes will raise
450 an error.
452 Non-array indexers (integer, slice, Ellipsis, tuples, ..) are passed
453 through as is.
455 Parameters
456 ----------
457 array : array-like
458 The array that is being indexed (only used for the length).
459 indexer : array-like, list-like, int, slice, or other indexer
460 The indexer used for indexing. Array-like and list-like inputs that
461 are not yet a numpy array or an ExtensionArray are converted to one.
462 Non-array indexers (int, slice, Ellipsis, tuples, etc.) are passed
463 through as is.
465 Returns
466 -------
467 numpy.ndarray
468 The validated indexer as a numpy array that can be used to index.
470 Raises
471 ------
472 IndexError
473 When the lengths don't match.
474 ValueError
475 When `indexer` cannot be converted to a numpy ndarray to index
476 (e.g. presence of missing values).
478 See Also
479 --------
480 api.types.is_bool_dtype : Check if `key` is of boolean dtype.
482 Examples
483 --------
484 When checking a boolean mask, a boolean ndarray is returned when the
485 arguments are all valid.
487 >>> mask = pd.array([True, False])
488 >>> arr = pd.array([1, 2])
489 >>> pd.api.indexers.check_array_indexer(arr, mask)
490 array([ True, False])
492 An IndexError is raised when the lengths don't match.
494 >>> mask = pd.array([True, False, True])
495 >>> pd.api.indexers.check_array_indexer(arr, mask)
496 Traceback (most recent call last):
497 ...
498 IndexError: Boolean index has wrong length: 3 instead of 2.
500 NA values in a boolean array are treated as False.
502 >>> mask = pd.array([True, pd.NA])
503 >>> pd.api.indexers.check_array_indexer(arr, mask)
504 array([ True, False])
506 A numpy boolean mask will get passed through (if the length is correct):
508 >>> mask = np.array([True, False])
509 >>> pd.api.indexers.check_array_indexer(arr, mask)
510 array([ True, False])
512 Integer and slice indexers are passed through as is:
514 >>> pd.api.indexers.check_array_indexer(arr, 1)
515 1
516 >>> pd.api.indexers.check_array_indexer(arr, slice(0, 1, 1))
517 slice(0, 1, 1)
519 Similarly for integer indexers, an integer ndarray is returned when it is
520 a valid indexer, otherwise an error is (for integer indexers, a matching
521 length is not required):
523 >>> indexer = pd.array([0, 2], dtype="Int64")
524 >>> arr = pd.array([1, 2, 3])
525 >>> pd.api.indexers.check_array_indexer(arr, indexer)
526 array([0, 2])
528 >>> indexer = pd.array([0, pd.NA], dtype="Int64")
529 >>> pd.api.indexers.check_array_indexer(arr, indexer)
530 Traceback (most recent call last):
531 ...
532 ValueError: Cannot index with an integer indexer containing NA values
534 For non-integer/boolean dtypes, an appropriate error is raised:
536 >>> indexer = np.array([0.0, 2.0], dtype="float64")
537 >>> pd.api.indexers.check_array_indexer(arr, indexer)
538 Traceback (most recent call last):
539 ...
540 IndexError: arrays used as indices must be of integer or boolean type
541 """
542 from pandas.core.construction import array as pd_array
544 # whatever is not an array-like is returned as-is (possible valid array
545 # indexers that are not array-like: integer, slice, Ellipsis, None)
546 # In this context, tuples are not considered as array-like, as they have
547 # a specific meaning in indexing (multi-dimensional indexing)
548 if is_list_like(indexer):
549 if isinstance(indexer, tuple):
550 return indexer
551 else:
552 return indexer
554 # convert list-likes to array
555 if not is_array_like(indexer):
556 indexer = pd_array(indexer)
557 if len(indexer) == 0:
558 # empty list is converted to float array by pd.array
559 indexer = np.array([], dtype=np.intp)
561 dtype = indexer.dtype
562 if is_bool_dtype(dtype):
563 if isinstance(dtype, ExtensionDtype):
564 indexer = indexer.to_numpy(dtype=bool, na_value=False)
565 else:
566 indexer = np.asarray(indexer, dtype=bool)
568 # GH26658
569 if len(indexer) != len(array):
570 raise IndexError(
571 f"Boolean index has wrong length: "
572 f"{len(indexer)} instead of {len(array)}"
573 )
574 elif is_integer_dtype(dtype):
575 try:
576 indexer = np.asarray(indexer, dtype=np.intp)
577 except ValueError as err:
578 raise ValueError(
579 "Cannot index with an integer indexer containing NA values"
580 ) from err
581 else:
582 raise IndexError("arrays used as indices must be of integer or boolean type")
584 return indexer