Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/common.py: 30%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Misc tools for implementing data structures
4Note: pandas.core.common is *not* part of the public API.
5"""
7from __future__ import annotations
9import builtins
10from collections import (
11 abc,
12 defaultdict,
13)
14from collections.abc import (
15 Callable,
16 Collection,
17 Generator,
18 Hashable,
19 Iterable,
20 Sequence,
21)
22import contextlib
23from functools import partial
24import inspect
25import sys
26from typing import (
27 TYPE_CHECKING,
28 Any,
29 Concatenate,
30 TypeVar,
31 cast,
32 overload,
33)
35import numpy as np
37from pandas._libs import lib
39from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
40from pandas.core.dtypes.common import (
41 is_bool_dtype,
42 is_integer,
43)
44from pandas.core.dtypes.generic import (
45 ABCExtensionArray,
46 ABCIndex,
47 ABCMultiIndex,
48 ABCNumpyExtensionArray,
49 ABCSeries,
50)
51from pandas.core.dtypes.inference import iterable_not_string
53from pandas.core.col import Expression
55if TYPE_CHECKING:
56 from pandas._typing import (
57 AnyArrayLike,
58 ArrayLike,
59 NpDtype,
60 P,
61 RandomState,
62 T,
63 )
65 from pandas import Index
68def flatten(line):
69 """
70 Flatten an arbitrarily nested sequence.
72 Parameters
73 ----------
74 line : sequence
75 The non string sequence to flatten
77 Notes
78 -----
79 This doesn't consider strings sequences.
81 Returns
82 -------
83 flattened : generator
84 """
85 for element in line:
86 if iterable_not_string(element):
87 yield from flatten(element)
88 else:
89 yield element
92def consensus_name_attr(objs):
93 name = objs[0].name
94 for obj in objs[1:]:
95 try:
96 if obj.name != name:
97 name = None
98 break
99 except ValueError:
100 name = None
101 break
102 return name
105def is_bool_indexer(key: Any) -> bool:
106 """
107 Check whether `key` is a valid boolean indexer.
109 Parameters
110 ----------
111 key : Any
112 Only list-likes may be considered boolean indexers.
113 All other types are not considered a boolean indexer.
114 For array-like input, boolean ndarrays or ExtensionArrays
115 with ``_is_boolean`` set are considered boolean indexers.
117 Returns
118 -------
119 bool
120 Whether `key` is a valid boolean indexer.
122 Raises
123 ------
124 ValueError
125 When the array is an object-dtype ndarray or ExtensionArray
126 and contains missing values.
128 See Also
129 --------
130 check_array_indexer : Check that `key` is a valid array to index,
131 and convert to an ndarray.
132 """
133 if isinstance(
134 key,
135 (ABCSeries, np.ndarray, ABCIndex, ABCExtensionArray, ABCNumpyExtensionArray),
136 ) and not isinstance(key, ABCMultiIndex):
137 if key.dtype == np.object_:
138 key_array = np.asarray(key)
140 if not lib.is_bool_array(key_array):
141 na_msg = "Cannot mask with non-boolean array containing NA / NaN values"
142 if lib.is_bool_array(key_array, skipna=True):
143 # Don't raise on e.g. ["A", "B", np.nan], see
144 # test_loc_getitem_list_of_labels_categoricalindex_with_na
145 raise ValueError(na_msg)
146 return False
147 return True
148 elif is_bool_dtype(key.dtype):
149 return True
150 elif isinstance(key, list):
151 # check if np.array(key).dtype would be bool
152 if len(key) > 0:
153 if type(key) is not list:
154 # GH#42461 cython will raise TypeError if we pass a subclass
155 key = list(key)
156 return lib.is_bool_list(key)
158 return False
161def cast_scalar_indexer(val):
162 """
163 Disallow indexing with a float key, even if that key is a round number.
165 Parameters
166 ----------
167 val : scalar
169 Returns
170 -------
171 outval : scalar
172 """
173 # assumes lib.is_scalar(val)
174 if lib.is_float(val) and val.is_integer():
175 raise IndexError(
176 # GH#34193
177 "Indexing with a float is no longer supported. Manually convert "
178 "to an integer key instead."
179 )
180 return val
183def not_none(*args):
184 """
185 Returns a generator consisting of the arguments that are not None.
186 """
187 return (arg for arg in args if arg is not None)
190def any_none(*args) -> bool:
191 """
192 Returns a boolean indicating if any argument is None.
193 """
194 return any(arg is None for arg in args)
197def all_none(*args) -> bool:
198 """
199 Returns a boolean indicating if all arguments are None.
200 """
201 return all(arg is None for arg in args)
204def any_not_none(*args) -> bool:
205 """
206 Returns a boolean indicating if any argument is not None.
207 """
208 return any(arg is not None for arg in args)
211def all_not_none(*args) -> bool:
212 """
213 Returns a boolean indicating if all arguments are not None.
214 """
215 return all(arg is not None for arg in args)
218def count_not_none(*args) -> int:
219 """
220 Returns the count of arguments that are not None.
221 """
222 return sum(x is not None for x in args)
225@overload
226def asarray_tuplesafe(
227 values: ArrayLike | list | tuple | zip, dtype: NpDtype | None = ...
228) -> np.ndarray:
229 # ExtensionArray can only be returned when values is an Index, all other iterables
230 # will return np.ndarray. Unfortunately "all other" cannot be encoded in a type
231 # signature, so instead we special-case some common types.
232 ...
235@overload
236def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = ...) -> ArrayLike: ...
239def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = None) -> ArrayLike:
240 if not (isinstance(values, (list, tuple)) or hasattr(values, "__array__")):
241 values = list(values)
242 elif isinstance(values, ABCIndex):
243 return values._values
244 elif isinstance(values, ABCSeries):
245 return values._values
247 if isinstance(values, list) and dtype in [np.object_, object]:
248 return construct_1d_object_array_from_listlike(values)
250 try:
251 result = np.asarray(values, dtype=dtype)
252 except ValueError:
253 # Using try/except since it's more performant than checking is_list_like
254 # over each element
255 # error: Argument 1 to "construct_1d_object_array_from_listlike"
256 # has incompatible type "Iterable[Any]"; expected "Sized"
257 return construct_1d_object_array_from_listlike(values) # type: ignore[arg-type]
259 if issubclass(result.dtype.type, str):
260 result = np.asarray(values, dtype=object)
262 if result.ndim == 2:
263 # Avoid building an array of arrays:
264 values = [tuple(x) for x in values]
265 result = construct_1d_object_array_from_listlike(values)
267 return result
270def index_labels_to_array(
271 labels: np.ndarray | Iterable, dtype: NpDtype | None = None
272) -> np.ndarray:
273 """
274 Transform label or iterable of labels to array, for use in Index.
276 Parameters
277 ----------
278 dtype : dtype
279 If specified, use as dtype of the resulting array, otherwise infer.
281 Returns
282 -------
283 array
284 """
285 if isinstance(labels, (str, tuple)):
286 labels = [labels]
288 if not isinstance(labels, (list, np.ndarray)):
289 try:
290 labels = list(labels)
291 except TypeError: # non-iterable
292 labels = [labels]
294 rlabels = asarray_tuplesafe(labels, dtype=dtype)
296 return rlabels
299def maybe_make_list(obj):
300 if obj is not None and not isinstance(obj, (tuple, list)):
301 return [obj]
302 return obj
305def maybe_iterable_to_list(obj: Iterable[T] | T) -> Collection[T] | T:
306 """
307 If obj is Iterable but not list-like, consume into list.
308 """
309 if isinstance(obj, abc.Iterable) and not isinstance(obj, abc.Sized):
310 return list(obj)
311 obj = cast(Collection, obj)
312 return obj
315def is_null_slice(obj) -> bool:
316 """
317 We have a null slice.
318 """
319 return (
320 isinstance(obj, slice)
321 and obj.start is None
322 and obj.stop is None
323 and obj.step is None
324 )
327def is_empty_slice(obj) -> bool:
328 """
329 We have an empty slice, e.g. no values are selected.
330 """
331 return (
332 isinstance(obj, slice)
333 and obj.start is not None
334 and obj.stop is not None
335 and obj.start == obj.stop
336 )
339def is_true_slices(line: abc.Iterable) -> abc.Generator[bool, None, None]:
340 """
341 Find non-trivial slices in "line": yields a bool.
342 """
343 for k in line:
344 yield isinstance(k, slice) and not is_null_slice(k)
347# TODO: used only once in indexing; belongs elsewhere?
348def is_full_slice(obj, line: int) -> bool:
349 """
350 We have a full length slice.
351 """
352 return (
353 isinstance(obj, slice)
354 and obj.start == 0
355 and obj.stop == line
356 and obj.step is None
357 )
360def get_callable_name(obj):
361 # typical case has name
362 if hasattr(obj, "__name__"):
363 return obj.__name__
364 # some objects don't; could recurse
365 if isinstance(obj, partial):
366 return get_callable_name(obj.func)
367 # fall back to class name
368 if callable(obj):
369 return type(obj).__name__
370 # everything failed (probably because the argument
371 # wasn't actually callable); we return None
372 # instead of the empty string in this case to allow
373 # distinguishing between no name and a name of ''
374 return None
377def apply_if_callable(maybe_callable, obj, **kwargs):
378 """
379 Evaluate possibly callable input using obj and kwargs if it is callable,
380 otherwise return as it is.
382 Parameters
383 ----------
384 maybe_callable : possibly a callable
385 obj : NDFrame
386 **kwargs
387 """
388 if isinstance(maybe_callable, Expression):
389 return maybe_callable._eval_expression(obj, **kwargs)
390 elif callable(maybe_callable):
391 return maybe_callable(obj, **kwargs)
393 return maybe_callable
396def standardize_mapping(into):
397 """
398 Helper function to standardize a supplied mapping.
400 Parameters
401 ----------
402 into : instance or subclass of collections.abc.Mapping
403 Must be a class, an initialized collections.defaultdict,
404 or an instance of a collections.abc.Mapping subclass.
406 Returns
407 -------
408 mapping : a collections.abc.Mapping subclass or other constructor
409 a callable object that can accept an iterator to create
410 the desired Mapping.
412 See Also
413 --------
414 DataFrame.to_dict
415 Series.to_dict
416 """
417 if not inspect.isclass(into):
418 if isinstance(into, defaultdict):
419 return partial(defaultdict, into.default_factory)
420 into = type(into)
421 if not issubclass(into, abc.Mapping):
422 raise TypeError(f"unsupported type: {into}")
423 if into == defaultdict:
424 raise TypeError("to_dict() only accepts initialized defaultdicts")
425 return into
428@overload
429def random_state(state: np.random.Generator) -> np.random.Generator: ...
432@overload
433def random_state(
434 state: int | np.ndarray | np.random.BitGenerator | np.random.RandomState | None,
435) -> np.random.RandomState: ...
438def random_state(state: RandomState | None = None):
439 """
440 Helper function for processing random_state arguments.
442 Parameters
443 ----------
444 state : int, array-like, BitGenerator, Generator, np.random.RandomState, None.
445 If receives an int, array-like, or BitGenerator, passes to
446 np.random.RandomState() as seed.
447 If receives an np.random RandomState or Generator, just returns that unchanged.
448 If receives `None`, returns np.random.
449 If receives anything else, raises an informative ValueError.
451 Default None.
453 Returns
454 -------
455 np.random.RandomState or np.random.Generator. If state is None, returns np.random
457 """
458 if is_integer(state) or isinstance(state, (np.ndarray, np.random.BitGenerator)):
459 return np.random.RandomState(state)
460 elif isinstance(state, np.random.RandomState):
461 return state
462 elif isinstance(state, np.random.Generator):
463 return state
464 elif state is None:
465 return np.random
466 else:
467 raise ValueError(
468 "random_state must be an integer, array-like, a BitGenerator, Generator, "
469 "a numpy RandomState, or None"
470 )
473_T = TypeVar("_T") # Secondary TypeVar for use in pipe's type hints
476@overload
477def pipe(
478 obj: _T,
479 func: Callable[Concatenate[_T, P], T],
480 *args: P.args,
481 **kwargs: P.kwargs,
482) -> T: ...
485@overload
486def pipe(
487 obj: Any,
488 func: tuple[Callable[..., T], str],
489 *args: Any,
490 **kwargs: Any,
491) -> T: ...
494def pipe(
495 obj: _T,
496 func: Callable[Concatenate[_T, P], T] | tuple[Callable[..., T], str],
497 *args: Any,
498 **kwargs: Any,
499) -> T:
500 """
501 Apply a function ``func`` to object ``obj`` either by passing obj as the
502 first argument to the function or, in the case that the func is a tuple,
503 interpret the first element of the tuple as a function and pass the obj to
504 that function as a keyword argument whose key is the value of the second
505 element of the tuple.
507 Parameters
508 ----------
509 func : callable or tuple of (callable, str)
510 Function to apply to this object or, alternatively, a
511 ``(callable, data_keyword)`` tuple where ``data_keyword`` is a
512 string indicating the keyword of ``callable`` that expects the
513 object.
514 *args : iterable, optional
515 Positional arguments passed into ``func``.
516 **kwargs : dict, optional
517 A dictionary of keyword arguments passed into ``func``.
519 Returns
520 -------
521 object : the return type of ``func``.
522 """
523 if isinstance(func, tuple):
524 # Assigning to func_ so pyright understands that it's a callable
525 func_, target = func
526 if target in kwargs:
527 msg = f"{target} is both the pipe target and a keyword argument"
528 raise ValueError(msg)
529 kwargs[target] = obj
530 return func_(*args, **kwargs)
531 else:
532 return func(obj, *args, **kwargs)
535def get_rename_function(mapper):
536 """
537 Returns a function that will map names/labels, dependent if mapper
538 is a dict, Series or just a function.
539 """
541 def f(x):
542 if x in mapper:
543 return mapper[x]
544 else:
545 return x
547 return f if isinstance(mapper, (abc.Mapping, ABCSeries)) else mapper
550def convert_to_list_like(
551 values: Hashable | Iterable | AnyArrayLike,
552) -> list | AnyArrayLike:
553 """
554 Convert list-like or scalar input to list-like. List, numpy and pandas array-like
555 inputs are returned unmodified whereas others are converted to list.
556 """
557 if isinstance(values, (list, np.ndarray, ABCIndex, ABCSeries, ABCExtensionArray)):
558 return values
559 elif isinstance(values, abc.Iterable) and not isinstance(values, str):
560 return list(values)
562 return [values]
565@contextlib.contextmanager
566def temp_setattr(obj, attr: str, value, condition: bool = True) -> Generator[None]:
567 """
568 Temporarily set attribute on an object.
570 Parameters
571 ----------
572 obj : object
573 Object whose attribute will be modified.
574 attr : str
575 Attribute to modify.
576 value : Any
577 Value to temporarily set attribute to.
578 condition : bool, default True
579 Whether to set the attribute. Provided in order to not have to
580 conditionally use this context manager.
582 Yields
583 ------
584 object : obj with modified attribute.
585 """
586 if condition:
587 old_value = getattr(obj, attr)
588 setattr(obj, attr, value)
589 try:
590 yield obj
591 finally:
592 if condition:
593 setattr(obj, attr, old_value)
596def require_length_match(data, index: Index) -> None:
597 """
598 Check the length of data matches the length of the index.
599 """
600 if len(data) != len(index):
601 raise ValueError(
602 "Length of values "
603 f"({len(data)}) "
604 "does not match length of index "
605 f"({len(index)})"
606 )
609_cython_table = {
610 builtins.sum: "sum",
611 builtins.max: "max",
612 builtins.min: "min",
613 np.all: "all",
614 np.any: "any",
615 np.sum: "sum",
616 np.nansum: "sum",
617 np.mean: "mean",
618 np.nanmean: "mean",
619 np.prod: "prod",
620 np.nanprod: "prod",
621 np.std: "std",
622 np.nanstd: "std",
623 np.var: "var",
624 np.nanvar: "var",
625 np.median: "median",
626 np.nanmedian: "median",
627 np.max: "max",
628 np.nanmax: "max",
629 np.min: "min",
630 np.nanmin: "min",
631 np.cumprod: "cumprod",
632 np.nancumprod: "cumprod",
633 np.cumsum: "cumsum",
634 np.nancumsum: "cumsum",
635}
638def get_cython_func(arg: Callable) -> str | None:
639 """
640 if we define an internal function for this argument, return it
641 """
642 return _cython_table.get(arg)
645def fill_missing_names(names: Sequence[Hashable | None]) -> list[Hashable]:
646 """
647 If a name is missing then replace it by level_n, where n is the count
649 Parameters
650 ----------
651 names : list-like
652 list of column names or None values.
654 Returns
655 -------
656 list
657 list of column names with the None values replaced.
658 """
659 return [f"level_{i}" if name is None else name for i, name in enumerate(names)]
662def is_local_in_caller_frame(obj):
663 """
664 Helper function used in detecting chained assignment.
666 If the pandas object (DataFrame/Series) is a local variable
667 in the caller's frame, it should not be a case of chained
668 assignment or method call.
670 For example:
672 def test():
673 df = pd.DataFrame(...)
674 df["a"] = 1 # not chained assignment
676 Inside ``df.__setitem__``, we call this function to check whether `df`
677 (`self`) is a local variable in `test` frame (the frame calling setitem). If
678 so, we know it is not a case of chained assignment (even when the refcount
679 of `df` is below the threshold due to optimization of local variables).
680 """
681 frame = sys._getframe(2)
682 for v in frame.f_locals.values():
683 if v is obj:
684 return True
685 return False