Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/common.py: 30%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Misc tools for implementing data structures
4Note: pandas.core.common is *not* part of the public API.
5"""
7from __future__ import annotations
9import builtins
10from collections import (
11 abc,
12 defaultdict,
13)
14import contextlib
15import dis
16from functools import partial
17import inspect
18import sys
19from typing import (
20 TYPE_CHECKING,
21 Any,
22 Concatenate,
23 TypeVar,
24 cast,
25 overload,
26)
28import numpy as np
30from pandas._libs import lib
32from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
33from pandas.core.dtypes.common import (
34 is_bool_dtype,
35 is_integer,
36)
37from pandas.core.dtypes.generic import (
38 ABCExtensionArray,
39 ABCIndex,
40 ABCMultiIndex,
41 ABCNumpyExtensionArray,
42 ABCSeries,
43)
44from pandas.core.dtypes.inference import iterable_not_string
46from pandas.core.col import Expression
48if TYPE_CHECKING:
49 from collections.abc import (
50 Callable,
51 Collection,
52 Generator,
53 Hashable,
54 Iterable,
55 Sequence,
56 )
58 from pandas._typing import (
59 AnyArrayLike,
60 ArrayLike,
61 NpDtype,
62 P,
63 RandomState,
64 T,
65 )
67 from pandas import (
68 DataFrame,
69 Index,
70 Series,
71 )
72 from pandas.core.generic import NDFrame
75def flatten(line: Iterable) -> Generator[Any]:
76 """
77 Flatten an arbitrarily nested sequence.
79 Parameters
80 ----------
81 line : sequence
82 The non string sequence to flatten
84 Notes
85 -----
86 This doesn't consider strings sequences.
88 Returns
89 -------
90 flattened : generator
91 """
92 for element in line:
93 if iterable_not_string(element):
94 yield from flatten(element)
95 else:
96 yield element
99def consensus_name_attr(objs: list[Series | DataFrame]) -> Hashable | None:
100 name = objs[0].name
101 for obj in objs[1:]:
102 try:
103 if obj.name != name:
104 name = None
105 break
106 except ValueError:
107 name = None
108 break
109 return name
112def is_bool_indexer(key: Any) -> bool:
113 """
114 Check whether `key` is a valid boolean indexer.
116 Parameters
117 ----------
118 key : Any
119 Only list-likes may be considered boolean indexers.
120 All other types are not considered a boolean indexer.
121 For array-like input, boolean ndarrays or ExtensionArrays
122 with ``_is_boolean`` set are considered boolean indexers.
124 Returns
125 -------
126 bool
127 Whether `key` is a valid boolean indexer.
129 Raises
130 ------
131 ValueError
132 When the array is an object-dtype ndarray or ExtensionArray
133 and contains missing values.
135 See Also
136 --------
137 check_array_indexer : Check that `key` is a valid array to index,
138 and convert to an ndarray.
139 """
140 if isinstance(
141 key,
142 (ABCSeries, np.ndarray, ABCIndex, ABCExtensionArray, ABCNumpyExtensionArray),
143 ) and not isinstance(key, ABCMultiIndex):
144 if key.dtype == np.object_:
145 key_array = np.asarray(key)
147 if not lib.is_bool_array(key_array):
148 na_msg = "Cannot mask with non-boolean array containing NA / NaN values"
149 if lib.is_bool_array(key_array, skipna=True):
150 # Don't raise on e.g. ["A", "B", np.nan], see
151 # test_loc_getitem_list_of_labels_categoricalindex_with_na
152 raise ValueError(na_msg)
153 return False
154 return True
155 elif is_bool_dtype(key.dtype):
156 return True
157 elif isinstance(key, list):
158 # check if np.array(key).dtype would be bool
159 if len(key) > 0:
160 if type(key) is not list:
161 # GH#42461 cython will raise TypeError if we pass a subclass
162 key = list(key)
163 return lib.is_bool_list(key)
165 return False
168def cast_scalar_indexer(val: Any) -> Any:
169 """
170 Disallow indexing with a float key, even if that key is a round number.
172 Parameters
173 ----------
174 val : scalar
176 Returns
177 -------
178 outval : scalar
179 """
180 # assumes lib.is_scalar(val)
181 if lib.is_float(val) and val.is_integer():
182 raise IndexError(
183 # GH#34193
184 "Indexing with a float is no longer supported. Manually convert "
185 "to an integer key instead."
186 )
187 return val
190def not_none(*args: object) -> Generator[object]:
191 """
192 Returns a generator consisting of the arguments that are not None.
193 """
194 return (arg for arg in args if arg is not None)
197def any_none(*args: object) -> bool:
198 """
199 Returns a boolean indicating if any argument is None.
200 """
201 return any(arg is None for arg in args)
204def all_none(*args: object) -> bool:
205 """
206 Returns a boolean indicating if all arguments are None.
207 """
208 return all(arg is None for arg in args)
211def any_not_none(*args: object) -> bool:
212 """
213 Returns a boolean indicating if any argument is not None.
214 """
215 return any(arg is not None for arg in args)
218def all_not_none(*args: object) -> bool:
219 """
220 Returns a boolean indicating if all arguments are not None.
221 """
222 return all(arg is not None for arg in args)
225def count_not_none(*args: object) -> int:
226 """
227 Returns the count of arguments that are not None.
228 """
229 return sum(x is not None for x in args)
232@overload
233def asarray_tuplesafe(
234 values: ArrayLike | list | tuple | zip, dtype: NpDtype | None = ...
235) -> np.ndarray:
236 # ExtensionArray can only be returned when values is an Index, all other iterables
237 # will return np.ndarray. Unfortunately "all other" cannot be encoded in a type
238 # signature, so instead we special-case some common types.
239 ...
242@overload
243def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = ...) -> ArrayLike: ...
246def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = None) -> ArrayLike:
247 if not (isinstance(values, (list, tuple)) or hasattr(values, "__array__")):
248 values = list(values)
249 elif isinstance(values, ABCIndex):
250 return values._values
251 elif isinstance(values, ABCSeries):
252 return values._values
254 if isinstance(values, list) and dtype in [np.object_, object]:
255 return construct_1d_object_array_from_listlike(values)
257 try:
258 result = np.asarray(values, dtype=dtype)
259 except ValueError:
260 # Using try/except since it's more performant than checking is_list_like
261 # over each element
262 # error: Argument 1 to "construct_1d_object_array_from_listlike"
263 # has incompatible type "Iterable[Any]"; expected "Sized"
264 return construct_1d_object_array_from_listlike(values) # type: ignore[arg-type]
266 if issubclass(result.dtype.type, str):
267 result = np.asarray(values, dtype=object)
269 if result.ndim == 2:
270 # Avoid building an array of arrays:
271 values = [tuple(x) for x in values]
272 result = construct_1d_object_array_from_listlike(values)
274 return result
277def index_labels_to_array(
278 labels: np.ndarray | Iterable, dtype: NpDtype | None = None
279) -> np.ndarray:
280 """
281 Transform label or iterable of labels to array, for use in Index.
283 Parameters
284 ----------
285 dtype : dtype
286 If specified, use as dtype of the resulting array, otherwise infer.
288 Returns
289 -------
290 array
291 """
292 if isinstance(labels, (str, tuple)):
293 labels = [labels]
295 if not isinstance(labels, (list, np.ndarray)):
296 try:
297 labels = list(labels)
298 except TypeError: # non-iterable
299 labels = [labels]
301 rlabels = asarray_tuplesafe(labels, dtype=dtype)
303 return rlabels
306def maybe_make_list(obj: Any) -> list | tuple | None:
307 if obj is not None and not isinstance(obj, (tuple, list)):
308 return [obj]
309 return obj
312def maybe_iterable_to_list(obj: Iterable[T] | T) -> Collection[T] | T:
313 """
314 If obj is Iterable but not list-like, consume into list.
315 """
316 if isinstance(obj, abc.Iterable) and not isinstance(obj, abc.Sized):
317 return list(obj)
318 obj = cast("Collection", obj)
319 return obj
322def is_null_slice(obj: object) -> bool:
323 """
324 We have a null slice.
325 """
326 return (
327 isinstance(obj, slice)
328 and obj.start is None
329 and obj.stop is None
330 and obj.step is None
331 )
334def is_empty_slice(obj: object) -> bool:
335 """
336 We have an empty slice, e.g. no values are selected.
337 """
338 return (
339 isinstance(obj, slice)
340 and obj.start is not None
341 and obj.stop is not None
342 and obj.start == obj.stop
343 )
346def is_true_slices(line: abc.Iterable) -> abc.Generator[bool, None, None]:
347 """
348 Find non-trivial slices in "line": yields a bool.
349 """
350 for k in line:
351 yield isinstance(k, slice) and not is_null_slice(k)
354# TODO: used only once in indexing; belongs elsewhere?
355def is_full_slice(obj: object, line: int) -> bool:
356 """
357 We have a full length slice.
358 """
359 return (
360 isinstance(obj, slice)
361 and obj.start == 0
362 and obj.stop == line
363 and obj.step is None
364 )
367def get_callable_name(obj: object) -> str | None:
368 # typical case has name
369 if hasattr(obj, "__name__"):
370 return obj.__name__ # pyright: ignore[reportAttributeAccessIssue]
371 # some objects don't; could recurse
372 if isinstance(obj, partial):
373 return get_callable_name(obj.func)
374 # fall back to class name
375 if callable(obj):
376 return type(obj).__name__
377 # everything failed (probably because the argument
378 # wasn't actually callable); we return None
379 # instead of the empty string in this case to allow
380 # distinguishing between no name and a name of ''
381 return None
384def apply_if_callable(maybe_callable: Any, obj: Any, **kwargs: Any) -> Any:
385 """
386 Evaluate possibly callable input using obj and kwargs if it is callable,
387 otherwise return as it is.
389 Parameters
390 ----------
391 maybe_callable : possibly a callable
392 obj : NDFrame
393 **kwargs
394 """
395 if isinstance(maybe_callable, Expression):
396 return maybe_callable._eval_expression(obj, **kwargs)
397 elif callable(maybe_callable):
398 return maybe_callable(obj, **kwargs)
400 return maybe_callable
403def standardize_mapping(into: type | abc.Mapping) -> type | partial:
404 """
405 Helper function to standardize a supplied mapping.
407 Parameters
408 ----------
409 into : instance or subclass of collections.abc.Mapping
410 Must be a class, an initialized collections.defaultdict,
411 or an instance of a collections.abc.Mapping subclass.
413 Returns
414 -------
415 mapping : a collections.abc.Mapping subclass or other constructor
416 a callable object that can accept an iterator to create
417 the desired Mapping.
419 See Also
420 --------
421 DataFrame.to_dict
422 Series.to_dict
423 """
424 if not inspect.isclass(into):
425 if isinstance(into, defaultdict):
426 return partial(defaultdict, into.default_factory)
427 into = type(into)
428 if not issubclass(into, abc.Mapping):
429 raise TypeError(f"unsupported type: {into}")
430 if into == defaultdict:
431 raise TypeError("to_dict() only accepts initialized defaultdicts")
432 return into
435@overload
436def random_state(state: np.random.Generator) -> np.random.Generator: ...
439@overload
440def random_state(
441 state: int | np.ndarray | np.random.BitGenerator | np.random.RandomState | None,
442) -> np.random.RandomState: ...
445def random_state(
446 state: RandomState | None = None,
447) -> np.random.RandomState | np.random.Generator:
448 """
449 Helper function for processing random_state arguments.
451 Parameters
452 ----------
453 state : int, array-like, BitGenerator, Generator, np.random.RandomState, None.
454 If receives an int, array-like, or BitGenerator, passes to
455 np.random.RandomState() as seed.
456 If receives an np.random RandomState or Generator, just returns that unchanged.
457 If receives `None`, returns np.random.
458 If receives anything else, raises an informative ValueError.
460 Default None.
462 Returns
463 -------
464 np.random.RandomState or np.random.Generator. If state is None, returns np.random
466 """
467 if is_integer(state) or isinstance(state, (np.ndarray, np.random.BitGenerator)):
468 return np.random.RandomState(state)
469 elif isinstance(state, np.random.RandomState):
470 return state
471 elif isinstance(state, np.random.Generator):
472 return state
473 elif state is None:
474 return np.random # type: ignore[return-value]
475 else:
476 raise ValueError(
477 "random_state must be an integer, array-like, a BitGenerator, Generator, "
478 "a numpy RandomState, or None"
479 )
482_T = TypeVar("_T") # Secondary TypeVar for use in pipe's type hints
485@overload
486def pipe(
487 obj: _T,
488 func: Callable[Concatenate[_T, P], T],
489 *args: P.args,
490 **kwargs: P.kwargs,
491) -> T: ...
494@overload
495def pipe(
496 obj: Any,
497 func: tuple[Callable[..., T], str],
498 *args: Any,
499 **kwargs: Any,
500) -> T: ...
503def pipe(
504 obj: _T,
505 func: Callable[Concatenate[_T, P], T] | tuple[Callable[..., T], str],
506 *args: Any,
507 **kwargs: Any,
508) -> T:
509 """
510 Apply a function ``func`` to object ``obj`` either by passing obj as the
511 first argument to the function or, in the case that the func is a tuple,
512 interpret the first element of the tuple as a function and pass the obj to
513 that function as a keyword argument whose key is the value of the second
514 element of the tuple.
516 Parameters
517 ----------
518 func : callable or tuple of (callable, str)
519 Function to apply to this object or, alternatively, a
520 ``(callable, data_keyword)`` tuple where ``data_keyword`` is a
521 string indicating the keyword of ``callable`` that expects the
522 object.
523 *args : iterable, optional
524 Positional arguments passed into ``func``.
525 **kwargs : dict, optional
526 A dictionary of keyword arguments passed into ``func``.
528 Returns
529 -------
530 object : the return type of ``func``.
531 """
532 if isinstance(func, tuple):
533 # Assigning to func_ so pyright understands that it's a callable
534 func_, target = func
535 if target in kwargs:
536 msg = f"{target} is both the pipe target and a keyword argument"
537 raise ValueError(msg)
538 kwargs[target] = obj
539 return func_(*args, **kwargs)
540 else:
541 return func(obj, *args, **kwargs)
544def get_rename_function(mapper: Any) -> Callable:
545 """
546 Returns a function that will map names/labels, dependent if mapper
547 is a dict, Series or just a function.
548 """
550 def f(x: Hashable) -> Any:
551 if x in mapper:
552 return mapper[x]
553 else:
554 return x
556 return f if isinstance(mapper, (abc.Mapping, ABCSeries)) else mapper
559def convert_to_list_like(
560 values: Hashable | Iterable | AnyArrayLike,
561) -> list | AnyArrayLike:
562 """
563 Convert list-like or scalar input to list-like. List, numpy and pandas array-like
564 inputs are returned unmodified whereas others are converted to list.
565 """
566 if isinstance(values, (list, np.ndarray, ABCIndex, ABCSeries, ABCExtensionArray)):
567 return values
568 elif isinstance(values, abc.Iterable) and not isinstance(values, str):
569 return list(values)
571 return [values]
574@contextlib.contextmanager
575def temp_setattr(
576 obj: Any, attr: str, value: Any, condition: bool = True
577) -> Generator[Any]:
578 """
579 Temporarily set attribute on an object.
581 Parameters
582 ----------
583 obj : object
584 Object whose attribute will be modified.
585 attr : str
586 Attribute to modify.
587 value : Any
588 Value to temporarily set attribute to.
589 condition : bool, default True
590 Whether to set the attribute. Provided in order to not have to
591 conditionally use this context manager.
593 Yields
594 ------
595 object : obj with modified attribute.
596 """
597 if condition:
598 old_value = getattr(obj, attr)
599 setattr(obj, attr, value)
600 try:
601 yield obj
602 finally:
603 if condition:
604 setattr(obj, attr, old_value)
607def require_length_match(data: Any, index: Index) -> None:
608 """
609 Check the length of data matches the length of the index.
610 """
611 if len(data) != len(index):
612 raise ValueError(
613 "Length of values "
614 f"({len(data)}) "
615 "does not match length of index "
616 f"({len(index)})"
617 )
620_cython_table = {
621 builtins.sum: "sum",
622 builtins.max: "max",
623 builtins.min: "min",
624 np.all: "all",
625 np.any: "any",
626 np.sum: "sum",
627 np.nansum: "sum",
628 np.mean: "mean",
629 np.nanmean: "mean",
630 np.prod: "prod",
631 np.nanprod: "prod",
632 np.std: "std",
633 np.nanstd: "std",
634 np.var: "var",
635 np.nanvar: "var",
636 np.median: "median",
637 np.nanmedian: "median",
638 np.max: "max",
639 np.nanmax: "max",
640 np.min: "min",
641 np.nanmin: "min",
642 np.cumprod: "cumprod",
643 np.nancumprod: "cumprod",
644 np.cumsum: "cumsum",
645 np.nancumsum: "cumsum",
646}
649def get_cython_func(arg: Callable) -> str | None:
650 """
651 if we define an internal function for this argument, return it
652 """
653 return _cython_table.get(arg)
656def fill_missing_names(names: Sequence[Hashable | None]) -> list[Hashable]:
657 """
658 If a name is missing then replace it by level_n, where n is the count
660 Parameters
661 ----------
662 names : list-like
663 list of column names or None values.
665 Returns
666 -------
667 list
668 list of column names with the None values replaced.
669 """
670 return [f"level_{i}" if name is None else name for i, name in enumerate(names)]
673# opcodes through which the interpreter calls ``__setitem__``
674# (STORE_SLICE only exists on Python 3.12+, before that slices went through
675# BUILD_SLICE + STORE_SUBSCR)
676_SETITEM_OPCODES = frozenset(
677 dis.opmap[name] for name in ("STORE_SUBSCR", "STORE_SLICE") if name in dis.opmap
678)
681def is_setitem_syntax_in_caller_frame() -> bool:
682 """
683 Helper function used in detecting chained assignment.
685 Whether the frame calling ``__setitem__`` is executing a subscript
686 assignment (``obj[key] = value``) right now.
688 The refcount check can only tell a temporary apart from a named object for
689 calls coming from the interpreter, because it relies on the extra
690 reference the interpreter holds on its value stack. Code compiled with
691 Cython calls ``PyObject_SetItem`` directly, so it has one reference less
692 and a perfectly valid ``df[col] = value`` there looks exactly like chained
693 assignment (GH#51315). Such a call leaves no Python frame of its own, and
694 the nearest Python frame is executing a call rather than a subscript
695 assignment, so checking the opcode filters those out.
696 """
697 try:
698 frame = sys._getframe(2)
699 except ValueError:
700 return False
701 return frame.f_code.co_code[frame.f_lasti] in _SETITEM_OPCODES
704def is_local_in_caller_frame(obj: NDFrame) -> bool:
705 """
706 Helper function used in detecting chained assignment.
708 If the pandas object (DataFrame/Series) is a local variable
709 in the caller's frame, it should not be a case of chained
710 assignment or method call.
712 For example:
714 def test():
715 df = pd.DataFrame(...)
716 df["a"] = 1 # not chained assignment
718 Inside ``df.__setitem__``, we call this function to check whether `df`
719 (`self`) is a local variable in `test` frame (the frame calling setitem). If
720 so, we know it is not a case of chained assignment (even when the refcount
721 of `df` is below the threshold due to optimization of local variables).
722 """
723 frame = sys._getframe(2)
724 for v in frame.f_locals.values():
725 if v is obj:
726 return True
727 return False