Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import inspect
4import re
5from typing import (
6 TYPE_CHECKING,
7 Any,
8 Literal,
9 Self,
10 cast,
11 final,
12)
13import warnings
15import numpy as np
17from pandas._libs import (
18 NaT,
19 algos as libalgos,
20 internals as libinternals,
21 lib,
22)
23from pandas._libs.internals import (
24 BlockPlacement,
25 BlockValuesRefs,
26)
27from pandas._libs.missing import NA
28from pandas.errors import (
29 AbstractMethodError,
30 OutOfBoundsDatetime,
31 Pandas4Warning,
32)
33from pandas.util._decorators import cache_readonly
34from pandas.util._exceptions import find_stack_level
35from pandas.util._validators import validate_bool_kwarg
37from pandas.core.dtypes.astype import (
38 astype_array_safe,
39 astype_is_view,
40)
41from pandas.core.dtypes.cast import (
42 LossySetitemError,
43 can_hold_element,
44 construct_1d_object_array_from_listlike,
45 convert_dtypes,
46 find_result_type,
47 np_can_hold_element,
48)
49from pandas.core.dtypes.common import (
50 is_1d_only_ea_dtype,
51 is_float_dtype,
52 is_integer_dtype,
53 is_list_like,
54 is_scalar,
55 is_string_dtype,
56)
57from pandas.core.dtypes.dtypes import (
58 CategoricalDtype,
59 DatetimeTZDtype,
60 ExtensionDtype,
61 IntervalDtype,
62 NumpyEADtype,
63 PeriodDtype,
64)
65from pandas.core.dtypes.generic import (
66 ABCDataFrame,
67 ABCIndex,
68 ABCNumpyExtensionArray,
69 ABCSeries,
70)
71from pandas.core.dtypes.inference import is_re
72from pandas.core.dtypes.missing import (
73 is_valid_na_for_dtype,
74 isna,
75 na_value_for_dtype,
76)
78from pandas.core import missing
79import pandas.core.algorithms as algos
80from pandas.core.array_algos.putmask import (
81 extract_bool_array,
82 putmask_inplace,
83 putmask_without_repeat,
84 setitem_datetimelike_compat,
85 validate_putmask,
86)
87from pandas.core.array_algos.quantile import quantile_compat
88from pandas.core.array_algos.replace import (
89 compare_or_regex_search,
90 replace_regex,
91 should_use_regex,
92)
93from pandas.core.array_algos.transforms import shift
94from pandas.core.arrays import (
95 DatetimeArray,
96 ExtensionArray,
97 IntervalArray,
98 NumpyExtensionArray,
99 PeriodArray,
100 TimedeltaArray,
101)
102from pandas.core.arrays.string_ import StringDtype
103from pandas.core.base import PandasObject
104import pandas.core.common as com
105from pandas.core.computation import expressions
106from pandas.core.construction import (
107 ensure_wrapped_if_datetimelike,
108 extract_array,
109)
110from pandas.core.indexers import (
111 check_setitem_lengths,
112 is_scalar_indexer,
113 length_of_indexer,
114)
115from pandas.core.indexes.base import get_values_for_csv
117if TYPE_CHECKING:
118 from collections.abc import (
119 Callable,
120 Generator,
121 Iterable,
122 Sequence,
123 )
125 from pandas._typing import (
126 ArrayLike,
127 AxisInt,
128 DtypeBackend,
129 DtypeObj,
130 FillnaOptions,
131 IgnoreRaise,
132 InterpolateOptions,
133 QuantileInterpolation,
134 Shape,
135 npt,
136 )
138 from pandas.core.api import Index
139 from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
141# comparison is faster than is_object_dtype
142_dtype_obj = np.dtype("object")
145class Block(PandasObject, libinternals.Block):
146 """
147 Canonical n-dimensional unit of homogeneous dtype contained in a pandas
148 data structure
150 Index-ignorant; let the container take care of that
151 """
153 values: np.ndarray | ExtensionArray
154 ndim: int
155 refs: BlockValuesRefs
156 __init__: Callable[..., None]
158 __slots__ = ()
159 is_numeric = False
161 @final
162 @cache_readonly
163 def _validate_ndim(self) -> bool:
164 """
165 We validate dimension for blocks that can hold 2D values, which for now
166 means numpy dtypes or EA dtypes like DatetimeTZDtype and PeriodDtype.
167 """
168 return not is_1d_only_ea_dtype(self.dtype)
170 @final
171 @cache_readonly
172 def is_object(self) -> bool:
173 return self.values.dtype == _dtype_obj
175 @final
176 @cache_readonly
177 def is_extension(self) -> bool:
178 return not lib.is_np_dtype(self.values.dtype)
180 @final
181 @cache_readonly
182 def _can_consolidate(self) -> bool:
183 # We _could_ consolidate for DatetimeTZDtype but don't for now.
184 return not self.is_extension
186 @final
187 @cache_readonly
188 def _consolidate_key(self):
189 return self._can_consolidate, self.dtype.name
191 @final
192 @cache_readonly
193 def _can_hold_na(self) -> bool:
194 """
195 Can we store NA values in this Block?
196 """
197 dtype = self.dtype
198 if isinstance(dtype, np.dtype):
199 return dtype.kind not in "iub"
200 return dtype._can_hold_na
202 @final
203 @property
204 def is_bool(self) -> bool:
205 """
206 We can be bool if a) we are bool dtype or b) object dtype with bool objects.
207 """
208 return self.values.dtype == np.dtype(bool)
210 @final
211 def external_values(self):
212 return external_values(self.values)
214 @final
215 @cache_readonly
216 def fill_value(self):
217 # Used in reindex_indexer
218 return na_value_for_dtype(self.dtype, compat=False)
220 @final
221 def _standardize_fill_value(self, value):
222 # if we are passed a scalar None, convert it here
223 if self.dtype != _dtype_obj and is_valid_na_for_dtype(value, self.dtype):
224 value = self.fill_value
225 return value
227 @property
228 def mgr_locs(self) -> BlockPlacement:
229 return self._mgr_locs
231 @mgr_locs.setter
232 def mgr_locs(self, new_mgr_locs: BlockPlacement) -> None:
233 self._mgr_locs = new_mgr_locs
235 @final
236 def make_block(
237 self,
238 values,
239 placement: BlockPlacement | None = None,
240 refs: BlockValuesRefs | None = None,
241 ) -> Block:
242 """
243 Create a new block, with type inference propagate any values that are
244 not specified
245 """
246 if placement is None:
247 placement = self._mgr_locs
248 if self.is_extension:
249 values = ensure_block_shape(values, ndim=self.ndim)
251 return new_block(values, placement=placement, ndim=self.ndim, refs=refs)
253 @final
254 def make_block_same_class(
255 self,
256 values,
257 placement: BlockPlacement | None = None,
258 refs: BlockValuesRefs | None = None,
259 ) -> Self:
260 """Wrap given values in a block of same type as self."""
261 # Pre-2.0 we called ensure_wrapped_if_datetimelike because fastparquet
262 # relied on it, as of 2.0 the caller is responsible for this.
263 if placement is None:
264 placement = self._mgr_locs
266 # We assume maybe_coerce_values has already been called
267 return type(self)(values, placement=placement, ndim=self.ndim, refs=refs)
269 @final
270 def __repr__(self) -> str:
271 # don't want to print out all of the items here
272 name = type(self).__name__
273 if self.ndim == 1:
274 result = f"{name}: {len(self)} dtype: {self.dtype}"
275 else:
276 shape = " x ".join([str(s) for s in self.shape])
277 result = f"{name}: {self.mgr_locs.indexer}, {shape}, dtype: {self.dtype}"
279 return result
281 @final
282 def __len__(self) -> int:
283 return len(self.values)
285 @final
286 def slice_block_columns(self, slc: slice) -> Self:
287 """
288 Perform __getitem__-like, return result as block.
289 """
290 new_mgr_locs = self._mgr_locs[slc]
292 new_values = self._slice(slc)
293 refs = self.refs
294 return type(self)(new_values, new_mgr_locs, self.ndim, refs=refs)
296 @final
297 def take_block_columns(self, indices: npt.NDArray[np.intp]) -> Self:
298 """
299 Perform __getitem__-like, return result as block.
301 Only supports slices that preserve dimensionality.
302 """
303 # Note: only called from is from internals.concat, and we can verify
304 # that never happens with 1-column blocks, i.e. never for ExtensionBlock.
306 new_mgr_locs = self._mgr_locs[indices]
308 new_values = self._slice(indices)
309 return type(self)(new_values, new_mgr_locs, self.ndim, refs=None)
311 @final
312 def getitem_block_columns(
313 self, slicer: slice, new_mgr_locs: BlockPlacement, ref_inplace_op: bool = False
314 ) -> Self:
315 """
316 Perform __getitem__-like, return result as block.
318 Only supports slices that preserve dimensionality.
319 """
320 new_values = self._slice(slicer)
321 refs = self.refs if not ref_inplace_op or self.refs.has_reference() else None
322 return type(self)(new_values, new_mgr_locs, self.ndim, refs=refs)
324 @final
325 def _can_hold_element(self, element: Any) -> bool:
326 """require the same dtype as ourselves"""
327 element = extract_array(element, extract_numpy=True)
328 return can_hold_element(self.values, element)
330 @final
331 def should_store(self, value: ArrayLike) -> bool:
332 """
333 Should we set self.values[indexer] = value inplace or do we need to cast?
335 Parameters
336 ----------
337 value : np.ndarray or ExtensionArray
339 Returns
340 -------
341 bool
342 """
343 return value.dtype == self.dtype
345 # ---------------------------------------------------------------------
346 # Apply/Reduce and Helpers
348 @final
349 def apply(self, func, **kwargs) -> list[Block]:
350 """
351 apply the function to my values; return a block if we are not
352 one
353 """
354 result = func(self.values, **kwargs)
356 result = maybe_coerce_values(result)
357 return self._split_op_result(result)
359 @final
360 def reduce(self, func) -> Block:
361 # We will apply the function and reshape the result into a single-row
362 # Block with the same mgr_locs; squeezing will be done at a higher level
363 assert self.ndim == 2
365 result = func(self.values)
367 if self.values.ndim == 1:
368 res_values = result
369 else:
370 res_values = result.reshape(-1, 1)
372 return self.make_block(res_values)
374 @final
375 def _split_op_result(self, result: ArrayLike) -> list[Block]:
376 # See also: split_and_operate
377 if result.ndim > 1 and isinstance(result.dtype, ExtensionDtype):
378 # TODO(EA2D): unnecessary with 2D EAs
379 # if we get a 2D ExtensionArray, we need to split it into 1D pieces
380 nbs = []
381 for i, loc in enumerate(self._mgr_locs):
382 if not is_1d_only_ea_dtype(result.dtype):
383 vals = result[i : i + 1]
384 else:
385 vals = result[i]
387 bp = BlockPlacement(loc)
388 block = self.make_block(values=vals, placement=bp)
389 nbs.append(block)
390 return nbs
392 nb = self.make_block(result)
394 return [nb]
396 @final
397 def _split(self) -> Generator[Block]:
398 """
399 Split a block into a list of single-column blocks.
400 """
401 assert self.ndim == 2
403 for i, ref_loc in enumerate(self._mgr_locs):
404 vals = self.values[slice(i, i + 1)]
406 bp = BlockPlacement(ref_loc)
407 nb = type(self)(vals, placement=bp, ndim=2, refs=self.refs)
408 yield nb
410 @final
411 def split_and_operate(self, func, *args, **kwargs) -> list[Block]:
412 """
413 Split the block and apply func column-by-column.
415 Parameters
416 ----------
417 func : Block method
418 *args
419 **kwargs
421 Returns
422 -------
423 List[Block]
424 """
425 assert self.ndim == 2 and self.shape[0] != 1
427 res_blocks = []
428 for nb in self._split():
429 rbs = func(nb, *args, **kwargs)
430 res_blocks.extend(rbs)
431 return res_blocks
433 # ---------------------------------------------------------------------
434 # Up/Down-casting
436 @final
437 def coerce_to_target_dtype(self, other, raise_on_upcast: bool) -> Block:
438 """
439 coerce the current block to a dtype compat for other
440 we will return a block, possibly object, and not raise
442 we can also safely try to coerce to the same dtype
443 and will receive the same block
444 """
445 new_dtype = find_result_type(self.values.dtype, other)
446 if new_dtype == self.dtype:
447 if lib.is_np_dtype(new_dtype, "mM"):
448 # GH#61671 e.g. datetime64[ns] column and a replacement value
449 # outside ns range. find_common_type picked the highest
450 # resolution which can't represent the value.
451 raise OutOfBoundsDatetime(
452 f"Incompatible (high-resolution) value for "
453 f"dtype='{self.dtype}'. "
454 "Explicitly cast before operating."
455 )
456 # GH#52927 avoid RecursionError
457 raise AssertionError(
458 "Something has gone wrong, please report a bug at "
459 "https://github.com/pandas-dev/pandas/issues"
460 )
462 # In a future version of pandas, the default will be that
463 # setting `nan` into an integer series won't raise.
464 if (
465 is_scalar(other)
466 and is_integer_dtype(self.values.dtype)
467 and isna(other)
468 and other is not NaT
469 and not (
470 isinstance(other, (np.datetime64, np.timedelta64)) and np.isnat(other)
471 )
472 ):
473 raise_on_upcast = False
474 elif (
475 isinstance(other, np.ndarray)
476 and other.ndim == 1
477 and is_integer_dtype(self.values.dtype)
478 and is_float_dtype(other.dtype)
479 and lib.has_only_ints_or_nan(other)
480 ):
481 raise_on_upcast = False
483 if raise_on_upcast:
484 raise TypeError(f"Invalid value '{other}' for dtype '{self.values.dtype}'")
485 if self.values.dtype == new_dtype:
486 raise AssertionError(
487 f"Did not expect new dtype {new_dtype} to equal self.dtype "
488 f"{self.values.dtype}. Please report a bug at "
489 "https://github.com/pandas-dev/pandas/issues."
490 )
491 try:
492 return self.astype(new_dtype)
493 except OutOfBoundsDatetime as err:
494 # e.g. GH#56419 if self.dtype is a low-resolution dt64 and we try to
495 # upcast to a higher-resolution dt64, we may have entries that are
496 # out of bounds for the higher resolution.
497 # Re-raise with a more informative message.
498 raise OutOfBoundsDatetime(
499 f"Incompatible (high-resolution) value for dtype='{self.dtype}'. "
500 "Explicitly cast before operating."
501 ) from err
503 @final
504 def convert(self) -> list[Block]:
505 """
506 Attempt to coerce any object types to better types. Return a copy
507 of the block (if copy = True).
508 """
509 if not self.is_object:
510 return [self.copy(deep=False)]
512 if self.ndim != 1 and self.shape[0] != 1:
513 blocks = self.split_and_operate(Block.convert)
514 if all(blk.dtype.kind == "O" for blk in blocks):
515 # Avoid fragmenting the block if convert is a no-op
516 return [self.copy(deep=False)]
517 return blocks
519 values = self.values
520 if values.ndim == 2:
521 # the check above ensures we only get here with values.shape[0] == 1,
522 # avoid doing .ravel as that might make a copy
523 values = values[0]
525 res_values = lib.maybe_convert_objects(
526 values, # type: ignore[arg-type]
527 convert_non_numeric=True,
528 )
529 refs = None
530 if res_values is values or (
531 isinstance(res_values, NumpyExtensionArray)
532 and res_values._ndarray is values
533 ):
534 refs = self.refs
536 res_values = ensure_block_shape(res_values, self.ndim)
537 res_values = maybe_coerce_values(res_values)
538 return [self.make_block(res_values, refs=refs)]
540 def convert_dtypes(
541 self,
542 infer_objects: bool = True,
543 convert_string: bool = True,
544 convert_integer: bool = True,
545 convert_boolean: bool = True,
546 convert_floating: bool = True,
547 dtype_backend: DtypeBackend = "numpy_nullable",
548 ) -> list[Block]:
549 if infer_objects and self.is_object:
550 blks = self.convert()
551 else:
552 blks = [self]
554 if not any(
555 [convert_floating, convert_integer, convert_boolean, convert_string]
556 ):
557 return [b.copy(deep=False) for b in blks]
559 rbs = []
560 for blk in blks:
561 # Determine dtype column by column
562 sub_blks = (
563 [blk] if blk.ndim == 1 or blk.shape[0] == 1 else list(blk._split())
564 )
565 dtypes = [
566 convert_dtypes(
567 b.values,
568 convert_string,
569 convert_integer,
570 convert_boolean,
571 convert_floating,
572 infer_objects,
573 dtype_backend,
574 )
575 for b in sub_blks
576 ]
577 if all(dtype == blk.dtype for dtype in dtypes):
578 # Avoid block splitting if no dtype changes
579 rbs.append(blk.copy(deep=False))
580 continue
582 for dtype, b in zip(dtypes, sub_blks, strict=True):
583 rbs.append(b.astype(dtype=dtype, squeeze=b.ndim != 1))
584 return rbs
586 # ---------------------------------------------------------------------
587 # Array-Like Methods
589 @final
590 @cache_readonly
591 def dtype(self) -> DtypeObj:
592 return self.values.dtype
594 @final
595 def astype(
596 self,
597 dtype: DtypeObj,
598 errors: IgnoreRaise = "raise",
599 squeeze: bool = False,
600 ) -> Block:
601 """
602 Coerce to the new dtype.
604 Parameters
605 ----------
606 dtype : np.dtype or ExtensionDtype
607 errors : str, {'raise', 'ignore'}, default 'raise'
608 - ``raise`` : allow exceptions to be raised
609 - ``ignore`` : suppress exceptions. On error return original object
610 squeeze : bool, default False
611 squeeze values to ndim=1 if only one column is given
613 Returns
614 -------
615 Block
616 """
617 values = self.values
618 if squeeze and values.ndim == 2 and is_1d_only_ea_dtype(dtype):
619 if values.shape[0] != 1:
620 raise ValueError("Can not squeeze with more than one column.")
621 values = values[0, :] # type: ignore[call-overload]
623 new_values = astype_array_safe(values, dtype, errors=errors)
625 new_values = maybe_coerce_values(new_values)
627 refs = None
628 if astype_is_view(values.dtype, new_values.dtype):
629 refs = self.refs
631 newb = self.make_block(new_values, refs=refs)
632 if newb.shape != self.shape:
633 raise TypeError(
634 f"cannot set astype for dtype "
635 f"({self.dtype.name} [{self.shape}]) to different shape "
636 f"({newb.dtype.name} [{newb.shape}])"
637 )
638 return newb
640 @final
641 def get_values_for_csv(
642 self, *, float_format, date_format, decimal, na_rep: str = "nan", quoting=None
643 ) -> Block:
644 """convert to our native types format"""
645 result = get_values_for_csv(
646 self.values,
647 na_rep=na_rep,
648 quoting=quoting,
649 float_format=float_format,
650 date_format=date_format,
651 decimal=decimal,
652 )
653 return self.make_block(result)
655 @final
656 def copy(self, *, deep: bool) -> Self:
657 """copy constructor"""
658 values = self.values
659 refs: BlockValuesRefs | None
660 if deep:
661 if isinstance(values, np.ndarray):
662 # "K" preserves memory layout; default "C" flips contiguity
663 # for non-C-order blocks (GH#60469).
664 values = values.copy(order="K")
665 else:
666 values = values.copy()
667 refs = None
668 else:
669 values = values.view()
670 refs = self.refs
671 return type(self)(values, placement=self._mgr_locs, ndim=self.ndim, refs=refs)
673 # ---------------------------------------------------------------------
674 # Copy-on-Write Helpers
676 def _maybe_copy(self, inplace: bool, deep: bool = True) -> Self:
677 if inplace and not self.refs.has_reference():
678 return self
679 return self.copy(deep=deep)
681 @final
682 def _get_refs_and_copy(self, inplace: bool):
683 refs = None
684 copy = not inplace
685 if inplace:
686 if self.refs.has_reference():
687 copy = True
688 else:
689 refs = self.refs
690 return copy, refs
692 # ---------------------------------------------------------------------
693 # Replace
695 @final
696 def replace(
697 self,
698 to_replace,
699 value,
700 inplace: bool = False,
701 # mask may be pre-computed if we're called from replace_list
702 mask: npt.NDArray[np.bool_] | None = None,
703 ) -> list[Block]:
704 """
705 replace the to_replace value with value, possible to create new
706 blocks here this is just a call to putmask.
707 """
709 # Note: the checks we do in NDFrame.replace ensure we never get
710 # here with listlike to_replace or value, as those cases
711 # go through replace_list
712 values = self.values
714 if not self._can_hold_element(to_replace):
715 # We cannot hold `to_replace`, so we know immediately that
716 # replacing it is a no-op.
717 # Note: If to_replace were a list, NDFrame.replace would call
718 # replace_list instead of replace.
719 return [self._maybe_copy(inplace, deep=False)]
721 if mask is None:
722 mask = missing.mask_missing(values, to_replace)
723 if not mask.any():
724 # Note: we get here with test_replace_extension_other incorrectly
725 # bc _can_hold_element is incorrect.
726 return [self._maybe_copy(inplace, deep=False)]
728 elif self._can_hold_element(value) or (self.dtype == "string" and is_re(value)):
729 # TODO(CoW): Maybe split here as well into columns where mask has True
730 # and rest?
731 blk = self._maybe_copy(inplace)
732 putmask_inplace(blk.values, mask, value)
733 return [blk]
735 elif self.ndim == 1 or self.shape[0] == 1:
736 if value is None or value is NA:
737 blk = self.astype(np.dtype(object))
738 else:
739 blk = self.coerce_to_target_dtype(value, raise_on_upcast=False)
740 return blk.replace(
741 to_replace=to_replace,
742 value=value,
743 inplace=True,
744 mask=mask,
745 )
747 else:
748 # split so that we only upcast where necessary
749 blocks = []
750 for i, nb in enumerate(self._split()):
751 blocks.extend(
752 type(self).replace(
753 nb,
754 to_replace=to_replace,
755 value=value,
756 inplace=True,
757 mask=mask[i : i + 1],
758 )
759 )
760 return blocks
762 @final
763 def _replace_regex(
764 self,
765 to_replace,
766 value,
767 inplace: bool = False,
768 mask=None,
769 ) -> list[Block]:
770 """
771 Replace elements by the given value.
773 Parameters
774 ----------
775 to_replace : object or pattern
776 Scalar to replace or regular expression to match.
777 value : object
778 Replacement object.
779 inplace : bool, default False
780 Perform inplace modification.
781 mask : array-like of bool, optional
782 True indicate corresponding element is ignored.
784 Returns
785 -------
786 List[Block]
787 """
788 if not is_re(to_replace) and not self._can_hold_element(to_replace):
789 # i.e. only if self.is_object is True, but could in principle include a
790 # String ExtensionBlock
791 return [self.copy(deep=False)]
793 if is_re(to_replace) and _regex_target_dtype(self.dtype) not in [
794 object,
795 "string",
796 ]:
797 # only object or string dtype can hold strings, and a regex object
798 # will only match strings
799 return [self.copy(deep=False)]
801 if not (
802 self._can_hold_element(value) or (self.dtype == "string" and is_re(value))
803 ):
804 # GH#57733 - astype to object may return a block sharing memory
805 # with self (e.g. StringArray backed by object ndarray). Since
806 # replace_regex mutates values in-place, we must ensure the
807 # block's data is independent.
808 block = self.astype(np.dtype(object))
809 if astype_is_view(self.dtype, np.dtype(object)):
810 block = block._maybe_copy(inplace=True)
811 else:
812 block = self._maybe_copy(inplace)
814 rx = re.compile(to_replace)
816 replace_regex(block.values, rx, value, mask)
817 return [block]
819 @final
820 def replace_list(
821 self,
822 src_list: Iterable[Any],
823 dest_list: Sequence[Any],
824 inplace: bool = False,
825 regex: bool = False,
826 ) -> list[Block]:
827 """
828 See BlockManager.replace_list docstring.
829 """
830 values = self.values
832 # Exclude anything that we know we won't contain
833 pairs = [
834 (x, y)
835 for x, y in zip(src_list, dest_list, strict=True)
836 if (self._can_hold_element(x) or (self.dtype == "string" and is_re(x)))
837 ]
838 if not pairs:
839 return [self.copy(deep=False)]
841 src_len = len(pairs) - 1
843 if is_string_dtype(values.dtype) or (
844 regex and is_string_dtype(_regex_target_dtype(values.dtype))
845 ):
846 # Calculate the mask once, prior to the call of comp
847 # in order to avoid repeating the same computations
848 na_mask = ~isna(values)
849 masks: Iterable[npt.NDArray[np.bool_]] = (
850 extract_bool_array(
851 compare_or_regex_search(values, s[0], regex=regex, mask=na_mask),
852 )
853 for s in pairs
854 )
855 else:
856 # GH#38086 faster if we know we dont need to check for regex
857 masks = (missing.mask_missing(values, s[0]) for s in pairs)
858 # Materialize if inplace = True, since the masks can change
859 # as we replace
860 if inplace:
861 masks = list(masks)
863 # Don't set up refs here, otherwise we will think that we have
864 # references when we check again later
865 rb = [self]
867 for i, ((src, dest), mask) in enumerate(zip(pairs, masks, strict=True)):
868 new_rb: list[Block] = []
870 # GH-39338: _replace_coerce can split a block into
871 # single-column blocks, so track the index so we know
872 # where to index into the mask
873 for blk_num, blk in enumerate(rb):
874 if len(rb) == 1:
875 m = mask
876 else:
877 mib = mask
878 assert not isinstance(mib, bool)
879 m = mib[blk_num : blk_num + 1]
881 # error: Argument "mask" to "_replace_coerce" of "Block" has
882 # incompatible type "Union[ExtensionArray, ndarray[Any, Any], bool]";
883 # expected "ndarray[Any, dtype[bool_]]"
884 result = blk._replace_coerce(
885 to_replace=src,
886 value=dest,
887 mask=m,
888 inplace=inplace,
889 regex=regex,
890 )
892 if i != src_len:
893 # This is ugly, but we have to get rid of intermediate refs.
894 self_blk_ids = {
895 id(ref()): idx
896 for idx, ref in enumerate(self.refs.referenced_blocks)
897 }
898 # Blocks still sharing memory with self are temporary, so drop
899 # them from self's refs. Pop highest index first: popping in
900 # ascending order shifts every later index down, so with two or
901 # more of them the later pops hit the wrong entry or ran off the
902 # end (GH#61972).
903 drop = {
904 self_blk_ids[id(res_blk)]
905 for res_blk in result
906 if res_blk.refs is self.refs
907 and id(res_blk) in self_blk_ids
908 and res_blk is not self
909 }
910 for idx in sorted(drop, reverse=True):
911 self.refs.referenced_blocks.pop(idx)
912 # Note: blocks whose refs are not self.refs are left alone. Those
913 # refs still track every live block sharing that data, which may
914 # include sibling sub-blocks from _split on an intermediate
915 # block, and always include the block itself. Emptying them would
916 # leave those blocks untracked, so writes through a later view
917 # would not trigger copy-on-write (GH#58966).
918 new_rb.extend(result)
919 rb = new_rb
921 # The drop above assumes every block it removes is a short-lived intermediate,
922 # but a block can survive to the end untouched (a later pair whose mask is
923 # empty hands it straight back). Re-register whatever we are actually
924 # returning, otherwise it stays invisible to has_reference() and writes
925 # through a later view skip copy-on-write (GH#58966).
926 # Blocks split off the same parent share one refs, so cache the ids per refs
927 # rather than rescanning the list for each of them.
928 registered: dict[int, set[int]] = {}
929 for ret_blk in rb:
930 refs = ret_blk.refs
931 ids = registered.get(id(refs))
932 if ids is None:
933 ids = {id(ref()) for ref in refs.referenced_blocks}
934 registered[id(refs)] = ids
935 if id(ret_blk) not in ids:
936 refs.add_reference(ret_blk)
937 ids.add(id(ret_blk))
938 return rb
940 @final
941 def _replace_coerce(
942 self,
943 to_replace,
944 value,
945 mask: npt.NDArray[np.bool_],
946 inplace: bool = True,
947 regex: bool = False,
948 ) -> list[Block]:
949 """
950 Replace value corresponding to the given boolean array with another
951 value.
953 Parameters
954 ----------
955 to_replace : object or pattern
956 Scalar to replace or regular expression to match.
957 value : object
958 Replacement object.
959 mask : np.ndarray[bool]
960 True indicate corresponding element is ignored.
961 inplace : bool, default True
962 Perform inplace modification.
963 regex : bool, default False
964 If true, perform regular expression substitution.
966 Returns
967 -------
968 List[Block]
969 """
970 if should_use_regex(regex, to_replace):
971 return self._replace_regex(
972 to_replace,
973 value,
974 inplace=inplace,
975 mask=mask,
976 )
977 else:
978 if value is None:
979 # gh-45601, gh-45836, gh-46634
980 if mask.any():
981 has_ref = self.refs.has_reference()
982 nb = self.astype(np.dtype(object))
983 if not inplace:
984 nb = nb.copy(deep=True)
985 elif inplace and has_ref and nb.refs.has_reference():
986 # no copy in astype and we had refs before
987 nb = nb.copy(deep=True)
988 putmask_inplace(nb.values, mask, value)
989 return [nb]
990 return [self.copy(deep=False)]
991 return self.replace(
992 to_replace=to_replace,
993 value=value,
994 inplace=inplace,
995 mask=mask,
996 )
998 # ---------------------------------------------------------------------
999 # 2D Methods - Shared by NumpyBlock and NDArrayBackedExtensionBlock
1000 # but not ExtensionBlock
1002 def _maybe_squeeze_arg(self, arg: np.ndarray) -> np.ndarray:
1003 """
1004 For compatibility with 1D-only ExtensionArrays.
1005 """
1006 return arg
1008 def _unwrap_setitem_indexer(self, indexer):
1009 """
1010 For compatibility with 1D-only ExtensionArrays.
1011 """
1012 return indexer
1014 # NB: this cannot be made cache_readonly because in mgr.set_values we pin
1015 # new .values that can have different shape GH#42631
1016 @property
1017 def shape(self) -> Shape:
1018 return self.values.shape
1020 def iget(self, i: int | tuple[int, int] | tuple[slice, int]) -> np.ndarray:
1021 # In the case where we have a tuple[slice, int], the slice will always
1022 # be slice(None)
1023 # Note: only reached with self.ndim == 2
1024 # Invalid index type "Union[int, Tuple[int, int], Tuple[slice, int]]"
1025 # for "Union[ndarray[Any, Any], ExtensionArray]"; expected type
1026 # "Union[int, integer[Any]]"
1027 return self.values[i] # type: ignore[index]
1029 def _slice(
1030 self, slicer: slice | npt.NDArray[np.bool_] | npt.NDArray[np.intp]
1031 ) -> ArrayLike:
1032 """return a slice of my values"""
1034 return self.values[slicer]
1036 def set_inplace(self, locs, values: ArrayLike, copy: bool = False) -> None:
1037 """
1038 Modify block values in-place with new item value.
1040 If copy=True, first copy the underlying values in place before modifying
1041 (for Copy-on-Write).
1043 Notes
1044 -----
1045 `set_inplace` never creates a new array or new Block, whereas `setitem`
1046 _may_ create a new array and always creates a new Block.
1048 Caller is responsible for checking values.dtype == self.dtype.
1049 """
1050 if copy:
1051 self.values = self.values.copy()
1052 self.values[locs] = values
1054 @final
1055 def take_nd(
1056 self,
1057 indexer: npt.NDArray[np.intp],
1058 axis: AxisInt,
1059 new_mgr_locs: BlockPlacement | None = None,
1060 fill_value=lib.no_default,
1061 ) -> Block:
1062 """
1063 Take values according to indexer and return them as a block.
1064 """
1065 values = self.values
1067 if fill_value is lib.no_default:
1068 fill_value = self.fill_value
1069 allow_fill = False
1070 else:
1071 allow_fill = True
1073 # Note: algos.take_nd has upcast logic similar to coerce_to_target_dtype
1074 new_values = algos.take_nd(
1075 values, indexer, axis=axis, allow_fill=allow_fill, fill_value=fill_value
1076 )
1078 # Called from three places in managers, all of which satisfy
1079 # these assertions
1080 if isinstance(self, ExtensionBlock):
1081 # NB: in this case, the 'axis' kwarg will be ignored in the
1082 # algos.take_nd call above.
1083 assert not (self.ndim == 1 and new_mgr_locs is None)
1084 assert not (axis == 0 and new_mgr_locs is None)
1086 if new_mgr_locs is None:
1087 new_mgr_locs = self._mgr_locs
1089 if new_values.dtype != self.dtype:
1090 return self.make_block(new_values, new_mgr_locs)
1091 else:
1092 return self.make_block_same_class(new_values, new_mgr_locs)
1094 def _unstack(
1095 self,
1096 unstacker,
1097 fill_value,
1098 new_placement: npt.NDArray[np.intp],
1099 needs_masking: npt.NDArray[np.bool_],
1100 ):
1101 """
1102 Return a list of unstacked blocks of self
1104 Parameters
1105 ----------
1106 unstacker : reshape._Unstacker
1107 fill_value : int
1108 Only used in ExtensionBlock._unstack
1109 new_placement : np.ndarray[np.intp]
1110 needs_masking : np.ndarray[bool]
1111 Only used in ExtensionBlock._unstack
1113 Returns
1114 -------
1115 blocks : list of Block
1116 New blocks of unstacked values.
1117 """
1118 new_values = unstacker.get_new_values(self.values.T, fill_value=fill_value)
1119 new_values = new_values.T
1121 # get_new_values can return a view of self.values on the identity-indexer
1122 # fast path (see _Unstacker._make_sorted_values); every other path copies
1123 # via take_nd. Gate on that, then the O(1) may_share_memory, so CoW
1124 # tracks the reference. GH#65107. For NDArrayBacked EAs (e.g. tz-aware
1125 # datetime, period) np.asarray boxes to object, so may_share_memory on
1126 # the EA objects is always False; compare the backing ._ndarray instead
1127 # as get_result does, else the shared view goes untracked. GH#65748
1128 if isinstance(self.values, np.ndarray):
1129 source, unstacked = self.values, new_values
1130 else:
1131 values = cast("NDArrayBackedExtensionArray", self.values)
1132 source, unstacked = values._ndarray, new_values._ndarray
1134 if unstacker._indexer_is_identity and np.may_share_memory(unstacked, source):
1135 refs = self.refs
1136 else:
1137 refs = None
1139 bp = BlockPlacement(new_placement)
1140 blocks = [new_block_2d(new_values, placement=bp, refs=refs)]
1141 return blocks
1143 # ---------------------------------------------------------------------
1145 def setitem(self, indexer, value) -> Block:
1146 """
1147 Attempt self.values[indexer] = value, possibly creating a new array.
1149 Parameters
1150 ----------
1151 indexer : tuple, list-like, array-like, slice, int
1152 The subset of self.values to set
1153 value : object
1154 The value being set
1156 Returns
1157 -------
1158 Block
1160 Notes
1161 -----
1162 `indexer` is a direct slice/positional indexer. `value` must
1163 be a compatible shape.
1164 """
1166 value = self._standardize_fill_value(value)
1168 values = cast("np.ndarray", self.values)
1169 if self.ndim == 2:
1170 values = values.T
1172 # length checking
1173 check_setitem_lengths(indexer, value, values)
1175 if self.dtype != _dtype_obj:
1176 # GH48933: extract_array would convert a pd.Series value to np.ndarray
1177 value = extract_array(value, extract_numpy=True)
1178 try:
1179 casted = np_can_hold_element(values.dtype, value)
1180 except LossySetitemError:
1181 # current dtype cannot store value, coerce to common dtype
1182 nb = self.coerce_to_target_dtype(value, raise_on_upcast=True)
1183 return nb.setitem(indexer, value)
1184 else:
1185 if self.dtype == _dtype_obj and not is_scalar_indexer(indexer, values.ndim):
1186 if isinstance(
1187 indexer, (ABCSeries, ABCIndex, np.ndarray, list, range, slice)
1188 ):
1189 num_set = length_of_indexer(indexer, values)
1190 else:
1191 num_set = len(values[indexer])
1192 casted = setitem_datetimelike_compat(values, num_set, casted)
1194 if (
1195 self.ndim == 1
1196 and isinstance(casted, list)
1197 and len(casted) > 0
1198 and isinstance(casted[0], (tuple, list, np.ndarray))
1199 ):
1200 # Prevent numpy from unpacking nested containers
1201 # (e.g. tuples) into a scalar cell during assignment. GH#37629
1202 # Only for 1D blocks: on a 2D block a list of lists/tuples is
1203 # a genuine 2D value whose rows must not be wrapped. GH#65264
1204 casted = construct_1d_object_array_from_listlike(casted)
1206 self = self._maybe_copy(inplace=True)
1207 values = cast("np.ndarray", self.values.T)
1208 if isinstance(casted, np.ndarray) and casted.ndim == 1 and len(casted) == 1:
1209 # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615
1210 casted = casted[0, ...]
1211 try:
1212 values[indexer] = casted
1213 except (TypeError, ValueError) as err:
1214 if is_list_like(casted):
1215 raise ValueError(
1216 "setting an array element with a sequence."
1217 ) from err
1218 raise
1219 return self
1221 def putmask(self, mask, new) -> list[Block]:
1222 """
1223 putmask the data to the block; it is possible that we may create a
1224 new dtype of block
1226 Return the resulting block(s).
1228 Parameters
1229 ----------
1230 mask : np.ndarray[bool], SparseArray[bool], or BooleanArray
1231 new : an ndarray/object
1233 Returns
1234 -------
1235 List[Block]
1236 """
1237 orig_mask = mask
1238 values = cast("np.ndarray", self.values)
1239 mask, noop = validate_putmask(values.T, mask)
1240 assert not isinstance(new, (ABCIndex, ABCSeries, ABCDataFrame))
1242 if new is lib.no_default:
1243 new = self.fill_value
1245 new = self._standardize_fill_value(new)
1246 new = extract_array(new, extract_numpy=True)
1248 if noop:
1249 return [self.copy(deep=False)]
1251 try:
1252 casted = np_can_hold_element(values.dtype, new)
1254 self = self._maybe_copy(inplace=True)
1255 values = cast("np.ndarray", self.values)
1257 putmask_without_repeat(values.T, mask, casted)
1258 return [self]
1259 except LossySetitemError:
1260 if self.ndim == 1 or self.shape[0] == 1:
1261 # no need to split columns
1263 if not is_list_like(new):
1264 # using just new[indexer] can't save us the need to cast
1265 return self.coerce_to_target_dtype(
1266 new, raise_on_upcast=True
1267 ).putmask(mask, new)
1268 else:
1269 indexer = mask.nonzero()[0]
1270 nb = self.setitem(indexer, new[indexer])
1271 return [nb]
1273 else:
1274 is_array = isinstance(new, np.ndarray)
1276 res_blocks = []
1277 for i, nb in enumerate(self._split()):
1278 n = new
1279 if is_array:
1280 # we have a different value per-column
1281 n = new[:, i : i + 1]
1283 submask = orig_mask[:, i : i + 1]
1284 rbs = nb.putmask(submask, n)
1285 res_blocks.extend(rbs)
1286 return res_blocks
1288 def where(self, other, cond) -> list[Block]:
1289 """
1290 evaluate the block; return result block(s) from the result
1292 Parameters
1293 ----------
1294 other : an ndarray/object
1295 cond : np.ndarray[bool], SparseArray[bool], or BooleanArray
1297 Returns
1298 -------
1299 List[Block]
1300 """
1301 assert cond.ndim == self.ndim
1302 assert not isinstance(other, (ABCIndex, ABCSeries, ABCDataFrame))
1304 transpose = self.ndim == 2
1306 cond = extract_bool_array(cond)
1308 # EABlocks override where
1309 values = cast("np.ndarray", self.values)
1310 orig_other = other
1311 if transpose:
1312 values = values.T
1314 icond, noop = validate_putmask(values, ~cond)
1315 if noop:
1316 return [self.copy(deep=False)]
1318 if other is lib.no_default:
1319 other = self.fill_value
1321 other = self._standardize_fill_value(other)
1323 try:
1324 # try/except here is equivalent to a self._can_hold_element check,
1325 # but this gets us back 'casted' which we will reuse below;
1326 # without using 'casted', expressions.where may do unwanted upcasts.
1327 casted = np_can_hold_element(values.dtype, other)
1328 except (ValueError, TypeError, LossySetitemError):
1329 # we cannot coerce, return a compat dtype
1331 if self.ndim == 1 or self.shape[0] == 1:
1332 # no need to split columns
1334 block = self.coerce_to_target_dtype(other, raise_on_upcast=False)
1335 return block.where(orig_other, cond)
1337 else:
1338 is_array = isinstance(other, (np.ndarray, ExtensionArray))
1340 res_blocks = []
1341 for i, nb in enumerate(self._split()):
1342 oth = other
1343 if is_array:
1344 # we have a different value per-column
1345 oth = other[:, i : i + 1]
1347 submask = cond[:, i : i + 1]
1348 rbs = nb.where(oth, submask)
1349 res_blocks.extend(rbs)
1350 return res_blocks
1352 else:
1353 other = casted
1354 alt = setitem_datetimelike_compat(values, icond.sum(), other) # type: ignore[arg-type]
1355 if isinstance(other, tuple) and values.dtype == _dtype_obj:
1356 # GH#37681 np.where/np.putmask unpack tuples, so wrap in an
1357 # object array to ensure the tuple is treated as a scalar.
1358 fill_arr = np.empty(1, dtype=object)
1359 fill_arr[0] = other
1360 result = values.copy()
1361 np.putmask(result, icond, fill_arr)
1362 elif alt is not other:
1363 if is_list_like(other) and len(other) < len(values):
1364 # call np.where with other to get the appropriate ValueError
1365 np.where(~icond, values, other)
1366 raise NotImplementedError(
1367 "This should not be reached; call to np.where above is "
1368 "expected to raise ValueError. Please report a bug at "
1369 "github.com/pandas-dev/pandas"
1370 )
1371 result = values.copy()
1372 np.putmask(result, icond, alt)
1373 else:
1374 # By the time we get here, we should have all Series/Index
1375 # args extracted to ndarray
1376 if (
1377 is_list_like(other)
1378 and not isinstance(other, np.ndarray)
1379 and len(other) == self.shape[-1]
1380 ):
1381 # If we don't do this broadcasting here, then expressions.where
1382 # will broadcast a 1D other to be row-like instead of
1383 # column-like.
1384 other = np.array(other).reshape(values.shape)
1385 # If lengths don't match (or len(other)==1), we will raise
1386 # inside expressions.where, see test_series_where
1388 # Note: expressions.where may upcast.
1389 result = expressions.where(~icond, values, other)
1390 # The np_can_hold_element check _should_ ensure that we always
1391 # have result.dtype == self.dtype here.
1393 if transpose:
1394 result = result.T
1396 return [self.make_block(result)]
1398 def fillna(
1399 self,
1400 value,
1401 limit: int | None = None,
1402 inplace: bool = False,
1403 ) -> list[Block]:
1404 """
1405 fillna on the block with the value. If we fail, then convert to
1406 block to hold objects instead and try again
1407 """
1408 # Caller is responsible for validating limit; if int it is strictly positive.
1409 # Caller is also responsible for unboxing Series/Index values, GH#22954
1410 inplace = validate_bool_kwarg(inplace, "inplace")
1412 if not self._can_hold_na:
1413 # can short-circuit the isna call
1414 return [self.copy(deep=False)]
1416 # Fast path: single-pass Cython for scalar fill values on
1417 # float and object dtypes with limit or inplace. Avoids
1418 # mask allocation and cumsum computation. GH#42147
1419 if (
1420 is_scalar(value)
1421 and self.dtype
1422 in (np.dtype("float64"), np.dtype("float32"), np.dtype("object"))
1423 and (limit is not None or inplace)
1424 ):
1425 values = cast("np.ndarray", self.values)
1426 try:
1427 casted = np_can_hold_element(values.dtype, value)
1428 except (LossySetitemError, NotImplementedError):
1429 pass
1430 else:
1431 copy, refs = self._get_refs_and_copy(inplace)
1432 new_values = values.copy() if copy else values
1434 if new_values.ndim == 1:
1435 filled = libalgos.scalar_fillna_inplace(
1436 new_values, casted, limit=limit
1437 )
1438 else:
1439 filled = libalgos.scalar_fillna_2d_inplace(
1440 new_values, casted, limit=limit
1441 )
1443 if filled == 0:
1444 return [self.copy(deep=False)]
1446 return [self.make_block_same_class(new_values, refs=refs)]
1448 # Standard path: generate mask, then apply via putmask/where
1449 mask = isna(self.values)
1450 mask, noop = validate_putmask(self.values, mask)
1451 if noop:
1452 return [self.copy(deep=False)]
1454 if not self._can_hold_element(value) and not is_valid_na_for_dtype(
1455 value, self.dtype
1456 ):
1457 # GH#45153 fillna with incompatible value requiring any
1458 # dtype casting is deprecated.
1459 warnings.warn(
1460 f"'{type(value).__name__}' is not supported as a "
1461 f"fill value for {self.dtype} dtype. In a future "
1462 f"version, calling fillna with an incompatible "
1463 f"value will raise. Explicitly cast to a common "
1464 f"dtype before filling.",
1465 Pandas4Warning,
1466 stacklevel=find_stack_level(),
1467 )
1469 if limit is not None:
1470 mask[mask.cumsum(self.values.ndim - 1) > limit] = False
1472 if inplace:
1473 nbs = self.putmask(mask.T, value)
1474 else:
1475 nbs = self.where(value, ~mask.T)
1476 return extend_blocks(nbs)
1478 def pad_or_backfill(
1479 self,
1480 *,
1481 method: FillnaOptions,
1482 inplace: bool = False,
1483 limit: int | None = None,
1484 limit_area: Literal["inside", "outside"] | None = None,
1485 ) -> list[Block]:
1486 if not self._can_hold_na:
1487 # If there are no NAs, then interpolate is a no-op
1488 return [self.copy(deep=False)]
1490 copy, refs = self._get_refs_and_copy(inplace)
1492 # Dispatch to the NumpyExtensionArray method.
1493 # We know self.array_values is a NumpyExtensionArray bc EABlock overrides
1494 vals = cast("NumpyExtensionArray", self.array_values)
1495 new_values = vals.T._pad_or_backfill(
1496 method=method,
1497 limit=limit,
1498 limit_area=limit_area,
1499 copy=copy,
1500 ).T
1502 data = extract_array(new_values, extract_numpy=True)
1503 return [self.make_block_same_class(data, refs=refs)]
1505 @final
1506 def interpolate(
1507 self,
1508 *,
1509 method: InterpolateOptions,
1510 index: Index,
1511 inplace: bool = False,
1512 limit: int | None = None,
1513 limit_direction: Literal["forward", "backward", "both"] = "forward",
1514 limit_area: Literal["inside", "outside"] | None = None,
1515 **kwargs,
1516 ) -> list[Block]:
1517 inplace = validate_bool_kwarg(inplace, "inplace")
1519 if not self._can_hold_na:
1520 # If there are no NAs, then interpolate is a no-op
1521 return [self.copy(deep=False)]
1523 if self.dtype == _dtype_obj:
1524 # GH#53631
1525 name = {1: "Series", 2: "DataFrame"}[self.ndim]
1526 raise TypeError(f"{name} cannot interpolate with object dtype.")
1528 copy, refs = self._get_refs_and_copy(inplace)
1530 # Dispatch to the EA method.
1531 new_values = self.array_values.interpolate(
1532 method=method,
1533 axis=self.ndim - 1,
1534 index=index,
1535 limit=limit,
1536 limit_direction=limit_direction,
1537 limit_area=limit_area,
1538 copy=copy,
1539 **kwargs,
1540 )
1541 data = extract_array(new_values, extract_numpy=True)
1542 return [self.make_block_same_class(data, refs=refs)]
1544 @final
1545 def diff(self, n: int) -> list[Block]:
1546 """return block for the diff of the values"""
1547 # only reached with ndim == 2
1548 new_values = algos.diff(self.values, n, axis=self.values.ndim - 1)
1549 return [self.make_block(values=new_values)]
1551 def shift(self, periods: int, fill_value: Any = None) -> list[Block]:
1552 """shift the block by periods, possibly upcast"""
1553 # convert integer to float if necessary. need to do a lot more than
1554 # that, handle boolean etc also
1556 # Note: periods is never 0 here, as that is handled at the top of
1557 # NDFrame.shift. If that ever changes, we can do a check for periods=0
1558 # and possibly avoid coercing.
1560 if not lib.is_scalar(fill_value) and self.dtype != _dtype_obj:
1561 # with object dtype there is nothing to promote, and the user can
1562 # pass pretty much any weird fill_value they like
1563 # see test_shift_object_non_scalar_fill
1564 raise ValueError("fill_value must be a scalar")
1566 fill_value = self._standardize_fill_value(fill_value)
1568 try:
1569 # error: Argument 1 to "np_can_hold_element" has incompatible type
1570 # "Union[dtype[Any], ExtensionDtype]"; expected "dtype[Any]"
1571 casted = np_can_hold_element(
1572 self.dtype, # type: ignore[arg-type]
1573 fill_value,
1574 )
1575 except LossySetitemError:
1576 if self.dtype.kind not in "iubSUV" or not is_valid_na_for_dtype(
1577 fill_value, self.dtype
1578 ):
1579 # GH#53802
1580 warnings.warn(
1581 "shifting with a fill value that cannot be held by "
1582 "original dtype is deprecated and will raise in a future "
1583 "version. Explicitly cast to the desired dtype before "
1584 "shifting instead.",
1585 Pandas4Warning,
1586 stacklevel=find_stack_level(),
1587 )
1588 nb = self.coerce_to_target_dtype(fill_value, raise_on_upcast=False)
1589 return nb.shift(periods, fill_value=fill_value)
1591 else:
1592 values = cast("np.ndarray", self.values)
1593 new_values = shift(values, periods, casted)
1594 return [self.make_block_same_class(new_values)]
1596 @final
1597 def quantile(
1598 self,
1599 qs: Index, # with dtype float64
1600 interpolation: QuantileInterpolation = "linear",
1601 ) -> Block:
1602 """
1603 compute the quantiles of the
1605 Parameters
1606 ----------
1607 qs : Index
1608 The quantiles to be computed in float64.
1609 interpolation : str, default 'linear'
1610 Type of interpolation.
1612 Returns
1613 -------
1614 Block
1615 """
1616 # We should always have ndim == 2 because Series dispatches to DataFrame
1617 assert self.ndim == 2
1618 assert is_list_like(qs) # caller is responsible for this
1620 result = quantile_compat(self.values, np.asarray(qs._values), interpolation)
1621 # ensure_block_shape needed for cases where we start with EA and result
1622 # is ndarray, e.g. IntegerArray, SparseArray
1623 result = ensure_block_shape(result, ndim=2)
1624 return new_block_2d(result, placement=self._mgr_locs)
1626 @final
1627 def round(self, decimals: int) -> Self:
1628 """
1629 Rounds the values.
1630 If the block is not of an integer or float dtype, nothing happens.
1631 This is consistent with DataFrame.round behavior.
1632 (Note: Series.round would raise)
1634 Parameters
1635 ----------
1636 decimals: int,
1637 Number of decimal places to round to.
1638 Caller is responsible for validating this
1639 """
1640 if not self.is_numeric or self.is_bool:
1641 if isinstance(self.values, (DatetimeArray, TimedeltaArray, PeriodArray)):
1642 # GH#57781
1643 # TODO: also the ArrowDtype analogues?
1644 warnings.warn(
1645 "obj.round has no effect with datetime, timedelta, "
1646 "or period dtypes. Use obj.dt.round(...) instead.",
1647 UserWarning,
1648 stacklevel=find_stack_level(),
1649 )
1650 return self.copy(deep=False)
1651 values = self.values.round(decimals)
1653 refs = None
1654 if values is self.values:
1655 refs = self.refs
1657 return self.make_block_same_class(values, refs=refs)
1659 # ---------------------------------------------------------------------
1660 # Abstract Methods Overridden By EABackedBlock and NumpyBlock
1662 def delete(self, loc) -> list[Block]:
1663 """Deletes the locs from the block.
1665 We split the block to avoid copying the underlying data. We create new
1666 blocks for every connected segment of the initial block that is not deleted.
1667 The new blocks point to the initial array.
1669 Assumes `loc` is strictly increasing when list-like.
1670 """
1671 if not is_list_like(loc):
1672 loc = [loc]
1674 if self.ndim == 1:
1675 values = cast("np.ndarray", self.values)
1676 values = np.delete(values, loc)
1677 mgr_locs = self._mgr_locs.delete(loc)
1678 return [type(self)(values, placement=mgr_locs, ndim=self.ndim)]
1680 if np.max(loc) >= self.values.shape[0]:
1681 raise IndexError
1683 # Add one out-of-bounds indexer as maximum to collect
1684 # all columns after our last indexer if any
1685 loc = np.concatenate([loc, [self.values.shape[0]]])
1686 mgr_locs_arr = self._mgr_locs.as_array
1687 new_blocks: list[Block] = []
1689 previous_loc = -1
1690 # TODO(CoW): This is tricky, if parent block goes out of scope
1691 # all split blocks are referencing each other even though they
1692 # don't share data
1693 refs = self.refs if self.refs.has_reference() else None
1694 for idx in loc:
1695 if idx == previous_loc + 1:
1696 # There is no column between current and last idx
1697 pass
1698 else:
1699 # No overload variant of "__getitem__" of "ExtensionArray" matches
1700 # argument type "Tuple[slice, slice]"
1701 values = self.values[previous_loc + 1 : idx, :] # type: ignore[call-overload]
1702 locs = mgr_locs_arr[previous_loc + 1 : idx]
1703 nb = type(self)(
1704 values, placement=BlockPlacement(locs), ndim=self.ndim, refs=refs
1705 )
1706 new_blocks.append(nb)
1708 previous_loc = idx
1710 return new_blocks
1712 @property
1713 def is_view(self) -> bool:
1714 """return a boolean if I am possibly a view"""
1715 raise AbstractMethodError(self)
1717 @property
1718 def array_values(self) -> ExtensionArray:
1719 """
1720 The array that Series.array returns. Always an ExtensionArray.
1721 """
1722 raise AbstractMethodError(self)
1724 def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray:
1725 """
1726 return an internal format, currently just the ndarray
1727 this is often overridden to handle to_dense like operations
1728 """
1729 raise AbstractMethodError(self)
1732class EABackedBlock(Block):
1733 """
1734 Mixin for Block subclasses backed by ExtensionArray.
1735 """
1737 values: ExtensionArray
1739 @final
1740 def shift(self, periods: int, fill_value: Any = None) -> list[Block]:
1741 """
1742 Shift the block by `periods`.
1744 Dispatches to underlying ExtensionArray and re-boxes in an
1745 ExtensionBlock.
1746 """
1747 new_values = self.values.shift(periods=periods, fill_value=fill_value)
1748 return [self.make_block_same_class(new_values)]
1750 @final
1751 def setitem(self, indexer, value):
1752 """
1753 Attempt self.values[indexer] = value, possibly creating a new array.
1755 This differs from Block.setitem by not allowing setitem to change
1756 the dtype of the Block.
1758 Parameters
1759 ----------
1760 indexer : tuple, list-like, array-like, slice, int
1761 The subset of self.values to set
1762 value : object
1763 The value being set
1765 Returns
1766 -------
1767 Block
1769 Notes
1770 -----
1771 `indexer` is a direct slice/positional indexer. `value` must
1772 be a compatible shape.
1773 """
1774 orig_indexer = indexer
1775 orig_value = value
1777 indexer = self._unwrap_setitem_indexer(indexer)
1778 value = self._maybe_squeeze_arg(value)
1780 values = self.values
1781 if values._readonly:
1782 # GH#38547 raise here instead of letting the ValueError from
1783 # EA.__setitem__ reach the except clause below, which would
1784 # misinterpret it as a cast failure. Also covers 3rd-party EAs,
1785 # whose __setitem__ does not check the flag.
1786 raise ValueError("Cannot modify read-only array")
1787 target = values
1788 if values.ndim == 2:
1789 target, indexer, value = self._setitem_target(indexer, value)
1790 check_setitem_lengths(indexer, value, values)
1792 try:
1793 target[indexer] = value
1794 except (ValueError, TypeError):
1795 if isinstance(self.dtype, IntervalDtype):
1796 # see TestSetitemFloatIntervalWithIntIntervalValues
1797 nb = self.coerce_to_target_dtype(orig_value, raise_on_upcast=True)
1798 return nb.setitem(orig_indexer, orig_value)
1800 elif isinstance(self, NDArrayBackedExtensionBlock):
1801 if values.ndim == 2 and _unbroadcastable_shape(target, indexer, value):
1802 # GH#68521 not for a 1D block: that would change the
1803 # exception type of Series setitem; see
1804 # test_iloc_setitem_1d_ea_block_shape_mismatch_keeps_its_message
1805 # target is values.T, already in frame order, so the
1806 # error's own message reports the shape mismatch correctly
1807 raise
1808 nb = self.coerce_to_target_dtype(orig_value, raise_on_upcast=True)
1809 return nb.setitem(orig_indexer, orig_value)
1811 else:
1812 raise
1814 else:
1815 return self
1817 def _setitem_target(self, indexer, value):
1818 """
1819 Get the array, indexer and value for setitem on a 2-D block.
1820 """
1821 # GH#45419 Adapt indexer/value to storage layout (nblocks, nrows)
1822 # instead of transposing values, since EA.T may not be a view.
1823 if not isinstance(indexer, tuple):
1824 indexer = (indexer, slice(None))
1825 if len(indexer) == 2:
1826 indexer = indexer[::-1]
1827 if isinstance(value, np.ndarray) and value.ndim == 2:
1828 value = value.T
1829 return self.values, indexer, value
1831 @final
1832 def where(self, other, cond) -> list[Block]:
1833 arr = self.values.T
1835 cond = extract_bool_array(cond)
1837 orig_other = other
1838 orig_cond = cond
1839 other = self._maybe_squeeze_arg(other)
1840 cond = self._maybe_squeeze_arg(cond)
1842 if other is lib.no_default:
1843 other = self.fill_value
1845 icond, noop = validate_putmask(arr, ~cond)
1846 if noop:
1847 # GH#44181, GH#45135
1848 # Avoid a) raising for Interval/PeriodDtype and b) unnecessary object upcast
1849 return [self.copy(deep=False)]
1851 try:
1852 res_values = arr._where(cond, other).T
1853 except OutOfBoundsDatetime:
1854 raise
1855 except (ValueError, TypeError):
1856 if self.ndim == 1 or self.shape[0] == 1:
1857 if isinstance(self.dtype, (IntervalDtype, StringDtype)):
1858 # TestSetitemFloatIntervalWithIntIntervalValues
1859 blk = self.coerce_to_target_dtype(orig_other, raise_on_upcast=False)
1860 if (
1861 self.ndim == 2
1862 and isinstance(orig_cond, np.ndarray)
1863 and orig_cond.ndim == 1
1864 and not is_1d_only_ea_dtype(blk.dtype)
1865 ):
1866 orig_cond = orig_cond[:, None]
1867 return blk.where(orig_other, orig_cond)
1869 elif isinstance(self, NDArrayBackedExtensionBlock):
1870 # NB: not (yet) the same as
1871 # isinstance(values, NDArrayBackedExtensionArray)
1872 blk = self.coerce_to_target_dtype(orig_other, raise_on_upcast=False)
1873 return blk.where(orig_other, orig_cond)
1875 else:
1876 raise
1878 else:
1879 # Same pattern we use in Block.putmask
1880 is_array = isinstance(orig_other, (np.ndarray, ExtensionArray))
1882 res_blocks = []
1883 for i, nb in enumerate(self._split()):
1884 n = orig_other
1885 if is_array:
1886 # we have a different value per-column
1887 n = orig_other[:, i : i + 1]
1889 submask = orig_cond[:, i : i + 1]
1890 rbs = nb.where(n, submask)
1891 res_blocks.extend(rbs)
1892 return res_blocks
1894 nb = self.make_block_same_class(res_values)
1895 return [nb]
1897 @final
1898 def putmask(self, mask, new) -> list[Block]:
1899 """
1900 See Block.putmask.__doc__
1901 """
1902 mask = extract_bool_array(mask)
1903 if new is lib.no_default:
1904 new = self.fill_value
1906 orig_new = new
1907 orig_mask = mask
1908 new = self._maybe_squeeze_arg(new)
1909 mask = self._maybe_squeeze_arg(mask)
1911 if not mask.any():
1912 return [self.copy(deep=False)]
1914 self = self._maybe_copy(inplace=True)
1915 values = self.values
1916 if values._readonly:
1917 # GH#38547 as in EABackedBlock.setitem, raise instead of letting
1918 # the except clause below misinterpret the EA-level ValueError
1919 # as a cast failure.
1920 raise ValueError("Cannot modify read-only array")
1921 if values.ndim == 2:
1922 # GH#64620 Reorient the read-only inputs to the block's storage
1923 # layout and putmask into self.values in place. We must not
1924 # transpose self.values: EA.T is only zero-copy when
1925 # dtype._can_fast_transpose, so a transpose could yield a copy and
1926 # _putmask would silently mutate that throwaway instead of self.
1927 # mask/new are read-only, so transposing them is always correct
1928 # (mirrors value = value.T in EABackedBlock.setitem); failing to
1929 # transpose new fills masked cells from the wrong column.
1930 mask = mask.T
1931 if isinstance(new, (np.ndarray, ExtensionArray)) and new.ndim == 2:
1932 new = new.T
1934 try:
1935 # Caller is responsible for ensuring matching lengths
1936 values._putmask(mask, new)
1937 except OutOfBoundsDatetime:
1938 raise
1939 except (TypeError, ValueError):
1940 if self.ndim == 1 or self.shape[0] == 1:
1941 if isinstance(self.dtype, IntervalDtype):
1942 # Discussion about what we want to support in the general
1943 # case GH#39584
1944 blk = self.coerce_to_target_dtype(orig_new, raise_on_upcast=True)
1945 return blk.putmask(orig_mask, orig_new)
1947 elif isinstance(self, NDArrayBackedExtensionBlock):
1948 # NB: not (yet) the same as
1949 # isinstance(values, NDArrayBackedExtensionArray)
1950 blk = self.coerce_to_target_dtype(orig_new, raise_on_upcast=True)
1951 return blk.putmask(orig_mask, orig_new)
1953 else:
1954 raise
1956 else:
1957 # Same pattern we use in Block.putmask
1958 is_array = isinstance(orig_new, (np.ndarray, ExtensionArray))
1960 res_blocks = []
1961 for i, nb in enumerate(self._split()):
1962 n = orig_new
1963 if is_array:
1964 # we have a different value per-column
1965 n = orig_new[:, i : i + 1]
1967 submask = orig_mask[:, i : i + 1]
1968 rbs = nb.putmask(submask, n)
1969 res_blocks.extend(rbs)
1970 return res_blocks
1972 return [self]
1974 @final
1975 def delete(self, loc) -> list[Block]:
1976 # This will be unnecessary if/when __array_function__ is implemented
1977 if self.ndim == 1:
1978 values = self.values.delete(loc)
1979 mgr_locs = self._mgr_locs.delete(loc)
1980 return [type(self)(values, placement=mgr_locs, ndim=self.ndim)]
1981 elif self.values.ndim == 1:
1982 # We get here through to_stata
1983 return []
1984 return super().delete(loc)
1986 @final
1987 @cache_readonly
1988 def array_values(self) -> ExtensionArray:
1989 return self.values
1991 @final
1992 def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray:
1993 """
1994 return object dtype as boxed values, such as Timestamps/Timedelta
1995 """
1996 values: ArrayLike = self.values
1997 if dtype == _dtype_obj:
1998 values = values.astype(object)
1999 # TODO(EA2D): reshape not needed with 2D EAs
2000 return np.asarray(values).reshape(self.shape)
2002 @final
2003 def pad_or_backfill(
2004 self,
2005 *,
2006 method: FillnaOptions,
2007 inplace: bool = False,
2008 limit: int | None = None,
2009 limit_area: Literal["inside", "outside"] | None = None,
2010 ) -> list[Block]:
2011 values = self.values
2013 kwargs: dict[str, Any] = {"method": method, "limit": limit}
2014 if "limit_area" in inspect.signature(values._pad_or_backfill).parameters:
2015 kwargs["limit_area"] = limit_area
2016 elif limit_area is not None:
2017 raise NotImplementedError(
2018 f"{type(values).__name__} does not implement limit_area "
2019 "(added in pandas 2.2). 3rd-party ExtensionArray authors "
2020 "need to add this argument to _pad_or_backfill."
2021 )
2023 if values.ndim == 2:
2024 # NDArrayBackedExtensionArray.fillna assumes axis=0
2025 new_values = values.T._pad_or_backfill(**kwargs).T
2026 else:
2027 new_values = values._pad_or_backfill(**kwargs)
2028 return [self.make_block_same_class(new_values)]
2031class ExtensionBlock(EABackedBlock):
2032 """
2033 Block for holding extension types.
2035 Notes
2036 -----
2037 This holds all 3rd-party extension array types. It's also the immediate
2038 parent class for our internal extension types' blocks.
2040 ExtensionArrays are limited to 1-D.
2041 """
2043 values: ExtensionArray
2045 def fillna(
2046 self,
2047 value,
2048 limit: int | None = None,
2049 inplace: bool = False,
2050 ) -> list[Block]:
2051 # Caller is responsible for unboxing Series/Index values, GH#22954
2052 if isinstance(self.dtype, (IntervalDtype, StringDtype)):
2053 # Block.fillna handles coercion (test_fillna_interval)
2054 if isinstance(self.dtype, IntervalDtype) and limit is not None:
2055 raise ValueError("limit must be None")
2056 return super().fillna(
2057 value=value,
2058 limit=limit,
2059 inplace=inplace,
2060 )
2061 if self._can_hold_na and not self.values._hasna:
2062 refs = self.refs
2063 new_values = self.values
2064 else:
2065 copy, refs = self._get_refs_and_copy(inplace)
2067 try:
2068 new_values = self.values.fillna(value=value, limit=limit, copy=copy)
2069 except TypeError:
2070 # 3rd party EA that has not implemented copy keyword yet
2071 refs = None
2072 new_values = self.values.fillna(value=value, limit=limit)
2073 # issue the warning *after* retrying, in case the TypeError
2074 # was caused by an invalid fill_value
2075 warnings.warn(
2076 # GH#53278
2077 "ExtensionArray.fillna added a 'copy' keyword in pandas "
2078 "2.1.0. In a future version, ExtensionArray subclasses will "
2079 "need to implement this keyword or an exception will be "
2080 "raised. In the interim, the keyword is ignored by "
2081 f"{type(self.values).__name__}.",
2082 Pandas4Warning,
2083 stacklevel=find_stack_level(),
2084 )
2086 return [self.make_block_same_class(new_values, refs=refs)]
2088 @cache_readonly
2089 def shape(self) -> Shape:
2090 # TODO(EA2D): override unnecessary with 2D EAs
2091 if self.ndim == 1:
2092 return (len(self.values),)
2093 return len(self._mgr_locs), len(self.values)
2095 def iget(self, i: int | tuple[int, int] | tuple[slice, int]):
2096 # In the case where we have a tuple[slice, int], the slice will always
2097 # be slice(None)
2098 # We _could_ make the annotation more specific, but mypy would
2099 # complain about override mismatch:
2100 # Literal[0] | tuple[Literal[0], int] | tuple[slice, int]
2102 # Note: only reached with self.ndim == 2
2104 if isinstance(i, tuple):
2105 # TODO(EA2D): unnecessary with 2D EAs
2106 col, loc = i
2107 if not com.is_null_slice(col) and col != 0:
2108 raise IndexError(f"{self} only contains one item")
2109 if isinstance(col, slice):
2110 # the is_null_slice check above assures that col is slice(None)
2111 # so what we want is a view on all our columns and row loc
2112 if loc < 0:
2113 loc += len(self.values)
2114 # Note: loc:loc+1 vs [[loc]] makes a difference when called
2115 # from fast_xs because we want to get a view back.
2116 return self.values[loc : loc + 1]
2117 return self.values[loc]
2118 else:
2119 if i != 0:
2120 raise IndexError(f"{self} only contains one item")
2121 return self.values
2123 def set_inplace(self, locs, values: ArrayLike, copy: bool = False) -> None:
2124 # When an ndarray, we should have locs.tolist() == [0]
2125 # When a BlockPlacement we should have list(locs) == [0]
2126 if copy:
2127 self.values = self.values.copy()
2128 self.values[:] = values
2130 def _maybe_squeeze_arg(self, arg):
2131 """
2132 If necessary, squeeze a (N, 1) ndarray to (N,)
2133 """
2134 # e.g. if we are passed a 2D mask for putmask
2135 if (
2136 isinstance(arg, (np.ndarray, ExtensionArray))
2137 and arg.ndim == self.values.ndim + 1
2138 ):
2139 # TODO(EA2D): unnecessary with 2D EAs
2140 assert arg.shape[1] == 1
2141 # error: No overload variant of "__getitem__" of "ExtensionArray"
2142 # matches argument type "Tuple[slice, int]"
2143 arg = arg[:, 0] # type: ignore[call-overload]
2144 elif isinstance(arg, ABCDataFrame):
2145 # 2022-01-06 only reached for setitem
2146 # TODO: should we avoid getting here with DataFrame?
2147 assert arg.shape[1] == 1
2148 arg = arg._ixs(0, axis=1)._values
2150 return arg
2152 def _unwrap_setitem_indexer(self, indexer):
2153 """
2154 Adapt a 2D-indexer to our 1D values.
2156 This is intended for 'setitem', not 'iget' or '_slice'.
2157 """
2158 # TODO: ATM this doesn't work for iget/_slice, can we change that?
2160 if isinstance(indexer, tuple) and len(indexer) == 2:
2161 # TODO(EA2D): not needed with 2D EAs
2162 # Should never have length > 2. Caller is responsible for checking.
2163 # Length 1 is reached vis setitem_single_block and setitem_single_column
2164 # each of which pass indexer=(pi,)
2165 if all(isinstance(x, np.ndarray) and x.ndim == 2 for x in indexer):
2166 # GH#44703 went through indexing.maybe_convert_ix
2167 first, second = indexer
2168 if not (
2169 second.size == 1 and (second == 0).all() and first.shape[1] == 1
2170 ):
2171 raise NotImplementedError(
2172 "This should not be reached. Please report a bug at "
2173 "github.com/pandas-dev/pandas/"
2174 )
2175 indexer = first[:, 0]
2177 elif lib.is_integer(indexer[1]) and indexer[1] == 0:
2178 # reached via setitem_single_block passing the whole indexer
2179 indexer = indexer[0]
2181 elif com.is_null_slice(indexer[1]):
2182 indexer = indexer[0]
2184 elif com.is_bool_indexer(indexer[1]) and len(indexer[1]) == 1:
2185 if indexer[1][0]:
2186 indexer = indexer[0]
2187 else:
2188 indexer = []
2190 elif is_list_like(indexer[1]) and indexer[1][0] == 0:
2191 indexer = indexer[0]
2193 else:
2194 raise NotImplementedError(
2195 "This should not be reached. Please report a bug at "
2196 "github.com/pandas-dev/pandas/"
2197 )
2198 return indexer
2200 @property
2201 def is_view(self) -> bool:
2202 """Extension arrays are never treated as views."""
2203 return False
2205 # error: Cannot override writeable attribute with read-only property
2206 @cache_readonly
2207 def is_numeric(self) -> bool: # type: ignore[override]
2208 return self.values.dtype._is_numeric
2210 def _slice(
2211 self, slicer: slice | npt.NDArray[np.bool_] | npt.NDArray[np.intp]
2212 ) -> ExtensionArray:
2213 """
2214 Return a slice of my values.
2216 Parameters
2217 ----------
2218 slicer : slice, ndarray[int], or ndarray[bool]
2219 Valid (non-reducing) indexer for self.values.
2221 Returns
2222 -------
2223 ExtensionArray
2224 """
2225 # Notes: ndarray[bool] is only reachable when via get_rows_with_mask, which
2226 # is only for Series, i.e. self.ndim == 1.
2228 # return same dims as we currently have
2229 if self.ndim == 2:
2230 # reached via getitem_block via _slice_take_blocks_ax0
2231 # TODO(EA2D): won't be necessary with 2D EAs
2233 if not isinstance(slicer, slice):
2234 raise AssertionError(
2235 "invalid slicing for a 1-ndim ExtensionArray", slicer
2236 )
2237 # GH#32959 only full-slicers along fake-dim0 are valid
2238 # TODO(EA2D): won't be necessary with 2D EAs
2239 # range(1) instead of self._mgr_locs to avoid exception on [::-1]
2240 # see test_iloc_getitem_slice_negative_step_ea_block
2241 new_locs = range(1)[slicer]
2242 if not len(new_locs):
2243 raise AssertionError(
2244 "invalid slicing for a 1-ndim ExtensionArray", slicer
2245 )
2246 slicer = slice(None)
2248 return self.values[slicer]
2250 @final
2251 def slice_block_rows(self, slicer: slice) -> Self:
2252 """
2253 Perform __getitem__-like specialized to slicing along index.
2254 """
2255 # GH#42787 in principle this is equivalent to values[..., slicer], but we don't
2256 # require subclasses of ExtensionArray to support that form (for now).
2257 new_values = self.values[slicer]
2258 return type(self)(new_values, self._mgr_locs, ndim=self.ndim, refs=self.refs)
2260 def _unstack(
2261 self,
2262 unstacker,
2263 fill_value,
2264 new_placement: npt.NDArray[np.intp],
2265 needs_masking: npt.NDArray[np.bool_],
2266 ):
2267 # ExtensionArray-safe unstack.
2268 # We override Block._unstack, which unstacks directly on the
2269 # values of the array. For EA-backed blocks, this would require
2270 # converting to a 2-D ndarray of objects.
2271 # Instead, we unstack an ndarray of integer positions, followed by
2272 # a `take` on the actual values.
2274 # Caller is responsible for ensuring self.shape[-1] == len(unstacker.index)
2275 new_values = unstacker.arange_result
2277 # needs_masking[i] calculated once in BlockManager.unstack tells
2278 # us if there are any -1s in the relevant indices. When False,
2279 # that allows us to go through a faster path in 'take', among
2280 # other things avoiding e.g. Categorical._validate_scalar.
2281 blocks = [
2282 # TODO: could cast to object depending on fill_value?
2283 type(self)(
2284 self.values.take(
2285 indices, allow_fill=needs_masking[i], fill_value=fill_value
2286 ),
2287 BlockPlacement(place),
2288 ndim=2,
2289 )
2290 for i, (indices, place) in enumerate(
2291 zip(new_values.T, new_placement, strict=True)
2292 )
2293 ]
2294 return blocks
2297class NumpyBlock(Block):
2298 values: np.ndarray
2299 __slots__ = ()
2301 @property
2302 def is_view(self) -> bool:
2303 """return a boolean if I am possibly a view"""
2304 return self.values.base is not None
2306 @property
2307 def array_values(self) -> ExtensionArray:
2308 return NumpyExtensionArray(self.values)
2310 def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray:
2311 if dtype == _dtype_obj:
2312 return self.values.astype(_dtype_obj)
2313 return self.values
2315 @cache_readonly
2316 def is_numeric(self) -> bool: # type: ignore[override]
2317 dtype = self.values.dtype
2318 kind = dtype.kind
2320 return kind in "fciub"
2323class NDArrayBackedExtensionBlock(EABackedBlock):
2324 """
2325 Block backed by an NDArrayBackedExtensionArray
2326 """
2328 values: NDArrayBackedExtensionArray
2330 @property
2331 def is_view(self) -> bool:
2332 """return a boolean if I am possibly a view"""
2333 # check the ndarray values of the DatetimeIndex values
2334 return self.values._ndarray.base is not None
2336 def _setitem_target(self, indexer, value):
2337 # GH#68521 .T is a view here, so write through it with the key
2338 # in frame order and let numpy do the broadcasting
2339 if not isinstance(indexer, tuple):
2340 # a 0-d ndarray key breaks EA.__setitem__; a tuple avoids it
2341 indexer = (indexer,)
2342 return self.values.T, indexer, value
2345class DatetimeLikeBlock(NDArrayBackedExtensionBlock):
2346 """Block for datetime64[ns], timedelta64[ns]."""
2348 __slots__ = ()
2349 is_numeric = False
2350 values: DatetimeArray | TimedeltaArray
2353def _unbroadcastable_shape(values: ArrayLike, indexer, value) -> bool:
2354 """
2355 Whether ``value`` cannot be broadcast into ``values[indexer]``.
2357 Returns False both when the value does fit and when that cannot be
2358 determined, so a caller can only use a True result to conclude a failed
2359 setitem is a shape problem rather than a dtype one.
2360 """
2361 try:
2362 target_shape = np.shape(values[indexer])
2363 value_shape = np.shape(value)
2364 except (IndexError, TypeError, ValueError):
2365 return False
2366 # assignment also drops leading length-1 axes of the value, which
2367 # broadcasting on its own does not, e.g. ``arr[0] = np.array([x])``
2368 while len(value_shape) > len(target_shape) and value_shape[0] == 1:
2369 value_shape = value_shape[1:]
2370 try:
2371 return np.broadcast_shapes(target_shape, value_shape) != target_shape
2372 except ValueError:
2373 return True
2376# -----------------------------------------------------------------
2377# Constructor Helpers
2380def maybe_coerce_values(values: ArrayLike) -> ArrayLike:
2381 """
2382 Input validation for values passed to __init__. Ensure that
2383 any datetime64/timedelta64 dtypes are in nanoseconds. Ensure
2384 that we do not have string dtypes.
2386 Parameters
2387 ----------
2388 values : np.ndarray or ExtensionArray
2390 Returns
2391 -------
2392 values : np.ndarray or ExtensionArray
2393 """
2394 # Caller is responsible for ensuring NumpyExtensionArray is already extracted.
2396 if isinstance(values, np.ndarray):
2397 values = ensure_wrapped_if_datetimelike(values)
2399 if issubclass(values.dtype.type, str):
2400 values = np.array(values, dtype=object)
2402 return values
2405def _regex_target_dtype(dtype: DtypeObj) -> DtypeObj:
2406 """
2407 The dtype a regex is actually matched against. A Categorical's elements have
2408 its categories' dtype (GH#38447).
2409 """
2410 if isinstance(dtype, CategoricalDtype):
2411 return dtype.categories.dtype
2412 return dtype
2415def get_block_type(dtype: DtypeObj) -> type[Block]:
2416 """
2417 Find the appropriate Block subclass to use for the given values and dtype.
2419 Parameters
2420 ----------
2421 dtype : numpy or pandas dtype
2423 Returns
2424 -------
2425 cls : class, subclass of Block
2426 """
2427 if isinstance(dtype, DatetimeTZDtype):
2428 return DatetimeLikeBlock
2429 elif isinstance(dtype, PeriodDtype):
2430 return NDArrayBackedExtensionBlock
2431 elif isinstance(dtype, ExtensionDtype):
2432 # Note: need to be sure NumpyExtensionArray is unwrapped before we get here
2433 return ExtensionBlock
2435 # We use kind checks because it is much more performant
2436 # than is_foo_dtype
2437 kind = dtype.kind
2438 if kind in "Mm":
2439 return DatetimeLikeBlock
2441 return NumpyBlock
2444def new_block_2d(
2445 values: ArrayLike, placement: BlockPlacement, refs: BlockValuesRefs | None = None
2446) -> Block:
2447 # new_block specialized to case with
2448 # ndim=2
2449 # isinstance(placement, BlockPlacement)
2450 # check_ndim/ensure_block_shape already checked
2451 klass = get_block_type(values.dtype)
2453 values = maybe_coerce_values(values)
2454 return klass(values, ndim=2, placement=placement, refs=refs)
2457def new_block(
2458 values,
2459 placement: BlockPlacement,
2460 *,
2461 ndim: int,
2462 refs: BlockValuesRefs | None = None,
2463) -> Block:
2464 # caller is responsible for ensuring:
2465 # - values is NOT a NumpyExtensionArray
2466 # - check_ndim/ensure_block_shape already checked
2467 # - maybe_coerce_values already called/unnecessary
2468 klass = get_block_type(values.dtype)
2469 return klass(values, ndim=ndim, placement=placement, refs=refs)
2472def check_ndim(values, placement: BlockPlacement, ndim: int) -> None:
2473 """
2474 ndim inference and validation.
2476 Validates that values.ndim and ndim are consistent.
2477 Validates that len(values) and len(placement) are consistent.
2479 Parameters
2480 ----------
2481 values : array-like
2482 placement : BlockPlacement
2483 ndim : int
2485 Raises
2486 ------
2487 ValueError : the number of dimensions do not match
2488 """
2490 if values.ndim > ndim:
2491 # Check for both np.ndarray and ExtensionArray
2492 raise ValueError(
2493 f"Wrong number of dimensions. values.ndim > ndim [{values.ndim} > {ndim}]"
2494 )
2496 if not is_1d_only_ea_dtype(values.dtype):
2497 # TODO(EA2D): special case not needed with 2D EAs
2498 if values.ndim != ndim:
2499 raise ValueError(
2500 "Wrong number of dimensions. "
2501 f"values.ndim != ndim [{values.ndim} != {ndim}]"
2502 )
2503 if len(placement) != len(values):
2504 raise ValueError(
2505 f"Wrong number of items passed {len(values)}, "
2506 f"placement implies {len(placement)}"
2507 )
2508 elif ndim == 2 and len(placement) != 1:
2509 # TODO(EA2D): special case unnecessary with 2D EAs
2510 raise ValueError("need to split")
2513def extract_pandas_array(
2514 values: ArrayLike, dtype: DtypeObj | None, ndim: int | None
2515) -> tuple[ArrayLike, DtypeObj | None]:
2516 """
2517 Ensure that we don't allow NumpyExtensionArray / NumpyEADtype in internals.
2518 """
2519 # For now, blocks should be backed by ndarrays when possible.
2520 if isinstance(values, ABCNumpyExtensionArray):
2521 values = values.to_numpy()
2522 if ndim and ndim > 1:
2523 # TODO(EA2D): special case not needed with 2D EAs
2524 values = np.atleast_2d(values)
2526 if isinstance(dtype, NumpyEADtype):
2527 dtype = dtype.numpy_dtype
2529 return values, dtype
2532# -----------------------------------------------------------------
2535def extend_blocks(result, blocks=None) -> list[Block]:
2536 """return a new extended blocks, given the result"""
2537 if blocks is None:
2538 blocks = []
2539 if isinstance(result, list):
2540 for r in result:
2541 if isinstance(r, list):
2542 blocks.extend(r)
2543 else:
2544 blocks.append(r)
2545 else:
2546 assert isinstance(result, Block), type(result)
2547 blocks.append(result)
2548 return blocks
2551def ensure_block_shape(values: ArrayLike, ndim: int = 1) -> ArrayLike:
2552 """
2553 Reshape if possible to have values.ndim == ndim.
2554 """
2556 if values.ndim < ndim:
2557 if not is_1d_only_ea_dtype(values.dtype):
2558 # TODO(EA2D): https://github.com/pandas-dev/pandas/issues/23023
2559 # block.shape is incorrect for "2D" ExtensionArrays
2560 # We can't, and don't need to, reshape.
2561 values = cast("np.ndarray | DatetimeArray | TimedeltaArray", values)
2562 values = values.reshape(1, -1)
2564 return values
2567def external_values(values: ArrayLike) -> ArrayLike:
2568 """
2569 The array that Series.values returns (public attribute).
2571 This has some historical constraints, and is overridden in block
2572 subclasses to return the correct array (e.g. period returns
2573 object ndarray and datetimetz a datetime64[ns] ndarray instead of
2574 proper extension array).
2575 """
2576 if isinstance(values, (PeriodArray, IntervalArray)):
2577 return values.astype(object)
2578 elif isinstance(values, (DatetimeArray, TimedeltaArray)):
2579 # NB: for datetime64tz this is different from np.asarray(values), since
2580 # that returns an object-dtype ndarray of Timestamps.
2581 # Avoid raising in .astype in casting from dt64tz to dt64
2582 values = values._ndarray
2584 if isinstance(values, np.ndarray):
2585 values = values.view()
2586 values.flags.writeable = False
2587 else:
2588 # ExtensionArrays
2589 # TODO decide on read-only https://github.com/pandas-dev/pandas/issues/63099
2590 # values = values.view()
2591 # values._readonly = True
2592 pass
2594 return values