Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import inspect
4import re
5from typing import (
6 TYPE_CHECKING,
7 Any,
8 Literal,
9 Self,
10 cast,
11 final,
12)
13import warnings
15import numpy as np
17from pandas._libs import (
18 NaT,
19 internals as libinternals,
20 lib,
21)
22from pandas._libs.internals import (
23 BlockPlacement,
24 BlockValuesRefs,
25)
26from pandas._libs.missing import NA
27from pandas.errors import (
28 AbstractMethodError,
29 OutOfBoundsDatetime,
30 Pandas4Warning,
31)
32from pandas.util._decorators import cache_readonly
33from pandas.util._exceptions import find_stack_level
34from pandas.util._validators import validate_bool_kwarg
36from pandas.core.dtypes.astype import (
37 astype_array_safe,
38 astype_is_view,
39)
40from pandas.core.dtypes.cast import (
41 LossySetitemError,
42 can_hold_element,
43 convert_dtypes,
44 find_result_type,
45 np_can_hold_element,
46)
47from pandas.core.dtypes.common import (
48 is_1d_only_ea_dtype,
49 is_float_dtype,
50 is_integer_dtype,
51 is_list_like,
52 is_scalar,
53 is_string_dtype,
54)
55from pandas.core.dtypes.dtypes import (
56 DatetimeTZDtype,
57 ExtensionDtype,
58 IntervalDtype,
59 NumpyEADtype,
60 PeriodDtype,
61)
62from pandas.core.dtypes.generic import (
63 ABCDataFrame,
64 ABCIndex,
65 ABCNumpyExtensionArray,
66 ABCSeries,
67)
68from pandas.core.dtypes.inference import is_re
69from pandas.core.dtypes.missing import (
70 is_valid_na_for_dtype,
71 isna,
72 na_value_for_dtype,
73)
75from pandas.core import missing
76import pandas.core.algorithms as algos
77from pandas.core.array_algos.putmask import (
78 extract_bool_array,
79 putmask_inplace,
80 putmask_without_repeat,
81 setitem_datetimelike_compat,
82 validate_putmask,
83)
84from pandas.core.array_algos.quantile import quantile_compat
85from pandas.core.array_algos.replace import (
86 compare_or_regex_search,
87 replace_regex,
88 should_use_regex,
89)
90from pandas.core.array_algos.transforms import shift
91from pandas.core.arrays import (
92 DatetimeArray,
93 ExtensionArray,
94 IntervalArray,
95 NumpyExtensionArray,
96 PeriodArray,
97 TimedeltaArray,
98)
99from pandas.core.arrays.string_ import StringDtype
100from pandas.core.base import PandasObject
101import pandas.core.common as com
102from pandas.core.computation import expressions
103from pandas.core.construction import (
104 ensure_wrapped_if_datetimelike,
105 extract_array,
106)
107from pandas.core.indexers import check_setitem_lengths
108from pandas.core.indexes.base import get_values_for_csv
110if TYPE_CHECKING:
111 from collections.abc import (
112 Callable,
113 Generator,
114 Iterable,
115 Sequence,
116 )
118 from pandas._typing import (
119 ArrayLike,
120 AxisInt,
121 DtypeBackend,
122 DtypeObj,
123 FillnaOptions,
124 IgnoreRaise,
125 InterpolateOptions,
126 QuantileInterpolation,
127 Shape,
128 npt,
129 )
131 from pandas.core.api import Index
132 from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
134# comparison is faster than is_object_dtype
135_dtype_obj = np.dtype("object")
138class Block(PandasObject, libinternals.Block):
139 """
140 Canonical n-dimensional unit of homogeneous dtype contained in a pandas
141 data structure
143 Index-ignorant; let the container take care of that
144 """
146 values: np.ndarray | ExtensionArray
147 ndim: int
148 refs: BlockValuesRefs
149 __init__: Callable
151 __slots__ = ()
152 is_numeric = False
154 @final
155 @cache_readonly
156 def _validate_ndim(self) -> bool:
157 """
158 We validate dimension for blocks that can hold 2D values, which for now
159 means numpy dtypes or EA dtypes like DatetimeTZDtype and PeriodDtype.
160 """
161 return not is_1d_only_ea_dtype(self.dtype)
163 @final
164 @cache_readonly
165 def is_object(self) -> bool:
166 return self.values.dtype == _dtype_obj
168 @final
169 @cache_readonly
170 def is_extension(self) -> bool:
171 return not lib.is_np_dtype(self.values.dtype)
173 @final
174 @cache_readonly
175 def _can_consolidate(self) -> bool:
176 # We _could_ consolidate for DatetimeTZDtype but don't for now.
177 return not self.is_extension
179 @final
180 @cache_readonly
181 def _consolidate_key(self):
182 return self._can_consolidate, self.dtype.name
184 @final
185 @cache_readonly
186 def _can_hold_na(self) -> bool:
187 """
188 Can we store NA values in this Block?
189 """
190 dtype = self.dtype
191 if isinstance(dtype, np.dtype):
192 return dtype.kind not in "iub"
193 return dtype._can_hold_na
195 @final
196 @property
197 def is_bool(self) -> bool:
198 """
199 We can be bool if a) we are bool dtype or b) object dtype with bool objects.
200 """
201 return self.values.dtype == np.dtype(bool)
203 @final
204 def external_values(self):
205 return external_values(self.values)
207 @final
208 @cache_readonly
209 def fill_value(self):
210 # Used in reindex_indexer
211 return na_value_for_dtype(self.dtype, compat=False)
213 @final
214 def _standardize_fill_value(self, value):
215 # if we are passed a scalar None, convert it here
216 if self.dtype != _dtype_obj and is_valid_na_for_dtype(value, self.dtype):
217 value = self.fill_value
218 return value
220 @property
221 def mgr_locs(self) -> BlockPlacement:
222 return self._mgr_locs
224 @mgr_locs.setter
225 def mgr_locs(self, new_mgr_locs: BlockPlacement) -> None:
226 self._mgr_locs = new_mgr_locs
228 @final
229 def make_block(
230 self,
231 values,
232 placement: BlockPlacement | None = None,
233 refs: BlockValuesRefs | None = None,
234 ) -> Block:
235 """
236 Create a new block, with type inference propagate any values that are
237 not specified
238 """
239 if placement is None:
240 placement = self._mgr_locs
241 if self.is_extension:
242 values = ensure_block_shape(values, ndim=self.ndim)
244 return new_block(values, placement=placement, ndim=self.ndim, refs=refs)
246 @final
247 def make_block_same_class(
248 self,
249 values,
250 placement: BlockPlacement | None = None,
251 refs: BlockValuesRefs | None = None,
252 ) -> Self:
253 """Wrap given values in a block of same type as self."""
254 # Pre-2.0 we called ensure_wrapped_if_datetimelike because fastparquet
255 # relied on it, as of 2.0 the caller is responsible for this.
256 if placement is None:
257 placement = self._mgr_locs
259 # We assume maybe_coerce_values has already been called
260 return type(self)(values, placement=placement, ndim=self.ndim, refs=refs)
262 @final
263 def __repr__(self) -> str:
264 # don't want to print out all of the items here
265 name = type(self).__name__
266 if self.ndim == 1:
267 result = f"{name}: {len(self)} dtype: {self.dtype}"
268 else:
269 shape = " x ".join([str(s) for s in self.shape])
270 result = f"{name}: {self.mgr_locs.indexer}, {shape}, dtype: {self.dtype}"
272 return result
274 @final
275 def __len__(self) -> int:
276 return len(self.values)
278 @final
279 def slice_block_columns(self, slc: slice) -> Self:
280 """
281 Perform __getitem__-like, return result as block.
282 """
283 new_mgr_locs = self._mgr_locs[slc]
285 new_values = self._slice(slc)
286 refs = self.refs
287 return type(self)(new_values, new_mgr_locs, self.ndim, refs=refs)
289 @final
290 def take_block_columns(self, indices: npt.NDArray[np.intp]) -> Self:
291 """
292 Perform __getitem__-like, return result as block.
294 Only supports slices that preserve dimensionality.
295 """
296 # Note: only called from is from internals.concat, and we can verify
297 # that never happens with 1-column blocks, i.e. never for ExtensionBlock.
299 new_mgr_locs = self._mgr_locs[indices]
301 new_values = self._slice(indices)
302 return type(self)(new_values, new_mgr_locs, self.ndim, refs=None)
304 @final
305 def getitem_block_columns(
306 self, slicer: slice, new_mgr_locs: BlockPlacement, ref_inplace_op: bool = False
307 ) -> Self:
308 """
309 Perform __getitem__-like, return result as block.
311 Only supports slices that preserve dimensionality.
312 """
313 new_values = self._slice(slicer)
314 refs = self.refs if not ref_inplace_op or self.refs.has_reference() else None
315 return type(self)(new_values, new_mgr_locs, self.ndim, refs=refs)
317 @final
318 def _can_hold_element(self, element: Any) -> bool:
319 """require the same dtype as ourselves"""
320 element = extract_array(element, extract_numpy=True)
321 return can_hold_element(self.values, element)
323 @final
324 def should_store(self, value: ArrayLike) -> bool:
325 """
326 Should we set self.values[indexer] = value inplace or do we need to cast?
328 Parameters
329 ----------
330 value : np.ndarray or ExtensionArray
332 Returns
333 -------
334 bool
335 """
336 return value.dtype == self.dtype
338 # ---------------------------------------------------------------------
339 # Apply/Reduce and Helpers
341 @final
342 def apply(self, func, **kwargs) -> list[Block]:
343 """
344 apply the function to my values; return a block if we are not
345 one
346 """
347 result = func(self.values, **kwargs)
349 result = maybe_coerce_values(result)
350 return self._split_op_result(result)
352 @final
353 def reduce(self, func) -> Block:
354 # We will apply the function and reshape the result into a single-row
355 # Block with the same mgr_locs; squeezing will be done at a higher level
356 assert self.ndim == 2
358 result = func(self.values)
360 if self.values.ndim == 1:
361 res_values = result
362 else:
363 res_values = result.reshape(-1, 1)
365 return self.make_block(res_values)
367 @final
368 def _split_op_result(self, result: ArrayLike) -> list[Block]:
369 # See also: split_and_operate
370 if result.ndim > 1 and isinstance(result.dtype, ExtensionDtype):
371 # TODO(EA2D): unnecessary with 2D EAs
372 # if we get a 2D ExtensionArray, we need to split it into 1D pieces
373 nbs = []
374 for i, loc in enumerate(self._mgr_locs):
375 if not is_1d_only_ea_dtype(result.dtype):
376 vals = result[i : i + 1]
377 else:
378 vals = result[i]
380 bp = BlockPlacement(loc)
381 block = self.make_block(values=vals, placement=bp)
382 nbs.append(block)
383 return nbs
385 nb = self.make_block(result)
387 return [nb]
389 @final
390 def _split(self) -> Generator[Block]:
391 """
392 Split a block into a list of single-column blocks.
393 """
394 assert self.ndim == 2
396 for i, ref_loc in enumerate(self._mgr_locs):
397 vals = self.values[slice(i, i + 1)]
399 bp = BlockPlacement(ref_loc)
400 nb = type(self)(vals, placement=bp, ndim=2, refs=self.refs)
401 yield nb
403 @final
404 def split_and_operate(self, func, *args, **kwargs) -> list[Block]:
405 """
406 Split the block and apply func column-by-column.
408 Parameters
409 ----------
410 func : Block method
411 *args
412 **kwargs
414 Returns
415 -------
416 List[Block]
417 """
418 assert self.ndim == 2 and self.shape[0] != 1
420 res_blocks = []
421 for nb in self._split():
422 rbs = func(nb, *args, **kwargs)
423 res_blocks.extend(rbs)
424 return res_blocks
426 # ---------------------------------------------------------------------
427 # Up/Down-casting
429 @final
430 def coerce_to_target_dtype(self, other, raise_on_upcast: bool) -> Block:
431 """
432 coerce the current block to a dtype compat for other
433 we will return a block, possibly object, and not raise
435 we can also safely try to coerce to the same dtype
436 and will receive the same block
437 """
438 new_dtype = find_result_type(self.values.dtype, other)
439 if new_dtype == self.dtype:
440 # GH#52927 avoid RecursionError
441 raise AssertionError(
442 "Something has gone wrong, please report a bug at "
443 "https://github.com/pandas-dev/pandas/issues"
444 )
446 # In a future version of pandas, the default will be that
447 # setting `nan` into an integer series won't raise.
448 if (
449 is_scalar(other)
450 and is_integer_dtype(self.values.dtype)
451 and isna(other)
452 and other is not NaT
453 and not (
454 isinstance(other, (np.datetime64, np.timedelta64)) and np.isnat(other)
455 )
456 ):
457 raise_on_upcast = False
458 elif (
459 isinstance(other, np.ndarray)
460 and other.ndim == 1
461 and is_integer_dtype(self.values.dtype)
462 and is_float_dtype(other.dtype)
463 and lib.has_only_ints_or_nan(other)
464 ):
465 raise_on_upcast = False
467 if raise_on_upcast:
468 raise TypeError(f"Invalid value '{other}' for dtype '{self.values.dtype}'")
469 if self.values.dtype == new_dtype:
470 raise AssertionError(
471 f"Did not expect new dtype {new_dtype} to equal self.dtype "
472 f"{self.values.dtype}. Please report a bug at "
473 "https://github.com/pandas-dev/pandas/issues."
474 )
475 try:
476 return self.astype(new_dtype)
477 except OutOfBoundsDatetime as err:
478 # e.g. GH#56419 if self.dtype is a low-resolution dt64 and we try to
479 # upcast to a higher-resolution dt64, we may have entries that are
480 # out of bounds for the higher resolution.
481 # Re-raise with a more informative message.
482 raise OutOfBoundsDatetime(
483 f"Incompatible (high-resolution) value for dtype='{self.dtype}'. "
484 "Explicitly cast before operating."
485 ) from err
487 @final
488 def convert(self) -> list[Block]:
489 """
490 Attempt to coerce any object types to better types. Return a copy
491 of the block (if copy = True).
492 """
493 if not self.is_object:
494 return [self.copy(deep=False)]
496 if self.ndim != 1 and self.shape[0] != 1:
497 blocks = self.split_and_operate(Block.convert)
498 if all(blk.dtype.kind == "O" for blk in blocks):
499 # Avoid fragmenting the block if convert is a no-op
500 return [self.copy(deep=False)]
501 return blocks
503 values = self.values
504 if values.ndim == 2:
505 # the check above ensures we only get here with values.shape[0] == 1,
506 # avoid doing .ravel as that might make a copy
507 values = values[0]
509 res_values = lib.maybe_convert_objects(
510 values, # type: ignore[arg-type]
511 convert_non_numeric=True,
512 )
513 refs = None
514 if res_values is values or (
515 isinstance(res_values, NumpyExtensionArray)
516 and res_values._ndarray is values
517 ):
518 refs = self.refs
520 res_values = ensure_block_shape(res_values, self.ndim)
521 res_values = maybe_coerce_values(res_values)
522 return [self.make_block(res_values, refs=refs)]
524 def convert_dtypes(
525 self,
526 infer_objects: bool = True,
527 convert_string: bool = True,
528 convert_integer: bool = True,
529 convert_boolean: bool = True,
530 convert_floating: bool = True,
531 dtype_backend: DtypeBackend = "numpy_nullable",
532 ) -> list[Block]:
533 if infer_objects and self.is_object:
534 blks = self.convert()
535 else:
536 blks = [self]
538 if not any(
539 [convert_floating, convert_integer, convert_boolean, convert_string]
540 ):
541 return [b.copy(deep=False) for b in blks]
543 rbs = []
544 for blk in blks:
545 # Determine dtype column by column
546 sub_blks = (
547 [blk] if blk.ndim == 1 or blk.shape[0] == 1 else list(blk._split())
548 )
549 dtypes = [
550 convert_dtypes(
551 b.values,
552 convert_string,
553 convert_integer,
554 convert_boolean,
555 convert_floating,
556 infer_objects,
557 dtype_backend,
558 )
559 for b in sub_blks
560 ]
561 if all(dtype == blk.dtype for dtype in dtypes):
562 # Avoid block splitting if no dtype changes
563 rbs.append(blk.copy(deep=False))
564 continue
566 for dtype, b in zip(dtypes, sub_blks, strict=True):
567 rbs.append(b.astype(dtype=dtype, squeeze=b.ndim != 1))
568 return rbs
570 # ---------------------------------------------------------------------
571 # Array-Like Methods
573 @final
574 @cache_readonly
575 def dtype(self) -> DtypeObj:
576 return self.values.dtype
578 @final
579 def astype(
580 self,
581 dtype: DtypeObj,
582 errors: IgnoreRaise = "raise",
583 squeeze: bool = False,
584 ) -> Block:
585 """
586 Coerce to the new dtype.
588 Parameters
589 ----------
590 dtype : np.dtype or ExtensionDtype
591 errors : str, {'raise', 'ignore'}, default 'raise'
592 - ``raise`` : allow exceptions to be raised
593 - ``ignore`` : suppress exceptions. On error return original object
594 squeeze : bool, default False
595 squeeze values to ndim=1 if only one column is given
597 Returns
598 -------
599 Block
600 """
601 values = self.values
602 if squeeze and values.ndim == 2 and is_1d_only_ea_dtype(dtype):
603 if values.shape[0] != 1:
604 raise ValueError("Can not squeeze with more than one column.")
605 values = values[0, :] # type: ignore[call-overload]
607 new_values = astype_array_safe(values, dtype, errors=errors)
609 new_values = maybe_coerce_values(new_values)
611 refs = None
612 if astype_is_view(values.dtype, new_values.dtype):
613 refs = self.refs
615 newb = self.make_block(new_values, refs=refs)
616 if newb.shape != self.shape:
617 raise TypeError(
618 f"cannot set astype for dtype "
619 f"({self.dtype.name} [{self.shape}]) to different shape "
620 f"({newb.dtype.name} [{newb.shape}])"
621 )
622 return newb
624 @final
625 def get_values_for_csv(
626 self, *, float_format, date_format, decimal, na_rep: str = "nan", quoting=None
627 ) -> Block:
628 """convert to our native types format"""
629 result = get_values_for_csv(
630 self.values,
631 na_rep=na_rep,
632 quoting=quoting,
633 float_format=float_format,
634 date_format=date_format,
635 decimal=decimal,
636 )
637 return self.make_block(result)
639 @final
640 def copy(self, *, deep: bool) -> Self:
641 """copy constructor"""
642 values = self.values
643 refs: BlockValuesRefs | None
644 if deep:
645 values = values.copy()
646 refs = None
647 else:
648 values = values.view()
649 refs = self.refs
650 return type(self)(values, placement=self._mgr_locs, ndim=self.ndim, refs=refs)
652 # ---------------------------------------------------------------------
653 # Copy-on-Write Helpers
655 def _maybe_copy(self, inplace: bool, deep: bool = True) -> Self:
656 if inplace and not self.refs.has_reference():
657 return self
658 return self.copy(deep=deep)
660 @final
661 def _get_refs_and_copy(self, inplace: bool):
662 refs = None
663 copy = not inplace
664 if inplace:
665 if self.refs.has_reference():
666 copy = True
667 else:
668 refs = self.refs
669 return copy, refs
671 # ---------------------------------------------------------------------
672 # Replace
674 @final
675 def replace(
676 self,
677 to_replace,
678 value,
679 inplace: bool = False,
680 # mask may be pre-computed if we're called from replace_list
681 mask: npt.NDArray[np.bool_] | None = None,
682 ) -> list[Block]:
683 """
684 replace the to_replace value with value, possible to create new
685 blocks here this is just a call to putmask.
686 """
688 # Note: the checks we do in NDFrame.replace ensure we never get
689 # here with listlike to_replace or value, as those cases
690 # go through replace_list
691 values = self.values
693 if not self._can_hold_element(to_replace):
694 # We cannot hold `to_replace`, so we know immediately that
695 # replacing it is a no-op.
696 # Note: If to_replace were a list, NDFrame.replace would call
697 # replace_list instead of replace.
698 return [self._maybe_copy(inplace, deep=False)]
700 if mask is None:
701 mask = missing.mask_missing(values, to_replace)
702 if not mask.any():
703 # Note: we get here with test_replace_extension_other incorrectly
704 # bc _can_hold_element is incorrect.
705 return [self._maybe_copy(inplace, deep=False)]
707 elif self._can_hold_element(value) or (self.dtype == "string" and is_re(value)):
708 # TODO(CoW): Maybe split here as well into columns where mask has True
709 # and rest?
710 blk = self._maybe_copy(inplace)
711 putmask_inplace(blk.values, mask, value)
712 return [blk]
714 elif self.ndim == 1 or self.shape[0] == 1:
715 if value is None or value is NA:
716 blk = self.astype(np.dtype(object))
717 else:
718 blk = self.coerce_to_target_dtype(value, raise_on_upcast=False)
719 return blk.replace(
720 to_replace=to_replace,
721 value=value,
722 inplace=True,
723 mask=mask,
724 )
726 else:
727 # split so that we only upcast where necessary
728 blocks = []
729 for i, nb in enumerate(self._split()):
730 blocks.extend(
731 type(self).replace(
732 nb,
733 to_replace=to_replace,
734 value=value,
735 inplace=True,
736 mask=mask[i : i + 1],
737 )
738 )
739 return blocks
741 @final
742 def _replace_regex(
743 self,
744 to_replace,
745 value,
746 inplace: bool = False,
747 mask=None,
748 ) -> list[Block]:
749 """
750 Replace elements by the given value.
752 Parameters
753 ----------
754 to_replace : object or pattern
755 Scalar to replace or regular expression to match.
756 value : object
757 Replacement object.
758 inplace : bool, default False
759 Perform inplace modification.
760 mask : array-like of bool, optional
761 True indicate corresponding element is ignored.
763 Returns
764 -------
765 List[Block]
766 """
767 if not is_re(to_replace) and not self._can_hold_element(to_replace):
768 # i.e. only if self.is_object is True, but could in principle include a
769 # String ExtensionBlock
770 return [self.copy(deep=False)]
772 if is_re(to_replace) and self.dtype not in [object, "string"]:
773 # only object or string dtype can hold strings, and a regex object
774 # will only match strings
775 return [self.copy(deep=False)]
777 if not (
778 self._can_hold_element(value) or (self.dtype == "string" and is_re(value))
779 ):
780 block = self.astype(np.dtype(object))
781 else:
782 block = self._maybe_copy(inplace)
784 rx = re.compile(to_replace)
786 replace_regex(block.values, rx, value, mask)
787 return [block]
789 @final
790 def replace_list(
791 self,
792 src_list: Iterable[Any],
793 dest_list: Sequence[Any],
794 inplace: bool = False,
795 regex: bool = False,
796 ) -> list[Block]:
797 """
798 See BlockManager.replace_list docstring.
799 """
800 values = self.values
802 # Exclude anything that we know we won't contain
803 pairs = [
804 (x, y)
805 for x, y in zip(src_list, dest_list, strict=True)
806 if (self._can_hold_element(x) or (self.dtype == "string" and is_re(x)))
807 ]
808 if not pairs:
809 return [self.copy(deep=False)]
811 src_len = len(pairs) - 1
813 if is_string_dtype(values.dtype):
814 # Calculate the mask once, prior to the call of comp
815 # in order to avoid repeating the same computations
816 na_mask = ~isna(values)
817 masks: Iterable[npt.NDArray[np.bool_]] = (
818 extract_bool_array(
819 compare_or_regex_search(values, s[0], regex=regex, mask=na_mask),
820 )
821 for s in pairs
822 )
823 else:
824 # GH#38086 faster if we know we dont need to check for regex
825 masks = (missing.mask_missing(values, s[0]) for s in pairs)
826 # Materialize if inplace = True, since the masks can change
827 # as we replace
828 if inplace:
829 masks = list(masks)
831 # Don't set up refs here, otherwise we will think that we have
832 # references when we check again later
833 rb = [self]
835 for i, ((src, dest), mask) in enumerate(zip(pairs, masks, strict=True)):
836 new_rb: list[Block] = []
838 # GH-39338: _replace_coerce can split a block into
839 # single-column blocks, so track the index so we know
840 # where to index into the mask
841 for blk_num, blk in enumerate(rb):
842 if len(rb) == 1:
843 m = mask
844 else:
845 mib = mask
846 assert not isinstance(mib, bool)
847 m = mib[blk_num : blk_num + 1]
849 # error: Argument "mask" to "_replace_coerce" of "Block" has
850 # incompatible type "Union[ExtensionArray, ndarray[Any, Any], bool]";
851 # expected "ndarray[Any, dtype[bool_]]"
852 result = blk._replace_coerce(
853 to_replace=src,
854 value=dest,
855 mask=m,
856 inplace=inplace,
857 regex=regex,
858 )
860 if i != src_len:
861 # This is ugly, but we have to get rid of intermediate refs. We
862 # can simply clear the referenced_blocks if we already copied,
863 # otherwise we have to remove ourselves
864 self_blk_ids = {
865 id(b()): i for i, b in enumerate(self.refs.referenced_blocks)
866 }
867 for b in result:
868 if b.refs is self.refs:
869 # We are still sharing memory with self
870 if id(b) in self_blk_ids and b is not self:
871 # Remove ourselves from the refs; we are temporary
872 self.refs.referenced_blocks.pop(self_blk_ids[id(b)])
873 else:
874 # We have already copied, so we can clear the refs to avoid
875 # future copies
876 b.refs.referenced_blocks.clear()
877 new_rb.extend(result)
878 rb = new_rb
879 return rb
881 @final
882 def _replace_coerce(
883 self,
884 to_replace,
885 value,
886 mask: npt.NDArray[np.bool_],
887 inplace: bool = True,
888 regex: bool = False,
889 ) -> list[Block]:
890 """
891 Replace value corresponding to the given boolean array with another
892 value.
894 Parameters
895 ----------
896 to_replace : object or pattern
897 Scalar to replace or regular expression to match.
898 value : object
899 Replacement object.
900 mask : np.ndarray[bool]
901 True indicate corresponding element is ignored.
902 inplace : bool, default True
903 Perform inplace modification.
904 regex : bool, default False
905 If true, perform regular expression substitution.
907 Returns
908 -------
909 List[Block]
910 """
911 if should_use_regex(regex, to_replace):
912 return self._replace_regex(
913 to_replace,
914 value,
915 inplace=inplace,
916 mask=mask,
917 )
918 else:
919 if value is None:
920 # gh-45601, gh-45836, gh-46634
921 if mask.any():
922 has_ref = self.refs.has_reference()
923 nb = self.astype(np.dtype(object))
924 if not inplace:
925 nb = nb.copy(deep=True)
926 elif inplace and has_ref and nb.refs.has_reference():
927 # no copy in astype and we had refs before
928 nb = nb.copy(deep=True)
929 putmask_inplace(nb.values, mask, value)
930 return [nb]
931 return [self.copy(deep=False)]
932 return self.replace(
933 to_replace=to_replace,
934 value=value,
935 inplace=inplace,
936 mask=mask,
937 )
939 # ---------------------------------------------------------------------
940 # 2D Methods - Shared by NumpyBlock and NDArrayBackedExtensionBlock
941 # but not ExtensionBlock
943 def _maybe_squeeze_arg(self, arg: np.ndarray) -> np.ndarray:
944 """
945 For compatibility with 1D-only ExtensionArrays.
946 """
947 return arg
949 def _unwrap_setitem_indexer(self, indexer):
950 """
951 For compatibility with 1D-only ExtensionArrays.
952 """
953 return indexer
955 # NB: this cannot be made cache_readonly because in mgr.set_values we pin
956 # new .values that can have different shape GH#42631
957 @property
958 def shape(self) -> Shape:
959 return self.values.shape
961 def iget(self, i: int | tuple[int, int] | tuple[slice, int]) -> np.ndarray:
962 # In the case where we have a tuple[slice, int], the slice will always
963 # be slice(None)
964 # Note: only reached with self.ndim == 2
965 # Invalid index type "Union[int, Tuple[int, int], Tuple[slice, int]]"
966 # for "Union[ndarray[Any, Any], ExtensionArray]"; expected type
967 # "Union[int, integer[Any]]"
968 return self.values[i] # type: ignore[index]
970 def _slice(
971 self, slicer: slice | npt.NDArray[np.bool_] | npt.NDArray[np.intp]
972 ) -> ArrayLike:
973 """return a slice of my values"""
975 return self.values[slicer]
977 def set_inplace(self, locs, values: ArrayLike, copy: bool = False) -> None:
978 """
979 Modify block values in-place with new item value.
981 If copy=True, first copy the underlying values in place before modifying
982 (for Copy-on-Write).
984 Notes
985 -----
986 `set_inplace` never creates a new array or new Block, whereas `setitem`
987 _may_ create a new array and always creates a new Block.
989 Caller is responsible for checking values.dtype == self.dtype.
990 """
991 if copy:
992 self.values = self.values.copy()
993 self.values[locs] = values
995 @final
996 def take_nd(
997 self,
998 indexer: npt.NDArray[np.intp],
999 axis: AxisInt,
1000 new_mgr_locs: BlockPlacement | None = None,
1001 fill_value=lib.no_default,
1002 ) -> Block:
1003 """
1004 Take values according to indexer and return them as a block.
1005 """
1006 values = self.values
1008 if fill_value is lib.no_default:
1009 fill_value = self.fill_value
1010 allow_fill = False
1011 else:
1012 allow_fill = True
1014 # Note: algos.take_nd has upcast logic similar to coerce_to_target_dtype
1015 new_values = algos.take_nd(
1016 values, indexer, axis=axis, allow_fill=allow_fill, fill_value=fill_value
1017 )
1019 # Called from three places in managers, all of which satisfy
1020 # these assertions
1021 if isinstance(self, ExtensionBlock):
1022 # NB: in this case, the 'axis' kwarg will be ignored in the
1023 # algos.take_nd call above.
1024 assert not (self.ndim == 1 and new_mgr_locs is None)
1025 assert not (axis == 0 and new_mgr_locs is None)
1027 if new_mgr_locs is None:
1028 new_mgr_locs = self._mgr_locs
1030 if new_values.dtype != self.dtype:
1031 return self.make_block(new_values, new_mgr_locs)
1032 else:
1033 return self.make_block_same_class(new_values, new_mgr_locs)
1035 def _unstack(
1036 self,
1037 unstacker,
1038 fill_value,
1039 new_placement: npt.NDArray[np.intp],
1040 needs_masking: npt.NDArray[np.bool_],
1041 ):
1042 """
1043 Return a list of unstacked blocks of self
1045 Parameters
1046 ----------
1047 unstacker : reshape._Unstacker
1048 fill_value : int
1049 Only used in ExtensionBlock._unstack
1050 new_placement : np.ndarray[np.intp]
1051 allow_fill : bool
1052 needs_masking : np.ndarray[bool]
1054 Returns
1055 -------
1056 blocks : list of Block
1057 New blocks of unstacked values.
1058 mask : array-like of bool
1059 The mask of columns of `blocks` we should keep.
1060 """
1061 new_values, mask = unstacker.get_new_values(
1062 self.values.T, fill_value=fill_value
1063 )
1065 mask = mask.any(0)
1066 # TODO: in all tests we have mask.all(); can we rely on that?
1068 # Note: these next two lines ensure that
1069 # mask.sum() == sum(len(nb.mgr_locs) for nb in blocks)
1070 # which the calling function needs in order to pass verify_integrity=False
1071 # to the BlockManager constructor
1072 new_values = new_values.T[mask]
1073 new_placement = new_placement[mask]
1075 bp = BlockPlacement(new_placement)
1076 blocks = [new_block_2d(new_values, placement=bp)]
1077 return blocks, mask
1079 # ---------------------------------------------------------------------
1081 def setitem(self, indexer, value) -> Block:
1082 """
1083 Attempt self.values[indexer] = value, possibly creating a new array.
1085 Parameters
1086 ----------
1087 indexer : tuple, list-like, array-like, slice, int
1088 The subset of self.values to set
1089 value : object
1090 The value being set
1092 Returns
1093 -------
1094 Block
1096 Notes
1097 -----
1098 `indexer` is a direct slice/positional indexer. `value` must
1099 be a compatible shape.
1100 """
1102 value = self._standardize_fill_value(value)
1104 values = cast(np.ndarray, self.values)
1105 if self.ndim == 2:
1106 values = values.T
1108 # length checking
1109 check_setitem_lengths(indexer, value, values)
1111 if self.dtype != _dtype_obj:
1112 # GH48933: extract_array would convert a pd.Series value to np.ndarray
1113 value = extract_array(value, extract_numpy=True)
1114 try:
1115 casted = np_can_hold_element(values.dtype, value)
1116 except LossySetitemError:
1117 # current dtype cannot store value, coerce to common dtype
1118 nb = self.coerce_to_target_dtype(value, raise_on_upcast=True)
1119 return nb.setitem(indexer, value)
1120 else:
1121 if self.dtype == _dtype_obj:
1122 # TODO: avoid having to construct values[indexer]
1123 vi = values[indexer]
1124 if lib.is_list_like(vi):
1125 # checking lib.is_scalar here fails on
1126 # test_iloc_setitem_custom_object
1127 casted = setitem_datetimelike_compat(values, len(vi), casted)
1129 self = self._maybe_copy(inplace=True)
1130 values = cast(np.ndarray, self.values.T)
1131 if isinstance(casted, np.ndarray) and casted.ndim == 1 and len(casted) == 1:
1132 # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615
1133 casted = casted[0, ...]
1134 try:
1135 values[indexer] = casted
1136 except (TypeError, ValueError) as err:
1137 if is_list_like(casted):
1138 raise ValueError(
1139 "setting an array element with a sequence."
1140 ) from err
1141 raise
1142 return self
1144 def putmask(self, mask, new) -> list[Block]:
1145 """
1146 putmask the data to the block; it is possible that we may create a
1147 new dtype of block
1149 Return the resulting block(s).
1151 Parameters
1152 ----------
1153 mask : np.ndarray[bool], SparseArray[bool], or BooleanArray
1154 new : an ndarray/object
1156 Returns
1157 -------
1158 List[Block]
1159 """
1160 orig_mask = mask
1161 values = cast(np.ndarray, self.values)
1162 mask, noop = validate_putmask(values.T, mask)
1163 assert not isinstance(new, (ABCIndex, ABCSeries, ABCDataFrame))
1165 if new is lib.no_default:
1166 new = self.fill_value
1168 new = self._standardize_fill_value(new)
1169 new = extract_array(new, extract_numpy=True)
1171 if noop:
1172 return [self.copy(deep=False)]
1174 try:
1175 casted = np_can_hold_element(values.dtype, new)
1177 self = self._maybe_copy(inplace=True)
1178 values = cast(np.ndarray, self.values)
1180 putmask_without_repeat(values.T, mask, casted)
1181 return [self]
1182 except LossySetitemError:
1183 if self.ndim == 1 or self.shape[0] == 1:
1184 # no need to split columns
1186 if not is_list_like(new):
1187 # using just new[indexer] can't save us the need to cast
1188 return self.coerce_to_target_dtype(
1189 new, raise_on_upcast=True
1190 ).putmask(mask, new)
1191 else:
1192 indexer = mask.nonzero()[0]
1193 nb = self.setitem(indexer, new[indexer])
1194 return [nb]
1196 else:
1197 is_array = isinstance(new, np.ndarray)
1199 res_blocks = []
1200 for i, nb in enumerate(self._split()):
1201 n = new
1202 if is_array:
1203 # we have a different value per-column
1204 n = new[:, i : i + 1]
1206 submask = orig_mask[:, i : i + 1]
1207 rbs = nb.putmask(submask, n)
1208 res_blocks.extend(rbs)
1209 return res_blocks
1211 def where(self, other, cond) -> list[Block]:
1212 """
1213 evaluate the block; return result block(s) from the result
1215 Parameters
1216 ----------
1217 other : an ndarray/object
1218 cond : np.ndarray[bool], SparseArray[bool], or BooleanArray
1220 Returns
1221 -------
1222 List[Block]
1223 """
1224 assert cond.ndim == self.ndim
1225 assert not isinstance(other, (ABCIndex, ABCSeries, ABCDataFrame))
1227 transpose = self.ndim == 2
1229 cond = extract_bool_array(cond)
1231 # EABlocks override where
1232 values = cast(np.ndarray, self.values)
1233 orig_other = other
1234 if transpose:
1235 values = values.T
1237 icond, noop = validate_putmask(values, ~cond)
1238 if noop:
1239 return [self.copy(deep=False)]
1241 if other is lib.no_default:
1242 other = self.fill_value
1244 other = self._standardize_fill_value(other)
1246 try:
1247 # try/except here is equivalent to a self._can_hold_element check,
1248 # but this gets us back 'casted' which we will reuse below;
1249 # without using 'casted', expressions.where may do unwanted upcasts.
1250 casted = np_can_hold_element(values.dtype, other)
1251 except (ValueError, TypeError, LossySetitemError):
1252 # we cannot coerce, return a compat dtype
1254 if self.ndim == 1 or self.shape[0] == 1:
1255 # no need to split columns
1257 block = self.coerce_to_target_dtype(other, raise_on_upcast=False)
1258 return block.where(orig_other, cond)
1260 else:
1261 is_array = isinstance(other, (np.ndarray, ExtensionArray))
1263 res_blocks = []
1264 for i, nb in enumerate(self._split()):
1265 oth = other
1266 if is_array:
1267 # we have a different value per-column
1268 oth = other[:, i : i + 1]
1270 submask = cond[:, i : i + 1]
1271 rbs = nb.where(oth, submask)
1272 res_blocks.extend(rbs)
1273 return res_blocks
1275 else:
1276 other = casted
1277 alt = setitem_datetimelike_compat(values, icond.sum(), other)
1278 if alt is not other:
1279 if is_list_like(other) and len(other) < len(values):
1280 # call np.where with other to get the appropriate ValueError
1281 np.where(~icond, values, other)
1282 raise NotImplementedError(
1283 "This should not be reached; call to np.where above is "
1284 "expected to raise ValueError. Please report a bug at "
1285 "github.com/pandas-dev/pandas"
1286 )
1287 result = values.copy()
1288 np.putmask(result, icond, alt)
1289 else:
1290 # By the time we get here, we should have all Series/Index
1291 # args extracted to ndarray
1292 if (
1293 is_list_like(other)
1294 and not isinstance(other, np.ndarray)
1295 and len(other) == self.shape[-1]
1296 ):
1297 # If we don't do this broadcasting here, then expressions.where
1298 # will broadcast a 1D other to be row-like instead of
1299 # column-like.
1300 other = np.array(other).reshape(values.shape)
1301 # If lengths don't match (or len(other)==1), we will raise
1302 # inside expressions.where, see test_series_where
1304 # Note: expressions.where may upcast.
1305 result = expressions.where(~icond, values, other)
1306 # The np_can_hold_element check _should_ ensure that we always
1307 # have result.dtype == self.dtype here.
1309 if transpose:
1310 result = result.T
1312 return [self.make_block(result)]
1314 def fillna(
1315 self,
1316 value,
1317 limit: int | None = None,
1318 inplace: bool = False,
1319 ) -> list[Block]:
1320 """
1321 fillna on the block with the value. If we fail, then convert to
1322 block to hold objects instead and try again
1323 """
1324 # Caller is responsible for validating limit; if int it is strictly positive
1325 inplace = validate_bool_kwarg(inplace, "inplace")
1327 if not self._can_hold_na:
1328 # can short-circuit the isna call
1329 noop = True
1330 else:
1331 mask = isna(self.values)
1332 mask, noop = validate_putmask(self.values, mask)
1334 if noop:
1335 # we can't process the value, but nothing to do
1336 return [self.copy(deep=False)]
1338 if limit is not None:
1339 mask[mask.cumsum(self.values.ndim - 1) > limit] = False
1341 if inplace:
1342 nbs = self.putmask(mask.T, value)
1343 else:
1344 nbs = self.where(value, ~mask.T)
1345 return extend_blocks(nbs)
1347 def pad_or_backfill(
1348 self,
1349 *,
1350 method: FillnaOptions,
1351 inplace: bool = False,
1352 limit: int | None = None,
1353 limit_area: Literal["inside", "outside"] | None = None,
1354 ) -> list[Block]:
1355 if not self._can_hold_na:
1356 # If there are no NAs, then interpolate is a no-op
1357 return [self.copy(deep=False)]
1359 copy, refs = self._get_refs_and_copy(inplace)
1361 # Dispatch to the NumpyExtensionArray method.
1362 # We know self.array_values is a NumpyExtensionArray bc EABlock overrides
1363 vals = cast(NumpyExtensionArray, self.array_values)
1364 new_values = vals.T._pad_or_backfill(
1365 method=method,
1366 limit=limit,
1367 limit_area=limit_area,
1368 copy=copy,
1369 ).T
1371 data = extract_array(new_values, extract_numpy=True)
1372 return [self.make_block_same_class(data, refs=refs)]
1374 @final
1375 def interpolate(
1376 self,
1377 *,
1378 method: InterpolateOptions,
1379 index: Index,
1380 inplace: bool = False,
1381 limit: int | None = None,
1382 limit_direction: Literal["forward", "backward", "both"] = "forward",
1383 limit_area: Literal["inside", "outside"] | None = None,
1384 **kwargs,
1385 ) -> list[Block]:
1386 inplace = validate_bool_kwarg(inplace, "inplace")
1387 # error: Non-overlapping equality check [...]
1388 if method == "asfreq": # type: ignore[comparison-overlap]
1389 # clean_fill_method used to allow this
1390 missing.clean_fill_method(method)
1392 if not self._can_hold_na:
1393 # If there are no NAs, then interpolate is a no-op
1394 return [self.copy(deep=False)]
1396 if self.dtype == _dtype_obj:
1397 # GH#53631
1398 name = {1: "Series", 2: "DataFrame"}[self.ndim]
1399 raise TypeError(f"{name} cannot interpolate with object dtype.")
1401 copy, refs = self._get_refs_and_copy(inplace)
1403 # Dispatch to the EA method.
1404 new_values = self.array_values.interpolate(
1405 method=method,
1406 axis=self.ndim - 1,
1407 index=index,
1408 limit=limit,
1409 limit_direction=limit_direction,
1410 limit_area=limit_area,
1411 copy=copy,
1412 **kwargs,
1413 )
1414 data = extract_array(new_values, extract_numpy=True)
1415 return [self.make_block_same_class(data, refs=refs)]
1417 @final
1418 def diff(self, n: int) -> list[Block]:
1419 """return block for the diff of the values"""
1420 # only reached with ndim == 2
1421 # TODO(EA2D): transpose will be unnecessary with 2D EAs
1422 new_values = algos.diff(self.values.T, n, axis=0).T
1423 return [self.make_block(values=new_values)]
1425 def shift(self, periods: int, fill_value: Any = None) -> list[Block]:
1426 """shift the block by periods, possibly upcast"""
1427 # convert integer to float if necessary. need to do a lot more than
1428 # that, handle boolean etc also
1429 axis = self.ndim - 1
1431 # Note: periods is never 0 here, as that is handled at the top of
1432 # NDFrame.shift. If that ever changes, we can do a check for periods=0
1433 # and possibly avoid coercing.
1435 if not lib.is_scalar(fill_value) and self.dtype != _dtype_obj:
1436 # with object dtype there is nothing to promote, and the user can
1437 # pass pretty much any weird fill_value they like
1438 # see test_shift_object_non_scalar_fill
1439 raise ValueError("fill_value must be a scalar")
1441 fill_value = self._standardize_fill_value(fill_value)
1443 try:
1444 # error: Argument 1 to "np_can_hold_element" has incompatible type
1445 # "Union[dtype[Any], ExtensionDtype]"; expected "dtype[Any]"
1446 casted = np_can_hold_element(
1447 self.dtype, # type: ignore[arg-type]
1448 fill_value,
1449 )
1450 except LossySetitemError:
1451 if self.dtype.kind not in "iub" or not is_valid_na_for_dtype(
1452 fill_value, self.dtype
1453 ):
1454 # GH#53802
1455 warnings.warn(
1456 "shifting with a fill value that cannot be held by "
1457 "original dtype is deprecated and will raise in a future "
1458 "version. Explicitly cast to the desired dtype before "
1459 "shifting instead.",
1460 Pandas4Warning,
1461 stacklevel=find_stack_level(),
1462 )
1463 nb = self.coerce_to_target_dtype(fill_value, raise_on_upcast=False)
1464 return nb.shift(periods, fill_value=fill_value)
1466 else:
1467 values = cast(np.ndarray, self.values)
1468 new_values = shift(values, periods, axis, casted)
1469 return [self.make_block_same_class(new_values)]
1471 @final
1472 def quantile(
1473 self,
1474 qs: Index, # with dtype float64
1475 interpolation: QuantileInterpolation = "linear",
1476 ) -> Block:
1477 """
1478 compute the quantiles of the
1480 Parameters
1481 ----------
1482 qs : Index
1483 The quantiles to be computed in float64.
1484 interpolation : str, default 'linear'
1485 Type of interpolation.
1487 Returns
1488 -------
1489 Block
1490 """
1491 # We should always have ndim == 2 because Series dispatches to DataFrame
1492 assert self.ndim == 2
1493 assert is_list_like(qs) # caller is responsible for this
1495 result = quantile_compat(self.values, np.asarray(qs._values), interpolation)
1496 # ensure_block_shape needed for cases where we start with EA and result
1497 # is ndarray, e.g. IntegerArray, SparseArray
1498 result = ensure_block_shape(result, ndim=2)
1499 return new_block_2d(result, placement=self._mgr_locs)
1501 @final
1502 def round(self, decimals: int) -> Self:
1503 """
1504 Rounds the values.
1505 If the block is not of an integer or float dtype, nothing happens.
1506 This is consistent with DataFrame.round behavior.
1507 (Note: Series.round would raise)
1509 Parameters
1510 ----------
1511 decimals: int,
1512 Number of decimal places to round to.
1513 Caller is responsible for validating this
1514 """
1515 if not self.is_numeric or self.is_bool:
1516 if isinstance(self.values, (DatetimeArray, TimedeltaArray, PeriodArray)):
1517 # GH#57781
1518 # TODO: also the ArrowDtype analogues?
1519 warnings.warn(
1520 "obj.round has no effect with datetime, timedelta, "
1521 "or period dtypes. Use obj.dt.round(...) instead.",
1522 UserWarning,
1523 stacklevel=find_stack_level(),
1524 )
1525 return self.copy(deep=False)
1526 # TODO: round only defined on BaseMaskedArray
1527 # Series also does this, so would need to fix both places
1528 # error: Item "ExtensionArray" of "Union[ndarray[Any, Any], ExtensionArray]"
1529 # has no attribute "round"
1530 values = self.values.round(decimals) # type: ignore[union-attr]
1532 refs = None
1533 if values is self.values:
1534 refs = self.refs
1536 return self.make_block_same_class(values, refs=refs)
1538 # ---------------------------------------------------------------------
1539 # Abstract Methods Overridden By EABackedBlock and NumpyBlock
1541 def delete(self, loc) -> list[Block]:
1542 """Deletes the locs from the block.
1544 We split the block to avoid copying the underlying data. We create new
1545 blocks for every connected segment of the initial block that is not deleted.
1546 The new blocks point to the initial array.
1548 Assumes `loc` is strictly increasing when list-like.
1549 """
1550 if not is_list_like(loc):
1551 loc = [loc]
1553 if self.ndim == 1:
1554 values = cast(np.ndarray, self.values)
1555 values = np.delete(values, loc)
1556 mgr_locs = self._mgr_locs.delete(loc)
1557 return [type(self)(values, placement=mgr_locs, ndim=self.ndim)]
1559 if np.max(loc) >= self.values.shape[0]:
1560 raise IndexError
1562 # Add one out-of-bounds indexer as maximum to collect
1563 # all columns after our last indexer if any
1564 loc = np.concatenate([loc, [self.values.shape[0]]])
1565 mgr_locs_arr = self._mgr_locs.as_array
1566 new_blocks: list[Block] = []
1568 previous_loc = -1
1569 # TODO(CoW): This is tricky, if parent block goes out of scope
1570 # all split blocks are referencing each other even though they
1571 # don't share data
1572 refs = self.refs if self.refs.has_reference() else None
1573 for idx in loc:
1574 if idx == previous_loc + 1:
1575 # There is no column between current and last idx
1576 pass
1577 else:
1578 # No overload variant of "__getitem__" of "ExtensionArray" matches
1579 # argument type "Tuple[slice, slice]"
1580 values = self.values[previous_loc + 1 : idx, :] # type: ignore[call-overload]
1581 locs = mgr_locs_arr[previous_loc + 1 : idx]
1582 nb = type(self)(
1583 values, placement=BlockPlacement(locs), ndim=self.ndim, refs=refs
1584 )
1585 new_blocks.append(nb)
1587 previous_loc = idx
1589 return new_blocks
1591 @property
1592 def is_view(self) -> bool:
1593 """return a boolean if I am possibly a view"""
1594 raise AbstractMethodError(self)
1596 @property
1597 def array_values(self) -> ExtensionArray:
1598 """
1599 The array that Series.array returns. Always an ExtensionArray.
1600 """
1601 raise AbstractMethodError(self)
1603 def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray:
1604 """
1605 return an internal format, currently just the ndarray
1606 this is often overridden to handle to_dense like operations
1607 """
1608 raise AbstractMethodError(self)
1611class EABackedBlock(Block):
1612 """
1613 Mixin for Block subclasses backed by ExtensionArray.
1614 """
1616 values: ExtensionArray
1618 @final
1619 def shift(self, periods: int, fill_value: Any = None) -> list[Block]:
1620 """
1621 Shift the block by `periods`.
1623 Dispatches to underlying ExtensionArray and re-boxes in an
1624 ExtensionBlock.
1625 """
1626 # Transpose since EA.shift is always along axis=0, while we want to shift
1627 # along rows.
1628 new_values = self.values.T.shift(periods=periods, fill_value=fill_value).T
1629 return [self.make_block_same_class(new_values)]
1631 @final
1632 def setitem(self, indexer, value):
1633 """
1634 Attempt self.values[indexer] = value, possibly creating a new array.
1636 This differs from Block.setitem by not allowing setitem to change
1637 the dtype of the Block.
1639 Parameters
1640 ----------
1641 indexer : tuple, list-like, array-like, slice, int
1642 The subset of self.values to set
1643 value : object
1644 The value being set
1646 Returns
1647 -------
1648 Block
1650 Notes
1651 -----
1652 `indexer` is a direct slice/positional indexer. `value` must
1653 be a compatible shape.
1654 """
1655 orig_indexer = indexer
1656 orig_value = value
1658 indexer = self._unwrap_setitem_indexer(indexer)
1659 value = self._maybe_squeeze_arg(value)
1661 values = self.values
1662 if values.ndim == 2:
1663 # TODO(GH#45419): string[pyarrow] tests break if we transpose
1664 # unconditionally
1665 values = values.T
1666 check_setitem_lengths(indexer, value, values)
1668 try:
1669 values[indexer] = value
1670 except (ValueError, TypeError):
1671 if isinstance(self.dtype, IntervalDtype):
1672 # see TestSetitemFloatIntervalWithIntIntervalValues
1673 nb = self.coerce_to_target_dtype(orig_value, raise_on_upcast=True)
1674 return nb.setitem(orig_indexer, orig_value)
1676 elif isinstance(self, NDArrayBackedExtensionBlock):
1677 nb = self.coerce_to_target_dtype(orig_value, raise_on_upcast=True)
1678 return nb.setitem(orig_indexer, orig_value)
1680 else:
1681 raise
1683 else:
1684 return self
1686 @final
1687 def where(self, other, cond) -> list[Block]:
1688 arr = self.values.T
1690 cond = extract_bool_array(cond)
1692 orig_other = other
1693 orig_cond = cond
1694 other = self._maybe_squeeze_arg(other)
1695 cond = self._maybe_squeeze_arg(cond)
1697 if other is lib.no_default:
1698 other = self.fill_value
1700 icond, noop = validate_putmask(arr, ~cond)
1701 if noop:
1702 # GH#44181, GH#45135
1703 # Avoid a) raising for Interval/PeriodDtype and b) unnecessary object upcast
1704 return [self.copy(deep=False)]
1706 try:
1707 res_values = arr._where(cond, other).T
1708 except OutOfBoundsDatetime:
1709 raise
1710 except (ValueError, TypeError):
1711 if self.ndim == 1 or self.shape[0] == 1:
1712 if isinstance(self.dtype, (IntervalDtype, StringDtype)):
1713 # TestSetitemFloatIntervalWithIntIntervalValues
1714 blk = self.coerce_to_target_dtype(orig_other, raise_on_upcast=False)
1715 if (
1716 self.ndim == 2
1717 and isinstance(orig_cond, np.ndarray)
1718 and orig_cond.ndim == 1
1719 and not is_1d_only_ea_dtype(blk.dtype)
1720 ):
1721 orig_cond = orig_cond[:, None]
1722 return blk.where(orig_other, orig_cond)
1724 elif isinstance(self, NDArrayBackedExtensionBlock):
1725 # NB: not (yet) the same as
1726 # isinstance(values, NDArrayBackedExtensionArray)
1727 blk = self.coerce_to_target_dtype(orig_other, raise_on_upcast=False)
1728 return blk.where(orig_other, orig_cond)
1730 else:
1731 raise
1733 else:
1734 # Same pattern we use in Block.putmask
1735 is_array = isinstance(orig_other, (np.ndarray, ExtensionArray))
1737 res_blocks = []
1738 for i, nb in enumerate(self._split()):
1739 n = orig_other
1740 if is_array:
1741 # we have a different value per-column
1742 n = orig_other[:, i : i + 1]
1744 submask = orig_cond[:, i : i + 1]
1745 rbs = nb.where(n, submask)
1746 res_blocks.extend(rbs)
1747 return res_blocks
1749 nb = self.make_block_same_class(res_values)
1750 return [nb]
1752 @final
1753 def putmask(self, mask, new) -> list[Block]:
1754 """
1755 See Block.putmask.__doc__
1756 """
1757 mask = extract_bool_array(mask)
1758 if new is lib.no_default:
1759 new = self.fill_value
1761 orig_new = new
1762 orig_mask = mask
1763 new = self._maybe_squeeze_arg(new)
1764 mask = self._maybe_squeeze_arg(mask)
1766 if not mask.any():
1767 return [self.copy(deep=False)]
1769 self = self._maybe_copy(inplace=True)
1770 values = self.values
1771 if values.ndim == 2:
1772 values = values.T
1774 try:
1775 # Caller is responsible for ensuring matching lengths
1776 values._putmask(mask, new)
1777 except OutOfBoundsDatetime:
1778 raise
1779 except (TypeError, ValueError):
1780 if self.ndim == 1 or self.shape[0] == 1:
1781 if isinstance(self.dtype, IntervalDtype):
1782 # Discussion about what we want to support in the general
1783 # case GH#39584
1784 blk = self.coerce_to_target_dtype(orig_new, raise_on_upcast=True)
1785 return blk.putmask(orig_mask, orig_new)
1787 elif isinstance(self, NDArrayBackedExtensionBlock):
1788 # NB: not (yet) the same as
1789 # isinstance(values, NDArrayBackedExtensionArray)
1790 blk = self.coerce_to_target_dtype(orig_new, raise_on_upcast=True)
1791 return blk.putmask(orig_mask, orig_new)
1793 else:
1794 raise
1796 else:
1797 # Same pattern we use in Block.putmask
1798 is_array = isinstance(orig_new, (np.ndarray, ExtensionArray))
1800 res_blocks = []
1801 for i, nb in enumerate(self._split()):
1802 n = orig_new
1803 if is_array:
1804 # we have a different value per-column
1805 n = orig_new[:, i : i + 1]
1807 submask = orig_mask[:, i : i + 1]
1808 rbs = nb.putmask(submask, n)
1809 res_blocks.extend(rbs)
1810 return res_blocks
1812 return [self]
1814 @final
1815 def delete(self, loc) -> list[Block]:
1816 # This will be unnecessary if/when __array_function__ is implemented
1817 if self.ndim == 1:
1818 values = self.values.delete(loc)
1819 mgr_locs = self._mgr_locs.delete(loc)
1820 return [type(self)(values, placement=mgr_locs, ndim=self.ndim)]
1821 elif self.values.ndim == 1:
1822 # We get here through to_stata
1823 return []
1824 return super().delete(loc)
1826 @final
1827 @cache_readonly
1828 def array_values(self) -> ExtensionArray:
1829 return self.values
1831 @final
1832 def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray:
1833 """
1834 return object dtype as boxed values, such as Timestamps/Timedelta
1835 """
1836 values: ArrayLike = self.values
1837 if dtype == _dtype_obj:
1838 values = values.astype(object)
1839 # TODO(EA2D): reshape not needed with 2D EAs
1840 return np.asarray(values).reshape(self.shape)
1842 @final
1843 def pad_or_backfill(
1844 self,
1845 *,
1846 method: FillnaOptions,
1847 inplace: bool = False,
1848 limit: int | None = None,
1849 limit_area: Literal["inside", "outside"] | None = None,
1850 ) -> list[Block]:
1851 values = self.values
1853 kwargs: dict[str, Any] = {"method": method, "limit": limit}
1854 if "limit_area" in inspect.signature(values._pad_or_backfill).parameters:
1855 kwargs["limit_area"] = limit_area
1856 elif limit_area is not None:
1857 raise NotImplementedError(
1858 f"{type(values).__name__} does not implement limit_area "
1859 "(added in pandas 2.2). 3rd-party ExtensionArray authors "
1860 "need to add this argument to _pad_or_backfill."
1861 )
1863 if values.ndim == 2:
1864 # NDArrayBackedExtensionArray.fillna assumes axis=0
1865 new_values = values.T._pad_or_backfill(**kwargs).T
1866 else:
1867 new_values = values._pad_or_backfill(**kwargs)
1868 return [self.make_block_same_class(new_values)]
1871class ExtensionBlock(EABackedBlock):
1872 """
1873 Block for holding extension types.
1875 Notes
1876 -----
1877 This holds all 3rd-party extension array types. It's also the immediate
1878 parent class for our internal extension types' blocks.
1880 ExtensionArrays are limited to 1-D.
1881 """
1883 values: ExtensionArray
1885 def fillna(
1886 self,
1887 value,
1888 limit: int | None = None,
1889 inplace: bool = False,
1890 ) -> list[Block]:
1891 if isinstance(self.dtype, (IntervalDtype, StringDtype)):
1892 # Block.fillna handles coercion (test_fillna_interval)
1893 if isinstance(self.dtype, IntervalDtype) and limit is not None:
1894 raise ValueError("limit must be None")
1895 return super().fillna(
1896 value=value,
1897 limit=limit,
1898 inplace=inplace,
1899 )
1900 if self._can_hold_na and not self.values._hasna:
1901 refs = self.refs
1902 new_values = self.values
1903 else:
1904 copy, refs = self._get_refs_and_copy(inplace)
1906 try:
1907 new_values = self.values.fillna(value=value, limit=limit, copy=copy)
1908 except TypeError:
1909 # 3rd party EA that has not implemented copy keyword yet
1910 refs = None
1911 new_values = self.values.fillna(value=value, limit=limit)
1912 # issue the warning *after* retrying, in case the TypeError
1913 # was caused by an invalid fill_value
1914 warnings.warn(
1915 # GH#53278
1916 "ExtensionArray.fillna added a 'copy' keyword in pandas "
1917 "2.1.0. In a future version, ExtensionArray subclasses will "
1918 "need to implement this keyword or an exception will be "
1919 "raised. In the interim, the keyword is ignored by "
1920 f"{type(self.values).__name__}.",
1921 Pandas4Warning,
1922 stacklevel=find_stack_level(),
1923 )
1925 return [self.make_block_same_class(new_values, refs=refs)]
1927 @cache_readonly
1928 def shape(self) -> Shape:
1929 # TODO(EA2D): override unnecessary with 2D EAs
1930 if self.ndim == 1:
1931 return (len(self.values),)
1932 return len(self._mgr_locs), len(self.values)
1934 def iget(self, i: int | tuple[int, int] | tuple[slice, int]):
1935 # In the case where we have a tuple[slice, int], the slice will always
1936 # be slice(None)
1937 # We _could_ make the annotation more specific, but mypy would
1938 # complain about override mismatch:
1939 # Literal[0] | tuple[Literal[0], int] | tuple[slice, int]
1941 # Note: only reached with self.ndim == 2
1943 if isinstance(i, tuple):
1944 # TODO(EA2D): unnecessary with 2D EAs
1945 col, loc = i
1946 if not com.is_null_slice(col) and col != 0:
1947 raise IndexError(f"{self} only contains one item")
1948 if isinstance(col, slice):
1949 # the is_null_slice check above assures that col is slice(None)
1950 # so what we want is a view on all our columns and row loc
1951 if loc < 0:
1952 loc += len(self.values)
1953 # Note: loc:loc+1 vs [[loc]] makes a difference when called
1954 # from fast_xs because we want to get a view back.
1955 return self.values[loc : loc + 1]
1956 return self.values[loc]
1957 else:
1958 if i != 0:
1959 raise IndexError(f"{self} only contains one item")
1960 return self.values
1962 def set_inplace(self, locs, values: ArrayLike, copy: bool = False) -> None:
1963 # When an ndarray, we should have locs.tolist() == [0]
1964 # When a BlockPlacement we should have list(locs) == [0]
1965 if copy:
1966 self.values = self.values.copy()
1967 self.values[:] = values
1969 def _maybe_squeeze_arg(self, arg):
1970 """
1971 If necessary, squeeze a (N, 1) ndarray to (N,)
1972 """
1973 # e.g. if we are passed a 2D mask for putmask
1974 if (
1975 isinstance(arg, (np.ndarray, ExtensionArray))
1976 and arg.ndim == self.values.ndim + 1
1977 ):
1978 # TODO(EA2D): unnecessary with 2D EAs
1979 assert arg.shape[1] == 1
1980 # error: No overload variant of "__getitem__" of "ExtensionArray"
1981 # matches argument type "Tuple[slice, int]"
1982 arg = arg[:, 0] # type: ignore[call-overload]
1983 elif isinstance(arg, ABCDataFrame):
1984 # 2022-01-06 only reached for setitem
1985 # TODO: should we avoid getting here with DataFrame?
1986 assert arg.shape[1] == 1
1987 arg = arg._ixs(0, axis=1)._values
1989 return arg
1991 def _unwrap_setitem_indexer(self, indexer):
1992 """
1993 Adapt a 2D-indexer to our 1D values.
1995 This is intended for 'setitem', not 'iget' or '_slice'.
1996 """
1997 # TODO: ATM this doesn't work for iget/_slice, can we change that?
1999 if isinstance(indexer, tuple) and len(indexer) == 2:
2000 # TODO(EA2D): not needed with 2D EAs
2001 # Should never have length > 2. Caller is responsible for checking.
2002 # Length 1 is reached vis setitem_single_block and setitem_single_column
2003 # each of which pass indexer=(pi,)
2004 if all(isinstance(x, np.ndarray) and x.ndim == 2 for x in indexer):
2005 # GH#44703 went through indexing.maybe_convert_ix
2006 first, second = indexer
2007 if not (
2008 second.size == 1 and (second == 0).all() and first.shape[1] == 1
2009 ):
2010 raise NotImplementedError(
2011 "This should not be reached. Please report a bug at "
2012 "github.com/pandas-dev/pandas/"
2013 )
2014 indexer = first[:, 0]
2016 elif lib.is_integer(indexer[1]) and indexer[1] == 0:
2017 # reached via setitem_single_block passing the whole indexer
2018 indexer = indexer[0]
2020 elif com.is_null_slice(indexer[1]):
2021 indexer = indexer[0]
2023 elif is_list_like(indexer[1]) and indexer[1][0] == 0:
2024 indexer = indexer[0]
2026 else:
2027 raise NotImplementedError(
2028 "This should not be reached. Please report a bug at "
2029 "github.com/pandas-dev/pandas/"
2030 )
2031 return indexer
2033 @property
2034 def is_view(self) -> bool:
2035 """Extension arrays are never treated as views."""
2036 return False
2038 # error: Cannot override writeable attribute with read-only property
2039 @cache_readonly
2040 def is_numeric(self) -> bool: # type: ignore[override]
2041 return self.values.dtype._is_numeric
2043 def _slice(
2044 self, slicer: slice | npt.NDArray[np.bool_] | npt.NDArray[np.intp]
2045 ) -> ExtensionArray:
2046 """
2047 Return a slice of my values.
2049 Parameters
2050 ----------
2051 slicer : slice, ndarray[int], or ndarray[bool]
2052 Valid (non-reducing) indexer for self.values.
2054 Returns
2055 -------
2056 ExtensionArray
2057 """
2058 # Notes: ndarray[bool] is only reachable when via get_rows_with_mask, which
2059 # is only for Series, i.e. self.ndim == 1.
2061 # return same dims as we currently have
2062 if self.ndim == 2:
2063 # reached via getitem_block via _slice_take_blocks_ax0
2064 # TODO(EA2D): won't be necessary with 2D EAs
2066 if not isinstance(slicer, slice):
2067 raise AssertionError(
2068 "invalid slicing for a 1-ndim ExtensionArray", slicer
2069 )
2070 # GH#32959 only full-slicers along fake-dim0 are valid
2071 # TODO(EA2D): won't be necessary with 2D EAs
2072 # range(1) instead of self._mgr_locs to avoid exception on [::-1]
2073 # see test_iloc_getitem_slice_negative_step_ea_block
2074 new_locs = range(1)[slicer]
2075 if not len(new_locs):
2076 raise AssertionError(
2077 "invalid slicing for a 1-ndim ExtensionArray", slicer
2078 )
2079 slicer = slice(None)
2081 return self.values[slicer]
2083 @final
2084 def slice_block_rows(self, slicer: slice) -> Self:
2085 """
2086 Perform __getitem__-like specialized to slicing along index.
2087 """
2088 # GH#42787 in principle this is equivalent to values[..., slicer], but we don't
2089 # require subclasses of ExtensionArray to support that form (for now).
2090 new_values = self.values[slicer]
2091 return type(self)(new_values, self._mgr_locs, ndim=self.ndim, refs=self.refs)
2093 def _unstack(
2094 self,
2095 unstacker,
2096 fill_value,
2097 new_placement: npt.NDArray[np.intp],
2098 needs_masking: npt.NDArray[np.bool_],
2099 ):
2100 # ExtensionArray-safe unstack.
2101 # We override Block._unstack, which unstacks directly on the
2102 # values of the array. For EA-backed blocks, this would require
2103 # converting to a 2-D ndarray of objects.
2104 # Instead, we unstack an ndarray of integer positions, followed by
2105 # a `take` on the actual values.
2107 # Caller is responsible for ensuring self.shape[-1] == len(unstacker.index)
2108 new_values, mask = unstacker.arange_result
2110 # Note: these next two lines ensure that
2111 # mask.sum() == sum(len(nb.mgr_locs) for nb in blocks)
2112 # which the calling function needs in order to pass verify_integrity=False
2113 # to the BlockManager constructor
2114 new_values = new_values.T[mask]
2115 new_placement = new_placement[mask]
2117 # needs_masking[i] calculated once in BlockManager.unstack tells
2118 # us if there are any -1s in the relevant indices. When False,
2119 # that allows us to go through a faster path in 'take', among
2120 # other things avoiding e.g. Categorical._validate_scalar.
2121 blocks = [
2122 # TODO: could cast to object depending on fill_value?
2123 type(self)(
2124 self.values.take(
2125 indices, allow_fill=needs_masking[i], fill_value=fill_value
2126 ),
2127 BlockPlacement(place),
2128 ndim=2,
2129 )
2130 for i, (indices, place) in enumerate(
2131 zip(new_values, new_placement, strict=True)
2132 )
2133 ]
2134 return blocks, mask
2137class NumpyBlock(Block):
2138 values: np.ndarray
2139 __slots__ = ()
2141 @property
2142 def is_view(self) -> bool:
2143 """return a boolean if I am possibly a view"""
2144 return self.values.base is not None
2146 @property
2147 def array_values(self) -> ExtensionArray:
2148 return NumpyExtensionArray(self.values)
2150 def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray:
2151 if dtype == _dtype_obj:
2152 return self.values.astype(_dtype_obj)
2153 return self.values
2155 @cache_readonly
2156 def is_numeric(self) -> bool: # type: ignore[override]
2157 dtype = self.values.dtype
2158 kind = dtype.kind
2160 return kind in "fciub"
2163class NDArrayBackedExtensionBlock(EABackedBlock):
2164 """
2165 Block backed by an NDArrayBackedExtensionArray
2166 """
2168 values: NDArrayBackedExtensionArray
2170 @property
2171 def is_view(self) -> bool:
2172 """return a boolean if I am possibly a view"""
2173 # check the ndarray values of the DatetimeIndex values
2174 return self.values._ndarray.base is not None
2177class DatetimeLikeBlock(NDArrayBackedExtensionBlock):
2178 """Block for datetime64[ns], timedelta64[ns]."""
2180 __slots__ = ()
2181 is_numeric = False
2182 values: DatetimeArray | TimedeltaArray
2185# -----------------------------------------------------------------
2186# Constructor Helpers
2189def maybe_coerce_values(values: ArrayLike) -> ArrayLike:
2190 """
2191 Input validation for values passed to __init__. Ensure that
2192 any datetime64/timedelta64 dtypes are in nanoseconds. Ensure
2193 that we do not have string dtypes.
2195 Parameters
2196 ----------
2197 values : np.ndarray or ExtensionArray
2199 Returns
2200 -------
2201 values : np.ndarray or ExtensionArray
2202 """
2203 # Caller is responsible for ensuring NumpyExtensionArray is already extracted.
2205 if isinstance(values, np.ndarray):
2206 values = ensure_wrapped_if_datetimelike(values)
2208 if issubclass(values.dtype.type, str):
2209 values = np.array(values, dtype=object)
2211 if isinstance(values, (DatetimeArray, TimedeltaArray)) and values.freq is not None:
2212 # freq is only stored in DatetimeIndex/TimedeltaIndex, not in Series/DataFrame
2213 values = values._with_freq(None)
2215 return values
2218def get_block_type(dtype: DtypeObj) -> type[Block]:
2219 """
2220 Find the appropriate Block subclass to use for the given values and dtype.
2222 Parameters
2223 ----------
2224 dtype : numpy or pandas dtype
2226 Returns
2227 -------
2228 cls : class, subclass of Block
2229 """
2230 if isinstance(dtype, DatetimeTZDtype):
2231 return DatetimeLikeBlock
2232 elif isinstance(dtype, PeriodDtype):
2233 return NDArrayBackedExtensionBlock
2234 elif isinstance(dtype, ExtensionDtype):
2235 # Note: need to be sure NumpyExtensionArray is unwrapped before we get here
2236 return ExtensionBlock
2238 # We use kind checks because it is much more performant
2239 # than is_foo_dtype
2240 kind = dtype.kind
2241 if kind in "Mm":
2242 return DatetimeLikeBlock
2244 return NumpyBlock
2247def new_block_2d(
2248 values: ArrayLike, placement: BlockPlacement, refs: BlockValuesRefs | None = None
2249) -> Block:
2250 # new_block specialized to case with
2251 # ndim=2
2252 # isinstance(placement, BlockPlacement)
2253 # check_ndim/ensure_block_shape already checked
2254 klass = get_block_type(values.dtype)
2256 values = maybe_coerce_values(values)
2257 return klass(values, ndim=2, placement=placement, refs=refs)
2260def new_block(
2261 values,
2262 placement: BlockPlacement,
2263 *,
2264 ndim: int,
2265 refs: BlockValuesRefs | None = None,
2266) -> Block:
2267 # caller is responsible for ensuring:
2268 # - values is NOT a NumpyExtensionArray
2269 # - check_ndim/ensure_block_shape already checked
2270 # - maybe_coerce_values already called/unnecessary
2271 klass = get_block_type(values.dtype)
2272 return klass(values, ndim=ndim, placement=placement, refs=refs)
2275def check_ndim(values, placement: BlockPlacement, ndim: int) -> None:
2276 """
2277 ndim inference and validation.
2279 Validates that values.ndim and ndim are consistent.
2280 Validates that len(values) and len(placement) are consistent.
2282 Parameters
2283 ----------
2284 values : array-like
2285 placement : BlockPlacement
2286 ndim : int
2288 Raises
2289 ------
2290 ValueError : the number of dimensions do not match
2291 """
2293 if values.ndim > ndim:
2294 # Check for both np.ndarray and ExtensionArray
2295 raise ValueError(
2296 f"Wrong number of dimensions. values.ndim > ndim [{values.ndim} > {ndim}]"
2297 )
2299 if not is_1d_only_ea_dtype(values.dtype):
2300 # TODO(EA2D): special case not needed with 2D EAs
2301 if values.ndim != ndim:
2302 raise ValueError(
2303 "Wrong number of dimensions. "
2304 f"values.ndim != ndim [{values.ndim} != {ndim}]"
2305 )
2306 if len(placement) != len(values):
2307 raise ValueError(
2308 f"Wrong number of items passed {len(values)}, "
2309 f"placement implies {len(placement)}"
2310 )
2311 elif ndim == 2 and len(placement) != 1:
2312 # TODO(EA2D): special case unnecessary with 2D EAs
2313 raise ValueError("need to split")
2316def extract_pandas_array(
2317 values: ArrayLike, dtype: DtypeObj | None, ndim: int
2318) -> tuple[ArrayLike, DtypeObj | None]:
2319 """
2320 Ensure that we don't allow NumpyExtensionArray / NumpyEADtype in internals.
2321 """
2322 # For now, blocks should be backed by ndarrays when possible.
2323 if isinstance(values, ABCNumpyExtensionArray):
2324 values = values.to_numpy()
2325 if ndim and ndim > 1:
2326 # TODO(EA2D): special case not needed with 2D EAs
2327 values = np.atleast_2d(values)
2329 if isinstance(dtype, NumpyEADtype):
2330 dtype = dtype.numpy_dtype
2332 return values, dtype
2335# -----------------------------------------------------------------
2338def extend_blocks(result, blocks=None) -> list[Block]:
2339 """return a new extended blocks, given the result"""
2340 if blocks is None:
2341 blocks = []
2342 if isinstance(result, list):
2343 for r in result:
2344 if isinstance(r, list):
2345 blocks.extend(r)
2346 else:
2347 blocks.append(r)
2348 else:
2349 assert isinstance(result, Block), type(result)
2350 blocks.append(result)
2351 return blocks
2354def ensure_block_shape(values: ArrayLike, ndim: int = 1) -> ArrayLike:
2355 """
2356 Reshape if possible to have values.ndim == ndim.
2357 """
2359 if values.ndim < ndim:
2360 if not is_1d_only_ea_dtype(values.dtype):
2361 # TODO(EA2D): https://github.com/pandas-dev/pandas/issues/23023
2362 # block.shape is incorrect for "2D" ExtensionArrays
2363 # We can't, and don't need to, reshape.
2364 values = cast("np.ndarray | DatetimeArray | TimedeltaArray", values)
2365 values = values.reshape(1, -1)
2367 return values
2370def external_values(values: ArrayLike) -> ArrayLike:
2371 """
2372 The array that Series.values returns (public attribute).
2374 This has some historical constraints, and is overridden in block
2375 subclasses to return the correct array (e.g. period returns
2376 object ndarray and datetimetz a datetime64[ns] ndarray instead of
2377 proper extension array).
2378 """
2379 if isinstance(values, (PeriodArray, IntervalArray)):
2380 return values.astype(object)
2381 elif isinstance(values, (DatetimeArray, TimedeltaArray)):
2382 # NB: for datetime64tz this is different from np.asarray(values), since
2383 # that returns an object-dtype ndarray of Timestamps.
2384 # Avoid raising in .astype in casting from dt64tz to dt64
2385 values = values._ndarray
2387 if isinstance(values, np.ndarray):
2388 values = values.view()
2389 values.flags.writeable = False
2390 else:
2391 # ExtensionArrays
2392 # TODO decide on read-only https://github.com/pandas-dev/pandas/issues/63099
2393 # values = values.view()
2394 # values._readonly = True
2395 pass
2397 return values