Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import itertools
4from typing import (
5 TYPE_CHECKING,
6 Any,
7 Literal,
8 NoReturn,
9 Self,
10 cast,
11 final,
12)
13import warnings
15import numpy as np
17from pandas._config.config import _global_config as config
19from pandas._libs import (
20 algos as libalgos,
21 internals as libinternals,
22 lib,
23)
24from pandas._libs.internals import (
25 BlockPlacement,
26 BlockValuesRefs,
27)
28from pandas._libs.tslibs import Timestamp
29from pandas.errors import (
30 AbstractMethodError,
31 PerformanceWarning,
32)
33from pandas.util._decorators import cache_readonly
34from pandas.util._exceptions import find_stack_level
35from pandas.util._validators import validate_bool_kwarg
37from pandas.core.dtypes.cast import (
38 find_common_type,
39 infer_dtype_from_scalar,
40 np_can_hold_element,
41)
42from pandas.core.dtypes.common import (
43 ensure_platform_int,
44 is_1d_only_ea_dtype,
45 is_list_like,
46)
47from pandas.core.dtypes.dtypes import (
48 CategoricalDtype,
49 DatetimeTZDtype,
50 ExtensionDtype,
51 SparseDtype,
52)
53from pandas.core.dtypes.generic import (
54 ABCDataFrame,
55 ABCSeries,
56)
57from pandas.core.dtypes.missing import (
58 array_equals,
59 isna,
60)
62import pandas.core.algorithms as algos
63from pandas.core.arrays import DatetimeArray
64from pandas.core.base import PandasObject
65from pandas.core.construction import (
66 ensure_wrapped_if_datetimelike,
67 extract_array,
68)
69from pandas.core.indexers import maybe_convert_indices
70from pandas.core.indexes.api import (
71 Index,
72 default_index,
73 ensure_index,
74)
75from pandas.core.internals.blocks import (
76 Block,
77 NumpyBlock,
78 ensure_block_shape,
79 extend_blocks,
80 get_block_type,
81 maybe_coerce_values,
82 new_block,
83 new_block_2d,
84)
85from pandas.core.internals.ops import (
86 blockwise_all,
87 operate_blockwise,
88)
90if TYPE_CHECKING:
91 from collections.abc import (
92 Callable,
93 Generator,
94 Hashable,
95 Sequence,
96 )
98 from pandas._typing import (
99 ArrayLike,
100 AxisInt,
101 DtypeObj,
102 QuantileInterpolation,
103 Shape,
104 npt,
105 )
107 from pandas.api.extensions import ExtensionArray
108 from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
111def interleaved_dtype(dtypes: list[DtypeObj]) -> DtypeObj | None:
112 """
113 Find the common dtype for `dtypes`.
115 Parameters
116 ----------
117 dtypes : List[DtypeObj]
119 Returns
120 -------
121 dtype : np.dtype, ExtensionDtype, or None
122 None is returned when `dtypes` is empty.
123 """
124 if not len(dtypes):
125 return None
127 return find_common_type(dtypes)
130def ensure_np_dtype(dtype: DtypeObj) -> np.dtype:
131 # TODO: https://github.com/pandas-dev/pandas/issues/22791
132 # Give EAs some input on what happens here. Sparse needs this.
133 if isinstance(dtype, SparseDtype):
134 dtype = dtype.subtype
135 dtype = cast("np.dtype", dtype)
136 elif isinstance(dtype, ExtensionDtype):
137 dtype = np.dtype("object")
138 elif dtype == np.dtype(str):
139 dtype = np.dtype("object")
140 return dtype
143class BaseBlockManager(PandasObject):
144 """
145 Core internal data structure to implement DataFrame, Series, etc.
147 Manage a bunch of labeled 2D mixed-type ndarrays. Essentially it's a
148 lightweight blocked set of labeled data to be manipulated by the DataFrame
149 public API class
151 Attributes
152 ----------
153 shape
154 ndim
155 axes
156 values
157 items
159 Methods
160 -------
161 set_axis(axis, new_labels)
162 copy(deep=True)
164 get_dtypes
166 apply(func, axes, block_filter_fn)
168 get_bool_data
169 get_numeric_data
171 get_slice(slice_like, axis)
172 get(label)
173 iget(loc)
175 take(indexer, axis)
176 reindex_axis(new_labels, axis)
177 reindex_indexer(new_labels, indexer, axis)
179 delete(label)
180 insert(loc, label, value)
181 set(label, value)
183 Parameters
184 ----------
185 blocks: Sequence of Block
186 axes: Sequence of Index
187 verify_integrity: bool, default True
189 Notes
190 -----
191 This is *not* a public API class
192 """
194 __slots__ = ()
196 _blknos: npt.NDArray[np.intp]
197 _blklocs: npt.NDArray[np.intp]
198 # (blocks, value): an entry is valid only while its blocks are self.blocks
199 _dtypes_cache: tuple[tuple[Block, ...], npt.NDArray[np.object_]] | None
200 blocks: tuple[Block, ...]
201 axes: list[Index]
203 @property
204 def ndim(self) -> int:
205 raise NotImplementedError
207 _known_consolidated: bool
208 _is_consolidated: bool
210 def __init__(self, blocks, axes, verify_integrity: bool = True) -> None:
211 raise NotImplementedError
213 @final
214 def __len__(self) -> int:
215 return len(self.items)
217 @property
218 def shape(self) -> Shape:
219 return tuple(len(ax) for ax in self.axes)
221 @classmethod
222 def from_blocks(cls, blocks: list[Block], axes: list[Index]) -> Self:
223 raise NotImplementedError
225 @property
226 def blknos(self) -> npt.NDArray[np.intp]:
227 """
228 Suppose we want to find the array corresponding to our i'th column.
230 blknos[i] identifies the block from self.blocks that contains this column.
232 blklocs[i] identifies the column of interest within
233 self.blocks[self.blknos[i]]
234 """
235 if self._blknos is None:
236 # Note: these can be altered by other BlockManager methods.
237 self._rebuild_blknos_and_blklocs()
239 return self._blknos
241 @property
242 def blklocs(self) -> npt.NDArray[np.intp]:
243 """
244 See blknos.__doc__
245 """
246 if self._blklocs is None:
247 # Note: these can be altered by other BlockManager methods.
248 self._rebuild_blknos_and_blklocs()
250 return self._blklocs
252 def make_empty(self, axes=None) -> Self:
253 """return an empty BlockManager with the items axis of len 0"""
254 if axes is None:
255 # TODO shallow copy remaining axis?
256 axes = [default_index(0), *self.axes[1:]]
258 # preserve dtype if possible
259 if self.ndim == 1:
260 assert isinstance(self, SingleBlockManager) # for mypy
261 blk = self.blocks[0]
262 arr = blk.values[:0]
263 bp = BlockPlacement(slice(0, 0))
264 nb = blk.make_block_same_class(arr, placement=bp)
265 blocks = [nb]
266 else:
267 blocks = []
268 return type(self).from_blocks(blocks, axes)
270 def __bool__(self) -> bool:
271 return True
273 def set_axis(self, axis: AxisInt, new_labels: Index) -> None:
274 # Caller is responsible for ensuring we have an Index object.
275 self._validate_set_axis(axis, new_labels)
276 self.axes[axis] = new_labels
278 @final
279 def _validate_set_axis(self, axis: AxisInt, new_labels: Index) -> None:
280 # Caller is responsible for ensuring we have an Index object.
281 old_len = len(self.axes[axis])
282 new_len = len(new_labels)
284 if axis == 1 and len(self.items) == 0:
285 # If we are setting the index on a DataFrame with no columns,
286 # it is OK to change the length.
287 pass
289 elif new_len != old_len:
290 raise ValueError(
291 f"Length mismatch: Expected axis has {old_len} elements, new "
292 f"values have {new_len} elements"
293 )
295 @property
296 def is_single_block(self) -> bool:
297 # Assumes we are 2D; overridden by SingleBlockManager
298 return len(self.blocks) == 1
300 @property
301 def items(self) -> Index:
302 return self.axes[0]
304 def _has_no_reference(self, i: int) -> bool:
305 """
306 Check for column `i` if it has references.
307 (whether it references another array or is itself being referenced)
308 Returns True if the column has no references.
309 """
310 blkno = self.blknos[i]
311 return self._has_no_reference_block(blkno)
313 def _has_no_reference_block(self, blkno: int) -> bool:
314 """
315 Check for block `i` if it has references.
316 (whether it references another array or is itself being referenced)
317 Returns True if the block has no references.
318 """
319 return not self.blocks[blkno].refs.has_reference()
321 def add_references(self, mgr: BaseBlockManager) -> None:
322 """
323 Adds the references from one manager to another. We assume that both
324 managers have the same block structure.
325 """
326 if len(self.blocks) != len(mgr.blocks):
327 # If block structure changes, then we made a copy
328 return
329 for i, blk in enumerate(self.blocks):
330 blk.refs = mgr.blocks[i].refs
331 blk.refs.add_reference(blk)
333 def references_same_values(self, mgr: BaseBlockManager, blkno: int) -> bool:
334 """
335 Checks if two blocks from two different block managers reference the
336 same underlying values.
337 """
338 blk = self.blocks[blkno]
339 return any(blk is ref() for ref in mgr.blocks[blkno].refs.referenced_blocks)
341 def get_unique_dtypes(self) -> npt.NDArray[np.object_]:
342 return algos.unique(np.array([blk.dtype for blk in self.blocks], dtype=object))
344 def get_dtypes(self) -> npt.NDArray[np.object_]:
345 blocks = self.blocks
346 cache = self._dtypes_cache
347 # Validate on read rather than on store: a reader racing a write may
348 # store an entry for blocks that are already gone (GH#68446). Keying on
349 # blocks alone relies on writes that change column dtypes rebinding
350 # blocks after they update blknos.
351 if cache is None or cache[0] is not blocks:
352 dtypes = np.array([blk.dtype for blk in blocks], dtype=object)
353 cache = (blocks, dtypes.take(self.blknos))
354 self._dtypes_cache = cache
355 return cache[1].copy()
357 @property
358 def arrays(self) -> list[ArrayLike]:
359 """
360 Quick access to the backing arrays of the Blocks.
362 Only for compatibility with ArrayManager for testing convenience.
363 Not to be used in actual code, and return value is not the same as the
364 ArrayManager method (list of 1D arrays vs iterator of 2D ndarrays / 1D EAs).
366 Warning! The returned arrays don't handle Copy-on-Write, so this should
367 be used with caution (only in read-mode).
368 """
369 # TODO: Deprecate, usage in Dask
370 # https://github.com/dask/dask/blob/484fc3f1136827308db133cd256ba74df7a38d8c/dask/base.py#L1312
371 return [blk.values for blk in self.blocks]
373 def __repr__(self) -> str:
374 output = type(self).__name__
375 for i, ax in enumerate(self.axes):
376 if i == 0:
377 output += f"\nItems: {ax}"
378 else:
379 output += f"\nAxis {i}: {ax}"
381 for block in self.blocks:
382 output += f"\n{block}"
383 return output
385 def _equal_values(self, other: Self) -> bool:
386 """
387 To be implemented by the subclasses. Only check the column values
388 assuming shape and indexes have already been checked.
389 """
390 raise AbstractMethodError(self)
392 @final
393 def equals(self, other: object) -> bool:
394 """
395 Implementation for DataFrame.equals
396 """
397 if not isinstance(other, type(self)):
398 return False
400 self_axes, other_axes = self.axes, other.axes
401 if len(self_axes) != len(other_axes):
402 return False
403 if not all(
404 ax1.equals(ax2) for ax1, ax2 in zip(self_axes, other_axes, strict=True)
405 ):
406 return False
408 return self._equal_values(other)
410 def apply(
411 self,
412 f,
413 align_keys: list[str] | None = None,
414 **kwargs,
415 ) -> Self:
416 """
417 Iterate over the blocks, collect and create a new BlockManager.
419 Parameters
420 ----------
421 f : str or callable
422 Name of the Block method to apply.
423 align_keys: List[str] or None, default None
424 **kwargs
425 Keywords to pass to `f`
427 Returns
428 -------
429 BlockManager
430 """
431 assert "filter" not in kwargs
433 align_keys = align_keys or []
434 result_blocks: list[Block] = []
435 # fillna: Series/DataFrame is responsible for making sure value is aligned
437 aligned_args = {k: kwargs[k] for k in align_keys}
439 for b in self.blocks:
440 if aligned_args:
441 for k, obj in aligned_args.items():
442 if isinstance(obj, (ABCSeries, ABCDataFrame)):
443 # The caller is responsible for ensuring that
444 # obj.axes[-1].equals(self.items)
445 if obj.ndim == 1:
446 kwargs[k] = obj.iloc[b.mgr_locs.indexer]._values
447 else:
448 kwargs[k] = obj.iloc[:, b.mgr_locs.indexer]._values
449 else:
450 # otherwise we have an ndarray
451 kwargs[k] = obj[b.mgr_locs.indexer]
453 if callable(f):
454 applied = b.apply(f, **kwargs)
455 else:
456 applied = getattr(b, f)(**kwargs)
457 result_blocks = extend_blocks(applied, result_blocks)
459 out = type(self).from_blocks(result_blocks, [ax.view() for ax in self.axes])
460 return out
462 @final
463 def isna(self, func) -> Self:
464 return self.apply("apply", func=func)
466 @final
467 def fillna(self, value, limit: int | None, inplace: bool) -> Self:
468 if limit is not None:
469 # Do this validation even if we go through one of the no-op paths
470 limit = libalgos.validate_limit(None, limit=limit)
472 return self.apply(
473 "fillna",
474 value=value,
475 limit=limit,
476 inplace=inplace,
477 )
479 @final
480 def where(self, other, cond, align: bool) -> Self:
481 if align:
482 align_keys = ["other", "cond"]
483 else:
484 align_keys = ["cond"]
485 other = extract_array(other, extract_numpy=True)
487 return self.apply(
488 "where",
489 align_keys=align_keys,
490 other=other,
491 cond=cond,
492 )
494 @final
495 def putmask(self, mask, new, align: bool = True) -> Self:
496 if align:
497 align_keys = ["new", "mask"]
498 else:
499 align_keys = ["mask"]
500 new = extract_array(new, extract_numpy=True)
502 return self.apply(
503 "putmask",
504 align_keys=align_keys,
505 mask=mask,
506 new=new,
507 )
509 @final
510 def round(self, decimals: int) -> Self:
511 return self.apply("round", decimals=decimals)
513 @final
514 def replace(self, to_replace, value, inplace: bool) -> Self:
515 inplace = validate_bool_kwarg(inplace, "inplace")
516 # NDFrame.replace ensures the not-is_list_likes here
517 assert not lib.is_list_like(to_replace)
518 assert not lib.is_list_like(value)
519 return self.apply(
520 "replace",
521 to_replace=to_replace,
522 value=value,
523 inplace=inplace,
524 )
526 @final
527 def replace_regex(self, **kwargs) -> Self:
528 return self.apply("_replace_regex", **kwargs)
530 @final
531 def replace_list(
532 self,
533 src_list: list[Any],
534 dest_list: list[Any],
535 inplace: bool = False,
536 regex: bool = False,
537 ) -> Self:
538 """do a list replace"""
539 inplace = validate_bool_kwarg(inplace, "inplace")
541 bm = self.apply(
542 "replace_list",
543 src_list=src_list,
544 dest_list=dest_list,
545 inplace=inplace,
546 regex=regex,
547 )
548 bm._consolidate_inplace()
549 return bm
551 def interpolate(self, inplace: bool, **kwargs) -> Self:
552 return self.apply("interpolate", inplace=inplace, **kwargs)
554 def pad_or_backfill(self, inplace: bool, **kwargs) -> Self:
555 return self.apply("pad_or_backfill", inplace=inplace, **kwargs)
557 def shift(self, periods: int, fill_value) -> Self:
558 if fill_value is lib.no_default:
559 fill_value = None
561 return self.apply("shift", periods=periods, fill_value=fill_value)
563 def setitem(self, indexer, value) -> Self:
564 """
565 Set values with indexer.
567 For SingleBlockManager, this backs s[indexer] = value
568 """
569 if isinstance(indexer, np.ndarray) and indexer.ndim > self.ndim:
570 raise ValueError(f"Cannot set values with ndim > {self.ndim}")
572 if not self._has_no_reference(0):
573 # this method is only called if there is a single block -> hardcoded 0
574 # Split blocks to only copy the columns we want to modify
575 if self.ndim == 2 and isinstance(indexer, tuple):
576 blk_loc = self.blklocs[indexer[1]]
577 if is_list_like(blk_loc) and blk_loc.ndim == 2:
578 blk_loc = np.squeeze(blk_loc, axis=0)
579 elif not is_list_like(blk_loc):
580 # Keep dimension and copy data later
581 blk_loc = [blk_loc] # type: ignore[assignment]
582 if len(blk_loc) == 0:
583 return self.copy(deep=False)
585 values = self.blocks[0].values
586 if values.ndim == 2:
587 # np.unique dedupes locs so the placement built inside
588 # _iset_split_block doesn't repeat a column; inverse maps
589 # the requested column order onto the new block (GH#65446)
590 blk_loc, inverse = np.unique(blk_loc, return_inverse=True)
591 values = values[blk_loc]
592 # "T" has no attribute "_iset_split_block"
593 self._iset_split_block( # type: ignore[attr-defined]
594 0, blk_loc, values
595 )
597 indexer = list(indexer)
598 # The column indexer has to keep the kind the caller used:
599 # a slice takes the cross product with the row indexer,
600 # while an array broadcasts against it (GH#65446)
601 if indexer[1] is Ellipsis or indexer[1] is None:
602 # the new block holds every column in order, so the
603 # key applies unchanged
604 pass
605 elif lib.is_integer(indexer[1]):
606 indexer[1] = 0
607 elif isinstance(indexer[1], slice):
608 # the new block holds the columns sorted, so a slice
609 # over them differs from the caller's only in direction
610 step = -1 if (indexer[1].step or 1) < 0 else None
611 indexer[1] = slice(None, None, step)
612 else:
613 indexer[1] = inverse
615 if isinstance(indexer[0], np.ndarray) and len(indexer[0]) == 0:
616 # numpy does not like empty indexer combined with slice
617 # and we are setting nothing anyway
618 return self
619 self.blocks[0].setitem(tuple(indexer), value)
620 return self
621 # No need to split if we either set all columns or on a single block
622 # manager
623 self = self.copy(deep=True)
625 return self.apply("setitem", indexer=indexer, value=value)
627 def diff(self, n: int) -> Self:
628 # only reached with self.ndim == 2
629 return self.apply("diff", n=n)
631 def astype(self, dtype, errors: str = "raise") -> Self:
632 return self.apply("astype", dtype=dtype, errors=errors)
634 def convert(self) -> Self:
635 return self.apply("convert")
637 def convert_dtypes(self, **kwargs):
638 return self.apply("convert_dtypes", **kwargs)
640 def get_values_for_csv(
641 self, *, float_format, date_format, decimal, na_rep: str = "nan", quoting=None
642 ) -> Self:
643 """
644 Convert values to native types (strings / python objects) that are used
645 in formatting (repr / csv).
646 """
647 return self.apply(
648 "get_values_for_csv",
649 na_rep=na_rep,
650 quoting=quoting,
651 float_format=float_format,
652 date_format=date_format,
653 decimal=decimal,
654 )
656 @property
657 def any_extension_types(self) -> bool:
658 """Whether any of the blocks in this manager are extension blocks"""
659 return any(block.is_extension for block in self.blocks)
661 @property
662 def is_view(self) -> bool:
663 """return a boolean if we are a single block and are a view"""
664 if len(self.blocks) == 1:
665 return self.blocks[0].is_view
667 # It is technically possible to figure out which blocks are views
668 # e.g. [ b.values.base is not None for b in self.blocks ]
669 # but then we have the case of possibly some blocks being a view
670 # and some blocks not. setting in theory is possible on the non-view
671 # blocks. But this is a bit
672 # complicated
674 return False
676 def _get_data_subset(self, predicate: Callable) -> Self:
677 blocks = [blk for blk in self.blocks if predicate(blk.values)]
678 return self._combine(blocks)
680 def _get_data_subset_indices(self, predicate: Callable) -> np.ndarray:
681 blocks = [blk for blk in self.blocks if predicate(blk.values)]
682 indexer = np.sort(np.concatenate([b.mgr_locs.as_array for b in blocks]))
683 return indexer
685 def get_bool_data(self) -> Self:
686 """
687 Select blocks that are bool-dtype and columns from object-dtype blocks
688 that are all-bool.
689 """
691 new_blocks = []
693 for blk in self.blocks:
694 if blk.dtype == bool:
695 new_blocks.append(blk)
697 elif blk.is_object:
698 new_blocks.extend(nb for nb in blk._split() if nb.is_bool)
700 return self._combine(new_blocks)
702 def get_numeric_data(self) -> Self:
703 numeric_blocks = [blk for blk in self.blocks if blk.is_numeric]
704 if len(numeric_blocks) == len(self.blocks):
705 # Avoid somewhat expensive _combine
706 # TODO(CoW) need to return a shallow copy here?
707 return self
708 return self._combine(numeric_blocks)
710 def _combine(self, blocks: list[Block], index: Index | None = None) -> Self:
711 """return a new manager with the blocks"""
712 if len(blocks) == 0:
713 if self.ndim == 2:
714 # retain our own Index dtype
715 if index is not None:
716 axes = [self.items[:0], index]
717 else:
718 axes = [self.items[:0], *self.axes[1:]]
719 return self.make_empty(axes)
720 return self.make_empty()
722 indexer = np.sort(np.concatenate([b.mgr_locs.as_array for b in blocks]))
723 inv_indexer = lib.get_reverse_indexer(indexer, self.shape[0])
725 new_blocks: list[Block] = []
726 for b in blocks:
727 nb = b.copy(deep=False)
728 nb.mgr_locs = BlockPlacement(inv_indexer[nb.mgr_locs.indexer])
729 new_blocks.append(nb)
731 axes = list(self.axes)
732 # TODO shallow copy of axes?
733 if index is not None:
734 axes[-1] = index
735 axes[0] = self.items.take(indexer)
737 return type(self).from_blocks(new_blocks, axes)
739 @property
740 def nblocks(self) -> int:
741 return len(self.blocks)
743 def copy(self, *, deep: bool) -> Self:
744 """
745 Make deep or shallow copy of BlockManager
747 Parameters
748 ----------
749 deep : bool, string or None, default True
750 If False, return a shallow copy (do not copy data)
752 Returns
753 -------
754 BlockManager
755 """
756 # TODO: Should deep=True be respected for axes?
757 new_axes = [ax.view() for ax in self.axes]
759 res = self.apply("copy", deep=deep)
760 res.axes = new_axes
762 if self.ndim > 1:
763 # Avoid needing to re-compute these
764 blknos = self._blknos
765 if blknos is not None:
766 res._blknos = blknos.copy()
767 res._blklocs = self._blklocs.copy()
769 if deep:
770 res._consolidate_inplace()
771 return res
773 def is_consolidated(self) -> bool:
774 return True
776 def consolidate(self) -> Self:
777 """
778 Join together blocks having same dtype
780 Returns
781 -------
782 y : BlockManager
783 """
784 if self.is_consolidated():
785 return self
787 # TODO shallow copy is not needed here?
788 bm = type(self)(self.blocks, self.axes, verify_integrity=False)
789 bm._is_consolidated = False
790 bm._consolidate_inplace()
791 return bm
793 def _consolidate_inplace(self) -> None:
794 return
796 @final
797 def reindex_axis(
798 self,
799 new_index: Index,
800 axis: AxisInt,
801 fill_value=None,
802 only_slice: bool = False,
803 ) -> Self:
804 """
805 Conform data manager to new index.
806 """
807 new_index, indexer = self.axes[axis].reindex(new_index)
809 return self.reindex_indexer(
810 new_index,
811 indexer,
812 axis=axis,
813 fill_value=fill_value,
814 only_slice=only_slice,
815 )
817 def reindex_indexer(
818 self,
819 new_axis: Index,
820 indexer: npt.NDArray[np.intp] | None,
821 axis: AxisInt,
822 fill_value=None,
823 allow_dups: bool = False,
824 only_slice: bool = False,
825 *,
826 use_na_proxy: bool = False,
827 ) -> Self:
828 """
829 Parameters
830 ----------
831 new_axis : Index
832 indexer : ndarray[intp] or None
833 axis : int
834 fill_value : object, default None
835 allow_dups : bool, default False
836 only_slice : bool, default False
837 Whether to take views, not copies, along columns.
838 use_na_proxy : bool, default False
839 Whether to use an np.void ndarray for newly introduced columns.
841 pandas-indexer with -1's only.
842 """
843 if indexer is None:
844 if new_axis is self.axes[axis]:
845 # TODO(CoW) need to handle CoW?
846 return self
848 result = self.copy(deep=False)
849 result.axes = list(self.axes)
850 result.axes[axis] = new_axis
851 return result
853 # Should be intp, but in some cases we get int64 on 32bit builds
854 assert isinstance(indexer, np.ndarray)
856 # some axes don't allow reindexing with dups
857 if not allow_dups:
858 self.axes[axis]._validate_can_reindex(indexer)
860 if axis >= self.ndim:
861 raise IndexError("Requested axis not found in manager")
863 if axis == 0:
864 new_blocks = list(
865 self._slice_take_blocks_ax0(
866 indexer,
867 fill_value=fill_value,
868 only_slice=only_slice,
869 use_na_proxy=use_na_proxy,
870 )
871 )
872 else:
873 new_blocks = []
874 for blk in self.blocks:
875 if blk.dtype == np.void:
876 # GH#58316: np.void placeholders cast to b'' when
877 # reindexed; preserve np.void so _setitem_single_column
878 # can later infer the correct dtype
879 vals = np.empty((blk.values.shape[0], len(indexer)), dtype=np.void)
880 new_blocks.append(NumpyBlock(vals, blk.mgr_locs, ndim=2))
881 else:
882 new_blocks.append(
883 blk.take_nd(
884 indexer,
885 axis=1,
886 fill_value=(
887 fill_value if fill_value is not None else blk.fill_value
888 ),
889 )
890 )
892 new_axes = list(self.axes)
893 new_axes[axis] = new_axis
894 if self.ndim == 2:
895 new_axes[1 - axis] = self.axes[1 - axis].view()
897 new_mgr = type(self).from_blocks(new_blocks, new_axes)
898 if axis == 1:
899 # We can avoid the need to rebuild these
900 new_mgr._blknos = self.blknos.copy()
901 new_mgr._blklocs = self.blklocs.copy()
902 return new_mgr
904 def _slice_take_blocks_ax0(
905 self,
906 slice_or_indexer: slice | np.ndarray,
907 fill_value=lib.no_default,
908 only_slice: bool = False,
909 *,
910 use_na_proxy: bool = False,
911 ref_inplace_op: bool = False,
912 ) -> Generator[Block]:
913 """
914 Slice/take blocks along axis=0.
916 Overloaded for SingleBlock
918 Parameters
919 ----------
920 slice_or_indexer : slice or np.ndarray[int64]
921 fill_value : scalar, default lib.no_default
922 only_slice : bool, default False
923 If True, we always return views on existing arrays, never copies.
924 This is used when called from ops.blockwise.operate_blockwise.
925 use_na_proxy : bool, default False
926 Whether to use an np.void ndarray for newly introduced columns.
927 ref_inplace_op: bool, default False
928 Don't track refs if True because we operate inplace
930 Yields
931 ------
932 Block : New Block
933 """
934 allow_fill = fill_value is not lib.no_default
936 sl_type, slobj, sllen = _preprocess_slice_or_indexer(
937 slice_or_indexer, self.shape[0], allow_fill=allow_fill
938 )
940 if self.is_single_block:
941 blk = self.blocks[0]
943 if sl_type == "slice":
944 # GH#32959 EABlock would fail since we can't make 0-width
945 # TODO(EA2D): special casing unnecessary with 2D EAs
946 if sllen == 0:
947 return
948 bp = BlockPlacement(slice(0, sllen))
949 yield blk.getitem_block_columns(slobj, new_mgr_locs=bp)
950 return
951 elif not allow_fill or self.ndim == 1:
952 if allow_fill and fill_value is None:
953 fill_value = blk.fill_value
955 if not allow_fill and only_slice:
956 # GH#33597 slice instead of take, so we get
957 # views instead of copies
958 for i, ml in enumerate(slobj):
959 yield blk.getitem_block_columns(
960 slice(ml, ml + 1),
961 new_mgr_locs=BlockPlacement(i),
962 ref_inplace_op=ref_inplace_op,
963 )
964 else:
965 bp = BlockPlacement(slice(0, sllen))
966 yield blk.take_nd(
967 slobj,
968 axis=0,
969 new_mgr_locs=bp,
970 fill_value=fill_value,
971 )
972 return
974 if sl_type == "slice":
975 blknos = self.blknos[slobj]
976 blklocs = self.blklocs[slobj]
977 else:
978 blknos = algos.take_nd(
979 self.blknos, slobj, fill_value=-1, allow_fill=allow_fill
980 )
981 blklocs = algos.take_nd(
982 self.blklocs, slobj, fill_value=-1, allow_fill=allow_fill
983 )
985 # When filling blknos, make sure blknos is updated before appending to
986 # blocks list, that way new blkno is exactly len(blocks).
987 group = not only_slice
988 for blkno, mgr_locs in libinternals.get_blkno_placements(blknos, group=group):
989 if blkno == -1:
990 # If we've got here, fill_value was not lib.no_default
992 dtype, _ = infer_dtype_from_scalar(fill_value)
993 if is_1d_only_ea_dtype(dtype) and len(mgr_locs) > 1:
994 # Handle 1D-only extension dtypes by creating separate blocks
995 # (GH#63993)
996 placements = [BlockPlacement(col_idx) for col_idx in mgr_locs]
997 else:
998 placements = [mgr_locs]
1000 for placement in placements:
1001 yield self._make_na_block(
1002 placement=placement,
1003 fill_value=fill_value,
1004 use_na_proxy=use_na_proxy,
1005 )
1006 else:
1007 blk = self.blocks[blkno]
1009 # Otherwise, slicing along items axis is necessary.
1010 if not blk._can_consolidate and not blk._validate_ndim:
1011 # i.e. we dont go through here for DatetimeTZBlock
1012 # A non-consolidatable block, it's easy, because there's
1013 # only one item and each mgr loc is a copy of that single
1014 # item.
1015 deep = False
1016 for mgr_loc in mgr_locs:
1017 newblk = blk.copy(deep=deep)
1018 newblk.mgr_locs = BlockPlacement(slice(mgr_loc, mgr_loc + 1))
1019 yield newblk
1021 else:
1022 # GH#32779 to avoid the performance penalty of copying,
1023 # we may try to only slice
1024 taker = blklocs[mgr_locs.indexer]
1025 max_len = max(len(mgr_locs), taker.max() + 1)
1026 taker = lib.maybe_indices_to_slice(taker, max_len)
1028 if isinstance(taker, slice):
1029 nb = blk.getitem_block_columns(taker, new_mgr_locs=mgr_locs)
1030 yield nb
1031 elif only_slice:
1032 # GH#33597 slice instead of take, so we get
1033 # views instead of copies
1034 for i, ml in zip(taker, mgr_locs, strict=True):
1035 slc = slice(i, i + 1)
1036 bp = BlockPlacement(ml)
1037 nb = blk.getitem_block_columns(slc, new_mgr_locs=bp)
1038 # We have np.shares_memory(nb.values, blk.values)
1039 yield nb
1040 else:
1041 nb = blk.take_nd(taker, axis=0, new_mgr_locs=mgr_locs)
1042 yield nb
1044 def _make_na_block(
1045 self, placement: BlockPlacement, fill_value=None, use_na_proxy: bool = False
1046 ) -> Block:
1047 # Note: we only get here with self.ndim == 2
1049 if use_na_proxy:
1050 assert fill_value is None
1051 shape = (len(placement), self.shape[1])
1052 vals = np.empty(shape, dtype=np.void)
1053 nb = NumpyBlock(vals, placement, ndim=2)
1054 return nb
1056 if fill_value is None or fill_value is np.nan:
1057 fill_value = np.nan
1058 # GH45857 avoid unnecessary upcasting
1059 dtype = interleaved_dtype([blk.dtype for blk in self.blocks])
1060 if dtype is not None and np.issubdtype(dtype.type, np.floating):
1061 fill_value = dtype.type(fill_value)
1063 shape = (len(placement), self.shape[1])
1065 dtype, fill_value = infer_dtype_from_scalar(fill_value)
1066 block_values = make_na_array(dtype, shape, fill_value)
1067 return new_block_2d(block_values, placement=placement)
1069 def take(
1070 self,
1071 indexer: npt.NDArray[np.intp],
1072 axis: AxisInt = 1,
1073 verify: bool = True,
1074 ) -> Self:
1075 """
1076 Take items along any axis.
1078 indexer : np.ndarray[np.intp]
1079 axis : int, default 1
1080 verify : bool, default True
1081 Check that all entries are between 0 and len(self) - 1, inclusive.
1082 Pass verify=False if this check has been done by the caller.
1084 Returns
1085 -------
1086 BlockManager
1087 """
1088 # Caller is responsible for ensuring indexer annotation is accurate
1090 n = self.shape[axis]
1091 indexer = maybe_convert_indices(indexer, n, verify=verify)
1093 new_labels = self.axes[axis].take(indexer)
1094 return self.reindex_indexer(
1095 new_axis=new_labels,
1096 indexer=indexer,
1097 axis=axis,
1098 allow_dups=True,
1099 )
1102class BlockManager(libinternals.BlockManager, BaseBlockManager):
1103 """
1104 BaseBlockManager that holds 2D blocks.
1105 """
1107 ndim = 2
1109 # ----------------------------------------------------------------
1110 # Constructors
1112 def __init__(
1113 self,
1114 blocks: Sequence[Block],
1115 axes: Sequence[Index],
1116 verify_integrity: bool = True,
1117 ) -> None:
1118 if verify_integrity:
1119 # Assertion disabled for performance
1120 # assert all(isinstance(x, Index) for x in axes)
1122 for block in blocks:
1123 if self.ndim != block.ndim:
1124 raise AssertionError(
1125 f"Number of Block dimensions ({block.ndim}) must equal "
1126 f"number of axes ({self.ndim})"
1127 )
1128 # As of 2.0, the caller is responsible for ensuring that
1129 # DatetimeTZBlock with block.ndim == 2 has block.values.ndim ==2;
1130 # previously there was a special check for fastparquet compat.
1132 self._verify_integrity()
1134 def _verify_integrity(self) -> None:
1135 mgr_shape = self.shape
1136 tot_items = sum(len(x.mgr_locs) for x in self.blocks)
1137 for block in self.blocks:
1138 if block.shape[1:] != mgr_shape[1:]:
1139 raise_construction_error(tot_items, block.shape[1:], self.axes)
1140 if len(self.items) != tot_items:
1141 raise AssertionError(
1142 "Number of manager items must equal union of "
1143 f"block items\n# manager items: {len(self.items)}, # "
1144 f"tot_items: {tot_items}"
1145 )
1147 @classmethod
1148 def from_blocks(cls, blocks: list[Block], axes: list[Index]) -> Self:
1149 """
1150 Constructor for BlockManager and SingleBlockManager with same signature.
1151 """
1152 return cls(blocks, axes, verify_integrity=False)
1154 # ----------------------------------------------------------------
1155 # Indexing
1157 def _get_interleaved_dtype(self) -> DtypeObj | None:
1158 blocks = self.blocks
1159 cache = self._interleaved_dtype
1160 # validated on read for the same reason as in get_dtypes
1161 if cache is None or cache[0] is not blocks:
1162 cache = (blocks, interleaved_dtype([blk.dtype for blk in blocks]))
1163 self._interleaved_dtype = cache
1164 return cache[1]
1166 def fast_xs(self, loc: int) -> SingleBlockManager:
1167 """
1168 Return the array corresponding to `frame.iloc[loc]`.
1170 Parameters
1171 ----------
1172 loc : int
1174 Returns
1175 -------
1176 np.ndarray or ExtensionArray
1177 """
1178 if len(self.blocks) == 1:
1179 # TODO: this could be wrong if blk.mgr_locs is not slice(None)-like;
1180 # is this ruled out in the general case?
1181 result: np.ndarray | ExtensionArray = self.blocks[0].iget(
1182 (slice(None), loc)
1183 )
1184 # in the case of a single block, the new block is a view
1185 bp = BlockPlacement(slice(0, len(result)))
1186 block = new_block(
1187 result,
1188 placement=bp,
1189 ndim=1,
1190 refs=self.blocks[0].refs,
1191 )
1192 return SingleBlockManager(block, self.axes[0].view())
1194 # GH#62263 cache interleaved_dtype to avoid recomputing on
1195 # repeated fast_xs calls
1196 dtype = self._get_interleaved_dtype()
1198 n = len(self)
1200 if isinstance(dtype, ExtensionDtype):
1201 # TODO: use object dtype as workaround for non-performant
1202 # EA.__setitem__ methods. (primarily ArrowExtensionArray.__setitem__
1203 # when iteratively setting individual values)
1204 # https://github.com/pandas-dev/pandas/pull/54508#issuecomment-1675827918
1205 result = np.empty(n, dtype=object)
1206 else:
1207 result = np.empty(n, dtype=dtype)
1208 result = ensure_wrapped_if_datetimelike(result)
1210 for blk in self.blocks:
1211 # Such assignment may incorrectly coerce NaT to None
1212 # result[blk.mgr_locs] = blk._slice((slice(None), loc))
1213 for i, rl in enumerate(blk.mgr_locs):
1214 item = blk.iget((i, loc))
1215 if (
1216 result.dtype.kind in "iub"
1217 and lib.is_float(item)
1218 and isna(item)
1219 and isinstance(blk.dtype, CategoricalDtype)
1220 ):
1221 # GH#58954 caused bc interleaved_dtype is wrong for Categorical
1222 # TODO(GH#38240) this will be unnecessary
1223 # Note that doing this in a try/except would work for the
1224 # integer case, but not for bool, which will cast the NaN
1225 # entry to True.
1226 if result.dtype.kind == "b":
1227 new_dtype = object
1228 else:
1229 new_dtype = np.float64
1230 result = result.astype(new_dtype)
1231 result[rl] = item
1233 if isinstance(dtype, ExtensionDtype):
1234 cls = dtype.construct_array_type()
1235 result = cls._from_sequence(result, dtype=dtype)
1237 bp = BlockPlacement(slice(0, len(result)))
1238 block = new_block(result, placement=bp, ndim=1)
1239 return SingleBlockManager(block, self.axes[0].view())
1241 def iget(self, i: int, track_ref: bool = True) -> SingleBlockManager:
1242 """
1243 Return the data as a SingleBlockManager.
1244 """
1245 block = self.blocks[self.blknos[i]]
1246 values = block.iget(self.blklocs[i])
1248 # shortcut for select a single-dim from a 2-dim BM
1249 bp = BlockPlacement(slice(0, len(values)))
1250 nb = type(block)(
1251 values, placement=bp, ndim=1, refs=block.refs if track_ref else None
1252 )
1253 return SingleBlockManager(nb, self.axes[1].view())
1255 def iget_values(self, i: int) -> ArrayLike:
1256 """
1257 Return the data for column i as the values (ndarray or ExtensionArray).
1259 Warning! The returned array is a view but doesn't handle Copy-on-Write,
1260 so this should be used with caution.
1261 """
1262 # TODO(CoW) making the arrays read-only might make this safer to use?
1263 block = self.blocks[self.blknos[i]]
1264 values = block.iget(self.blklocs[i])
1265 return values
1267 @property
1268 def column_arrays(self) -> list[np.ndarray]:
1269 """
1270 Used in the JSON C code to access column arrays.
1271 This optimizes compared to using `iget_values` by converting each
1273 Warning! This doesn't handle Copy-on-Write, so should be used with
1274 caution (current use case of consuming this in the JSON code is fine).
1275 """
1276 # This is an optimized equivalent to
1277 # result = [self.iget_values(i) for i in range(len(self.items))]
1278 result: list[np.ndarray | None] = [None] * len(self.items)
1280 for blk in self.blocks:
1281 mgr_locs = blk._mgr_locs
1282 values = blk.array_values._values_for_json()
1283 if values.ndim == 1:
1284 # TODO(EA2D): special casing not needed with 2D EAs
1285 result[mgr_locs[0]] = values
1287 else:
1288 for i, loc in enumerate(mgr_locs):
1289 result[loc] = values[i]
1291 # error: Incompatible return value type (got "List[None]",
1292 # expected "List[ndarray[Any, Any]]")
1293 return result # type: ignore[return-value]
1295 def iset(
1296 self,
1297 loc: int | slice | np.ndarray,
1298 value: ArrayLike,
1299 inplace: bool = False,
1300 refs: BlockValuesRefs | None = None,
1301 ) -> None:
1302 """
1303 Set new item in-place. Does not consolidate. Adds new Block if not
1304 contained in the current set of items
1305 """
1306 if self._blklocs is None and self.ndim > 1:
1307 self._rebuild_blknos_and_blklocs()
1309 # Note: we exclude DTA/TDA here
1310 value_is_extension_type = is_1d_only_ea_dtype(value.dtype)
1311 if not value_is_extension_type:
1312 if value.ndim == 2:
1313 value = value.T
1314 else:
1315 value = ensure_block_shape(value, ndim=2)
1317 if value.shape[1:] != self.shape[1:]:
1318 raise AssertionError(
1319 "Shape of new values must be compatible with manager shape"
1320 )
1322 if lib.is_integer(loc):
1323 # We have 6 tests where loc is _not_ an int.
1324 # In this case, get_blkno_placements will yield only one tuple,
1325 # containing (self._blknos[loc], BlockPlacement(slice(0, 1, 1)))
1327 # Check if we can use _iset_single fastpath
1328 loc = cast("int", loc)
1329 if not value_is_extension_type and len(value) > 1:
1330 # GH#46544 setting a single column with a 2D value; matches the
1331 # check in self.insert. value has been transposed above, so
1332 # len(value) is the number of columns in the original value.
1333 raise ValueError(
1334 f"Expected a 1D array, got an array with shape {value.T.shape}"
1335 )
1336 blkno = self.blknos[loc]
1337 blk = self.blocks[blkno]
1338 if len(blk._mgr_locs) == 1: # TODO: fastest way to check this?
1339 return self._iset_single(
1340 loc,
1341 value,
1342 inplace=inplace,
1343 blkno=blkno,
1344 blk=blk,
1345 refs=refs,
1346 )
1348 # error: Incompatible types in assignment (expression has type
1349 # "List[Union[int, slice, ndarray]]", variable has type "Union[int,
1350 # slice, ndarray]")
1351 loc = [loc] # type: ignore[assignment]
1353 # categorical/sparse/datetimetz
1354 if value_is_extension_type:
1356 def value_getitem(placement): # pyright: ignore[reportRedeclaration]
1357 return value
1359 else:
1361 def value_getitem(placement):
1362 return value[placement.indexer]
1364 # Accessing public blknos ensures the public versions are initialized
1365 blknos = self.blknos[loc]
1366 blklocs = self.blklocs[loc].copy()
1368 unfit_mgr_locs = []
1369 unfit_val_locs = []
1370 removed_blknos = []
1371 for blkno_l, val_locs in libinternals.get_blkno_placements(blknos, group=True):
1372 blk = self.blocks[blkno_l]
1373 blk_locs = blklocs[val_locs.indexer]
1374 if inplace and blk.should_store(value):
1375 # Updating inplace -> check if we need to do Copy-on-Write
1376 if not self._has_no_reference_block(blkno_l):
1377 self._iset_split_block(
1378 blkno_l, blk_locs, value_getitem(val_locs), refs=refs
1379 )
1380 else:
1381 blk.set_inplace(blk_locs, value_getitem(val_locs))
1382 continue
1383 else:
1384 unfit_mgr_locs.append(blk.mgr_locs.as_array[blk_locs])
1385 unfit_val_locs.append(val_locs)
1387 # If all block items are unfit, schedule the block for removal.
1388 if len(val_locs) == len(blk.mgr_locs):
1389 removed_blknos.append(blkno_l)
1390 continue
1391 else:
1392 # Defer setting the new values to enable consolidation
1393 self._iset_split_block(blkno_l, blk_locs, refs=refs)
1395 if removed_blknos:
1396 # Remove blocks & update blknos accordingly
1397 is_deleted = np.zeros(self.nblocks, dtype=np.bool_)
1398 is_deleted[removed_blknos] = True
1400 new_blknos = np.empty(self.nblocks, dtype=np.intp)
1401 new_blknos.fill(-1)
1402 new_blknos[~is_deleted] = np.arange(self.nblocks - len(removed_blknos))
1403 self._blknos = new_blknos[self._blknos]
1404 self.blocks = tuple(
1405 blk for i, blk in enumerate(self.blocks) if i not in set(removed_blknos)
1406 )
1408 if unfit_val_locs:
1409 unfit_idxr = np.concatenate(unfit_mgr_locs)
1410 unfit_count = len(unfit_idxr)
1412 new_blocks: list[Block] = []
1413 if value_is_extension_type:
1414 # This code (ab-)uses the fact that EA blocks contain only
1415 # one item.
1416 # TODO(EA2D): special casing unnecessary with 2D EAs
1417 new_blocks.extend(
1418 new_block_2d(
1419 values=value,
1420 placement=BlockPlacement(slice(mgr_loc, mgr_loc + 1)),
1421 refs=refs,
1422 )
1423 for mgr_loc in unfit_idxr
1424 )
1426 self._blknos[unfit_idxr] = np.arange(unfit_count) + len(self.blocks)
1427 self._blklocs[unfit_idxr] = 0
1429 else:
1430 # unfit_val_locs contains BlockPlacement objects
1431 unfit_val_items = unfit_val_locs[0].append(unfit_val_locs[1:])
1433 new_blocks.append(
1434 new_block_2d(
1435 values=value_getitem(unfit_val_items),
1436 placement=BlockPlacement(unfit_idxr),
1437 refs=refs,
1438 )
1439 )
1441 self._blknos[unfit_idxr] = len(self.blocks)
1442 self._blklocs[unfit_idxr] = np.arange(unfit_count)
1444 # Not needed for correctness (see get_dtypes); releases the old blocks
1445 self._interleaved_dtype = None
1446 self._dtypes_cache = None
1447 self._known_consolidated = False
1449 self.blocks += tuple(new_blocks)
1451 def _iset_split_block(
1452 self,
1453 blkno_l: int,
1454 blk_locs: np.ndarray | list[int],
1455 value: ArrayLike | None = None,
1456 refs: BlockValuesRefs | None = None,
1457 ) -> None:
1458 """Removes columns from a block by splitting the block.
1460 Avoids copying the whole block through slicing and updates the manager
1461 after determining the new block structure. Optionally adds a new block,
1462 otherwise has to be done by the caller.
1464 Parameters
1465 ----------
1466 blkno_l: The block number to operate on, relevant for updating the manager
1467 blk_locs: The locations of our block that should be deleted.
1468 value: The value to set as a replacement.
1469 refs: The reference tracking object of the value to set.
1470 """
1471 blk = self.blocks[blkno_l]
1473 if self._blklocs is None:
1474 self._rebuild_blknos_and_blklocs()
1476 # Block.delete requires sorted unique locs; blk_locs keeps the caller's
1477 # order below, where it pairs with value.
1478 nbs_tup = tuple(blk.delete(np.unique(blk_locs)))
1479 if value is not None:
1480 locs = blk.mgr_locs.as_array[blk_locs]
1481 first_nb = new_block_2d(value, BlockPlacement(locs), refs=refs)
1482 else:
1483 first_nb = nbs_tup[0]
1484 nbs_tup = tuple(nbs_tup[1:])
1486 nr_blocks = len(self.blocks)
1487 blocks_tup = (
1488 *self.blocks[:blkno_l],
1489 first_nb,
1490 *self.blocks[blkno_l + 1 :],
1491 *nbs_tup,
1492 )
1493 self.blocks = blocks_tup
1494 self._blklocs[first_nb.mgr_locs.indexer] = np.arange(len(first_nb.mgr_locs))
1495 self._interleaved_dtype = None
1496 self._dtypes_cache = None
1498 if not nbs_tup and value is not None:
1499 # No need to update _blknos if split did not happen
1500 return
1502 for i, nb in enumerate(nbs_tup):
1503 self._blklocs[nb.mgr_locs.indexer] = np.arange(len(nb))
1504 self._blknos[nb.mgr_locs.indexer] = i + nr_blocks
1506 def _iset_single(
1507 self,
1508 loc: int,
1509 value: ArrayLike,
1510 inplace: bool,
1511 blkno: int,
1512 blk: Block,
1513 refs: BlockValuesRefs | None = None,
1514 ) -> None:
1515 """
1516 Fastpath for iset when we are only setting a single position and
1517 the Block currently in that position is itself single-column.
1519 In this case we can swap out the entire Block and blklocs and blknos
1520 are unaffected.
1521 """
1522 # Caller is responsible for verifying value.shape
1524 if inplace and blk.should_store(value):
1525 copy = not self._has_no_reference_block(blkno)
1526 iloc = self.blklocs[loc]
1527 blk.set_inplace(slice(iloc, iloc + 1), value, copy=copy)
1528 return
1530 nb = new_block_2d(value, placement=blk._mgr_locs, refs=refs)
1531 old_blocks = self.blocks
1532 new_blocks = (*old_blocks[:blkno], nb, *old_blocks[blkno + 1 :])
1533 # Not needed for correctness (see get_dtypes); releases the old blocks
1534 self._interleaved_dtype = None
1535 self._dtypes_cache = None
1536 self.blocks = new_blocks
1537 return
1539 def column_setitem(
1540 self, loc: int, idx: int | slice | np.ndarray, value, inplace_only: bool = False
1541 ) -> None:
1542 """
1543 Set values ("setitem") into a single column (not setting the full column).
1545 This is a method on the BlockManager level, to avoid creating an
1546 intermediate Series at the DataFrame level (`s = df[loc]; s[idx] = value`)
1547 """
1548 if not self._has_no_reference(loc):
1549 blkno = self.blknos[loc]
1550 # Split blocks to only copy the column we want to modify
1551 blk_loc = self.blklocs[loc]
1552 # Copy our values
1553 values = self.blocks[blkno].values
1554 if values.ndim == 1:
1555 values = values.copy()
1556 else:
1557 # Use [blk_loc] as indexer to keep ndim=2, this already results in a
1558 # copy
1559 values = values[[blk_loc]]
1560 self._iset_split_block(blkno, [blk_loc], values)
1562 # this manager is only created temporarily to mutate the values in place
1563 # so don't track references, otherwise the `setitem` would perform CoW again
1564 col_mgr = self.iget(loc, track_ref=False)
1565 if inplace_only:
1566 col_mgr.setitem_inplace(idx, value)
1567 else:
1568 new_mgr = col_mgr.setitem((idx,), value)
1569 self.iset(loc, new_mgr._block.values, inplace=True)
1571 def insert(self, loc: int, item: Hashable, value: ArrayLike, refs=None) -> None:
1572 """
1573 Insert item at selected position.
1575 Parameters
1576 ----------
1577 loc : int
1578 item : hashable
1579 value : np.ndarray or ExtensionArray
1580 refs : The reference tracking object of the value to set.
1581 """
1582 new_axis = self.items.insert(loc, item)
1584 if value.ndim == 2:
1585 value = value.T
1586 if len(value) > 1:
1587 raise ValueError(
1588 f"Expected a 1D array, got an array with shape {value.T.shape}"
1589 )
1590 else:
1591 value = ensure_block_shape(value, ndim=self.ndim)
1593 bp = BlockPlacement(slice(loc, loc + 1))
1594 block = new_block_2d(values=value, placement=bp, refs=refs)
1596 if not len(self.blocks):
1597 # Fastpath
1598 self._blklocs = np.array([0], dtype=np.intp)
1599 self._blknos = np.array([0], dtype=np.intp)
1600 else:
1601 self._insert_update_mgr_locs(loc)
1602 self._insert_update_blklocs_and_blknos(loc)
1604 self.axes[0] = new_axis
1605 # Not needed for correctness (see get_dtypes); releases the old blocks
1606 self._interleaved_dtype = None
1607 self._dtypes_cache = None
1608 self._known_consolidated = False
1609 self.blocks += (block,)
1611 # len check is a cheap O(1) short-circuit to avoid the O(n) sum
1612 # and the get_option overhead on every insert call (GH#57641)
1613 warn_threshold = 100
1614 if (
1615 len(self.blocks) > warn_threshold
1616 and config["mode"]["performance_warnings"]
1617 and sum(not block.is_extension for block in self.blocks) > warn_threshold
1618 ):
1619 warnings.warn(
1620 "DataFrame is highly fragmented. This is usually the result "
1621 "of calling `frame.insert` many times, which has poor performance. "
1622 "Consider joining all columns at once using pd.concat(axis=1) "
1623 "instead. To get a de-fragmented frame, use `newframe = frame.copy()`",
1624 PerformanceWarning,
1625 stacklevel=find_stack_level(),
1626 )
1628 def _insert_update_mgr_locs(self, loc) -> None:
1629 """
1630 When inserting a new Block at location 'loc', we increment
1631 all of the mgr_locs of blocks above that by one.
1632 """
1633 # Faster version of set(arr) for sequences of small numbers
1634 blknos = np.bincount(self.blknos[loc:]).nonzero()[0]
1635 for blkno in blknos:
1636 # .620 this way, .326 of which is in increment_above
1637 blk = self.blocks[blkno]
1638 blk._mgr_locs = blk._mgr_locs.increment_above(loc)
1640 def _insert_update_blklocs_and_blknos(self, loc) -> None:
1641 """
1642 When inserting a new Block at location 'loc', we update our
1643 _blklocs and _blknos.
1644 """
1646 # Accessing public blklocs ensures the public versions are initialized
1647 if loc == self.blklocs.shape[0]:
1648 # np.append is a lot faster, let's use it if we can.
1649 self._blklocs = np.append(self._blklocs, 0)
1650 self._blknos = np.append(self._blknos, len(self.blocks))
1651 elif loc == 0:
1652 # As of numpy 1.26.4, np.concatenate faster than np.append
1653 self._blklocs = np.concatenate([[0], self._blklocs])
1654 self._blknos = np.concatenate([[len(self.blocks)], self._blknos])
1655 else:
1656 new_blklocs, new_blknos = libinternals.update_blklocs_and_blknos(
1657 self.blklocs, self.blknos, loc, len(self.blocks)
1658 )
1659 self._blklocs = new_blklocs
1660 self._blknos = new_blknos
1662 def idelete(self, indexer) -> BlockManager:
1663 """
1664 Delete selected locations, returning a new BlockManager.
1665 """
1666 is_deleted = np.zeros(self.shape[0], dtype=np.bool_)
1667 is_deleted[indexer] = True
1668 taker = (~is_deleted).nonzero()[0]
1670 nbs = self._slice_take_blocks_ax0(taker, only_slice=True, ref_inplace_op=True)
1671 new_columns = self.items[~is_deleted]
1672 axes = [new_columns, self.axes[1]]
1673 return type(self)(tuple(nbs), axes, verify_integrity=False)
1675 # ----------------------------------------------------------------
1676 # Block-wise Operation
1678 def grouped_reduce(self, func: Callable) -> Self:
1679 """
1680 Apply grouped reduction function blockwise, returning a new BlockManager.
1682 Parameters
1683 ----------
1684 func : grouped reduction function
1686 Returns
1687 -------
1688 BlockManager
1689 """
1690 result_blocks: list[Block] = []
1692 for blk in self.blocks:
1693 if blk.is_object:
1694 # split on object-dtype blocks bc some columns may raise
1695 # while others do not.
1696 for sb in blk._split():
1697 applied = sb.apply(func)
1698 result_blocks = extend_blocks(applied, result_blocks)
1699 else:
1700 applied = blk.apply(func)
1701 result_blocks = extend_blocks(applied, result_blocks)
1703 if len(result_blocks) == 0:
1704 nrows = 0
1705 else:
1706 nrows = result_blocks[0].values.shape[-1]
1707 index = default_index(nrows)
1709 # TODO shallow copy columns?
1710 return type(self).from_blocks(result_blocks, [self.axes[0].view(), index])
1712 def reduce(self, func: Callable) -> Self:
1713 """
1714 Apply reduction function blockwise, returning a single-row BlockManager.
1716 Parameters
1717 ----------
1718 func : reduction function
1720 Returns
1721 -------
1722 BlockManager
1723 """
1724 # If 2D, we assume that we're operating column-wise
1725 assert self.ndim == 2
1727 res_blocks = [blk.reduce(func) for blk in self.blocks]
1728 index = default_index(1) # placeholder
1729 # shallow copy self.items not needed because DataFrame._reduce does a getitem
1730 new_mgr = type(self).from_blocks(res_blocks, [self.items, index])
1731 return new_mgr
1733 def operate_blockwise(self, other: BlockManager, array_op) -> BlockManager:
1734 """
1735 Apply array_op blockwise with another (aligned) BlockManager.
1736 """
1737 return operate_blockwise(self, other, array_op)
1739 def _equal_values(self: BlockManager, other: BlockManager) -> bool:
1740 """
1741 Used in .equals defined in base class. Only check the column values
1742 assuming shape and indexes have already been checked.
1743 """
1744 return blockwise_all(self, other, array_equals)
1746 def quantile(
1747 self,
1748 *,
1749 qs: Index, # with dtype float 64
1750 interpolation: QuantileInterpolation = "linear",
1751 ) -> Self:
1752 """
1753 Iterate over blocks applying quantile reduction.
1754 This routine is intended for reduction type operations and
1755 will do inference on the generated blocks.
1757 Parameters
1758 ----------
1759 interpolation : type of interpolation, default 'linear'
1760 qs : list of the quantiles to be computed
1762 Returns
1763 -------
1764 BlockManager
1765 """
1766 # Series dispatches to DataFrame for quantile, which allows us to
1767 # simplify some of the code here and in the blocks
1768 assert self.ndim >= 2
1769 assert is_list_like(qs) # caller is responsible for this
1771 new_axes = [self.axes[0].view(), Index(qs, dtype=np.float64)]
1773 blocks = [
1774 blk.quantile(qs=qs, interpolation=interpolation) for blk in self.blocks
1775 ]
1777 return type(self)(blocks, new_axes)
1779 # ----------------------------------------------------------------
1781 def unstack(self, unstacker, fill_value) -> BlockManager:
1782 """
1783 Return a BlockManager with all blocks unstacked.
1785 Parameters
1786 ----------
1787 unstacker : reshape._Unstacker
1788 fill_value : Any
1789 fill_value for newly introduced missing values.
1791 Returns
1792 -------
1793 unstacked : BlockManager
1794 """
1795 new_columns = unstacker.get_new_columns(self.items)
1796 new_index = unstacker.new_index
1798 if not unstacker.mask_all:
1799 # calculating the full mask once and passing it to
1800 # ExtensionBlock._unstack is faster than calculating
1801 # it in each repeated call
1802 new_mask2D = (~unstacker.mask).reshape(*unstacker.full_shape)
1803 needs_masking = new_mask2D.any(axis=0)
1804 else:
1805 needs_masking = np.zeros(unstacker.full_shape[1], dtype=bool)
1807 new_blocks: list[Block] = []
1809 if len(self.items) == 0:
1810 factor = 1
1811 else:
1812 fac = len(new_columns) / len(self.items)
1813 assert fac == int(fac)
1814 factor = int(fac)
1816 for blk in self.blocks:
1817 mgr_locs = blk.mgr_locs
1818 new_placement = mgr_locs.tile_for_unstack(factor)
1820 blocks = blk._unstack(
1821 unstacker,
1822 fill_value,
1823 new_placement=new_placement,
1824 needs_masking=needs_masking,
1825 )
1827 new_blocks.extend(blocks)
1829 bm = BlockManager(new_blocks, [new_columns, new_index], verify_integrity=False)
1830 return bm
1832 def to_iter_dict(self) -> Generator[tuple[str, Self]]:
1833 """
1834 Yield a tuple of (str(dtype), BlockManager)
1836 Returns
1837 -------
1838 values : a tuple of (str(dtype), BlockManager)
1839 """
1840 key = lambda block: str(block.dtype)
1841 for dtype, blocks in itertools.groupby(sorted(self.blocks, key=key), key=key):
1842 # TODO(EA2D): the combine will be unnecessary with 2D EAs
1843 yield dtype, self._combine(list(blocks))
1845 def as_array(
1846 self,
1847 dtype: np.dtype | None = None,
1848 copy: bool = False,
1849 na_value: object = lib.no_default,
1850 ) -> np.ndarray:
1851 """
1852 Convert the blockmanager data into a numpy array.
1854 Parameters
1855 ----------
1856 dtype : np.dtype or None, default None
1857 Data type of the return array.
1858 copy : bool, default False
1859 If True then guarantee that a copy is returned. A value of
1860 False does not guarantee that the underlying data is not
1861 copied.
1862 na_value : object, default lib.no_default
1863 Value to be used as the missing value sentinel.
1865 Returns
1866 -------
1867 arr : ndarray
1868 """
1869 passed_nan = lib.is_float(na_value) and isna(na_value)
1871 if len(self.blocks) == 0:
1872 arr = np.empty(self.shape, dtype=float)
1873 return arr.transpose()
1875 if self.is_single_block:
1876 blk = self.blocks[0]
1878 if na_value is not lib.no_default:
1879 # We want to copy when na_value is provided to avoid
1880 # mutating the original object
1881 if lib.is_np_dtype(blk.dtype, "f") and passed_nan:
1882 # We are already numpy-float and na_value=np.nan
1883 pass
1884 else:
1885 copy = True
1887 if blk.is_extension:
1888 # Avoid implicit conversion of extension blocks to object
1890 # error: Item "ndarray" of "Union[ndarray, ExtensionArray]" has no
1891 # attribute "to_numpy"
1892 arr = blk.values.to_numpy( # type: ignore[union-attr]
1893 dtype=dtype,
1894 na_value=na_value,
1895 copy=copy,
1896 ).reshape(blk.shape)
1897 elif not copy:
1898 arr = np.asarray(blk.values, dtype=dtype)
1899 else:
1900 arr = np.array(blk.values, dtype=dtype, copy=copy)
1901 if passed_nan and blk.dtype.kind in "mM":
1902 arr[isna(blk.values)] = na_value
1904 if not copy:
1905 arr = arr.view()
1906 arr.flags.writeable = False
1907 else:
1908 arr = self._interleave(dtype=dtype, na_value=na_value)
1909 # The underlying data was copied within _interleave, so no need
1910 # to further copy if copy=True or setting na_value
1912 if na_value is lib.no_default:
1913 pass
1914 elif arr.dtype.kind == "f" and passed_nan:
1915 pass
1916 else:
1917 # GH#56233 avoid raising in the case of an all false mask
1918 mask = isna(arr)
1919 if mask.any():
1920 arr[mask] = na_value
1922 return arr.transpose()
1924 def _interleave(
1925 self,
1926 dtype: np.dtype | None = None,
1927 na_value: object = lib.no_default,
1928 ) -> np.ndarray:
1929 """
1930 Return ndarray from blocks with specified item order
1931 Items must be contained in the blocks
1932 """
1933 if not dtype:
1934 # Incompatible types in assignment (expression has type
1935 # "Optional[Union[dtype[Any], ExtensionDtype]]", variable has
1936 # type "Optional[dtype[Any]]")
1937 # GH#62263 use cached interleaved_dtype
1938 dtype = self._get_interleaved_dtype() # type: ignore[assignment]
1940 # error: Argument 1 to "ensure_np_dtype" has incompatible type
1941 # "Optional[dtype[Any]]"; expected "Union[dtype[Any], ExtensionDtype]"
1942 dtype = ensure_np_dtype(dtype) # type: ignore[arg-type]
1943 result = np.empty(self.shape, dtype=dtype)
1945 itemmask = np.zeros(self.shape[0])
1947 if dtype == np.dtype("object") and na_value is lib.no_default:
1948 # much more performant than using to_numpy below
1949 for blk in self.blocks:
1950 rl = blk.mgr_locs
1951 arr = blk.get_values(dtype)
1952 result[rl.indexer] = arr
1953 itemmask[rl.indexer] = 1
1954 return result
1956 for blk in self.blocks:
1957 rl = blk.mgr_locs
1958 if blk.is_extension:
1959 # Avoid implicit conversion of extension blocks to object
1961 # error: Item "ndarray" of "Union[ndarray, ExtensionArray]" has no
1962 # attribute "to_numpy"
1963 arr = blk.values.to_numpy( # type: ignore[union-attr]
1964 dtype=dtype,
1965 na_value=na_value,
1966 )
1967 else:
1968 arr = blk.get_values(dtype)
1969 result[rl.indexer] = arr
1970 if na_value is not lib.no_default and blk.dtype.kind in "mM":
1971 result[rl.indexer][isna(arr)] = na_value
1972 itemmask[rl.indexer] = 1
1974 if not itemmask.all():
1975 raise AssertionError("Some items were not contained in blocks")
1977 return result
1979 # ----------------------------------------------------------------
1980 # Consolidation
1982 def is_consolidated(self) -> bool:
1983 """
1984 Return True if more than one block with the same dtype
1985 """
1986 if not self._known_consolidated:
1987 self._consolidate_check()
1988 return self._is_consolidated
1990 def _consolidate_check(self) -> None:
1991 if len(self.blocks) == 1:
1992 # fastpath
1993 self._is_consolidated = True
1994 self._known_consolidated = True
1995 return
1996 # Exit early on first duplicate dtype rather than collecting all dtypes
1997 dtypes: set[DtypeObj] = set()
1998 for blk in self.blocks:
1999 if blk._can_consolidate:
2000 dtype = blk.dtype
2001 if dtype in dtypes:
2002 self._is_consolidated = False
2003 self._known_consolidated = True
2004 return
2005 dtypes.add(dtype)
2006 self._is_consolidated = True
2007 self._known_consolidated = True
2009 def _consolidate_inplace(self) -> None:
2010 if not self.is_consolidated():
2011 self.blocks = _consolidate(self.blocks)
2012 self._is_consolidated = True
2013 self._known_consolidated = True
2014 self._rebuild_blknos_and_blklocs()
2016 # ----------------------------------------------------------------
2017 # Concatenation
2019 @classmethod
2020 def concat_horizontal(cls, mgrs: list[Self], axes: list[Index]) -> Self:
2021 """
2022 Concatenate uniformly-indexed BlockManagers horizontally.
2023 """
2024 offset = 0
2025 blocks: list[Block] = []
2026 for mgr in mgrs:
2027 for blk in mgr.blocks:
2028 # We need to do getitem_block here otherwise we would be altering
2029 # blk.mgr_locs in place, which would render it invalid. This is only
2030 # relevant in the copy=False case.
2031 nb = blk.slice_block_columns(slice(None))
2032 nb._mgr_locs = nb._mgr_locs.add(offset)
2033 blocks.append(nb)
2035 offset += len(mgr.items)
2037 # TODO relevant axis already shallow-copied at caller?
2038 new_mgr = cls(tuple(blocks), axes)
2039 return new_mgr
2041 @classmethod
2042 def concat_vertical(cls, mgrs: list[Self], axes: list[Index]) -> Self:
2043 """
2044 Concatenate uniformly-indexed BlockManagers vertically.
2045 """
2046 raise NotImplementedError("This logic lives (for now) in internals.concat")
2049class SingleBlockManager(BaseBlockManager):
2050 """manage a single block with"""
2052 @property
2053 def ndim(self) -> Literal[1]:
2054 return 1
2056 _is_consolidated = True
2057 _known_consolidated = True
2058 __slots__ = ()
2059 is_single_block = True
2061 def __init__(
2062 self,
2063 block: Block,
2064 axis: Index,
2065 verify_integrity: bool = False,
2066 ) -> None:
2067 # Assertions disabled for performance
2068 # assert isinstance(block, Block), type(block)
2069 # assert isinstance(axis, Index), type(axis)
2071 self.axes = [axis]
2072 self.blocks = (block,)
2074 @classmethod
2075 def from_blocks(
2076 cls,
2077 blocks: list[Block],
2078 axes: list[Index],
2079 ) -> Self:
2080 """
2081 Constructor for BlockManager and SingleBlockManager with same signature.
2082 """
2083 assert len(blocks) == 1
2084 assert len(axes) == 1
2085 return cls(blocks[0], axes[0], verify_integrity=False)
2087 @classmethod
2088 def from_array(
2089 cls, array: ArrayLike, index: Index, refs: BlockValuesRefs | None = None
2090 ) -> SingleBlockManager:
2091 """
2092 Constructor for if we have an array that is not yet a Block.
2093 """
2094 array = maybe_coerce_values(array)
2095 bp = BlockPlacement(slice(0, len(index)))
2096 block = new_block(array, placement=bp, ndim=1, refs=refs)
2097 return cls(block, index)
2099 def to_2d_mgr(self, columns: Index) -> BlockManager:
2100 """
2101 Manager analogue of Series.to_frame
2102 """
2103 blk = self.blocks[0]
2104 arr = ensure_block_shape(blk.values, ndim=2)
2105 bp = BlockPlacement(0)
2106 new_blk = type(blk)(arr, placement=bp, ndim=2, refs=blk.refs)
2107 axes = [columns, self.axes[0].view()]
2108 return BlockManager([new_blk], axes=axes, verify_integrity=False)
2110 def _has_no_reference(self, i: int = 0) -> bool:
2111 """
2112 Check for column `i` if it has references.
2113 (whether it references another array or is itself being referenced)
2114 Returns True if the column has no references.
2115 """
2116 return not self.blocks[0].refs.has_reference()
2118 def __getstate__(self):
2119 block_values = [b.values for b in self.blocks]
2120 block_items = [self.items[b.mgr_locs.indexer] for b in self.blocks]
2121 axes_array = list(self.axes)
2123 extra_state = {
2124 "0.14.1": {
2125 "axes": axes_array,
2126 "blocks": [
2127 {"values": b.values, "mgr_locs": b.mgr_locs.indexer}
2128 for b in self.blocks
2129 ],
2130 }
2131 }
2133 # First three elements of the state are to maintain forward
2134 # compatibility with 0.13.1.
2135 return axes_array, block_values, block_items, extra_state
2137 def __setstate__(self, state) -> None:
2138 def unpickle_block(values, mgr_locs, ndim: int) -> Block:
2139 # TODO(EA2D): ndim would be unnecessary with 2D EAs
2140 # older pickles may store e.g. DatetimeIndex instead of DatetimeArray
2141 values = extract_array(values, extract_numpy=True)
2142 if not isinstance(mgr_locs, BlockPlacement):
2143 mgr_locs = BlockPlacement(mgr_locs)
2145 values = maybe_coerce_values(values)
2146 return new_block(values, placement=mgr_locs, ndim=ndim)
2148 if isinstance(state, tuple) and len(state) >= 4 and "0.14.1" in state[3]:
2149 state = state[3]["0.14.1"]
2150 self.axes = [ensure_index(ax) for ax in state["axes"]]
2151 ndim = len(self.axes)
2152 self.blocks = tuple(
2153 unpickle_block(b["values"], b["mgr_locs"], ndim=ndim)
2154 for b in state["blocks"]
2155 )
2156 else:
2157 raise NotImplementedError("pre-0.14.1 pickles are no longer supported")
2159 self._post_setstate()
2161 def _post_setstate(self) -> None:
2162 pass
2164 @cache_readonly
2165 def _block(self) -> Block:
2166 return self.blocks[0]
2168 @final
2169 @property
2170 def array(self) -> ArrayLike:
2171 """
2172 Quick access to the backing array of the Block.
2173 """
2174 return self.blocks[0].values
2176 # error: Cannot override writeable attribute with read-only property
2177 @property
2178 def _blknos(self) -> None: # type: ignore[override]
2179 """compat with BlockManager"""
2180 return None
2182 # error: Cannot override writeable attribute with read-only property
2183 @property
2184 def _blklocs(self) -> None: # type: ignore[override]
2185 """compat with BlockManager"""
2186 return None
2188 def get_rows_with_mask(self, indexer: npt.NDArray[np.bool_]) -> Self:
2189 # similar to get_slice, but not restricted to slice indexer
2190 blk = self._block
2191 if len(indexer) > 0 and indexer.all():
2192 return type(self)(blk.copy(deep=False), self.index)
2193 array = blk.values[indexer]
2195 if isinstance(indexer, np.ndarray) and indexer.dtype.kind == "b":
2196 # boolean indexing always gives a copy with numpy
2197 refs = None
2198 else:
2199 # TODO(CoW) in theory only need to track reference if new_array is a view
2200 refs = blk.refs
2202 bp = BlockPlacement(slice(0, len(array)))
2203 block = type(blk)(array, placement=bp, ndim=1, refs=refs)
2205 new_idx = self.index[indexer]
2206 return type(self)(block, new_idx)
2208 def get_slice(
2209 self, slobj: slice, axis: AxisInt = 0, new_index: Index | None = None
2210 ) -> SingleBlockManager:
2211 # Assertion disabled for performance
2212 # assert isinstance(slobj, slice), type(slobj)
2213 if axis >= self.ndim:
2214 raise IndexError("Requested axis not found in manager")
2216 blk = self._block
2217 array = blk.values[slobj]
2218 bp = BlockPlacement(slice(0, len(array)))
2219 # TODO this method is only used in groupby SeriesSplitter at the moment,
2220 # so passing refs is not yet covered by the tests
2221 block = type(blk)(array, placement=bp, ndim=1, refs=blk.refs)
2222 if new_index is None:
2223 new_index = self.index._getitem_slice(slobj)
2224 return type(self)(block, new_index)
2226 @property
2227 def index(self) -> Index:
2228 return self.axes[0]
2230 @property
2231 def dtype(self) -> DtypeObj:
2232 return self._block.dtype
2234 def get_dtypes(self) -> npt.NDArray[np.object_]:
2235 return np.array([self._block.dtype], dtype=object)
2237 def external_values(self):
2238 """The array that Series.values returns"""
2239 return self._block.external_values()
2241 def internal_values(self):
2242 """The array that Series._values returns"""
2243 return self._block.values
2245 def array_values(self) -> ExtensionArray:
2246 """The array that Series.array returns"""
2247 return self._block.array_values
2249 def get_numeric_data(self) -> Self:
2250 if self._block.is_numeric:
2251 return self.copy(deep=False)
2252 return self.make_empty()
2254 @property
2255 def _can_hold_na(self) -> bool:
2256 return self._block._can_hold_na
2258 def setitem_inplace(self, indexer, value) -> None:
2259 """
2260 Set values with indexer.
2262 For SingleBlockManager, this backs s[indexer] = value
2264 This is an inplace version of `setitem()`, mutating the manager/values
2265 in place, not returning a new Manager (and Block), and thus never changing
2266 the dtype.
2267 """
2268 if not self._has_no_reference(0):
2269 self.blocks = (self._block.copy(deep=True),)
2270 self._reset_cache()
2272 arr = self.array
2274 # EAs will do this validation in their own __setitem__ methods.
2275 if isinstance(arr, np.ndarray):
2276 # Note: checking for ndarray instead of np.dtype means we exclude
2277 # dt64/td64, which do their own validation.
2278 value = np_can_hold_element(arr.dtype, value)
2280 if isinstance(value, np.ndarray) and value.ndim == 1 and len(value) == 1:
2281 # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615
2282 value = value[0, ...]
2284 arr[indexer] = value
2286 def idelete(self, indexer) -> SingleBlockManager:
2287 """
2288 Delete single location from SingleBlockManager.
2290 Ensures that self.blocks doesn't become empty.
2291 """
2292 nb = self._block.delete(indexer)[0]
2293 self.blocks = (nb,)
2294 self.axes[0] = self.axes[0].delete(indexer)
2295 self._reset_cache()
2296 return self
2298 def fast_xs(self, loc):
2299 """
2300 fast path for getting a cross-section
2301 return a view of the data
2302 """
2303 raise NotImplementedError("Use series._values[loc] instead")
2305 def set_values(self, values: ArrayLike) -> None:
2306 """
2307 Set the values of the single block in place.
2309 Use at your own risk! This does not check if the passed values are
2310 valid for the current Block/SingleBlockManager (length, dtype, etc),
2311 and this does not properly keep track of references.
2312 """
2313 # NOTE(CoW) Currently this is only used for FrameColumnApply.series_generator
2314 # which handles CoW by setting the refs manually if necessary
2315 self.blocks[0].values = values
2316 self.blocks[0]._mgr_locs = BlockPlacement(slice(len(values)))
2318 def _equal_values(self, other: Self) -> bool:
2319 """
2320 Used in .equals defined in base class. Only check the column values
2321 assuming shape and indexes have already been checked.
2322 """
2323 left = self.blocks[0].values
2324 right = other.blocks[0].values
2325 return array_equals(left, right)
2327 def grouped_reduce(self, func):
2328 arr = self.array
2329 res = func(arr)
2330 index = default_index(len(res))
2332 mgr = type(self).from_array(res, index)
2333 return mgr
2336# --------------------------------------------------------------------
2337# Constructor Helpers
2340def create_block_manager_from_blocks(
2341 blocks: list[Block],
2342 axes: list[Index],
2343 consolidate: bool = True,
2344 verify_integrity: bool = True,
2345) -> BlockManager:
2346 # If verify_integrity=False, then caller is responsible for checking
2347 # all(x.shape[-1] == len(axes[1]) for x in blocks)
2348 # sum(x.shape[0] for x in blocks) == len(axes[0])
2349 # set(x for blk in blocks for x in blk.mgr_locs) == set(range(len(axes[0])))
2350 # all(blk.ndim == 2 for blk in blocks)
2351 # This allows us to safely pass verify_integrity=False
2353 try:
2354 mgr = BlockManager(blocks, axes, verify_integrity=verify_integrity)
2356 except ValueError as err:
2357 arrays = [blk.values for blk in blocks]
2358 tot_items = sum(arr.shape[0] for arr in arrays)
2359 raise_construction_error(tot_items, arrays[0].shape[1:], axes, err)
2361 if consolidate:
2362 mgr._consolidate_inplace()
2363 return mgr
2366def create_block_manager_from_column_arrays(
2367 arrays: list[ArrayLike],
2368 axes: list[Index],
2369 consolidate: bool,
2370 refs: list,
2371) -> BlockManager:
2372 # Assertions disabled for performance (caller is responsible for verifying)
2373 # assert isinstance(axes, list)
2374 # assert all(isinstance(x, Index) for x in axes)
2375 # assert all(isinstance(x, (np.ndarray, ExtensionArray)) for x in arrays)
2376 # assert all(type(x) is not NumpyExtensionArray for x in arrays)
2377 # assert all(x.ndim == 1 for x in arrays)
2378 # assert all(len(x) == len(axes[1]) for x in arrays)
2379 # assert len(arrays) == len(axes[0])
2380 # These last three are sufficient to allow us to safely pass
2381 # verify_integrity=False below.
2383 try:
2384 blocks = _form_blocks(arrays, consolidate, refs)
2385 mgr = BlockManager(blocks, axes, verify_integrity=False)
2386 except ValueError as e:
2387 raise_construction_error(len(arrays), arrays[0].shape, axes, e)
2388 if consolidate:
2389 mgr._consolidate_inplace()
2390 return mgr
2393def raise_construction_error(
2394 tot_items: int,
2395 block_shape: Shape,
2396 axes: list[Index],
2397 e: ValueError | None = None,
2398) -> NoReturn:
2399 """raise a helpful message about our construction"""
2400 passed = tuple(map(int, [tot_items, *block_shape]))
2401 # Correcting the user facing error message during dataframe construction
2402 if len(passed) <= 2:
2403 passed = passed[::-1]
2405 implied = tuple(len(ax) for ax in axes)
2406 # Correcting the user facing error message during dataframe construction
2407 if len(implied) <= 2:
2408 implied = implied[::-1]
2410 # We return the exception object instead of raising it so that we
2411 # can raise it in the caller; mypy plays better with that
2412 if passed == implied and e is not None:
2413 raise e
2414 if block_shape[0] == 0:
2415 raise ValueError("Empty data passed with indices specified.")
2416 raise ValueError(f"Shape of passed values is {passed}, indices imply {implied}")
2419# -----------------------------------------------------------------------
2422def _form_blocks(arrays: list[ArrayLike], consolidate: bool, refs: list) -> list[Block]:
2423 tuples = enumerate(arrays)
2425 if not consolidate:
2426 return _tuples_to_blocks_no_consolidate(tuples, refs)
2428 # when consolidating, we can ignore refs (either stacking always copies,
2429 # or the EA is already copied in the calling dict_to_mgr)
2431 groups: dict[Hashable, list[tuple[int, ArrayLike]]] = {}
2432 for i, arr in tuples:
2433 dtype = arr.dtype
2434 # Extension dtypes each get their own block regardless, so use id()
2435 # to avoid a potentially expensive __hash__ (e.g. CategoricalDtype
2436 # hashes all categories).
2437 key = dtype if isinstance(dtype, np.dtype) else id(dtype)
2438 try:
2439 groups[key].append((i, arr))
2440 except KeyError:
2441 groups[key] = [(i, arr)]
2443 nbs: list[Block] = []
2444 for tup_block in groups.values():
2445 dtype = tup_block[0][1].dtype
2446 block_type = get_block_type(dtype)
2448 if isinstance(dtype, np.dtype):
2449 is_dtlike = dtype.kind in "mM"
2451 if issubclass(dtype.type, (str, bytes)):
2452 dtype = np.dtype(object)
2454 values, placement = _stack_arrays(tup_block, dtype)
2455 if is_dtlike:
2456 values = ensure_wrapped_if_datetimelike(values)
2457 blk = block_type(values, placement=BlockPlacement(placement), ndim=2)
2458 nbs.append(blk)
2460 elif is_1d_only_ea_dtype(dtype):
2461 dtype_blocks = [
2462 block_type(x[1], placement=BlockPlacement(x[0]), ndim=2)
2463 for x in tup_block
2464 ]
2465 nbs.extend(dtype_blocks)
2467 else:
2468 dtype_blocks = [
2469 block_type(
2470 ensure_block_shape(x[1], 2), placement=BlockPlacement(x[0]), ndim=2
2471 )
2472 for x in tup_block
2473 ]
2474 nbs.extend(dtype_blocks)
2475 return nbs
2478def _tuples_to_blocks_no_consolidate(tuples, refs) -> list[Block]:
2479 # tuples produced within _form_blocks are of the form (placement, array)
2480 return [
2481 new_block_2d(
2482 ensure_block_shape(arr, ndim=2), placement=BlockPlacement(i), refs=ref
2483 )
2484 for ((i, arr), ref) in zip(tuples, refs, strict=True)
2485 ]
2488def _stack_arrays(tuples, dtype: np.dtype):
2489 placement, arrays = zip(*tuples, strict=True)
2491 first = arrays[0]
2492 shape = (len(arrays), *first.shape)
2494 stacked = np.empty(shape, dtype=dtype)
2495 for i, arr in enumerate(arrays):
2496 stacked[i] = arr
2498 return stacked, placement
2501def _consolidate(blocks: tuple[Block, ...]) -> tuple[Block, ...]:
2502 """
2503 Merge blocks having same dtype, exclude non-consolidating blocks
2504 """
2505 # sort by _can_consolidate, dtype
2506 gkey = lambda x: x._consolidate_key
2507 grouper = itertools.groupby(sorted(blocks, key=gkey), gkey)
2509 new_blocks: list[Block] = []
2510 for (_can_consolidate, dtype), group_blocks in grouper:
2511 merged_blocks, _ = _merge_blocks(
2512 list(group_blocks), dtype=dtype, can_consolidate=_can_consolidate
2513 )
2514 new_blocks = extend_blocks(merged_blocks, new_blocks)
2515 return tuple(new_blocks)
2518def _merge_blocks(
2519 blocks: list[Block], dtype: DtypeObj, can_consolidate: bool
2520) -> tuple[list[Block], bool]:
2521 if len(blocks) == 1:
2522 return blocks, False
2524 if can_consolidate:
2525 # TODO: optimization potential in case all mgrs contain slices and
2526 # combination of those slices is a slice, too.
2527 new_mgr_locs = np.concatenate([b.mgr_locs.as_array for b in blocks])
2529 new_values: ArrayLike
2531 if isinstance(blocks[0].dtype, np.dtype):
2532 # Use np.concatenate directly instead of np.vstack to avoid the
2533 # overhead of atleast_2d calls (block values are always 2D)
2534 new_values = np.concatenate([b.values for b in blocks], axis=0)
2535 else:
2536 bvals = [blk.values for blk in blocks]
2537 bvals2 = cast("Sequence[NDArrayBackedExtensionArray]", bvals)
2538 new_values = bvals2[0]._concat_same_type(bvals2, axis=0)
2540 # Only sort if locations are not already in order
2541 if not libalgos.is_monotonic(new_mgr_locs, False)[0]:
2542 argsort = np.argsort(new_mgr_locs)
2543 new_values = new_values[argsort]
2544 new_mgr_locs = new_mgr_locs[argsort]
2546 bp = BlockPlacement(new_mgr_locs)
2547 return [new_block_2d(new_values, placement=bp)], True
2549 # can't consolidate --> no merge
2550 return blocks, False
2553def _preprocess_slice_or_indexer(
2554 slice_or_indexer: slice | np.ndarray, length: int, allow_fill: bool
2555):
2556 if isinstance(slice_or_indexer, slice):
2557 return (
2558 "slice",
2559 slice_or_indexer,
2560 libinternals.slice_len(slice_or_indexer, length),
2561 )
2562 else:
2563 if (
2564 not isinstance(slice_or_indexer, np.ndarray)
2565 or slice_or_indexer.dtype.kind != "i"
2566 ):
2567 dtype = getattr(slice_or_indexer, "dtype", None)
2568 raise TypeError(type(slice_or_indexer), dtype)
2570 indexer = ensure_platform_int(slice_or_indexer)
2571 if not allow_fill:
2572 indexer = maybe_convert_indices(indexer, length)
2573 return "fancy", indexer, len(indexer)
2576def make_na_array(dtype: DtypeObj, shape: Shape, fill_value) -> ArrayLike:
2577 if isinstance(dtype, DatetimeTZDtype):
2578 # NB: exclude e.g. pyarrow[dt64tz] dtypes
2579 ts = Timestamp(fill_value).as_unit(dtype.unit)
2580 i8values = np.full(shape, ts._value)
2581 dt64values = i8values.view(f"M8[{dtype.unit}]")
2582 return DatetimeArray._simple_new(dt64values, dtype=dtype)
2584 elif is_1d_only_ea_dtype(dtype):
2585 dtype = cast("ExtensionDtype", dtype)
2586 cls = dtype.construct_array_type()
2588 missing_arr = cls._from_sequence([], dtype=dtype)
2589 ncols, nrows = shape
2590 assert ncols == 1, ncols
2591 empty_arr = -1 * np.ones((nrows,), dtype=np.intp)
2592 return missing_arr.take(empty_arr, allow_fill=True, fill_value=fill_value)
2593 elif isinstance(dtype, ExtensionDtype):
2594 # TODO: no tests get here, a handful would if we disabled
2595 # the dt64tz special-case above (which is faster)
2596 cls = dtype.construct_array_type()
2597 missing_arr = cls._empty(shape=shape, dtype=dtype)
2598 missing_arr[:] = fill_value
2599 return missing_arr
2600 else:
2601 # NB: we should never get here with dtype integer or bool;
2602 # if we did, the missing_arr.fill would cast to gibberish
2603 missing_arr_np = np.empty(shape, dtype=dtype)
2604 missing_arr_np.fill(fill_value)
2606 if dtype.kind in "mM":
2607 missing_arr_np = ensure_wrapped_if_datetimelike(missing_arr_np)
2608 return missing_arr_np