Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from collections.abc import (
4 Callable,
5 Hashable,
6 Sequence,
7)
8import itertools
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 Literal,
13 NoReturn,
14 Self,
15 cast,
16 final,
17)
18import warnings
20import numpy as np
22from pandas._config.config import get_option
24from pandas._libs import (
25 algos as libalgos,
26 internals as libinternals,
27 lib,
28)
29from pandas._libs.internals import (
30 BlockPlacement,
31 BlockValuesRefs,
32)
33from pandas._libs.tslibs import Timestamp
34from pandas.errors import (
35 AbstractMethodError,
36 PerformanceWarning,
37)
38from pandas.util._decorators import cache_readonly
39from pandas.util._exceptions import find_stack_level
40from pandas.util._validators import validate_bool_kwarg
42from pandas.core.dtypes.cast import (
43 find_common_type,
44 infer_dtype_from_scalar,
45 np_can_hold_element,
46)
47from pandas.core.dtypes.common import (
48 ensure_platform_int,
49 is_1d_only_ea_dtype,
50 is_list_like,
51)
52from pandas.core.dtypes.dtypes import (
53 CategoricalDtype,
54 DatetimeTZDtype,
55 ExtensionDtype,
56 SparseDtype,
57)
58from pandas.core.dtypes.generic import (
59 ABCDataFrame,
60 ABCSeries,
61)
62from pandas.core.dtypes.missing import (
63 array_equals,
64 isna,
65)
67import pandas.core.algorithms as algos
68from pandas.core.arrays import DatetimeArray
69from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
70from pandas.core.base import PandasObject
71from pandas.core.construction import (
72 ensure_wrapped_if_datetimelike,
73 extract_array,
74)
75from pandas.core.indexers import maybe_convert_indices
76from pandas.core.indexes.api import (
77 Index,
78 default_index,
79 ensure_index,
80)
81from pandas.core.internals.blocks import (
82 Block,
83 NumpyBlock,
84 ensure_block_shape,
85 extend_blocks,
86 get_block_type,
87 maybe_coerce_values,
88 new_block,
89 new_block_2d,
90)
91from pandas.core.internals.ops import (
92 blockwise_all,
93 operate_blockwise,
94)
96if TYPE_CHECKING:
97 from collections.abc import Generator
99 from pandas._typing import (
100 ArrayLike,
101 AxisInt,
102 DtypeObj,
103 QuantileInterpolation,
104 Shape,
105 npt,
106 )
108 from pandas.api.extensions import ExtensionArray
111def interleaved_dtype(dtypes: list[DtypeObj]) -> DtypeObj | None:
112 """
113 Find the common dtype for `blocks`.
115 Parameters
116 ----------
117 blocks : List[DtypeObj]
119 Returns
120 -------
121 dtype : np.dtype, ExtensionDtype, or None
122 None is returned when `blocks` is empty.
123 """
124 if not len(dtypes):
125 return None
127 return find_common_type(dtypes)
130def ensure_np_dtype(dtype: DtypeObj) -> np.dtype:
131 # TODO: https://github.com/pandas-dev/pandas/issues/22791
132 # Give EAs some input on what happens here. Sparse needs this.
133 if isinstance(dtype, SparseDtype):
134 dtype = dtype.subtype
135 dtype = cast(np.dtype, dtype)
136 elif isinstance(dtype, ExtensionDtype):
137 dtype = np.dtype("object")
138 elif dtype == np.dtype(str):
139 dtype = np.dtype("object")
140 return dtype
143class BaseBlockManager(PandasObject):
144 """
145 Core internal data structure to implement DataFrame, Series, etc.
147 Manage a bunch of labeled 2D mixed-type ndarrays. Essentially it's a
148 lightweight blocked set of labeled data to be manipulated by the DataFrame
149 public API class
151 Attributes
152 ----------
153 shape
154 ndim
155 axes
156 values
157 items
159 Methods
160 -------
161 set_axis(axis, new_labels)
162 copy(deep=True)
164 get_dtypes
166 apply(func, axes, block_filter_fn)
168 get_bool_data
169 get_numeric_data
171 get_slice(slice_like, axis)
172 get(label)
173 iget(loc)
175 take(indexer, axis)
176 reindex_axis(new_labels, axis)
177 reindex_indexer(new_labels, indexer, axis)
179 delete(label)
180 insert(loc, label, value)
181 set(label, value)
183 Parameters
184 ----------
185 blocks: Sequence of Block
186 axes: Sequence of Index
187 verify_integrity: bool, default True
189 Notes
190 -----
191 This is *not* a public API class
192 """
194 __slots__ = ()
196 _blknos: npt.NDArray[np.intp]
197 _blklocs: npt.NDArray[np.intp]
198 blocks: tuple[Block, ...]
199 axes: list[Index]
201 @property
202 def ndim(self) -> int:
203 raise NotImplementedError
205 _known_consolidated: bool
206 _is_consolidated: bool
208 def __init__(self, blocks, axes, verify_integrity: bool = True) -> None:
209 raise NotImplementedError
211 @final
212 def __len__(self) -> int:
213 return len(self.items)
215 @property
216 def shape(self) -> Shape:
217 return tuple(len(ax) for ax in self.axes)
219 @classmethod
220 def from_blocks(cls, blocks: list[Block], axes: list[Index]) -> Self:
221 raise NotImplementedError
223 @property
224 def blknos(self) -> npt.NDArray[np.intp]:
225 """
226 Suppose we want to find the array corresponding to our i'th column.
228 blknos[i] identifies the block from self.blocks that contains this column.
230 blklocs[i] identifies the column of interest within
231 self.blocks[self.blknos[i]]
232 """
233 if self._blknos is None:
234 # Note: these can be altered by other BlockManager methods.
235 self._rebuild_blknos_and_blklocs()
237 return self._blknos
239 @property
240 def blklocs(self) -> npt.NDArray[np.intp]:
241 """
242 See blknos.__doc__
243 """
244 if self._blklocs is None:
245 # Note: these can be altered by other BlockManager methods.
246 self._rebuild_blknos_and_blklocs()
248 return self._blklocs
250 def make_empty(self, axes=None) -> Self:
251 """return an empty BlockManager with the items axis of len 0"""
252 if axes is None:
253 # TODO shallow copy remaining axis?
254 axes = [default_index(0), *self.axes[1:]]
256 # preserve dtype if possible
257 if self.ndim == 1:
258 assert isinstance(self, SingleBlockManager) # for mypy
259 blk = self.blocks[0]
260 arr = blk.values[:0]
261 bp = BlockPlacement(slice(0, 0))
262 nb = blk.make_block_same_class(arr, placement=bp)
263 blocks = [nb]
264 else:
265 blocks = []
266 return type(self).from_blocks(blocks, axes)
268 def __bool__(self) -> bool:
269 return True
271 def set_axis(self, axis: AxisInt, new_labels: Index) -> None:
272 # Caller is responsible for ensuring we have an Index object.
273 self._validate_set_axis(axis, new_labels)
274 self.axes[axis] = new_labels
276 @final
277 def _validate_set_axis(self, axis: AxisInt, new_labels: Index) -> None:
278 # Caller is responsible for ensuring we have an Index object.
279 old_len = len(self.axes[axis])
280 new_len = len(new_labels)
282 if axis == 1 and len(self.items) == 0:
283 # If we are setting the index on a DataFrame with no columns,
284 # it is OK to change the length.
285 pass
287 elif new_len != old_len:
288 raise ValueError(
289 f"Length mismatch: Expected axis has {old_len} elements, new "
290 f"values have {new_len} elements"
291 )
293 @property
294 def is_single_block(self) -> bool:
295 # Assumes we are 2D; overridden by SingleBlockManager
296 return len(self.blocks) == 1
298 @property
299 def items(self) -> Index:
300 return self.axes[0]
302 def _has_no_reference(self, i: int) -> bool:
303 """
304 Check for column `i` if it has references.
305 (whether it references another array or is itself being referenced)
306 Returns True if the column has no references.
307 """
308 blkno = self.blknos[i]
309 return self._has_no_reference_block(blkno)
311 def _has_no_reference_block(self, blkno: int) -> bool:
312 """
313 Check for block `i` if it has references.
314 (whether it references another array or is itself being referenced)
315 Returns True if the block has no references.
316 """
317 return not self.blocks[blkno].refs.has_reference()
319 def add_references(self, mgr: BaseBlockManager) -> None:
320 """
321 Adds the references from one manager to another. We assume that both
322 managers have the same block structure.
323 """
324 if len(self.blocks) != len(mgr.blocks):
325 # If block structure changes, then we made a copy
326 return
327 for i, blk in enumerate(self.blocks):
328 blk.refs = mgr.blocks[i].refs
329 blk.refs.add_reference(blk)
331 def references_same_values(self, mgr: BaseBlockManager, blkno: int) -> bool:
332 """
333 Checks if two blocks from two different block managers reference the
334 same underlying values.
335 """
336 blk = self.blocks[blkno]
337 return any(blk is ref() for ref in mgr.blocks[blkno].refs.referenced_blocks)
339 def get_unique_dtypes(self) -> npt.NDArray[np.object_]:
340 return algos.unique(np.array([blk.dtype for blk in self.blocks], dtype=object))
342 def get_dtypes(self) -> npt.NDArray[np.object_]:
343 dtypes = np.array([blk.dtype for blk in self.blocks], dtype=object)
344 return dtypes.take(self.blknos)
346 @property
347 def arrays(self) -> list[ArrayLike]:
348 """
349 Quick access to the backing arrays of the Blocks.
351 Only for compatibility with ArrayManager for testing convenience.
352 Not to be used in actual code, and return value is not the same as the
353 ArrayManager method (list of 1D arrays vs iterator of 2D ndarrays / 1D EAs).
355 Warning! The returned arrays don't handle Copy-on-Write, so this should
356 be used with caution (only in read-mode).
357 """
358 # TODO: Deprecate, usage in Dask
359 # https://github.com/dask/dask/blob/484fc3f1136827308db133cd256ba74df7a38d8c/dask/base.py#L1312
360 return [blk.values for blk in self.blocks]
362 def __repr__(self) -> str:
363 output = type(self).__name__
364 for i, ax in enumerate(self.axes):
365 if i == 0:
366 output += f"\nItems: {ax}"
367 else:
368 output += f"\nAxis {i}: {ax}"
370 for block in self.blocks:
371 output += f"\n{block}"
372 return output
374 def _equal_values(self, other: Self) -> bool:
375 """
376 To be implemented by the subclasses. Only check the column values
377 assuming shape and indexes have already been checked.
378 """
379 raise AbstractMethodError(self)
381 @final
382 def equals(self, other: object) -> bool:
383 """
384 Implementation for DataFrame.equals
385 """
386 if not isinstance(other, type(self)):
387 return False
389 self_axes, other_axes = self.axes, other.axes
390 if len(self_axes) != len(other_axes):
391 return False
392 if not all(
393 ax1.equals(ax2) for ax1, ax2 in zip(self_axes, other_axes, strict=True)
394 ):
395 return False
397 return self._equal_values(other)
399 def apply(
400 self,
401 f,
402 align_keys: list[str] | None = None,
403 **kwargs,
404 ) -> Self:
405 """
406 Iterate over the blocks, collect and create a new BlockManager.
408 Parameters
409 ----------
410 f : str or callable
411 Name of the Block method to apply.
412 align_keys: List[str] or None, default None
413 **kwargs
414 Keywords to pass to `f`
416 Returns
417 -------
418 BlockManager
419 """
420 assert "filter" not in kwargs
422 align_keys = align_keys or []
423 result_blocks: list[Block] = []
424 # fillna: Series/DataFrame is responsible for making sure value is aligned
426 aligned_args = {k: kwargs[k] for k in align_keys}
428 for b in self.blocks:
429 if aligned_args:
430 for k, obj in aligned_args.items():
431 if isinstance(obj, (ABCSeries, ABCDataFrame)):
432 # The caller is responsible for ensuring that
433 # obj.axes[-1].equals(self.items)
434 if obj.ndim == 1:
435 kwargs[k] = obj.iloc[b.mgr_locs.indexer]._values
436 else:
437 kwargs[k] = obj.iloc[:, b.mgr_locs.indexer]._values
438 else:
439 # otherwise we have an ndarray
440 kwargs[k] = obj[b.mgr_locs.indexer]
442 if callable(f):
443 applied = b.apply(f, **kwargs)
444 else:
445 applied = getattr(b, f)(**kwargs)
446 result_blocks = extend_blocks(applied, result_blocks)
448 out = type(self).from_blocks(result_blocks, [ax.view() for ax in self.axes])
449 return out
451 @final
452 def isna(self, func) -> Self:
453 return self.apply("apply", func=func)
455 @final
456 def fillna(self, value, limit: int | None, inplace: bool) -> Self:
457 if limit is not None:
458 # Do this validation even if we go through one of the no-op paths
459 limit = libalgos.validate_limit(None, limit=limit)
461 return self.apply(
462 "fillna",
463 value=value,
464 limit=limit,
465 inplace=inplace,
466 )
468 @final
469 def where(self, other, cond, align: bool) -> Self:
470 if align:
471 align_keys = ["other", "cond"]
472 else:
473 align_keys = ["cond"]
474 other = extract_array(other, extract_numpy=True)
476 return self.apply(
477 "where",
478 align_keys=align_keys,
479 other=other,
480 cond=cond,
481 )
483 @final
484 def putmask(self, mask, new, align: bool = True) -> Self:
485 if align:
486 align_keys = ["new", "mask"]
487 else:
488 align_keys = ["mask"]
489 new = extract_array(new, extract_numpy=True)
491 return self.apply(
492 "putmask",
493 align_keys=align_keys,
494 mask=mask,
495 new=new,
496 )
498 @final
499 def round(self, decimals: int) -> Self:
500 return self.apply("round", decimals=decimals)
502 @final
503 def replace(self, to_replace, value, inplace: bool) -> Self:
504 inplace = validate_bool_kwarg(inplace, "inplace")
505 # NDFrame.replace ensures the not-is_list_likes here
506 assert not lib.is_list_like(to_replace)
507 assert not lib.is_list_like(value)
508 return self.apply(
509 "replace",
510 to_replace=to_replace,
511 value=value,
512 inplace=inplace,
513 )
515 @final
516 def replace_regex(self, **kwargs) -> Self:
517 return self.apply("_replace_regex", **kwargs)
519 @final
520 def replace_list(
521 self,
522 src_list: list[Any],
523 dest_list: list[Any],
524 inplace: bool = False,
525 regex: bool = False,
526 ) -> Self:
527 """do a list replace"""
528 inplace = validate_bool_kwarg(inplace, "inplace")
530 bm = self.apply(
531 "replace_list",
532 src_list=src_list,
533 dest_list=dest_list,
534 inplace=inplace,
535 regex=regex,
536 )
537 bm._consolidate_inplace()
538 return bm
540 def interpolate(self, inplace: bool, **kwargs) -> Self:
541 return self.apply("interpolate", inplace=inplace, **kwargs)
543 def pad_or_backfill(self, inplace: bool, **kwargs) -> Self:
544 return self.apply("pad_or_backfill", inplace=inplace, **kwargs)
546 def shift(self, periods: int, fill_value) -> Self:
547 if fill_value is lib.no_default:
548 fill_value = None
550 return self.apply("shift", periods=periods, fill_value=fill_value)
552 def setitem(self, indexer, value) -> Self:
553 """
554 Set values with indexer.
556 For SingleBlockManager, this backs s[indexer] = value
557 """
558 if isinstance(indexer, np.ndarray) and indexer.ndim > self.ndim:
559 raise ValueError(f"Cannot set values with ndim > {self.ndim}")
561 if not self._has_no_reference(0):
562 # this method is only called if there is a single block -> hardcoded 0
563 # Split blocks to only copy the columns we want to modify
564 if self.ndim == 2 and isinstance(indexer, tuple):
565 blk_loc = self.blklocs[indexer[1]]
566 if is_list_like(blk_loc) and blk_loc.ndim == 2:
567 blk_loc = np.squeeze(blk_loc, axis=0)
568 elif not is_list_like(blk_loc):
569 # Keep dimension and copy data later
570 blk_loc = [blk_loc] # type: ignore[assignment]
571 if len(blk_loc) == 0:
572 return self.copy(deep=False)
574 values = self.blocks[0].values
575 if values.ndim == 2:
576 # Block.delete in _iset_split_block requires sorted unique
577 # locs; inverse maps the requested column order onto the
578 # new block (GH#65446)
579 blk_loc, inverse = np.unique(blk_loc, return_inverse=True)
580 values = values[blk_loc]
581 # "T" has no attribute "_iset_split_block"
582 self._iset_split_block( # type: ignore[attr-defined]
583 0, blk_loc, values
584 )
586 indexer = list(indexer)
587 # first block equals values we are setting to -> set to all columns
588 if lib.is_integer(indexer[1]):
589 col_indexer = 0
590 elif len(inverse) > 1 and lib.is_range_indexer(
591 inverse, len(blk_loc)
592 ):
593 col_indexer = slice(None) # type: ignore[assignment]
594 else:
595 col_indexer = inverse # type: ignore[assignment]
596 indexer[1] = col_indexer
598 row_indexer = indexer[0]
599 if isinstance(col_indexer, np.ndarray):
600 if (
601 isinstance(row_indexer, np.ndarray)
602 and row_indexer.ndim == 1
603 ):
604 # GH#65446: Make the row indexer 2d to take a cross product
605 row_indexer = row_indexer[:, None]
606 elif isinstance(row_indexer, np.ndarray) and row_indexer.ndim == 2:
607 # numpy cannot handle a 2d indexer in combo with a slice
608 row_indexer = np.squeeze(row_indexer, axis=1)
609 if isinstance(row_indexer, np.ndarray) and len(row_indexer) == 0:
610 # numpy does not like empty indexer combined with slice
611 # and we are setting nothing anyway
612 return self
613 indexer[0] = row_indexer
614 self.blocks[0].setitem(tuple(indexer), value)
615 return self
616 # No need to split if we either set all columns or on a single block
617 # manager
618 self = self.copy(deep=True)
620 return self.apply("setitem", indexer=indexer, value=value)
622 def diff(self, n: int) -> Self:
623 # only reached with self.ndim == 2
624 return self.apply("diff", n=n)
626 def astype(self, dtype, errors: str = "raise") -> Self:
627 return self.apply("astype", dtype=dtype, errors=errors)
629 def convert(self) -> Self:
630 return self.apply("convert")
632 def convert_dtypes(self, **kwargs):
633 return self.apply("convert_dtypes", **kwargs)
635 def get_values_for_csv(
636 self, *, float_format, date_format, decimal, na_rep: str = "nan", quoting=None
637 ) -> Self:
638 """
639 Convert values to native types (strings / python objects) that are used
640 in formatting (repr / csv).
641 """
642 return self.apply(
643 "get_values_for_csv",
644 na_rep=na_rep,
645 quoting=quoting,
646 float_format=float_format,
647 date_format=date_format,
648 decimal=decimal,
649 )
651 @property
652 def any_extension_types(self) -> bool:
653 """Whether any of the blocks in this manager are extension blocks"""
654 return any(block.is_extension for block in self.blocks)
656 @property
657 def is_view(self) -> bool:
658 """return a boolean if we are a single block and are a view"""
659 if len(self.blocks) == 1:
660 return self.blocks[0].is_view
662 # It is technically possible to figure out which blocks are views
663 # e.g. [ b.values.base is not None for b in self.blocks ]
664 # but then we have the case of possibly some blocks being a view
665 # and some blocks not. setting in theory is possible on the non-view
666 # blocks. But this is a bit
667 # complicated
669 return False
671 def _get_data_subset(self, predicate: Callable) -> Self:
672 blocks = [blk for blk in self.blocks if predicate(blk.values)]
673 return self._combine(blocks)
675 def _get_data_subset_indices(self, predicate: Callable) -> np.ndarray:
676 blocks = [blk for blk in self.blocks if predicate(blk.values)]
677 indexer = np.sort(np.concatenate([b.mgr_locs.as_array for b in blocks]))
678 return indexer
680 def get_bool_data(self) -> Self:
681 """
682 Select blocks that are bool-dtype and columns from object-dtype blocks
683 that are all-bool.
684 """
686 new_blocks = []
688 for blk in self.blocks:
689 if blk.dtype == bool:
690 new_blocks.append(blk)
692 elif blk.is_object:
693 new_blocks.extend(nb for nb in blk._split() if nb.is_bool)
695 return self._combine(new_blocks)
697 def get_numeric_data(self) -> Self:
698 numeric_blocks = [blk for blk in self.blocks if blk.is_numeric]
699 if len(numeric_blocks) == len(self.blocks):
700 # Avoid somewhat expensive _combine
701 # TODO(CoW) need to return a shallow copy here?
702 return self
703 return self._combine(numeric_blocks)
705 def _combine(self, blocks: list[Block], index: Index | None = None) -> Self:
706 """return a new manager with the blocks"""
707 if len(blocks) == 0:
708 if self.ndim == 2:
709 # retain our own Index dtype
710 if index is not None:
711 axes = [self.items[:0], index]
712 else:
713 axes = [self.items[:0], *self.axes[1:]]
714 return self.make_empty(axes)
715 return self.make_empty()
717 # FIXME: optimization potential
718 indexer = np.sort(np.concatenate([b.mgr_locs.as_array for b in blocks]))
719 inv_indexer = lib.get_reverse_indexer(indexer, self.shape[0])
721 new_blocks: list[Block] = []
722 for b in blocks:
723 nb = b.copy(deep=False)
724 nb.mgr_locs = BlockPlacement(inv_indexer[nb.mgr_locs.indexer])
725 new_blocks.append(nb)
727 axes = list(self.axes)
728 # TODO shallow copy of axes?
729 if index is not None:
730 axes[-1] = index
731 axes[0] = self.items.take(indexer)
733 return type(self).from_blocks(new_blocks, axes)
735 @property
736 def nblocks(self) -> int:
737 return len(self.blocks)
739 def copy(self, *, deep: bool) -> Self:
740 """
741 Make deep or shallow copy of BlockManager
743 Parameters
744 ----------
745 deep : bool, string or None, default True
746 If False, return a shallow copy (do not copy data)
748 Returns
749 -------
750 BlockManager
751 """
752 # TODO: Should deep=True be respected for axes?
753 new_axes = [ax.view() for ax in self.axes]
755 res = self.apply("copy", deep=deep)
756 res.axes = new_axes
758 if self.ndim > 1:
759 # Avoid needing to re-compute these
760 blknos = self._blknos
761 if blknos is not None:
762 res._blknos = blknos.copy()
763 res._blklocs = self._blklocs.copy()
765 if deep:
766 res._consolidate_inplace()
767 return res
769 def is_consolidated(self) -> bool:
770 return True
772 def consolidate(self) -> Self:
773 """
774 Join together blocks having same dtype
776 Returns
777 -------
778 y : BlockManager
779 """
780 if self.is_consolidated():
781 return self
783 # TODO shallow copy is not needed here?
784 bm = type(self)(self.blocks, self.axes, verify_integrity=False)
785 bm._is_consolidated = False
786 bm._consolidate_inplace()
787 return bm
789 def _consolidate_inplace(self) -> None:
790 return
792 @final
793 def reindex_axis(
794 self,
795 new_index: Index,
796 axis: AxisInt,
797 fill_value=None,
798 only_slice: bool = False,
799 ) -> Self:
800 """
801 Conform data manager to new index.
802 """
803 new_index, indexer = self.axes[axis].reindex(new_index)
805 return self.reindex_indexer(
806 new_index,
807 indexer,
808 axis=axis,
809 fill_value=fill_value,
810 only_slice=only_slice,
811 )
813 def reindex_indexer(
814 self,
815 new_axis: Index,
816 indexer: npt.NDArray[np.intp] | None,
817 axis: AxisInt,
818 fill_value=None,
819 allow_dups: bool = False,
820 only_slice: bool = False,
821 *,
822 use_na_proxy: bool = False,
823 ) -> Self:
824 """
825 Parameters
826 ----------
827 new_axis : Index
828 indexer : ndarray[intp] or None
829 axis : int
830 fill_value : object, default None
831 allow_dups : bool, default False
832 only_slice : bool, default False
833 Whether to take views, not copies, along columns.
834 use_na_proxy : bool, default False
835 Whether to use an np.void ndarray for newly introduced columns.
837 pandas-indexer with -1's only.
838 """
839 if indexer is None:
840 if new_axis is self.axes[axis]:
841 # TODO(CoW) need to handle CoW?
842 return self
844 result = self.copy(deep=False)
845 result.axes = list(self.axes)
846 result.axes[axis] = new_axis
847 return result
849 # Should be intp, but in some cases we get int64 on 32bit builds
850 assert isinstance(indexer, np.ndarray)
852 # some axes don't allow reindexing with dups
853 if not allow_dups:
854 self.axes[axis]._validate_can_reindex(indexer)
856 if axis >= self.ndim:
857 raise IndexError("Requested axis not found in manager")
859 if axis == 0:
860 new_blocks = list(
861 self._slice_take_blocks_ax0(
862 indexer,
863 fill_value=fill_value,
864 only_slice=only_slice,
865 use_na_proxy=use_na_proxy,
866 )
867 )
868 else:
869 new_blocks = []
870 for blk in self.blocks:
871 if blk.dtype == np.void:
872 # GH#58316: np.void placeholders cast to b'' when
873 # reindexed; preserve np.void so _setitem_single_column
874 # can later infer the correct dtype
875 vals = np.empty((blk.values.shape[0], len(indexer)), dtype=np.void)
876 new_blocks.append(NumpyBlock(vals, blk.mgr_locs, ndim=2))
877 else:
878 new_blocks.append(
879 blk.take_nd(
880 indexer,
881 axis=1,
882 fill_value=(
883 fill_value if fill_value is not None else blk.fill_value
884 ),
885 )
886 )
888 new_axes = list(self.axes)
889 new_axes[axis] = new_axis
890 if self.ndim == 2:
891 new_axes[1 - axis] = self.axes[1 - axis].view()
893 new_mgr = type(self).from_blocks(new_blocks, new_axes)
894 if axis == 1:
895 # We can avoid the need to rebuild these
896 new_mgr._blknos = self.blknos.copy()
897 new_mgr._blklocs = self.blklocs.copy()
898 return new_mgr
900 def _slice_take_blocks_ax0(
901 self,
902 slice_or_indexer: slice | np.ndarray,
903 fill_value=lib.no_default,
904 only_slice: bool = False,
905 *,
906 use_na_proxy: bool = False,
907 ref_inplace_op: bool = False,
908 ) -> Generator[Block]:
909 """
910 Slice/take blocks along axis=0.
912 Overloaded for SingleBlock
914 Parameters
915 ----------
916 slice_or_indexer : slice or np.ndarray[int64]
917 fill_value : scalar, default lib.no_default
918 only_slice : bool, default False
919 If True, we always return views on existing arrays, never copies.
920 This is used when called from ops.blockwise.operate_blockwise.
921 use_na_proxy : bool, default False
922 Whether to use an np.void ndarray for newly introduced columns.
923 ref_inplace_op: bool, default False
924 Don't track refs if True because we operate inplace
926 Yields
927 ------
928 Block : New Block
929 """
930 allow_fill = fill_value is not lib.no_default
932 sl_type, slobj, sllen = _preprocess_slice_or_indexer(
933 slice_or_indexer, self.shape[0], allow_fill=allow_fill
934 )
936 if self.is_single_block:
937 blk = self.blocks[0]
939 if sl_type == "slice":
940 # GH#32959 EABlock would fail since we can't make 0-width
941 # TODO(EA2D): special casing unnecessary with 2D EAs
942 if sllen == 0:
943 return
944 bp = BlockPlacement(slice(0, sllen))
945 yield blk.getitem_block_columns(slobj, new_mgr_locs=bp)
946 return
947 elif not allow_fill or self.ndim == 1:
948 if allow_fill and fill_value is None:
949 fill_value = blk.fill_value
951 if not allow_fill and only_slice:
952 # GH#33597 slice instead of take, so we get
953 # views instead of copies
954 for i, ml in enumerate(slobj):
955 yield blk.getitem_block_columns(
956 slice(ml, ml + 1),
957 new_mgr_locs=BlockPlacement(i),
958 ref_inplace_op=ref_inplace_op,
959 )
960 else:
961 bp = BlockPlacement(slice(0, sllen))
962 yield blk.take_nd(
963 slobj,
964 axis=0,
965 new_mgr_locs=bp,
966 fill_value=fill_value,
967 )
968 return
970 if sl_type == "slice":
971 blknos = self.blknos[slobj]
972 blklocs = self.blklocs[slobj]
973 else:
974 blknos = algos.take_nd(
975 self.blknos, slobj, fill_value=-1, allow_fill=allow_fill
976 )
977 blklocs = algos.take_nd(
978 self.blklocs, slobj, fill_value=-1, allow_fill=allow_fill
979 )
981 # When filling blknos, make sure blknos is updated before appending to
982 # blocks list, that way new blkno is exactly len(blocks).
983 group = not only_slice
984 for blkno, mgr_locs in libinternals.get_blkno_placements(blknos, group=group):
985 if blkno == -1:
986 # If we've got here, fill_value was not lib.no_default
988 dtype, _ = infer_dtype_from_scalar(fill_value)
989 if is_1d_only_ea_dtype(dtype) and len(mgr_locs) > 1:
990 # Handle 1D-only extension dtypes by creating separate blocks
991 # (GH#63993)
992 placements = [BlockPlacement(col_idx) for col_idx in mgr_locs]
993 else:
994 placements = [mgr_locs]
996 for placement in placements:
997 yield self._make_na_block(
998 placement=placement,
999 fill_value=fill_value,
1000 use_na_proxy=use_na_proxy,
1001 )
1002 else:
1003 blk = self.blocks[blkno]
1005 # Otherwise, slicing along items axis is necessary.
1006 if not blk._can_consolidate and not blk._validate_ndim:
1007 # i.e. we dont go through here for DatetimeTZBlock
1008 # A non-consolidatable block, it's easy, because there's
1009 # only one item and each mgr loc is a copy of that single
1010 # item.
1011 deep = False
1012 for mgr_loc in mgr_locs:
1013 newblk = blk.copy(deep=deep)
1014 newblk.mgr_locs = BlockPlacement(slice(mgr_loc, mgr_loc + 1))
1015 yield newblk
1017 else:
1018 # GH#32779 to avoid the performance penalty of copying,
1019 # we may try to only slice
1020 taker = blklocs[mgr_locs.indexer]
1021 max_len = max(len(mgr_locs), taker.max() + 1)
1022 taker = lib.maybe_indices_to_slice(taker, max_len)
1024 if isinstance(taker, slice):
1025 nb = blk.getitem_block_columns(taker, new_mgr_locs=mgr_locs)
1026 yield nb
1027 elif only_slice:
1028 # GH#33597 slice instead of take, so we get
1029 # views instead of copies
1030 for i, ml in zip(taker, mgr_locs, strict=True):
1031 slc = slice(i, i + 1)
1032 bp = BlockPlacement(ml)
1033 nb = blk.getitem_block_columns(slc, new_mgr_locs=bp)
1034 # We have np.shares_memory(nb.values, blk.values)
1035 yield nb
1036 else:
1037 nb = blk.take_nd(taker, axis=0, new_mgr_locs=mgr_locs)
1038 yield nb
1040 def _make_na_block(
1041 self, placement: BlockPlacement, fill_value=None, use_na_proxy: bool = False
1042 ) -> Block:
1043 # Note: we only get here with self.ndim == 2
1045 if use_na_proxy:
1046 assert fill_value is None
1047 shape = (len(placement), self.shape[1])
1048 vals = np.empty(shape, dtype=np.void)
1049 nb = NumpyBlock(vals, placement, ndim=2)
1050 return nb
1052 if fill_value is None or fill_value is np.nan:
1053 fill_value = np.nan
1054 # GH45857 avoid unnecessary upcasting
1055 dtype = interleaved_dtype([blk.dtype for blk in self.blocks])
1056 if dtype is not None and np.issubdtype(dtype.type, np.floating):
1057 fill_value = dtype.type(fill_value)
1059 shape = (len(placement), self.shape[1])
1061 dtype, fill_value = infer_dtype_from_scalar(fill_value)
1062 block_values = make_na_array(dtype, shape, fill_value)
1063 return new_block_2d(block_values, placement=placement)
1065 def take(
1066 self,
1067 indexer: npt.NDArray[np.intp],
1068 axis: AxisInt = 1,
1069 verify: bool = True,
1070 ) -> Self:
1071 """
1072 Take items along any axis.
1074 indexer : np.ndarray[np.intp]
1075 axis : int, default 1
1076 verify : bool, default True
1077 Check that all entries are between 0 and len(self) - 1, inclusive.
1078 Pass verify=False if this check has been done by the caller.
1080 Returns
1081 -------
1082 BlockManager
1083 """
1084 # Caller is responsible for ensuring indexer annotation is accurate
1086 n = self.shape[axis]
1087 indexer = maybe_convert_indices(indexer, n, verify=verify)
1089 new_labels = self.axes[axis].take(indexer)
1090 return self.reindex_indexer(
1091 new_axis=new_labels,
1092 indexer=indexer,
1093 axis=axis,
1094 allow_dups=True,
1095 )
1098class BlockManager(libinternals.BlockManager, BaseBlockManager):
1099 """
1100 BaseBlockManager that holds 2D blocks.
1101 """
1103 ndim = 2
1105 # ----------------------------------------------------------------
1106 # Constructors
1108 def __init__(
1109 self,
1110 blocks: Sequence[Block],
1111 axes: Sequence[Index],
1112 verify_integrity: bool = True,
1113 ) -> None:
1114 if verify_integrity:
1115 # Assertion disabled for performance
1116 # assert all(isinstance(x, Index) for x in axes)
1118 for block in blocks:
1119 if self.ndim != block.ndim:
1120 raise AssertionError(
1121 f"Number of Block dimensions ({block.ndim}) must equal "
1122 f"number of axes ({self.ndim})"
1123 )
1124 # As of 2.0, the caller is responsible for ensuring that
1125 # DatetimeTZBlock with block.ndim == 2 has block.values.ndim ==2;
1126 # previously there was a special check for fastparquet compat.
1128 self._verify_integrity()
1130 def _verify_integrity(self) -> None:
1131 mgr_shape = self.shape
1132 tot_items = sum(len(x.mgr_locs) for x in self.blocks)
1133 for block in self.blocks:
1134 if block.shape[1:] != mgr_shape[1:]:
1135 raise_construction_error(tot_items, block.shape[1:], self.axes)
1136 if len(self.items) != tot_items:
1137 raise AssertionError(
1138 "Number of manager items must equal union of "
1139 f"block items\n# manager items: {len(self.items)}, # "
1140 f"tot_items: {tot_items}"
1141 )
1143 @classmethod
1144 def from_blocks(cls, blocks: list[Block], axes: list[Index]) -> Self:
1145 """
1146 Constructor for BlockManager and SingleBlockManager with same signature.
1147 """
1148 return cls(blocks, axes, verify_integrity=False)
1150 # ----------------------------------------------------------------
1151 # Indexing
1153 def fast_xs(self, loc: int) -> SingleBlockManager:
1154 """
1155 Return the array corresponding to `frame.iloc[loc]`.
1157 Parameters
1158 ----------
1159 loc : int
1161 Returns
1162 -------
1163 np.ndarray or ExtensionArray
1164 """
1165 if len(self.blocks) == 1:
1166 # TODO: this could be wrong if blk.mgr_locs is not slice(None)-like;
1167 # is this ruled out in the general case?
1168 result: np.ndarray | ExtensionArray = self.blocks[0].iget(
1169 (slice(None), loc)
1170 )
1171 # in the case of a single block, the new block is a view
1172 bp = BlockPlacement(slice(0, len(result)))
1173 block = new_block(
1174 result,
1175 placement=bp,
1176 ndim=1,
1177 refs=self.blocks[0].refs,
1178 )
1179 return SingleBlockManager(block, self.axes[0].view())
1181 dtype = interleaved_dtype([blk.dtype for blk in self.blocks])
1183 n = len(self)
1185 if isinstance(dtype, ExtensionDtype):
1186 # TODO: use object dtype as workaround for non-performant
1187 # EA.__setitem__ methods. (primarily ArrowExtensionArray.__setitem__
1188 # when iteratively setting individual values)
1189 # https://github.com/pandas-dev/pandas/pull/54508#issuecomment-1675827918
1190 result = np.empty(n, dtype=object)
1191 else:
1192 result = np.empty(n, dtype=dtype)
1193 result = ensure_wrapped_if_datetimelike(result)
1195 for blk in self.blocks:
1196 # Such assignment may incorrectly coerce NaT to None
1197 # result[blk.mgr_locs] = blk._slice((slice(None), loc))
1198 for i, rl in enumerate(blk.mgr_locs):
1199 item = blk.iget((i, loc))
1200 if (
1201 result.dtype.kind in "iub"
1202 and lib.is_float(item)
1203 and isna(item)
1204 and isinstance(blk.dtype, CategoricalDtype)
1205 ):
1206 # GH#58954 caused bc interleaved_dtype is wrong for Categorical
1207 # TODO(GH#38240) this will be unnecessary
1208 # Note that doing this in a try/except would work for the
1209 # integer case, but not for bool, which will cast the NaN
1210 # entry to True.
1211 if result.dtype.kind == "b":
1212 new_dtype = object
1213 else:
1214 new_dtype = np.float64
1215 result = result.astype(new_dtype)
1216 result[rl] = item
1218 if isinstance(dtype, ExtensionDtype):
1219 cls = dtype.construct_array_type()
1220 result = cls._from_sequence(result, dtype=dtype)
1222 bp = BlockPlacement(slice(0, len(result)))
1223 block = new_block(result, placement=bp, ndim=1)
1224 return SingleBlockManager(block, self.axes[0].view())
1226 def iget(self, i: int, track_ref: bool = True) -> SingleBlockManager:
1227 """
1228 Return the data as a SingleBlockManager.
1229 """
1230 block = self.blocks[self.blknos[i]]
1231 values = block.iget(self.blklocs[i])
1233 # shortcut for select a single-dim from a 2-dim BM
1234 bp = BlockPlacement(slice(0, len(values)))
1235 nb = type(block)(
1236 values, placement=bp, ndim=1, refs=block.refs if track_ref else None
1237 )
1238 return SingleBlockManager(nb, self.axes[1].view())
1240 def iget_values(self, i: int) -> ArrayLike:
1241 """
1242 Return the data for column i as the values (ndarray or ExtensionArray).
1244 Warning! The returned array is a view but doesn't handle Copy-on-Write,
1245 so this should be used with caution.
1246 """
1247 # TODO(CoW) making the arrays read-only might make this safer to use?
1248 block = self.blocks[self.blknos[i]]
1249 values = block.iget(self.blklocs[i])
1250 return values
1252 @property
1253 def column_arrays(self) -> list[np.ndarray]:
1254 """
1255 Used in the JSON C code to access column arrays.
1256 This optimizes compared to using `iget_values` by converting each
1258 Warning! This doesn't handle Copy-on-Write, so should be used with
1259 caution (current use case of consuming this in the JSON code is fine).
1260 """
1261 # This is an optimized equivalent to
1262 # result = [self.iget_values(i) for i in range(len(self.items))]
1263 result: list[np.ndarray | None] = [None] * len(self.items)
1265 for blk in self.blocks:
1266 mgr_locs = blk._mgr_locs
1267 values = blk.array_values._values_for_json()
1268 if values.ndim == 1:
1269 # TODO(EA2D): special casing not needed with 2D EAs
1270 result[mgr_locs[0]] = values
1272 else:
1273 for i, loc in enumerate(mgr_locs):
1274 result[loc] = values[i]
1276 # error: Incompatible return value type (got "List[None]",
1277 # expected "List[ndarray[Any, Any]]")
1278 return result # type: ignore[return-value]
1280 def iset(
1281 self,
1282 loc: int | slice | np.ndarray,
1283 value: ArrayLike,
1284 inplace: bool = False,
1285 refs: BlockValuesRefs | None = None,
1286 ) -> None:
1287 """
1288 Set new item in-place. Does not consolidate. Adds new Block if not
1289 contained in the current set of items
1290 """
1292 # FIXME: refactor, clearly separate broadcasting & zip-like assignment
1293 # can prob also fix the various if tests for sparse/categorical
1294 if self._blklocs is None and self.ndim > 1:
1295 self._rebuild_blknos_and_blklocs()
1297 # Note: we exclude DTA/TDA here
1298 value_is_extension_type = is_1d_only_ea_dtype(value.dtype)
1299 if not value_is_extension_type:
1300 if value.ndim == 2:
1301 value = value.T
1302 else:
1303 value = ensure_block_shape(value, ndim=2)
1305 if value.shape[1:] != self.shape[1:]:
1306 raise AssertionError(
1307 "Shape of new values must be compatible with manager shape"
1308 )
1310 if lib.is_integer(loc):
1311 # We have 6 tests where loc is _not_ an int.
1312 # In this case, get_blkno_placements will yield only one tuple,
1313 # containing (self._blknos[loc], BlockPlacement(slice(0, 1, 1)))
1315 # Check if we can use _iset_single fastpath
1316 loc = cast(int, loc)
1317 blkno = self.blknos[loc]
1318 blk = self.blocks[blkno]
1319 if len(blk._mgr_locs) == 1: # TODO: fastest way to check this?
1320 return self._iset_single(
1321 loc,
1322 value,
1323 inplace=inplace,
1324 blkno=blkno,
1325 blk=blk,
1326 refs=refs,
1327 )
1329 # error: Incompatible types in assignment (expression has type
1330 # "List[Union[int, slice, ndarray]]", variable has type "Union[int,
1331 # slice, ndarray]")
1332 loc = [loc] # type: ignore[assignment]
1334 # categorical/sparse/datetimetz
1335 if value_is_extension_type:
1337 def value_getitem(placement):
1338 return value
1340 else:
1342 def value_getitem(placement):
1343 return value[placement.indexer]
1345 # Accessing public blknos ensures the public versions are initialized
1346 blknos = self.blknos[loc]
1347 blklocs = self.blklocs[loc].copy()
1349 unfit_mgr_locs = []
1350 unfit_val_locs = []
1351 removed_blknos = []
1352 for blkno_l, val_locs in libinternals.get_blkno_placements(blknos, group=True):
1353 blk = self.blocks[blkno_l]
1354 blk_locs = blklocs[val_locs.indexer]
1355 if inplace and blk.should_store(value):
1356 # Updating inplace -> check if we need to do Copy-on-Write
1357 if not self._has_no_reference_block(blkno_l):
1358 self._iset_split_block(
1359 blkno_l, blk_locs, value_getitem(val_locs), refs=refs
1360 )
1361 else:
1362 blk.set_inplace(blk_locs, value_getitem(val_locs))
1363 continue
1364 else:
1365 unfit_mgr_locs.append(blk.mgr_locs.as_array[blk_locs])
1366 unfit_val_locs.append(val_locs)
1368 # If all block items are unfit, schedule the block for removal.
1369 if len(val_locs) == len(blk.mgr_locs):
1370 removed_blknos.append(blkno_l)
1371 continue
1372 else:
1373 # Defer setting the new values to enable consolidation
1374 self._iset_split_block(blkno_l, blk_locs, refs=refs)
1376 if removed_blknos:
1377 # Remove blocks & update blknos accordingly
1378 is_deleted = np.zeros(self.nblocks, dtype=np.bool_)
1379 is_deleted[removed_blknos] = True
1381 new_blknos = np.empty(self.nblocks, dtype=np.intp)
1382 new_blknos.fill(-1)
1383 new_blknos[~is_deleted] = np.arange(self.nblocks - len(removed_blknos))
1384 self._blknos = new_blknos[self._blknos]
1385 self.blocks = tuple(
1386 blk for i, blk in enumerate(self.blocks) if i not in set(removed_blknos)
1387 )
1389 if unfit_val_locs:
1390 unfit_idxr = np.concatenate(unfit_mgr_locs)
1391 unfit_count = len(unfit_idxr)
1393 new_blocks: list[Block] = []
1394 if value_is_extension_type:
1395 # This code (ab-)uses the fact that EA blocks contain only
1396 # one item.
1397 # TODO(EA2D): special casing unnecessary with 2D EAs
1398 new_blocks.extend(
1399 new_block_2d(
1400 values=value,
1401 placement=BlockPlacement(slice(mgr_loc, mgr_loc + 1)),
1402 refs=refs,
1403 )
1404 for mgr_loc in unfit_idxr
1405 )
1407 self._blknos[unfit_idxr] = np.arange(unfit_count) + len(self.blocks)
1408 self._blklocs[unfit_idxr] = 0
1410 else:
1411 # unfit_val_locs contains BlockPlacement objects
1412 unfit_val_items = unfit_val_locs[0].append(unfit_val_locs[1:])
1414 new_blocks.append(
1415 new_block_2d(
1416 values=value_getitem(unfit_val_items),
1417 placement=BlockPlacement(unfit_idxr),
1418 refs=refs,
1419 )
1420 )
1422 self._blknos[unfit_idxr] = len(self.blocks)
1423 self._blklocs[unfit_idxr] = np.arange(unfit_count)
1425 self.blocks += tuple(new_blocks)
1427 # Newly created block's dtype may already be present.
1428 self._known_consolidated = False
1430 def _iset_split_block(
1431 self,
1432 blkno_l: int,
1433 blk_locs: np.ndarray | list[int],
1434 value: ArrayLike | None = None,
1435 refs: BlockValuesRefs | None = None,
1436 ) -> None:
1437 """Removes columns from a block by splitting the block.
1439 Avoids copying the whole block through slicing and updates the manager
1440 after determining the new block structure. Optionally adds a new block,
1441 otherwise has to be done by the caller.
1443 Parameters
1444 ----------
1445 blkno_l: The block number to operate on, relevant for updating the manager
1446 blk_locs: The locations of our block that should be deleted.
1447 value: The value to set as a replacement.
1448 refs: The reference tracking object of the value to set.
1449 """
1450 blk = self.blocks[blkno_l]
1452 if self._blklocs is None:
1453 self._rebuild_blknos_and_blklocs()
1455 nbs_tup = tuple(blk.delete(blk_locs))
1456 if value is not None:
1457 locs = blk.mgr_locs.as_array[blk_locs]
1458 first_nb = new_block_2d(value, BlockPlacement(locs), refs=refs)
1459 else:
1460 first_nb = nbs_tup[0]
1461 nbs_tup = tuple(nbs_tup[1:])
1463 nr_blocks = len(self.blocks)
1464 blocks_tup = (
1465 *self.blocks[:blkno_l],
1466 first_nb,
1467 *self.blocks[blkno_l + 1 :],
1468 *nbs_tup,
1469 )
1470 self.blocks = blocks_tup
1472 if not nbs_tup and value is not None:
1473 # No need to update anything if split did not happen
1474 return
1476 self._blklocs[first_nb.mgr_locs.indexer] = np.arange(len(first_nb))
1478 for i, nb in enumerate(nbs_tup):
1479 self._blklocs[nb.mgr_locs.indexer] = np.arange(len(nb))
1480 self._blknos[nb.mgr_locs.indexer] = i + nr_blocks
1482 def _iset_single(
1483 self,
1484 loc: int,
1485 value: ArrayLike,
1486 inplace: bool,
1487 blkno: int,
1488 blk: Block,
1489 refs: BlockValuesRefs | None = None,
1490 ) -> None:
1491 """
1492 Fastpath for iset when we are only setting a single position and
1493 the Block currently in that position is itself single-column.
1495 In this case we can swap out the entire Block and blklocs and blknos
1496 are unaffected.
1497 """
1498 # Caller is responsible for verifying value.shape
1500 if inplace and blk.should_store(value):
1501 copy = not self._has_no_reference_block(blkno)
1502 iloc = self.blklocs[loc]
1503 blk.set_inplace(slice(iloc, iloc + 1), value, copy=copy)
1504 return
1506 nb = new_block_2d(value, placement=blk._mgr_locs, refs=refs)
1507 old_blocks = self.blocks
1508 new_blocks = (*old_blocks[:blkno], nb, *old_blocks[blkno + 1 :])
1509 self.blocks = new_blocks
1510 return
1512 def column_setitem(
1513 self, loc: int, idx: int | slice | np.ndarray, value, inplace_only: bool = False
1514 ) -> None:
1515 """
1516 Set values ("setitem") into a single column (not setting the full column).
1518 This is a method on the BlockManager level, to avoid creating an
1519 intermediate Series at the DataFrame level (`s = df[loc]; s[idx] = value`)
1520 """
1521 if not self._has_no_reference(loc):
1522 blkno = self.blknos[loc]
1523 # Split blocks to only copy the column we want to modify
1524 blk_loc = self.blklocs[loc]
1525 # Copy our values
1526 values = self.blocks[blkno].values
1527 if values.ndim == 1:
1528 values = values.copy()
1529 else:
1530 # Use [blk_loc] as indexer to keep ndim=2, this already results in a
1531 # copy
1532 values = values[[blk_loc]]
1533 self._iset_split_block(blkno, [blk_loc], values)
1535 # this manager is only created temporarily to mutate the values in place
1536 # so don't track references, otherwise the `setitem` would perform CoW again
1537 col_mgr = self.iget(loc, track_ref=False)
1538 if inplace_only:
1539 col_mgr.setitem_inplace(idx, value)
1540 else:
1541 new_mgr = col_mgr.setitem((idx,), value)
1542 self.iset(loc, new_mgr._block.values, inplace=True)
1544 def insert(self, loc: int, item: Hashable, value: ArrayLike, refs=None) -> None:
1545 """
1546 Insert item at selected position.
1548 Parameters
1549 ----------
1550 loc : int
1551 item : hashable
1552 value : np.ndarray or ExtensionArray
1553 refs : The reference tracking object of the value to set.
1554 """
1555 new_axis = self.items.insert(loc, item)
1557 if value.ndim == 2:
1558 value = value.T
1559 if len(value) > 1:
1560 raise ValueError(
1561 f"Expected a 1D array, got an array with shape {value.T.shape}"
1562 )
1563 else:
1564 value = ensure_block_shape(value, ndim=self.ndim)
1566 bp = BlockPlacement(slice(loc, loc + 1))
1567 block = new_block_2d(values=value, placement=bp, refs=refs)
1569 if not len(self.blocks):
1570 # Fastpath
1571 self._blklocs = np.array([0], dtype=np.intp)
1572 self._blknos = np.array([0], dtype=np.intp)
1573 else:
1574 self._insert_update_mgr_locs(loc)
1575 self._insert_update_blklocs_and_blknos(loc)
1577 self.axes[0] = new_axis
1578 self.blocks += (block,)
1580 self._known_consolidated = False
1582 if (
1583 get_option("performance_warnings")
1584 and sum(not block.is_extension for block in self.blocks) > 100
1585 ):
1586 warnings.warn(
1587 "DataFrame is highly fragmented. This is usually the result "
1588 "of calling `frame.insert` many times, which has poor performance. "
1589 "Consider joining all columns at once using pd.concat(axis=1) "
1590 "instead. To get a de-fragmented frame, use `newframe = frame.copy()`",
1591 PerformanceWarning,
1592 stacklevel=find_stack_level(),
1593 )
1595 def _insert_update_mgr_locs(self, loc) -> None:
1596 """
1597 When inserting a new Block at location 'loc', we increment
1598 all of the mgr_locs of blocks above that by one.
1599 """
1600 # Faster version of set(arr) for sequences of small numbers
1601 blknos = np.bincount(self.blknos[loc:]).nonzero()[0]
1602 for blkno in blknos:
1603 # .620 this way, .326 of which is in increment_above
1604 blk = self.blocks[blkno]
1605 blk._mgr_locs = blk._mgr_locs.increment_above(loc)
1607 def _insert_update_blklocs_and_blknos(self, loc) -> None:
1608 """
1609 When inserting a new Block at location 'loc', we update our
1610 _blklocs and _blknos.
1611 """
1613 # Accessing public blklocs ensures the public versions are initialized
1614 if loc == self.blklocs.shape[0]:
1615 # np.append is a lot faster, let's use it if we can.
1616 self._blklocs = np.append(self._blklocs, 0)
1617 self._blknos = np.append(self._blknos, len(self.blocks))
1618 elif loc == 0:
1619 # As of numpy 1.26.4, np.concatenate faster than np.append
1620 self._blklocs = np.concatenate([[0], self._blklocs])
1621 self._blknos = np.concatenate([[len(self.blocks)], self._blknos])
1622 else:
1623 new_blklocs, new_blknos = libinternals.update_blklocs_and_blknos(
1624 self.blklocs, self.blknos, loc, len(self.blocks)
1625 )
1626 self._blklocs = new_blklocs
1627 self._blknos = new_blknos
1629 def idelete(self, indexer) -> BlockManager:
1630 """
1631 Delete selected locations, returning a new BlockManager.
1632 """
1633 is_deleted = np.zeros(self.shape[0], dtype=np.bool_)
1634 is_deleted[indexer] = True
1635 taker = (~is_deleted).nonzero()[0]
1637 nbs = self._slice_take_blocks_ax0(taker, only_slice=True, ref_inplace_op=True)
1638 new_columns = self.items[~is_deleted]
1639 axes = [new_columns, self.axes[1]]
1640 return type(self)(tuple(nbs), axes, verify_integrity=False)
1642 # ----------------------------------------------------------------
1643 # Block-wise Operation
1645 def grouped_reduce(self, func: Callable) -> Self:
1646 """
1647 Apply grouped reduction function blockwise, returning a new BlockManager.
1649 Parameters
1650 ----------
1651 func : grouped reduction function
1653 Returns
1654 -------
1655 BlockManager
1656 """
1657 result_blocks: list[Block] = []
1659 for blk in self.blocks:
1660 if blk.is_object:
1661 # split on object-dtype blocks bc some columns may raise
1662 # while others do not.
1663 for sb in blk._split():
1664 applied = sb.apply(func)
1665 result_blocks = extend_blocks(applied, result_blocks)
1666 else:
1667 applied = blk.apply(func)
1668 result_blocks = extend_blocks(applied, result_blocks)
1670 if len(result_blocks) == 0:
1671 nrows = 0
1672 else:
1673 nrows = result_blocks[0].values.shape[-1]
1674 index = default_index(nrows)
1676 # TODO shallow copy columns?
1677 return type(self).from_blocks(result_blocks, [self.axes[0].view(), index])
1679 def reduce(self, func: Callable) -> Self:
1680 """
1681 Apply reduction function blockwise, returning a single-row BlockManager.
1683 Parameters
1684 ----------
1685 func : reduction function
1687 Returns
1688 -------
1689 BlockManager
1690 """
1691 # If 2D, we assume that we're operating column-wise
1692 assert self.ndim == 2
1694 res_blocks = [blk.reduce(func) for blk in self.blocks]
1695 index = default_index(1) # placeholder
1696 # shallow copy self.items not needed because DataFrame._reduce does a getitem
1697 new_mgr = type(self).from_blocks(res_blocks, [self.items, index])
1698 return new_mgr
1700 def operate_blockwise(self, other: BlockManager, array_op) -> BlockManager:
1701 """
1702 Apply array_op blockwise with another (aligned) BlockManager.
1703 """
1704 return operate_blockwise(self, other, array_op)
1706 def _equal_values(self: BlockManager, other: BlockManager) -> bool:
1707 """
1708 Used in .equals defined in base class. Only check the column values
1709 assuming shape and indexes have already been checked.
1710 """
1711 return blockwise_all(self, other, array_equals)
1713 def quantile(
1714 self,
1715 *,
1716 qs: Index, # with dtype float 64
1717 interpolation: QuantileInterpolation = "linear",
1718 ) -> Self:
1719 """
1720 Iterate over blocks applying quantile reduction.
1721 This routine is intended for reduction type operations and
1722 will do inference on the generated blocks.
1724 Parameters
1725 ----------
1726 interpolation : type of interpolation, default 'linear'
1727 qs : list of the quantiles to be computed
1729 Returns
1730 -------
1731 BlockManager
1732 """
1733 # Series dispatches to DataFrame for quantile, which allows us to
1734 # simplify some of the code here and in the blocks
1735 assert self.ndim >= 2
1736 assert is_list_like(qs) # caller is responsible for this
1738 new_axes = [self.axes[0].view(), Index(qs, dtype=np.float64)]
1740 blocks = [
1741 blk.quantile(qs=qs, interpolation=interpolation) for blk in self.blocks
1742 ]
1744 return type(self)(blocks, new_axes)
1746 # ----------------------------------------------------------------
1748 def unstack(self, unstacker, fill_value) -> BlockManager:
1749 """
1750 Return a BlockManager with all blocks unstacked.
1752 Parameters
1753 ----------
1754 unstacker : reshape._Unstacker
1755 fill_value : Any
1756 fill_value for newly introduced missing values.
1758 Returns
1759 -------
1760 unstacked : BlockManager
1761 """
1762 new_columns = unstacker.get_new_columns(self.items)
1763 new_index = unstacker.new_index
1765 allow_fill = not unstacker.mask_all
1766 if allow_fill:
1767 # calculating the full mask once and passing it to Block._unstack is
1768 # faster than letting calculating it in each repeated call
1769 new_mask2D = (~unstacker.mask).reshape(*unstacker.full_shape)
1770 needs_masking = new_mask2D.any(axis=0)
1771 else:
1772 needs_masking = np.zeros(unstacker.full_shape[1], dtype=bool)
1774 new_blocks: list[Block] = []
1775 columns_mask: list[np.ndarray] = []
1777 if len(self.items) == 0:
1778 factor = 1
1779 else:
1780 fac = len(new_columns) / len(self.items)
1781 assert fac == int(fac)
1782 factor = int(fac)
1784 for blk in self.blocks:
1785 mgr_locs = blk.mgr_locs
1786 new_placement = mgr_locs.tile_for_unstack(factor)
1788 blocks, mask = blk._unstack(
1789 unstacker,
1790 fill_value,
1791 new_placement=new_placement,
1792 needs_masking=needs_masking,
1793 )
1795 new_blocks.extend(blocks)
1796 columns_mask.extend(mask)
1798 # Block._unstack should ensure this holds,
1799 assert mask.sum() == sum(len(nb._mgr_locs) for nb in blocks)
1800 # In turn this ensures that in the BlockManager call below
1801 # we have len(new_columns) == sum(x.shape[0] for x in new_blocks)
1802 # which suffices to allow us to pass verify_inegrity=False
1804 new_columns = new_columns[columns_mask]
1806 bm = BlockManager(new_blocks, [new_columns, new_index], verify_integrity=False)
1807 return bm
1809 def to_iter_dict(self) -> Generator[tuple[str, Self]]:
1810 """
1811 Yield a tuple of (str(dtype), BlockManager)
1813 Returns
1814 -------
1815 values : a tuple of (str(dtype), BlockManager)
1816 """
1817 key = lambda block: str(block.dtype)
1818 for dtype, blocks in itertools.groupby(sorted(self.blocks, key=key), key=key):
1819 # TODO(EA2D): the combine will be unnecessary with 2D EAs
1820 yield dtype, self._combine(list(blocks))
1822 def as_array(
1823 self,
1824 dtype: np.dtype | None = None,
1825 copy: bool = False,
1826 na_value: object = lib.no_default,
1827 ) -> np.ndarray:
1828 """
1829 Convert the blockmanager data into a numpy array.
1831 Parameters
1832 ----------
1833 dtype : np.dtype or None, default None
1834 Data type of the return array.
1835 copy : bool, default False
1836 If True then guarantee that a copy is returned. A value of
1837 False does not guarantee that the underlying data is not
1838 copied.
1839 na_value : object, default lib.no_default
1840 Value to be used as the missing value sentinel.
1842 Returns
1843 -------
1844 arr : ndarray
1845 """
1846 passed_nan = lib.is_float(na_value) and isna(na_value)
1848 if len(self.blocks) == 0:
1849 arr = np.empty(self.shape, dtype=float)
1850 return arr.transpose()
1852 if self.is_single_block:
1853 blk = self.blocks[0]
1855 if na_value is not lib.no_default:
1856 # We want to copy when na_value is provided to avoid
1857 # mutating the original object
1858 if lib.is_np_dtype(blk.dtype, "f") and passed_nan:
1859 # We are already numpy-float and na_value=np.nan
1860 pass
1861 else:
1862 copy = True
1864 if blk.is_extension:
1865 # Avoid implicit conversion of extension blocks to object
1867 # error: Item "ndarray" of "Union[ndarray, ExtensionArray]" has no
1868 # attribute "to_numpy"
1869 arr = blk.values.to_numpy( # type: ignore[union-attr]
1870 dtype=dtype,
1871 na_value=na_value,
1872 copy=copy,
1873 ).reshape(blk.shape)
1874 elif not copy:
1875 arr = np.asarray(blk.values, dtype=dtype)
1876 else:
1877 arr = np.array(blk.values, dtype=dtype, copy=copy)
1878 if passed_nan and blk.dtype.kind in "mM":
1879 arr[isna(blk.values)] = na_value
1881 if not copy:
1882 arr = arr.view()
1883 arr.flags.writeable = False
1884 else:
1885 arr = self._interleave(dtype=dtype, na_value=na_value)
1886 # The underlying data was copied within _interleave, so no need
1887 # to further copy if copy=True or setting na_value
1889 if na_value is lib.no_default:
1890 pass
1891 elif arr.dtype.kind == "f" and passed_nan:
1892 pass
1893 else:
1894 arr[isna(arr)] = na_value
1896 return arr.transpose()
1898 def _interleave(
1899 self,
1900 dtype: np.dtype | None = None,
1901 na_value: object = lib.no_default,
1902 ) -> np.ndarray:
1903 """
1904 Return ndarray from blocks with specified item order
1905 Items must be contained in the blocks
1906 """
1907 if not dtype:
1908 # Incompatible types in assignment (expression has type
1909 # "Optional[Union[dtype[Any], ExtensionDtype]]", variable has
1910 # type "Optional[dtype[Any]]")
1911 dtype = interleaved_dtype( # type: ignore[assignment]
1912 [blk.dtype for blk in self.blocks]
1913 )
1915 # error: Argument 1 to "ensure_np_dtype" has incompatible type
1916 # "Optional[dtype[Any]]"; expected "Union[dtype[Any], ExtensionDtype]"
1917 dtype = ensure_np_dtype(dtype) # type: ignore[arg-type]
1918 result = np.empty(self.shape, dtype=dtype)
1920 itemmask = np.zeros(self.shape[0])
1922 if dtype == np.dtype("object") and na_value is lib.no_default:
1923 # much more performant than using to_numpy below
1924 for blk in self.blocks:
1925 rl = blk.mgr_locs
1926 arr = blk.get_values(dtype)
1927 result[rl.indexer] = arr
1928 itemmask[rl.indexer] = 1
1929 return result
1931 for blk in self.blocks:
1932 rl = blk.mgr_locs
1933 if blk.is_extension:
1934 # Avoid implicit conversion of extension blocks to object
1936 # error: Item "ndarray" of "Union[ndarray, ExtensionArray]" has no
1937 # attribute "to_numpy"
1938 arr = blk.values.to_numpy( # type: ignore[union-attr]
1939 dtype=dtype,
1940 na_value=na_value,
1941 )
1942 else:
1943 arr = blk.get_values(dtype)
1944 result[rl.indexer] = arr
1945 if na_value is not lib.no_default and blk.dtype.kind in "mM":
1946 result[rl.indexer][isna(arr)] = na_value
1947 itemmask[rl.indexer] = 1
1949 if not itemmask.all():
1950 raise AssertionError("Some items were not contained in blocks")
1952 return result
1954 # ----------------------------------------------------------------
1955 # Consolidation
1957 def is_consolidated(self) -> bool:
1958 """
1959 Return True if more than one block with the same dtype
1960 """
1961 if not self._known_consolidated:
1962 self._consolidate_check()
1963 return self._is_consolidated
1965 def _consolidate_check(self) -> None:
1966 if len(self.blocks) == 1:
1967 # fastpath
1968 self._is_consolidated = True
1969 self._known_consolidated = True
1970 return
1971 dtypes = [blk.dtype for blk in self.blocks if blk._can_consolidate]
1972 self._is_consolidated = len(dtypes) == len(set(dtypes))
1973 self._known_consolidated = True
1975 def _consolidate_inplace(self) -> None:
1976 if not self.is_consolidated():
1977 self.blocks = _consolidate(self.blocks)
1978 self._is_consolidated = True
1979 self._known_consolidated = True
1980 self._rebuild_blknos_and_blklocs()
1982 # ----------------------------------------------------------------
1983 # Concatenation
1985 @classmethod
1986 def concat_horizontal(cls, mgrs: list[Self], axes: list[Index]) -> Self:
1987 """
1988 Concatenate uniformly-indexed BlockManagers horizontally.
1989 """
1990 offset = 0
1991 blocks: list[Block] = []
1992 for mgr in mgrs:
1993 for blk in mgr.blocks:
1994 # We need to do getitem_block here otherwise we would be altering
1995 # blk.mgr_locs in place, which would render it invalid. This is only
1996 # relevant in the copy=False case.
1997 nb = blk.slice_block_columns(slice(None))
1998 nb._mgr_locs = nb._mgr_locs.add(offset)
1999 blocks.append(nb)
2001 offset += len(mgr.items)
2003 # TODO relevant axis already shallow-copied at caller?
2004 new_mgr = cls(tuple(blocks), axes)
2005 return new_mgr
2007 @classmethod
2008 def concat_vertical(cls, mgrs: list[Self], axes: list[Index]) -> Self:
2009 """
2010 Concatenate uniformly-indexed BlockManagers vertically.
2011 """
2012 raise NotImplementedError("This logic lives (for now) in internals.concat")
2015class SingleBlockManager(BaseBlockManager):
2016 """manage a single block with"""
2018 @property
2019 def ndim(self) -> Literal[1]:
2020 return 1
2022 _is_consolidated = True
2023 _known_consolidated = True
2024 __slots__ = ()
2025 is_single_block = True
2027 def __init__(
2028 self,
2029 block: Block,
2030 axis: Index,
2031 verify_integrity: bool = False,
2032 ) -> None:
2033 # Assertions disabled for performance
2034 # assert isinstance(block, Block), type(block)
2035 # assert isinstance(axis, Index), type(axis)
2037 self.axes = [axis]
2038 self.blocks = (block,)
2040 @classmethod
2041 def from_blocks(
2042 cls,
2043 blocks: list[Block],
2044 axes: list[Index],
2045 ) -> Self:
2046 """
2047 Constructor for BlockManager and SingleBlockManager with same signature.
2048 """
2049 assert len(blocks) == 1
2050 assert len(axes) == 1
2051 return cls(blocks[0], axes[0], verify_integrity=False)
2053 @classmethod
2054 def from_array(
2055 cls, array: ArrayLike, index: Index, refs: BlockValuesRefs | None = None
2056 ) -> SingleBlockManager:
2057 """
2058 Constructor for if we have an array that is not yet a Block.
2059 """
2060 array = maybe_coerce_values(array)
2061 bp = BlockPlacement(slice(0, len(index)))
2062 block = new_block(array, placement=bp, ndim=1, refs=refs)
2063 return cls(block, index)
2065 def to_2d_mgr(self, columns: Index) -> BlockManager:
2066 """
2067 Manager analogue of Series.to_frame
2068 """
2069 blk = self.blocks[0]
2070 arr = ensure_block_shape(blk.values, ndim=2)
2071 bp = BlockPlacement(0)
2072 new_blk = type(blk)(arr, placement=bp, ndim=2, refs=blk.refs)
2073 axes = [columns, self.axes[0].view()]
2074 return BlockManager([new_blk], axes=axes, verify_integrity=False)
2076 def _has_no_reference(self, i: int = 0) -> bool:
2077 """
2078 Check for column `i` if it has references.
2079 (whether it references another array or is itself being referenced)
2080 Returns True if the column has no references.
2081 """
2082 return not self.blocks[0].refs.has_reference()
2084 def __getstate__(self):
2085 block_values = [b.values for b in self.blocks]
2086 block_items = [self.items[b.mgr_locs.indexer] for b in self.blocks]
2087 axes_array = list(self.axes)
2089 extra_state = {
2090 "0.14.1": {
2091 "axes": axes_array,
2092 "blocks": [
2093 {"values": b.values, "mgr_locs": b.mgr_locs.indexer}
2094 for b in self.blocks
2095 ],
2096 }
2097 }
2099 # First three elements of the state are to maintain forward
2100 # compatibility with 0.13.1.
2101 return axes_array, block_values, block_items, extra_state
2103 def __setstate__(self, state) -> None:
2104 def unpickle_block(values, mgr_locs, ndim: int) -> Block:
2105 # TODO(EA2D): ndim would be unnecessary with 2D EAs
2106 # older pickles may store e.g. DatetimeIndex instead of DatetimeArray
2107 values = extract_array(values, extract_numpy=True)
2108 if not isinstance(mgr_locs, BlockPlacement):
2109 mgr_locs = BlockPlacement(mgr_locs)
2111 values = maybe_coerce_values(values)
2112 return new_block(values, placement=mgr_locs, ndim=ndim)
2114 if isinstance(state, tuple) and len(state) >= 4 and "0.14.1" in state[3]:
2115 state = state[3]["0.14.1"]
2116 self.axes = [ensure_index(ax) for ax in state["axes"]]
2117 ndim = len(self.axes)
2118 self.blocks = tuple(
2119 unpickle_block(b["values"], b["mgr_locs"], ndim=ndim)
2120 for b in state["blocks"]
2121 )
2122 else:
2123 raise NotImplementedError("pre-0.14.1 pickles are no longer supported")
2125 self._post_setstate()
2127 def _post_setstate(self) -> None:
2128 pass
2130 @cache_readonly
2131 def _block(self) -> Block:
2132 return self.blocks[0]
2134 @final
2135 @property
2136 def array(self) -> ArrayLike:
2137 """
2138 Quick access to the backing array of the Block.
2139 """
2140 return self.blocks[0].values
2142 # error: Cannot override writeable attribute with read-only property
2143 @property
2144 def _blknos(self) -> None: # type: ignore[override]
2145 """compat with BlockManager"""
2146 return None
2148 # error: Cannot override writeable attribute with read-only property
2149 @property
2150 def _blklocs(self) -> None: # type: ignore[override]
2151 """compat with BlockManager"""
2152 return None
2154 def get_rows_with_mask(self, indexer: npt.NDArray[np.bool_]) -> Self:
2155 # similar to get_slice, but not restricted to slice indexer
2156 blk = self._block
2157 if len(indexer) > 0 and indexer.all():
2158 return type(self)(blk.copy(deep=False), self.index)
2159 array = blk.values[indexer]
2161 if isinstance(indexer, np.ndarray) and indexer.dtype.kind == "b":
2162 # boolean indexing always gives a copy with numpy
2163 refs = None
2164 else:
2165 # TODO(CoW) in theory only need to track reference if new_array is a view
2166 refs = blk.refs
2168 bp = BlockPlacement(slice(0, len(array)))
2169 block = type(blk)(array, placement=bp, ndim=1, refs=refs)
2171 new_idx = self.index[indexer]
2172 return type(self)(block, new_idx)
2174 def get_slice(self, slobj: slice, axis: AxisInt = 0) -> SingleBlockManager:
2175 # Assertion disabled for performance
2176 # assert isinstance(slobj, slice), type(slobj)
2177 if axis >= self.ndim:
2178 raise IndexError("Requested axis not found in manager")
2180 blk = self._block
2181 array = blk.values[slobj]
2182 bp = BlockPlacement(slice(0, len(array)))
2183 # TODO this method is only used in groupby SeriesSplitter at the moment,
2184 # so passing refs is not yet covered by the tests
2185 block = type(blk)(array, placement=bp, ndim=1, refs=blk.refs)
2186 new_index = self.index._getitem_slice(slobj)
2187 return type(self)(block, new_index)
2189 @property
2190 def index(self) -> Index:
2191 return self.axes[0]
2193 @property
2194 def dtype(self) -> DtypeObj:
2195 return self._block.dtype
2197 def get_dtypes(self) -> npt.NDArray[np.object_]:
2198 return np.array([self._block.dtype], dtype=object)
2200 def external_values(self):
2201 """The array that Series.values returns"""
2202 return self._block.external_values()
2204 def internal_values(self):
2205 """The array that Series._values returns"""
2206 return self._block.values
2208 def array_values(self) -> ExtensionArray:
2209 """The array that Series.array returns"""
2210 return self._block.array_values
2212 def get_numeric_data(self) -> Self:
2213 if self._block.is_numeric:
2214 return self.copy(deep=False)
2215 return self.make_empty()
2217 @property
2218 def _can_hold_na(self) -> bool:
2219 return self._block._can_hold_na
2221 def setitem_inplace(self, indexer, value) -> None:
2222 """
2223 Set values with indexer.
2225 For SingleBlockManager, this backs s[indexer] = value
2227 This is an inplace version of `setitem()`, mutating the manager/values
2228 in place, not returning a new Manager (and Block), and thus never changing
2229 the dtype.
2230 """
2231 if not self._has_no_reference(0):
2232 self.blocks = (self._block.copy(deep=True),)
2233 self._reset_cache()
2235 arr = self.array
2237 # EAs will do this validation in their own __setitem__ methods.
2238 if isinstance(arr, np.ndarray):
2239 # Note: checking for ndarray instead of np.dtype means we exclude
2240 # dt64/td64, which do their own validation.
2241 value = np_can_hold_element(arr.dtype, value)
2243 if isinstance(value, np.ndarray) and value.ndim == 1 and len(value) == 1:
2244 # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615
2245 value = value[0, ...]
2247 arr[indexer] = value
2249 def idelete(self, indexer) -> SingleBlockManager:
2250 """
2251 Delete single location from SingleBlockManager.
2253 Ensures that self.blocks doesn't become empty.
2254 """
2255 nb = self._block.delete(indexer)[0]
2256 self.blocks = (nb,)
2257 self.axes[0] = self.axes[0].delete(indexer)
2258 self._reset_cache()
2259 return self
2261 def fast_xs(self, loc):
2262 """
2263 fast path for getting a cross-section
2264 return a view of the data
2265 """
2266 raise NotImplementedError("Use series._values[loc] instead")
2268 def set_values(self, values: ArrayLike) -> None:
2269 """
2270 Set the values of the single block in place.
2272 Use at your own risk! This does not check if the passed values are
2273 valid for the current Block/SingleBlockManager (length, dtype, etc),
2274 and this does not properly keep track of references.
2275 """
2276 # NOTE(CoW) Currently this is only used for FrameColumnApply.series_generator
2277 # which handles CoW by setting the refs manually if necessary
2278 self.blocks[0].values = values
2279 self.blocks[0]._mgr_locs = BlockPlacement(slice(len(values)))
2281 def _equal_values(self, other: Self) -> bool:
2282 """
2283 Used in .equals defined in base class. Only check the column values
2284 assuming shape and indexes have already been checked.
2285 """
2286 # For SingleBlockManager (i.e.Series)
2287 if other.ndim != 1:
2288 return False
2289 left = self.blocks[0].values
2290 right = other.blocks[0].values
2291 return array_equals(left, right)
2293 def grouped_reduce(self, func):
2294 arr = self.array
2295 res = func(arr)
2296 index = default_index(len(res))
2298 mgr = type(self).from_array(res, index)
2299 return mgr
2302# --------------------------------------------------------------------
2303# Constructor Helpers
2306def create_block_manager_from_blocks(
2307 blocks: list[Block],
2308 axes: list[Index],
2309 consolidate: bool = True,
2310 verify_integrity: bool = True,
2311) -> BlockManager:
2312 # If verify_integrity=False, then caller is responsible for checking
2313 # all(x.shape[-1] == len(axes[1]) for x in blocks)
2314 # sum(x.shape[0] for x in blocks) == len(axes[0])
2315 # set(x for blk in blocks for x in blk.mgr_locs) == set(range(len(axes[0])))
2316 # all(blk.ndim == 2 for blk in blocks)
2317 # This allows us to safely pass verify_integrity=False
2319 try:
2320 mgr = BlockManager(blocks, axes, verify_integrity=verify_integrity)
2322 except ValueError as err:
2323 arrays = [blk.values for blk in blocks]
2324 tot_items = sum(arr.shape[0] for arr in arrays)
2325 raise_construction_error(tot_items, arrays[0].shape[1:], axes, err)
2327 if consolidate:
2328 mgr._consolidate_inplace()
2329 return mgr
2332def create_block_manager_from_column_arrays(
2333 arrays: list[ArrayLike],
2334 axes: list[Index],
2335 consolidate: bool,
2336 refs: list,
2337) -> BlockManager:
2338 # Assertions disabled for performance (caller is responsible for verifying)
2339 # assert isinstance(axes, list)
2340 # assert all(isinstance(x, Index) for x in axes)
2341 # assert all(isinstance(x, (np.ndarray, ExtensionArray)) for x in arrays)
2342 # assert all(type(x) is not NumpyExtensionArray for x in arrays)
2343 # assert all(x.ndim == 1 for x in arrays)
2344 # assert all(len(x) == len(axes[1]) for x in arrays)
2345 # assert len(arrays) == len(axes[0])
2346 # These last three are sufficient to allow us to safely pass
2347 # verify_integrity=False below.
2349 try:
2350 blocks = _form_blocks(arrays, consolidate, refs)
2351 mgr = BlockManager(blocks, axes, verify_integrity=False)
2352 except ValueError as e:
2353 raise_construction_error(len(arrays), arrays[0].shape, axes, e)
2354 if consolidate:
2355 mgr._consolidate_inplace()
2356 return mgr
2359def raise_construction_error(
2360 tot_items: int,
2361 block_shape: Shape,
2362 axes: list[Index],
2363 e: ValueError | None = None,
2364) -> NoReturn:
2365 """raise a helpful message about our construction"""
2366 passed = tuple(map(int, [tot_items, *block_shape]))
2367 # Correcting the user facing error message during dataframe construction
2368 if len(passed) <= 2:
2369 passed = passed[::-1]
2371 implied = tuple(len(ax) for ax in axes)
2372 # Correcting the user facing error message during dataframe construction
2373 if len(implied) <= 2:
2374 implied = implied[::-1]
2376 # We return the exception object instead of raising it so that we
2377 # can raise it in the caller; mypy plays better with that
2378 if passed == implied and e is not None:
2379 raise e
2380 if block_shape[0] == 0:
2381 raise ValueError("Empty data passed with indices specified.")
2382 raise ValueError(f"Shape of passed values is {passed}, indices imply {implied}")
2385# -----------------------------------------------------------------------
2388def _grouping_func(tup: tuple[int, ArrayLike]) -> tuple[int, DtypeObj]:
2389 dtype = tup[1].dtype
2391 if is_1d_only_ea_dtype(dtype):
2392 # We know these won't be consolidated, so don't need to group these.
2393 # This avoids expensive comparisons of CategoricalDtype objects
2394 sep = id(dtype)
2395 else:
2396 sep = 0
2398 return sep, dtype
2401def _form_blocks(arrays: list[ArrayLike], consolidate: bool, refs: list) -> list[Block]:
2402 tuples = enumerate(arrays)
2404 if not consolidate:
2405 return _tuples_to_blocks_no_consolidate(tuples, refs)
2407 # when consolidating, we can ignore refs (either stacking always copies,
2408 # or the EA is already copied in the calling dict_to_mgr)
2410 # group by dtype
2411 grouper = itertools.groupby(tuples, _grouping_func)
2413 nbs: list[Block] = []
2414 for (_, dtype), tup_block in grouper:
2415 block_type = get_block_type(dtype)
2417 if isinstance(dtype, np.dtype):
2418 is_dtlike = dtype.kind in "mM"
2420 if issubclass(dtype.type, (str, bytes)):
2421 dtype = np.dtype(object)
2423 values, placement = _stack_arrays(tup_block, dtype)
2424 if is_dtlike:
2425 values = ensure_wrapped_if_datetimelike(values)
2426 blk = block_type(values, placement=BlockPlacement(placement), ndim=2)
2427 nbs.append(blk)
2429 elif is_1d_only_ea_dtype(dtype):
2430 dtype_blocks = [
2431 block_type(x[1], placement=BlockPlacement(x[0]), ndim=2)
2432 for x in tup_block
2433 ]
2434 nbs.extend(dtype_blocks)
2436 else:
2437 dtype_blocks = [
2438 block_type(
2439 ensure_block_shape(x[1], 2), placement=BlockPlacement(x[0]), ndim=2
2440 )
2441 for x in tup_block
2442 ]
2443 nbs.extend(dtype_blocks)
2444 return nbs
2447def _tuples_to_blocks_no_consolidate(tuples, refs) -> list[Block]:
2448 # tuples produced within _form_blocks are of the form (placement, array)
2449 return [
2450 new_block_2d(
2451 ensure_block_shape(arr, ndim=2), placement=BlockPlacement(i), refs=ref
2452 )
2453 for ((i, arr), ref) in zip(tuples, refs, strict=True)
2454 ]
2457def _stack_arrays(tuples, dtype: np.dtype):
2458 placement, arrays = zip(*tuples, strict=True)
2460 first = arrays[0]
2461 shape = (len(arrays), *first.shape)
2463 stacked = np.empty(shape, dtype=dtype)
2464 for i, arr in enumerate(arrays):
2465 stacked[i] = arr
2467 return stacked, placement
2470def _consolidate(blocks: tuple[Block, ...]) -> tuple[Block, ...]:
2471 """
2472 Merge blocks having same dtype, exclude non-consolidating blocks
2473 """
2474 # sort by _can_consolidate, dtype
2475 gkey = lambda x: x._consolidate_key
2476 grouper = itertools.groupby(sorted(blocks, key=gkey), gkey)
2478 new_blocks: list[Block] = []
2479 for (_can_consolidate, dtype), group_blocks in grouper:
2480 merged_blocks, _ = _merge_blocks(
2481 list(group_blocks), dtype=dtype, can_consolidate=_can_consolidate
2482 )
2483 new_blocks = extend_blocks(merged_blocks, new_blocks)
2484 return tuple(new_blocks)
2487def _merge_blocks(
2488 blocks: list[Block], dtype: DtypeObj, can_consolidate: bool
2489) -> tuple[list[Block], bool]:
2490 if len(blocks) == 1:
2491 return blocks, False
2493 if can_consolidate:
2494 # TODO: optimization potential in case all mgrs contain slices and
2495 # combination of those slices is a slice, too.
2496 new_mgr_locs = np.concatenate([b.mgr_locs.as_array for b in blocks])
2498 new_values: ArrayLike
2500 if isinstance(blocks[0].dtype, np.dtype):
2501 # error: List comprehension has incompatible type List[Union[ndarray,
2502 # ExtensionArray]]; expected List[Union[complex, generic,
2503 # Sequence[Union[int, float, complex, str, bytes, generic]],
2504 # Sequence[Sequence[Any]], SupportsArray]]
2505 new_values = np.vstack([b.values for b in blocks]) # type: ignore[misc]
2506 else:
2507 bvals = [blk.values for blk in blocks]
2508 bvals2 = cast(Sequence[NDArrayBackedExtensionArray], bvals)
2509 new_values = bvals2[0]._concat_same_type(bvals2, axis=0)
2511 argsort = np.argsort(new_mgr_locs)
2512 new_values = new_values[argsort]
2513 new_mgr_locs = new_mgr_locs[argsort]
2515 bp = BlockPlacement(new_mgr_locs)
2516 return [new_block_2d(new_values, placement=bp)], True
2518 # can't consolidate --> no merge
2519 return blocks, False
2522def _preprocess_slice_or_indexer(
2523 slice_or_indexer: slice | np.ndarray, length: int, allow_fill: bool
2524):
2525 if isinstance(slice_or_indexer, slice):
2526 return (
2527 "slice",
2528 slice_or_indexer,
2529 libinternals.slice_len(slice_or_indexer, length),
2530 )
2531 else:
2532 if (
2533 not isinstance(slice_or_indexer, np.ndarray)
2534 or slice_or_indexer.dtype.kind != "i"
2535 ):
2536 dtype = getattr(slice_or_indexer, "dtype", None)
2537 raise TypeError(type(slice_or_indexer), dtype)
2539 indexer = ensure_platform_int(slice_or_indexer)
2540 if not allow_fill:
2541 indexer = maybe_convert_indices(indexer, length)
2542 return "fancy", indexer, len(indexer)
2545def make_na_array(dtype: DtypeObj, shape: Shape, fill_value) -> ArrayLike:
2546 if isinstance(dtype, DatetimeTZDtype):
2547 # NB: exclude e.g. pyarrow[dt64tz] dtypes
2548 ts = Timestamp(fill_value).as_unit(dtype.unit)
2549 i8values = np.full(shape, ts._value)
2550 dt64values = i8values.view(f"M8[{dtype.unit}]")
2551 return DatetimeArray._simple_new(dt64values, dtype=dtype)
2553 elif is_1d_only_ea_dtype(dtype):
2554 dtype = cast(ExtensionDtype, dtype)
2555 cls = dtype.construct_array_type()
2557 missing_arr = cls._from_sequence([], dtype=dtype)
2558 ncols, nrows = shape
2559 assert ncols == 1, ncols
2560 empty_arr = -1 * np.ones((nrows,), dtype=np.intp)
2561 return missing_arr.take(empty_arr, allow_fill=True, fill_value=fill_value)
2562 elif isinstance(dtype, ExtensionDtype):
2563 # TODO: no tests get here, a handful would if we disabled
2564 # the dt64tz special-case above (which is faster)
2565 cls = dtype.construct_array_type()
2566 missing_arr = cls._empty(shape=shape, dtype=dtype)
2567 missing_arr[:] = fill_value
2568 return missing_arr
2569 else:
2570 # NB: we should never get here with dtype integer or bool;
2571 # if we did, the missing_arr.fill would cast to gibberish
2572 missing_arr_np = np.empty(shape, dtype=dtype)
2573 missing_arr_np.fill(fill_value)
2575 if dtype.kind in "mM":
2576 missing_arr_np = ensure_wrapped_if_datetimelike(missing_arr_np)
2577 return missing_arr_np