Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Functions for preparing various inputs passed to the DataFrame or Series
3constructors before passing them to a BlockManager.
4"""
6from __future__ import annotations
8from collections import abc
9from typing import (
10 TYPE_CHECKING,
11 Any,
12)
13import warnings
15import numpy as np
16from numpy import ma
18from pandas._config import using_string_dtype
20from pandas._libs import lib
21from pandas.errors import Pandas4Warning
22from pandas.util._exceptions import find_stack_level
24from pandas.core.dtypes.astype import astype_is_view
25from pandas.core.dtypes.cast import (
26 construct_1d_arraylike_from_scalar,
27 construct_1d_object_array_from_listlike,
28 dict_compat,
29 maybe_cast_to_datetime,
30)
31from pandas.core.dtypes.common import (
32 is_1d_only_ea_dtype,
33 is_integer_dtype,
34 is_list_like,
35 is_named_tuple,
36 is_object_dtype,
37 is_scalar,
38)
39from pandas.core.dtypes.dtypes import (
40 BaseMaskedDtype,
41 ExtensionDtype,
42)
43from pandas.core.dtypes.generic import (
44 ABCDataFrame,
45 ABCSeries,
46)
47from pandas.core.dtypes.missing import isna
49from pandas.core import (
50 algorithms,
51 common as com,
52)
53from pandas.core.arrays import ExtensionArray
54from pandas.core.arrays.string_ import StringDtype
55from pandas.core.construction import (
56 array as pd_array,
57 extract_array,
58 range_to_ndarray,
59 sanitize_array,
60)
61from pandas.core.indexes.api import (
62 DatetimeIndex,
63 Index,
64 MultiIndex,
65 TimedeltaIndex,
66 default_index,
67 ensure_index,
68 get_objs_combined_axis,
69 maybe_sequence_to_range,
70 union_indexes,
71)
72from pandas.core.internals.blocks import (
73 BlockPlacement,
74 ensure_block_shape,
75 new_block,
76 new_block_2d,
77)
78from pandas.core.internals.managers import (
79 create_block_manager_from_blocks,
80 create_block_manager_from_column_arrays,
81)
83if TYPE_CHECKING:
84 from collections.abc import (
85 Hashable,
86 Sequence,
87 )
89 from pandas._typing import (
90 ArrayLike,
91 DtypeObj,
92 Manager,
93 npt,
94 )
95# ---------------------------------------------------------------------
96# BlockManager Interface
99def arrays_to_mgr(
100 arrays,
101 columns: Index,
102 index,
103 *,
104 dtype: DtypeObj | None = None,
105 verify_integrity: bool = True,
106 consolidate: bool = True,
107) -> Manager:
108 """
109 Segregate Series based on type and coerce into matrices.
111 Needs to handle a lot of exceptional cases.
112 """
113 if verify_integrity:
114 # figure out the index, if necessary
115 if index is None:
116 index = _extract_index(arrays)
117 else:
118 index = ensure_index(index)
120 # don't force copy because getting jammed in an ndarray anyway
121 arrays, refs = _homogenize(arrays, index, dtype)
122 # _homogenize ensures
123 # - all(len(x) == len(index) for x in arrays)
124 # - all(x.ndim == 1 for x in arrays)
125 # - all(isinstance(x, (np.ndarray, ExtensionArray)) for x in arrays)
126 # - all(type(x) is not NumpyExtensionArray for x in arrays)
128 else:
129 index = ensure_index(index)
130 arrays = [extract_array(x, extract_numpy=True) for x in arrays]
131 # with _from_arrays, the passed arrays should never be Series objects
132 refs = [None] * len(arrays)
134 # Reached via DataFrame._from_arrays; we do minimal validation here
135 for arr in arrays:
136 if (
137 not isinstance(arr, (np.ndarray, ExtensionArray))
138 or arr.ndim != 1
139 or len(arr) != len(index)
140 ):
141 raise ValueError(
142 "Arrays must be 1-dimensional np.ndarray or ExtensionArray "
143 "with length matching len(index)"
144 )
146 columns = ensure_index(columns)
147 if len(columns) != len(arrays):
148 raise ValueError("len(arrays) must match len(columns)")
150 # from BlockManager perspective
151 axes = [columns, index]
153 return create_block_manager_from_column_arrays(
154 arrays, axes, consolidate=consolidate, refs=refs
155 )
158def rec_array_to_mgr(
159 data: np.rec.recarray | np.ndarray,
160 index,
161 columns,
162 dtype: DtypeObj | None,
163 copy: bool,
164) -> Manager:
165 """
166 Extract from a masked rec array and create the manager.
167 """
168 # essentially process a record array then fill it
169 fdata = ma.getdata(data)
170 if index is None:
171 index = default_index(len(fdata))
172 else:
173 index = ensure_index(index)
175 if columns is not None:
176 columns = ensure_index(columns)
177 arrays, arr_columns = to_arrays(fdata, columns)
179 # create the manager
181 arrays, arr_columns = reorder_arrays(arrays, arr_columns, columns, len(index))
182 if columns is None:
183 columns = arr_columns
185 mgr = arrays_to_mgr(arrays, columns, index, dtype=dtype)
187 if copy:
188 mgr = mgr.copy(deep=True)
189 return mgr
192# ---------------------------------------------------------------------
193# DataFrame Constructor Interface
196def ndarray_to_mgr(
197 values, index, columns, dtype: DtypeObj | None, copy: bool
198) -> Manager:
199 # used in DataFrame.__init__
200 # input must be an ndarray, list, Series, Index, ExtensionArray
201 infer_object = not isinstance(values, (ABCSeries, Index, ExtensionArray))
203 if isinstance(values, ABCSeries):
204 if columns is None:
205 if values.name is not None:
206 columns = Index([values.name])
207 if index is None:
208 index = values.index
209 else:
210 values = values.reindex(index)
212 # zero len case (GH #2234)
213 if not len(values) and columns is not None and len(columns):
214 values = np.empty((0, 1), dtype=object)
216 vdtype = getattr(values, "dtype", None)
217 refs = None
218 if is_1d_only_ea_dtype(vdtype) or is_1d_only_ea_dtype(dtype):
219 # GH#19157
221 if isinstance(values, (np.ndarray, ExtensionArray)) and values.ndim > 1:
222 # GH#12513 an EA dtype passed with a 2D array, split into
223 # multiple EAs that view the values
224 # error: No overload variant of "__getitem__" of "ExtensionArray"
225 # matches argument type "Tuple[slice, int]"
226 values = [
227 values[:, n] # type: ignore[call-overload]
228 for n in range(values.shape[1])
229 ]
230 else:
231 values = [values]
233 # Handle copy semantics: already copy 1d-only EA. Other arrays will
234 # be copied when consolidating the blocks
235 if copy:
236 values = [
237 (x.copy(deep=True) if isinstance(x, Index) else x.copy())
238 if isinstance(x, (ExtensionArray, Index, ABCSeries))
239 and is_1d_only_ea_dtype(x.dtype)
240 else x
241 for x in values
242 ]
244 if columns is None:
245 columns = Index(range(len(values)))
246 else:
247 columns = ensure_index(columns)
249 return arrays_to_mgr(values, columns, index, dtype=dtype, consolidate=copy)
251 if isinstance(values, (ABCSeries, Index)):
252 if not copy and (dtype is None or astype_is_view(values.dtype, dtype)):
253 refs = values._references
255 if isinstance(vdtype, ExtensionDtype):
256 # i.e. Datetime64TZ, PeriodDtype; cases with is_1d_only_ea_dtype(vdtype)
257 # are already caught above
258 values = extract_array(values, extract_numpy=True)
259 if copy:
260 values = values.copy()
261 if values.ndim == 1:
262 values = values.reshape(-1, 1)
264 elif isinstance(values, (ABCSeries, Index)):
265 if copy:
266 values = values._values.copy()
267 else:
268 values = values._values
270 values = _ensure_2d(values)
272 elif isinstance(values, (np.ndarray, ExtensionArray)):
273 # drop subclass info
274 if copy and (dtype is None or astype_is_view(values.dtype, dtype)):
275 # only force a copy now if copy=True was requested
276 # and a subsequent `astype` will not already result in a copy
277 values = np.array(values, copy=True, order="F")
278 else:
279 values = np.asarray(values)
280 values = _ensure_2d(values)
282 else:
283 # by definition an array here
284 # the dtypes will be coerced to a single dtype
285 values = _prep_ndarraylike(values, copy=copy)
287 if dtype is not None and values.dtype != dtype:
288 # GH#40110 see similar check inside sanitize_array
289 values = sanitize_array(
290 values,
291 None,
292 dtype=dtype,
293 copy=copy,
294 allow_2d=True,
295 )
297 # _prep_ndarraylike ensures that values.ndim == 2 at this point
298 index, columns = _get_axes(
299 values.shape[0], values.shape[1], index=index, columns=columns
300 )
302 _check_values_indices_shape_match(values, index, columns)
304 values = values.T
306 # if we don't have a dtype specified, then try to convert objects
307 # on the entire block; this is to convert if we have datetimelike's
308 # embedded in an object type
309 if dtype is None and infer_object and is_object_dtype(values.dtype):
310 obj_columns = list(values)
311 maybe_datetime = [
312 lib.maybe_convert_objects(
313 x,
314 # Here we do not convert numeric dtypes, as if we wanted that,
315 # numpy would have done it for us.
316 convert_numeric=False,
317 convert_non_numeric=True,
318 convert_to_nullable_dtype=False,
319 dtype_if_all_nat=np.dtype("M8[s]"),
320 )
321 for x in obj_columns
322 ]
323 # don't convert (and copy) the objects if no type inference occurs
324 if any(x is not y for x, y in zip(obj_columns, maybe_datetime, strict=True)):
325 block_values = [
326 new_block_2d(ensure_block_shape(dval, 2), placement=BlockPlacement(n))
327 for n, dval in enumerate(maybe_datetime)
328 ]
329 else:
330 bp = BlockPlacement(slice(len(columns)))
331 nb = new_block_2d(values, placement=bp, refs=refs)
332 block_values = [nb]
333 elif dtype is None and values.dtype.kind == "U" and using_string_dtype():
334 dtype = StringDtype(na_value=np.nan)
336 obj_columns = list(values)
337 block_values = [
338 new_block(
339 dtype.construct_array_type()._from_sequence(data, dtype=dtype),
340 BlockPlacement(slice(i, i + 1)),
341 ndim=2,
342 )
343 for i, data in enumerate(obj_columns)
344 ]
346 else:
347 bp = BlockPlacement(slice(len(columns)))
348 nb = new_block_2d(values, placement=bp, refs=refs)
349 block_values = [nb]
351 if len(columns) == 0:
352 # TODO: check len(values) == 0?
353 block_values = []
355 return create_block_manager_from_blocks(
356 block_values, [columns, index], verify_integrity=False
357 )
360def _check_values_indices_shape_match(
361 values: np.ndarray, index: Index, columns: Index
362) -> None:
363 """
364 Check that the shape implied by our axes matches the actual shape of the
365 data.
366 """
367 if values.shape[1] != len(columns) or values.shape[0] != len(index):
368 # Could let this raise in Block constructor, but we get a more
369 # helpful exception message this way.
370 if values.shape[0] == 0 < len(index):
371 raise ValueError("Empty data passed with indices specified.")
373 passed = values.shape
374 implied = (len(index), len(columns))
375 raise ValueError(f"Shape of passed values is {passed}, indices imply {implied}")
378def dict_to_mgr(
379 data: dict,
380 index,
381 columns,
382 *,
383 dtype: DtypeObj | None = None,
384 copy: bool = True,
385) -> Manager:
386 """
387 Segregate Series based on type and coerce into matrices.
388 Needs to handle a lot of exceptional cases.
390 Used in DataFrame.__init__
391 """
392 arrays: Sequence[Any]
394 if columns is not None:
395 columns = ensure_index(columns)
396 if dtype is not None and not isinstance(dtype, np.dtype):
397 # e.g. test_dataframe_from_dict_of_series
398 arrays = [dtype.na_value] * len(columns)
399 else:
400 arrays = [np.nan] * len(columns)
401 midxs = set()
402 data_keys = ensure_index(data.keys()) # type: ignore[arg-type]
403 data_values = list(data.values())
405 for i, column in enumerate(columns):
406 try:
407 idx = data_keys.get_loc(column)
408 except KeyError:
409 midxs.add(i)
410 continue
411 array = data_values[idx]
412 arrays[i] = array
413 if is_scalar(array) and isna(array):
414 midxs.add(i)
416 if index is None:
417 # GH10856
418 # raise ValueError if only scalars in dict
419 if midxs:
420 index = _extract_index(
421 [array for i, array in enumerate(arrays) if i not in midxs]
422 )
423 else:
424 index = _extract_index(arrays)
425 else:
426 index = ensure_index(index)
428 # no obvious "empty" int column
429 if midxs and not is_integer_dtype(dtype):
430 # GH#1783
431 for i in midxs:
432 arr = construct_1d_arraylike_from_scalar(
433 arrays[i],
434 len(index),
435 dtype if dtype is not None else np.dtype("object"),
436 )
437 arrays[i] = arr
439 else:
440 keys = maybe_sequence_to_range(list(data.keys()))
441 columns = Index(keys) if keys else default_index(0)
442 arrays = [com.maybe_iterable_to_list(data[k]) for k in keys]
444 if copy:
445 # We only need to copy arrays that will not get consolidated, i.e.
446 # only EA arrays
447 arrays = [
448 (
449 x.copy()
450 if isinstance(x, ExtensionArray)
451 else (
452 x.copy(deep=True)
453 if (
454 isinstance(x, Index)
455 or (isinstance(x, ABCSeries) and is_1d_only_ea_dtype(x.dtype))
456 )
457 else x
458 )
459 )
460 for x in arrays
461 ]
463 return arrays_to_mgr(arrays, columns, index, dtype=dtype, consolidate=copy)
466def nested_data_to_arrays(
467 data: Sequence,
468 columns: Index | None,
469 index: Index | None,
470 dtype: DtypeObj | None,
471) -> tuple[list[ArrayLike], Index, Index]:
472 """
473 Convert a single sequence of arrays to multiple arrays.
474 """
475 # By the time we get here we have already checked treat_as_nested(data)
477 if is_named_tuple(data[0]) and columns is None:
478 columns = ensure_index(data[0]._fields)
480 arrays, columns = to_arrays(data, columns, dtype=dtype)
481 columns = ensure_index(columns)
483 if index is None:
484 if isinstance(data[0], ABCSeries):
485 index = _get_names_from_index(data)
486 else:
487 index = default_index(len(data))
489 return arrays, columns, index
492def treat_as_nested(data) -> bool:
493 """
494 Check if we should use nested_data_to_arrays.
495 """
496 return (
497 len(data) > 0
498 and is_list_like(data[0])
499 and getattr(data[0], "ndim", 1) == 1
500 and not (isinstance(data, ExtensionArray) and data.ndim == 2)
501 )
504# ---------------------------------------------------------------------
507def _prep_ndarraylike(values, copy: bool = True) -> np.ndarray:
508 # values is specifically _not_ ndarray, EA, Index, or Series
509 # We only get here with `not treat_as_nested(values)`
511 if len(values) == 0:
512 # TODO: check for length-zero range, in which case return int64 dtype?
513 # TODO: reuse anything in try_cast?
514 return np.empty((0, 0), dtype=object)
515 elif isinstance(values, range):
516 arr = range_to_ndarray(values)
517 return arr[..., np.newaxis]
519 def convert(v):
520 if not is_list_like(v) or isinstance(v, ABCDataFrame):
521 return v
523 v = extract_array(v, extract_numpy=True)
524 if isinstance(v, (list, tuple, range)):
525 v = construct_1d_object_array_from_listlike(v)
526 if isinstance(v, np.ndarray) and v.dtype == object:
527 v = lib.maybe_convert_objects(v)
528 # We don't do maybe_infer_objects here bc we will end up doing
529 # it column-by-column in ndarray_to_mgr
530 return v
532 # we could have a 1-dim or 2-dim list here
533 # this is equiv of np.asarray, but does object conversion
534 # and platform dtype preservation
535 # does not convert e.g. [1, "a", True] to ["1", "a", "True"] like
536 # np.asarray would
537 if is_list_like(values[0]):
538 values = np.array([convert(v) for v in values])
539 elif isinstance(values[0], np.ndarray) and values[0].ndim == 0:
540 # GH#21861 see test_constructor_list_of_lists
541 values = np.array([convert(v) for v in values])
542 else:
543 values = convert(values)
545 return _ensure_2d(values)
548def _ensure_2d(values: np.ndarray) -> np.ndarray:
549 """
550 Reshape 1D values, raise on anything else other than 2D.
551 """
552 if values.ndim == 1:
553 values = values.reshape((values.shape[0], 1))
554 elif values.ndim != 2:
555 raise ValueError(f"Must pass 2-d input. shape={values.shape}")
556 return values
559def _homogenize(
560 data, index: Index, dtype: DtypeObj | None
561) -> tuple[list[ArrayLike], list[Any]]:
562 oindex = None
563 homogenized = []
564 # if the original array-like in `data` is a Series, keep track of this Series' refs
565 refs: list[Any] = []
567 for val in data:
568 if isinstance(val, (ABCSeries, Index)):
569 if dtype is not None:
570 val = val.astype(dtype)
571 if isinstance(val, ABCSeries) and val.index._id is not index._id:
572 # Forces alignment. No need to copy data since we
573 # are putting it into an ndarray later
574 val = val.reindex(index)
575 refs.append(val._references)
576 val = val._values
577 else:
578 if isinstance(val, dict):
579 # GH#41785 this _should_ be equivalent to (but faster than)
580 # val = Series(val, index=index)._values
581 if oindex is None:
582 oindex = index.astype("O")
584 if isinstance(index, (DatetimeIndex, TimedeltaIndex)):
585 # see test_constructor_dict_datetime64_index
586 val = dict_compat(val)
587 else:
588 # see test_constructor_subclass_dict
589 val = dict(val)
591 if not isinstance(index, MultiIndex) and index.hasnans:
592 # GH#63889 Check if dict has missing value keys that need special
593 # handling (i.e. None/np.nan/pd.NA might no longer be matched
594 # when using fast_multiget with processed object index values)
595 from pandas import Series
597 val = Series(val).reindex(index)._values
598 else:
599 # Fast path: use lib.fast_multiget for dicts without missing keys
600 val = lib.fast_multiget(val, oindex._values, default=np.nan) # type: ignore[arg-type]
602 val = sanitize_array(val, index, dtype=dtype, copy=False)
603 com.require_length_match(val, index)
604 refs.append(None)
606 homogenized.append(val)
608 return homogenized, refs
611def _extract_index(data) -> Index:
612 """
613 Try to infer an Index from the passed data, raise ValueError on failure.
614 """
615 index: Index
616 if len(data) == 0:
617 return default_index(0)
619 raw_lengths = set()
620 indexes: list[list[Hashable] | Index] = []
622 have_raw_arrays = False
623 have_series = False
624 have_dicts = False
626 for val in data:
627 if isinstance(val, ABCSeries):
628 have_series = True
629 indexes.append(val.index)
630 elif isinstance(val, dict):
631 have_dicts = True
632 indexes.append(list(val.keys()))
633 elif is_list_like(val) and getattr(val, "ndim", 1) == 1:
634 have_raw_arrays = True
635 raw_lengths.add(len(val))
636 elif isinstance(val, np.ndarray) and val.ndim > 1:
637 raise ValueError("Per-column arrays must each be 1-dimensional")
639 if not indexes and not raw_lengths:
640 raise ValueError("If using all scalar values, you must pass an index")
642 if have_series:
643 index, _ = union_indexes(indexes)
644 elif have_dicts:
645 index, _ = union_indexes(indexes, sort=False)
647 if have_raw_arrays:
648 if len(raw_lengths) > 1:
649 raise ValueError("All arrays must be of the same length")
651 if have_dicts:
652 raise ValueError(
653 "Mixing dicts with non-Series may lead to ambiguous ordering."
654 )
655 raw_length = raw_lengths.pop()
656 if have_series:
657 if raw_length != len(index):
658 msg = (
659 f"array length {raw_length} does not match index "
660 f"length {len(index)}"
661 )
662 raise ValueError(msg)
663 else:
664 index = default_index(raw_length)
666 return ensure_index(index)
669def reorder_arrays(
670 arrays: list[ArrayLike], arr_columns: Index, columns: Index | None, length: int
671) -> tuple[list[ArrayLike], Index]:
672 """
673 Preemptively (cheaply) reindex arrays with new columns.
674 """
675 # reorder according to the columns
676 if columns is not None:
677 if not columns.equals(arr_columns):
678 # if they are equal, there is nothing to do
679 new_arrays: list[ArrayLike] = []
680 indexer = arr_columns.get_indexer(columns)
681 for i, k in enumerate(indexer):
682 if k == -1:
683 # by convention default is all-NaN object dtype
684 arr = np.empty(length, dtype=object)
685 arr.fill(np.nan)
686 else:
687 arr = arrays[k]
688 new_arrays.append(arr)
690 arrays = new_arrays
691 arr_columns = columns
693 return arrays, arr_columns
696def _get_names_from_index(data) -> Index:
697 has_some_name = any(getattr(s, "name", None) is not None for s in data)
698 if not has_some_name:
699 return default_index(len(data))
701 index: list[Hashable] = list(range(len(data)))
702 count = 0
703 for i, s in enumerate(data):
704 n = getattr(s, "name", None)
705 if n is not None:
706 index[i] = n
707 else:
708 index[i] = f"Unnamed {count}"
709 count += 1
711 return Index(index)
714def _get_axes(
715 N: int, K: int, index: Index | None, columns: Index | None
716) -> tuple[Index, Index]:
717 # helper to create the axes as indexes
718 # return axes or defaults
720 if index is None:
721 index = default_index(N)
722 else:
723 index = ensure_index(index)
725 if columns is None:
726 columns = default_index(K)
727 else:
728 columns = ensure_index(columns)
729 return index, columns
732def dataclasses_to_dicts(data):
733 """
734 Converts a list of dataclass instances to a list of dictionaries.
736 Parameters
737 ----------
738 data : List[Type[dataclass]]
740 Returns
741 --------
742 list_dict : List[dict]
744 Examples
745 --------
746 >>> from dataclasses import dataclass
747 >>> @dataclass
748 ... class Point:
749 ... x: int
750 ... y: int
752 >>> dataclasses_to_dicts([Point(1, 2), Point(2, 3)])
753 [{'x': 1, 'y': 2}, {'x': 2, 'y': 3}]
755 """
756 from dataclasses import asdict
758 return list(map(asdict, data))
761# ---------------------------------------------------------------------
762# Conversion of Inputs to Arrays
765def to_arrays(
766 data, columns: Index | None, dtype: DtypeObj | None = None
767) -> tuple[list[ArrayLike], Index]:
768 """
769 Return list of arrays, columns.
771 Returns
772 -------
773 list[ArrayLike]
774 These will become columns in a DataFrame.
775 Index
776 This will become frame.columns.
778 Notes
779 -----
780 Ensures that len(result_arrays) == len(result_index).
781 """
783 if not len(data):
784 if isinstance(data, np.ndarray):
785 if data.dtype.names is not None:
786 # i.e. numpy structured array
787 columns = ensure_index(data.dtype.names)
788 arrays = [data[name] for name in columns]
790 if len(data) == 0:
791 # GH#42456 the indexing above results in list of 2D ndarrays
792 # TODO: is that an issue with numpy?
793 for i, arr in enumerate(arrays):
794 if arr.ndim == 2:
795 arrays[i] = arr[:, 0]
797 return arrays, columns
798 elif data.ndim == 2:
799 # GH#22025 - empty 2D ndarray
800 arrays = [data[:, idx] for idx in range(data.shape[1])]
801 if columns is None:
802 columns = default_index(data.shape[1])
803 return arrays, columns
804 return [], ensure_index([])
806 elif isinstance(data, np.ndarray) and data.dtype.names is not None:
807 # e.g. recarray
808 if columns is None:
809 columns = Index(data.dtype.names)
810 arrays = [data[k] for k in columns]
811 return arrays, columns
813 elif isinstance(data, np.ndarray) and data.ndim == 2:
814 # Plain 2D ndarray: slice columns directly instead of falling through
815 # to the "last ditch" path that converts each row to a tuple.
816 arrays = [data[:, idx] for idx in range(data.shape[1])]
817 if columns is None:
818 columns = default_index(data.shape[1])
819 return arrays, columns
821 if isinstance(data[0], (list, tuple)):
822 arr = _list_to_arrays(data)
823 elif isinstance(data[0], abc.Mapping):
824 arr, columns = _list_of_dict_to_arrays(data, columns)
825 elif isinstance(data[0], ABCSeries):
826 arr, columns = _list_of_series_to_arrays(data, columns)
827 else:
828 # last ditch effort
829 # GH#23985, GH#49593: if all rows are arrays with a uniform
830 # dtype, construct columns directly to preserve that dtype
831 # (e.g. timedelta64, pyarrow, Int64, Categorical). Only take this
832 # path for non-ragged rows; ragged rows fall through to the
833 # object/pad path below so they are padded to max width with NaN.
834 row_dtypes = {row.dtype for row in data if hasattr(row, "dtype")}
835 if (
836 len(row_dtypes) == 1
837 and all(hasattr(row, "dtype") for row in data)
838 and len({len(row) for row in data}) == 1
839 ):
840 common_dtype = row_dtypes.pop()
841 ncols = len(data[0])
842 arrays = [
843 pd_array([row[col_idx] for row in data], dtype=common_dtype)
844 for col_idx in range(ncols)
845 ]
846 columns = _validate_or_indexify_columns(arrays, columns)
847 return arrays, columns
849 data = [tuple(x) for x in data]
850 arr = _list_to_arrays(data)
852 content, columns = _finalize_columns_and_data(arr, columns, dtype)
853 return content, columns
856def _list_to_arrays(data: list[tuple | list]) -> np.ndarray:
857 # Returned np.ndarray has ndim = 2
858 # Note: we already check len(data) > 0 before getting hre
860 # GH#65751 shorter sequences get padded with NaN out to the longest one,
861 # which is deprecated. A null scalar counts as length 1, mirroring the
862 # handling in lib.to_object_array_tuples.
863 lengths = {1 if is_scalar(row) and isna(row) else len(row) for row in data}
864 if len(lengths) > 1:
865 warnings.warn(
866 "Constructing a DataFrame from a list of sequences with mismatched "
867 "lengths is deprecated and will raise in a future version. The "
868 "shorter sequences are currently padded with NaN; make all "
869 "sequences the same length before constructing instead.",
870 Pandas4Warning,
871 stacklevel=find_stack_level(),
872 )
874 if isinstance(data[0], tuple):
875 content = lib.to_object_array_tuples(data)
876 else:
877 # list of lists
878 content = lib.to_object_array(data)
879 return content
882def _list_of_series_to_arrays(
883 data: list,
884 columns: Index | None,
885) -> tuple[np.ndarray, Index]:
886 # returned np.ndarray has ndim == 2
888 if columns is None:
889 # We know pass_data is non-empty because data[0] is a Series
890 pass_data = [x for x in data if isinstance(x, (ABCSeries, ABCDataFrame))]
891 columns, _ = get_objs_combined_axis(pass_data, sort=False)
893 indexer_cache: dict[int, np.ndarray] = {}
895 aligned_values = []
896 for s in data:
897 index = getattr(s, "index", None)
898 if index is None:
899 index = default_index(len(s))
901 if id(index) in indexer_cache:
902 indexer = indexer_cache[id(index)]
903 else:
904 indexer = indexer_cache[id(index)] = index.get_indexer(columns)
906 values = extract_array(s, extract_numpy=True)
907 aligned_values.append(algorithms.take_nd(values, indexer))
909 content = np.vstack(aligned_values)
910 return content, columns
913def _list_of_dict_to_arrays(
914 data: list[dict],
915 columns: Index | None,
916) -> tuple[np.ndarray, Index]:
917 """
918 Convert list of dicts to numpy arrays
920 if `columns` is not passed, column names are inferred from the records
921 - for OrderedDict and dicts, the column names match
922 the key insertion-order from the first record to the last.
923 - For other kinds of dict-likes, the keys are lexically sorted.
925 Parameters
926 ----------
927 data : iterable
928 collection of records (OrderedDict, dict)
929 columns: iterables or None
931 Returns
932 -------
933 content : np.ndarray[object, ndim=2]
934 columns : Index
935 """
936 # assure that they are of the base dict class and not of derived
937 # classes
938 data = [d if type(d) is dict else dict(d) for d in data]
940 if columns is None:
941 gen = (list(x.keys()) for x in data)
942 sort = not any(isinstance(d, dict) for d in data)
943 pre_cols = lib.fast_unique_multiple_list_gen(gen, sort=sort)
944 columns = ensure_index(pre_cols)
946 # use pre_cols to preserve exact values that were present as dict keys
947 # (e.g. otherwise missing values might be coerced to the canonical repr)
948 content = lib.dicts_to_array(data, pre_cols)
949 else:
950 content = lib.dicts_to_array(data, list(columns))
952 return content, columns
955def _finalize_columns_and_data(
956 content: np.ndarray, # ndim == 2
957 columns: Index | None,
958 dtype: DtypeObj | None,
959) -> tuple[list[ArrayLike], Index]:
960 """
961 Ensure we have valid columns, cast object dtypes if possible.
962 """
963 contents = list(content.T)
965 try:
966 columns = _validate_or_indexify_columns(contents, columns)
967 except AssertionError as err:
968 # GH#26429 do not raise user-facing AssertionError
969 raise ValueError(err) from err
971 if contents and contents[0].dtype == np.object_:
972 contents = convert_object_array(contents, dtype=dtype)
974 return contents, columns
977def _validate_or_indexify_columns(
978 content: list[np.ndarray], columns: Index | None
979) -> Index:
980 """
981 If columns is None, make numbers as column names; Otherwise, validate that
982 columns have valid length.
984 Parameters
985 ----------
986 content : list of np.ndarrays
987 columns : Index or None
989 Returns
990 -------
991 Index
992 If columns is None, assign positional column index value as columns.
994 Raises
995 ------
996 AssertionError
997 When the number of columns does not match the number of arrays in
998 content.
999 """
1000 if columns is None:
1001 columns = default_index(len(content))
1002 elif len(columns) != len(content): # pragma: no cover
1003 # caller's responsibility to check for this...
1004 raise AssertionError(
1005 f"{len(columns)} columns passed, passed data had {len(content)} columns"
1006 )
1007 return columns
1010def convert_object_array(
1011 content: list[npt.NDArray[np.object_]],
1012 dtype: DtypeObj | None,
1013 dtype_backend: str = "numpy",
1014 coerce_float: bool = False,
1015) -> list[ArrayLike]:
1016 """
1017 Internal function to convert object array.
1019 Parameters
1020 ----------
1021 content: List[np.ndarray]
1022 dtype: np.dtype or ExtensionDtype
1023 dtype_backend: Controls if nullable/pyarrow dtypes are returned.
1024 coerce_float: Cast floats that are integers to int.
1026 Returns
1027 -------
1028 List[ArrayLike]
1029 """
1030 # provide soft conversion of object dtypes
1032 def convert(arr):
1033 if dtype != np.dtype("O"):
1034 # e.g. if dtype is UInt32 then we want to cast Nones to NA instead of
1035 # NaN in maybe_convert_objects.
1036 to_nullable = dtype_backend != "numpy" or isinstance(dtype, BaseMaskedDtype)
1037 arr = lib.maybe_convert_objects(
1038 arr,
1039 try_float=coerce_float,
1040 convert_to_nullable_dtype=to_nullable,
1041 )
1042 # Notes on cases that get here 2023-02-15
1043 # 1) we DO get here when arr is all Timestamps and dtype=None
1044 # 2) disabling this doesn't break the world, so this must be
1045 # getting caught at a higher level
1046 # 3) passing convert_non_numeric to maybe_convert_objects get this right
1047 # 4) convert_non_numeric?
1049 if dtype is None:
1050 if arr.dtype == np.dtype("O"):
1051 # i.e. maybe_convert_objects didn't convert
1052 convert_to_nullable_dtype = dtype_backend != "numpy"
1053 arr = lib.maybe_convert_objects(
1054 arr,
1055 # Here we do not convert numeric dtypes, as if we wanted that,
1056 # numpy would have done it for us.
1057 convert_numeric=False,
1058 convert_non_numeric=True,
1059 convert_to_nullable_dtype=convert_to_nullable_dtype,
1060 dtype_if_all_nat=np.dtype("M8[s]"),
1061 )
1062 if convert_to_nullable_dtype and arr.dtype == np.dtype("O"):
1063 new_dtype = StringDtype()
1064 arr_cls = new_dtype.construct_array_type()
1065 arr = arr_cls._from_sequence(arr, dtype=new_dtype)
1066 elif dtype_backend != "numpy" and isinstance(arr, np.ndarray):
1067 if arr.dtype.kind in "iufb":
1068 arr = pd_array(arr, copy=False)
1070 elif isinstance(dtype, ExtensionDtype):
1071 # TODO: test(s) that get here
1072 # TODO: try to de-duplicate this convert function with
1073 # core.construction functions
1074 cls = dtype.construct_array_type()
1075 arr = cls._from_sequence(arr, dtype=dtype, copy=False)
1076 elif dtype.kind in "mM":
1077 # This restriction is harmless bc these are the only cases
1078 # where maybe_cast_to_datetime is not a no-op.
1079 # Here we know:
1080 # 1) dtype.kind in "mM" and
1081 # 2) arr is either object or numeric dtype
1082 arr = maybe_cast_to_datetime(arr, dtype)
1084 return arr
1086 arrays = [convert(arr) for arr in content]
1088 return arrays