Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Concat routines.
3"""
5from __future__ import annotations
7from collections import abc
8from itertools import pairwise
9import types
10from typing import (
11 TYPE_CHECKING,
12 Literal,
13 cast,
14 overload,
15)
16import warnings
18import numpy as np
20from pandas._libs import lib
21from pandas.errors import Pandas4Warning
22from pandas.util._decorators import set_module
23from pandas.util._exceptions import find_stack_level
25from pandas.core.dtypes.common import (
26 is_bool,
27 is_scalar,
28)
29from pandas.core.dtypes.concat import concat_compat
30from pandas.core.dtypes.generic import (
31 ABCDataFrame,
32 ABCSeries,
33)
34from pandas.core.dtypes.missing import isna
36from pandas.core.arrays.categorical import (
37 factorize_from_iterable,
38 factorize_from_iterables,
39)
40import pandas.core.common as com
41from pandas.core.indexes.api import (
42 Index,
43 MultiIndex,
44 all_indexes_same,
45 default_index,
46 ensure_index,
47 get_objs_combined_axis,
48 get_unanimous_names,
49 union_indexes,
50)
51from pandas.core.indexes.datetimes import DatetimeIndex
52from pandas.core.internals import concatenate_managers
54if TYPE_CHECKING:
55 from collections.abc import (
56 Callable,
57 Hashable,
58 Iterable,
59 Mapping,
60 )
62 from pandas._typing import (
63 Axis,
64 AxisInt,
65 HashableT,
66 )
68 from pandas import (
69 DataFrame,
70 Series,
71 )
73# ---------------------------------------------------------------------
74# Concatenate DataFrame objects
77@overload
78def concat(
79 objs: Iterable[DataFrame] | Mapping[HashableT, DataFrame],
80 *,
81 axis: Literal[0, "index"] = ...,
82 join: str = ...,
83 ignore_index: bool = ...,
84 keys: Iterable[Hashable] | None = ...,
85 levels=...,
86 names: list[HashableT] | None = ...,
87 verify_integrity: bool = ...,
88 sort: bool = ...,
89 copy: bool | lib.NoDefault = ...,
90 union_categories: bool = ...,
91) -> DataFrame: ...
94@overload
95def concat(
96 objs: Iterable[Series] | Mapping[HashableT, Series],
97 *,
98 axis: Literal[0, "index"] = ...,
99 join: str = ...,
100 ignore_index: bool = ...,
101 keys: Iterable[Hashable] | None = ...,
102 levels=...,
103 names: list[HashableT] | None = ...,
104 verify_integrity: bool = ...,
105 sort: bool = ...,
106 copy: bool | lib.NoDefault = ...,
107 union_categories: bool = ...,
108) -> Series: ...
111@overload
112def concat(
113 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame],
114 *,
115 axis: Literal[0, "index"] = ...,
116 join: str = ...,
117 ignore_index: bool = ...,
118 keys: Iterable[Hashable] | None = ...,
119 levels=...,
120 names: list[HashableT] | None = ...,
121 verify_integrity: bool = ...,
122 sort: bool = ...,
123 copy: bool | lib.NoDefault = ...,
124 union_categories: bool = ...,
125) -> DataFrame | Series: ...
128@overload
129def concat(
130 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame],
131 *,
132 axis: Literal[1, "columns"],
133 join: str = ...,
134 ignore_index: bool = ...,
135 keys: Iterable[Hashable] | None = ...,
136 levels=...,
137 names: list[HashableT] | None = ...,
138 verify_integrity: bool = ...,
139 sort: bool = ...,
140 copy: bool | lib.NoDefault = ...,
141 union_categories: bool = ...,
142) -> DataFrame: ...
145@overload
146def concat(
147 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame],
148 *,
149 axis: Axis = ...,
150 join: str = ...,
151 ignore_index: bool = ...,
152 keys: Iterable[Hashable] | None = ...,
153 levels=...,
154 names: list[HashableT] | None = ...,
155 verify_integrity: bool = ...,
156 sort: bool = ...,
157 copy: bool | lib.NoDefault = ...,
158 union_categories: bool = ...,
159) -> DataFrame | Series: ...
162@set_module("pandas")
163def concat(
164 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame],
165 *,
166 axis: Axis = 0,
167 join: str = "outer",
168 ignore_index: bool = False,
169 keys: Iterable[Hashable] | None = None,
170 levels=None,
171 names: list[HashableT] | None = None,
172 verify_integrity: bool = False,
173 sort: bool | lib.NoDefault = lib.no_default,
174 copy: bool | lib.NoDefault = lib.no_default,
175 union_categories: bool = False,
176) -> DataFrame | Series:
177 """
178 Concatenate pandas objects along a particular axis.
180 Allows optional set logic along the other axes.
182 Can also add a layer of hierarchical indexing on the concatenation axis,
183 which may be useful if the labels are the same (or overlapping) on
184 the passed axis number.
186 Parameters
187 ----------
188 objs : an iterable or mapping of Series or DataFrame objects
189 If a mapping is passed, the keys will be used as the `keys`
190 argument, unless it is passed, in which case the values will be
191 selected (see below). Any None objects will be dropped silently unless
192 they are all None in which case a ValueError will be raised.
193 axis : {0/'index', 1/'columns'}, default 0
194 The axis to concatenate along.
195 join : {'inner', 'outer'}, default 'outer'
196 How to handle indexes on other axis (or axes).
197 ignore_index : bool, default False
198 If True, do not use the index values along the concatenation axis. The
199 resulting axis will be labeled 0, ..., n - 1. This is useful if you are
200 concatenating objects where the concatenation axis does not have
201 meaningful indexing information. Note the index values on the other
202 axes are still respected in the join.
203 keys : sequence, default None
204 If multiple levels passed, should contain tuples. Construct
205 hierarchical index using the passed keys as the outermost level.
206 levels : list of sequences, default None
207 Specific levels (unique values) to use for constructing a
208 MultiIndex. Otherwise they will be inferred from the keys.
209 names : list, default None
210 Names for the levels in the resulting hierarchical index.
211 verify_integrity : bool, default False
212 Check whether the new concatenated axis contains duplicates. This can
213 be very expensive relative to the actual data concatenation.
214 sort : bool, default False
215 Sort non-concatenation axis. One exception to this is when the
216 non-concatenation axis is a DatetimeIndex and join='outer' and the axis is
217 not already aligned. In that case, the non-concatenation axis is always
218 sorted lexicographically.
219 copy : bool, default False
220 This keyword is now ignored; changing its value will have no
221 impact on the method.
223 .. deprecated:: 3.0.0
225 This keyword is ignored and will be removed in pandas 4.0. Since
226 pandas 3.0, this method always returns a new object using a lazy
227 copy mechanism that defers copies until necessary
228 (Copy-on-Write). See the `user guide on Copy-on-Write
229 <https://pandas.pydata.org/docs/dev/user_guide/migration.html>`__
230 for more details.
232 union_categories : bool, default False
233 If True and all values for a given column have categorical dtype,
234 the resulting column will preserve categorical dtype with the
235 union of the categories, rather than casting to the underlying
236 dtype. Categories whose dtypes differ are cast to a common dtype.
237 The result is unordered unless every input is ordered with the
238 same categories (after any casting to a common dtype) in the same
239 order, in which case the result is ordered.
240 Has no effect when concatenating mixed categorical and
241 non-categorical data.
243 .. versionadded:: 3.1.0
245 Returns
246 -------
247 object, type of objs
248 When concatenating all ``Series`` along the index (axis=0), a
249 ``Series`` is returned. When ``objs`` contains at least one
250 ``DataFrame``, a ``DataFrame`` is returned. When concatenating along
251 the columns (axis=1), a ``DataFrame`` is returned.
253 See Also
254 --------
255 DataFrame.join : Join DataFrames using indexes.
256 DataFrame.merge : Merge DataFrames by indexes or columns.
258 Notes
259 -----
260 The keys, levels, and names arguments are all optional.
262 A walkthrough of how this method fits in with other tools for combining
263 pandas objects can be found `here
264 <https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html>`__.
266 It is not recommended to build DataFrames by adding single rows in a
267 for loop. Build a list of rows and make a DataFrame in a single concat.
269 Examples
270 --------
271 Combine two ``Series``.
273 >>> s1 = pd.Series(["a", "b"])
274 >>> s2 = pd.Series(["c", "d"])
275 >>> pd.concat([s1, s2])
276 0 a
277 1 b
278 0 c
279 1 d
280 dtype: str
282 Clear the existing index and reset it in the result
283 by setting the ``ignore_index`` option to ``True``.
285 >>> pd.concat([s1, s2], ignore_index=True)
286 0 a
287 1 b
288 2 c
289 3 d
290 dtype: str
292 Add a hierarchical index at the outermost level of
293 the data with the ``keys`` option.
295 >>> pd.concat([s1, s2], keys=["s1", "s2"])
296 s1 0 a
297 1 b
298 s2 0 c
299 1 d
300 dtype: str
302 Label the index keys you create with the ``names`` option.
304 >>> pd.concat([s1, s2], keys=["s1", "s2"], names=["Series name", "Row ID"])
305 Series name Row ID
306 s1 0 a
307 1 b
308 s2 0 c
309 1 d
310 dtype: str
312 Combine two ``DataFrame`` objects with identical columns.
314 >>> df1 = pd.DataFrame([["a", 1], ["b", 2]], columns=["letter", "number"])
315 >>> df1
316 letter number
317 0 a 1
318 1 b 2
319 >>> df2 = pd.DataFrame([["c", 3], ["d", 4]], columns=["letter", "number"])
320 >>> df2
321 letter number
322 0 c 3
323 1 d 4
324 >>> pd.concat([df1, df2])
325 letter number
326 0 a 1
327 1 b 2
328 0 c 3
329 1 d 4
331 Combine ``DataFrame`` objects with overlapping columns
332 and return everything. Columns outside the intersection will
333 be filled with ``NaN`` values.
335 >>> df3 = pd.DataFrame(
336 ... [["c", 3, "cat"], ["d", 4, "dog"]], columns=["letter", "number", "animal"]
337 ... )
338 >>> df3
339 letter number animal
340 0 c 3 cat
341 1 d 4 dog
342 >>> pd.concat([df1, df3], sort=False)
343 letter number animal
344 0 a 1 NaN
345 1 b 2 NaN
346 0 c 3 cat
347 1 d 4 dog
349 Combine ``DataFrame`` objects with overlapping columns
350 and return only those that are shared by passing ``inner`` to
351 the ``join`` keyword argument.
353 >>> pd.concat([df1, df3], join="inner")
354 letter number
355 0 a 1
356 1 b 2
357 0 c 3
358 1 d 4
360 Combine ``DataFrame`` objects horizontally along the x axis by
361 passing in ``axis=1``.
363 >>> df4 = pd.DataFrame(
364 ... [["bird", "polly"], ["monkey", "george"]], columns=["animal", "name"]
365 ... )
366 >>> pd.concat([df1, df4], axis=1)
367 letter number animal name
368 0 a 1 bird polly
369 1 b 2 monkey george
371 Prevent the result from including duplicate index values with the
372 ``verify_integrity`` option.
374 >>> df5 = pd.DataFrame([1], index=["a"])
375 >>> df5
376 0
377 a 1
378 >>> df6 = pd.DataFrame([2], index=["a"])
379 >>> df6
380 0
381 a 2
382 >>> pd.concat([df5, df6], verify_integrity=True)
383 Traceback (most recent call last):
384 ...
385 ValueError: Indexes have overlapping values: ['a']
387 Append a single row to the end of a ``DataFrame`` object.
389 >>> df7 = pd.DataFrame({"a": 1, "b": 2}, index=[0])
390 >>> df7
391 a b
392 0 1 2
393 >>> new_row = pd.Series({"a": 3, "b": 4})
394 >>> new_row
395 a 3
396 b 4
397 dtype: int64
398 >>> pd.concat([df7, new_row.to_frame().T], ignore_index=True)
399 a b
400 0 1 2
401 1 3 4
402 """
403 if ignore_index and keys is not None:
404 raise ValueError(
405 f"Cannot set {ignore_index=} and specify keys. Either should be used."
406 )
408 if copy is not lib.no_default:
409 warnings.warn(
410 "The copy keyword is deprecated and will be removed in a future "
411 "version. Copy-on-Write is active in pandas since 3.0 which utilizes "
412 "a lazy copy mechanism that defers copies until necessary. Use "
413 ".copy() to make an eager copy if necessary.",
414 Pandas4Warning,
415 stacklevel=find_stack_level(),
416 )
417 if join == "outer":
418 intersect = False
419 elif join == "inner":
420 intersect = True
421 else: # pragma: no cover
422 raise ValueError(
423 "Only can inner (intersect) or outer (union) join the other axis"
424 )
426 objs, keys, ndims = _clean_keys_and_objs(objs, keys)
428 if sort is lib.no_default:
429 if axis == 0:
430 non_concat_axis = [
431 obj.columns if isinstance(obj, ABCDataFrame) else Index([obj.name])
432 for obj in objs
433 ]
434 else:
435 non_concat_axis = [obj.index for obj in objs]
437 if (
438 intersect
439 or any(not isinstance(index, DatetimeIndex) for index in non_concat_axis)
440 or all(prev is curr for prev, curr in pairwise(non_concat_axis))
441 or (
442 all(
443 prev[-1] <= curr[0] and prev.is_monotonic_increasing
444 for prev, curr in pairwise(non_concat_axis)
445 if not prev.empty and not curr.empty
446 )
447 and non_concat_axis[-1].is_monotonic_increasing
448 )
449 ):
450 # Sorting or not will not impact the result.
451 sort = False
452 elif not is_bool(sort):
453 raise ValueError(
454 f"The 'sort' keyword only accepts boolean values; {sort} was passed."
455 )
456 else:
457 sort = bool(sort)
459 # select an object to be our result reference
460 sample, objs = _get_sample_object(objs, ndims, keys, names, levels, intersect)
462 # Standardize axis parameter to int
463 if sample.ndim == 1:
464 from pandas import DataFrame
466 bm_axis = DataFrame._get_axis_number(axis)
467 is_frame = False
468 is_series = True
469 else:
470 bm_axis = sample._get_axis_number(axis)
471 is_frame = True
472 is_series = False
474 # Need to flip BlockManager axis in the DataFrame special case
475 bm_axis = sample._get_block_manager_axis(bm_axis)
477 # if we have mixed ndims, then convert to highest ndim
478 # creating column numbers as needed
479 if len(ndims) > 1:
480 objs = _sanitize_mixed_ndim(objs, sample, ignore_index, bm_axis)
482 orig_axis = axis
483 axis = 1 - bm_axis if is_frame else 0
484 names = names or getattr(keys, "names", None)
485 result = _get_result(
486 objs,
487 is_series,
488 bm_axis,
489 ignore_index,
490 intersect,
491 sort,
492 keys,
493 levels,
494 verify_integrity,
495 names,
496 axis,
497 union_categories,
498 )
500 # When result is a Series, there is no other axis to sort.
501 if sort is lib.no_default and not isinstance(result, ABCSeries):
502 if orig_axis == 0:
503 non_concat_axis = [
504 obj.columns if isinstance(obj, ABCDataFrame) else Index([obj.name])
505 for obj in objs
506 ]
507 else:
508 non_concat_axis = [obj.index for obj in objs]
509 no_sort_result_index, _ = union_indexes(non_concat_axis, sort=False)
510 orig = result.index if orig_axis == 1 else result.columns
511 if not no_sort_result_index.equals(orig):
512 msg = (
513 "Sorting by default when concatenating all DatetimeIndex is "
514 "deprecated. In the future, pandas will respect the default "
515 "of `sort=False`. Specify `sort=True` or `sort=False` to "
516 "silence this message. If you see this warnings when not "
517 "directly calling concat, report a bug to pandas."
518 )
519 warnings.warn(msg, Pandas4Warning, stacklevel=find_stack_level())
521 return result
524def _sanitize_mixed_ndim(
525 objs: list[Series | DataFrame],
526 sample: Series | DataFrame,
527 ignore_index: bool,
528 axis: AxisInt,
529) -> list[Series | DataFrame]:
530 # if we have mixed ndims, then convert to highest ndim
531 # creating column numbers as needed
533 new_objs = []
535 current_column = 0
536 max_ndim = sample.ndim
537 for obj in objs:
538 ndim = obj.ndim
539 if ndim == max_ndim:
540 pass
542 elif ndim != max_ndim - 1:
543 raise ValueError(
544 "cannot concatenate unaligned mixed dimensional NDFrame objects"
545 )
547 else:
548 name = getattr(obj, "name", None)
549 rename_columns = False
550 if ignore_index or name is None:
551 if axis == 1:
552 # doing a row-wise concatenation so need everything
553 # to line up
554 if name is None:
555 name = 0
556 rename_columns = True
557 # doing a column-wise concatenation so need series
558 # to have unique names
559 elif name is None:
560 rename_columns = True
561 name = current_column
562 current_column += 1
563 obj = sample._constructor(obj, copy=False)
564 if isinstance(obj, ABCDataFrame) and rename_columns:
565 obj.columns = range(name, name + 1, 1)
566 else:
567 obj = sample._constructor({name: obj}, copy=False)
569 new_objs.append(obj)
571 return new_objs
574def _get_result(
575 objs: list[Series | DataFrame],
576 is_series: bool,
577 bm_axis: AxisInt,
578 ignore_index: bool,
579 intersect: bool,
580 sort: bool | lib.NoDefault,
581 keys: Iterable[Hashable] | None,
582 levels,
583 verify_integrity: bool,
584 names: list[HashableT] | None,
585 axis: AxisInt,
586 union_categories: bool = False,
587):
588 cons: Callable[..., DataFrame | Series]
589 sample: DataFrame | Series
591 # series only
592 if is_series:
593 sample = cast("Series", objs[0])
595 # stack blocks
596 if bm_axis == 0:
597 name = com.consensus_name_attr(objs)
598 cons = sample._constructor
600 arrs = [ser._values for ser in objs]
602 res = concat_compat(arrs, axis=0, union_categories=union_categories)
604 if ignore_index:
605 new_index: Index = default_index(len(res))
606 else:
607 new_index = _get_concat_axis_series(
608 objs,
609 ignore_index,
610 bm_axis,
611 keys,
612 levels,
613 verify_integrity,
614 names,
615 )
617 mgr = type(sample._mgr).from_array(res, index=new_index)
619 result = sample._constructor_from_mgr(mgr, axes=mgr.axes)
620 result._name = name
621 return result.__finalize__(
622 types.SimpleNamespace(input_objs=objs, objs=objs), method="concat"
623 )
625 # combine as columns in a frame
626 else:
627 data = dict(enumerate(objs))
629 # GH28330 Preserves subclassed objects through concat
630 cons = sample._constructor_expanddim
632 index, _ = get_objs_combined_axis(
633 objs,
634 axis=objs[0]._get_block_manager_axis(0),
635 intersect=intersect,
636 sort=sort,
637 )
638 columns = _get_concat_axis_series(
639 objs, ignore_index, bm_axis, keys, levels, verify_integrity, names
640 )
641 df = cons(data, index=index, copy=False)
642 df.columns = columns
643 return df.__finalize__(
644 types.SimpleNamespace(input_objs=objs, objs=objs), method="concat"
645 )
647 # combine block managers
648 else:
649 sample = cast("DataFrame", objs[0])
651 mgrs_indexers = []
652 result_axes, all_equal = new_axes(
653 objs,
654 bm_axis,
655 intersect,
656 sort,
657 keys,
658 names,
659 axis,
660 levels,
661 verify_integrity,
662 ignore_index,
663 )
664 for obj in objs:
665 indexers = {}
666 for ax, new_labels in enumerate(result_axes):
667 # ::-1 to convert BlockManager ax to DataFrame ax
668 if ax == bm_axis:
669 # Suppress reindexing on concat axis
670 continue
672 # 1-ax to convert BlockManager axis to DataFrame axis
673 obj_labels = obj.axes[1 - ax]
674 if not all_equal[ax] and not new_labels.equals(obj_labels):
675 indexers[ax] = obj_labels.get_indexer(new_labels)
677 mgrs_indexers.append((obj._mgr, indexers))
679 new_data = concatenate_managers(
680 mgrs_indexers,
681 result_axes,
682 concat_axis=bm_axis,
683 copy=False,
684 union_categories=union_categories,
685 )
687 out = sample._constructor_from_mgr(new_data, axes=new_data.axes)
688 return out.__finalize__(
689 types.SimpleNamespace(input_objs=objs, objs=objs), method="concat"
690 )
693def new_axes(
694 objs: list[Series | DataFrame],
695 bm_axis: AxisInt,
696 intersect: bool,
697 sort: bool | lib.NoDefault,
698 keys: Iterable[Hashable] | None,
699 names: list[HashableT] | None,
700 axis: AxisInt,
701 levels,
702 verify_integrity: bool,
703 ignore_index: bool,
704) -> tuple[list[Index], list[bool]]:
705 """Return the new [index, column] result for concat."""
706 result_axes: list[Index] = []
707 all_equal: list[bool] = []
709 for i in range(2):
710 if i == bm_axis:
711 result_axis = _get_concat_axis_dataframe(
712 objs,
713 axis,
714 ignore_index,
715 keys,
716 names,
717 levels,
718 verify_integrity,
719 )
720 is_all_equal = False
721 else:
722 result_axis, is_all_equal = get_objs_combined_axis(
723 objs,
724 axis=objs[0]._get_block_manager_axis(i),
725 intersect=intersect,
726 sort=sort,
727 )
729 result_axes.append(result_axis)
730 all_equal.append(is_all_equal)
732 return result_axes, all_equal
735def _get_concat_axis_series(
736 objs: list[Series | DataFrame],
737 ignore_index: bool,
738 bm_axis: AxisInt,
739 keys: Iterable[Hashable] | None,
740 levels,
741 verify_integrity: bool,
742 names: list[HashableT] | None,
743) -> Index:
744 """Return result concat axis when concatenating Series objects."""
745 if ignore_index:
746 return default_index(len(objs))
747 elif bm_axis == 0:
748 indexes = [x.index for x in objs]
749 if keys is None:
750 if levels is not None:
751 raise ValueError("levels supported only when keys is not None")
752 concat_axis = _concat_indexes(indexes)
753 else:
754 concat_axis = _make_concat_multiindex(indexes, keys, levels, names)
755 if verify_integrity and not concat_axis.is_unique:
756 overlap = concat_axis[concat_axis.duplicated()].unique()
757 raise ValueError(f"Indexes have overlapping values: {overlap}")
758 return concat_axis
759 elif keys is None:
760 result_names: list[Hashable] = [None] * len(objs)
761 num = 0
762 has_names = False
763 for i, x in enumerate(objs):
764 if x.ndim != 1:
765 raise TypeError(
766 f"Cannot concatenate type 'Series' with "
767 f"object of type '{type(x).__name__}'"
768 )
769 if x.name is not None:
770 result_names[i] = x.name
771 has_names = True
772 else:
773 result_names[i] = num
774 num += 1
775 if has_names:
776 return Index(result_names)
777 else:
778 return default_index(len(objs))
779 else:
780 return ensure_index(keys).set_names(names) # type: ignore[arg-type]
783def _get_concat_axis_dataframe(
784 objs: list[Series | DataFrame],
785 axis: AxisInt,
786 ignore_index: bool,
787 keys: Iterable[Hashable] | None,
788 names: list[HashableT] | None,
789 levels,
790 verify_integrity: bool,
791) -> Index:
792 """Return result concat axis when concatenating DataFrame objects."""
793 indexes_gen = (x.axes[axis] for x in objs)
795 if ignore_index:
796 return default_index(sum(len(i) for i in indexes_gen))
797 else:
798 indexes = list(indexes_gen)
800 if keys is None:
801 if levels is not None:
802 raise ValueError("levels supported only when keys is not None")
803 concat_axis = _concat_indexes(indexes)
804 else:
805 concat_axis = _make_concat_multiindex(indexes, keys, levels, names)
807 if verify_integrity and not concat_axis.is_unique:
808 overlap = concat_axis[concat_axis.duplicated()].unique()
809 raise ValueError(f"Indexes have overlapping values: {overlap}")
811 return concat_axis
814def _clean_keys_and_objs(
815 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame],
816 keys,
817) -> tuple[list[Series | DataFrame], Index | None, set[int]]:
818 """
819 Returns
820 -------
821 clean_objs : list[Series | DataFrame]
822 List of DataFrame and Series with Nones removed.
823 keys : Index | None
824 None if keys was None
825 Index if objs was a Mapping or keys was not None. Filtered where objs was None.
826 ndim : set[int]
827 Unique .ndim attribute of obj encountered.
828 """
829 if isinstance(objs, abc.Mapping):
830 if keys is None:
831 keys = objs.keys()
832 objs = [objs[k] for k in keys]
833 elif isinstance(objs, (ABCSeries, ABCDataFrame)) or is_scalar(objs):
834 raise TypeError(
835 "first argument must be an iterable of pandas "
836 f'objects, you passed an object of type "{type(objs).__name__}"'
837 )
838 elif not isinstance(objs, abc.Sized):
839 objs = list(objs)
841 if len(objs) == 0:
842 raise ValueError("No objects to concatenate")
844 if keys is not None:
845 if not isinstance(keys, Index):
846 keys = Index(keys)
847 if len(keys) != len(objs):
848 # GH#43485
849 raise ValueError(
850 f"The length of the keys ({len(keys)}) must match "
851 f"the length of the objects to concatenate ({len(objs)})"
852 )
854 # GH#1649
855 key_indices = []
856 clean_objs = []
857 ndims = set()
858 for i, obj in enumerate(objs):
859 if obj is None:
860 continue
861 elif isinstance(obj, (ABCSeries, ABCDataFrame)):
862 key_indices.append(i)
863 clean_objs.append(obj)
864 ndims.add(obj.ndim)
865 else:
866 msg = (
867 f"cannot concatenate object of type '{type(obj)}'; "
868 "only Series and DataFrame objs are valid"
869 )
870 raise TypeError(msg)
872 if keys is not None and len(key_indices) < len(keys):
873 keys = keys.take(key_indices)
875 if len(clean_objs) == 0:
876 raise ValueError("All objects passed were None")
878 return clean_objs, keys, ndims
881def _get_sample_object(
882 objs: list[Series | DataFrame],
883 ndims: set[int],
884 keys,
885 names,
886 levels,
887 intersect: bool,
888) -> tuple[Series | DataFrame, list[Series | DataFrame]]:
889 # get the sample
890 # want the highest ndim that we have, and must be non-empty
891 # unless all objs are empty
892 if len(ndims) > 1:
893 max_ndim = max(ndims)
894 for obj in objs:
895 if obj.ndim == max_ndim and sum(obj.shape): # type: ignore[arg-type]
896 return obj, objs
897 elif keys is None and names is None and levels is None and not intersect:
898 # filter out the empties if we have not multi-index possibilities
899 # note to keep empty Series as it affects the result columns / name
900 if ndims.pop() == 2:
901 non_empties = [obj for obj in objs if sum(obj.shape)]
902 else:
903 non_empties = objs
905 if len(non_empties):
906 return non_empties[0], non_empties
908 return objs[0], objs
911def _concat_indexes(indexes) -> Index:
912 return indexes[0].append(indexes[1:])
915def validate_unique_levels(levels: list[Index]) -> None:
916 for level in levels:
917 if not level.is_unique:
918 raise ValueError(f"Level values not unique: {level.tolist()}")
921def _make_concat_multiindex(indexes, keys, levels=None, names=None) -> MultiIndex:
922 if (levels is None and isinstance(keys[0], tuple)) or (
923 levels is not None and len(levels) > 1
924 ):
925 zipped = list(zip(*keys, strict=True))
926 if names is None:
927 names = [None] * len(zipped)
929 if levels is None:
930 _, levels = factorize_from_iterables(zipped)
931 else:
932 levels = [ensure_index(x) for x in levels]
933 validate_unique_levels(levels)
934 else:
935 zipped = [keys]
936 if names is None:
937 names = [None]
939 if levels is None:
940 levels = [ensure_index(keys).unique()]
941 else:
942 levels = [ensure_index(x) for x in levels]
943 validate_unique_levels(levels)
945 if not all_indexes_same(indexes):
946 codes_list = []
948 # things are potentially different sizes, so compute the exact codes
949 # for each level and pass those to MultiIndex.from_arrays
951 for hlevel, level in zip(zipped, levels, strict=True):
952 to_concat = []
953 if isinstance(hlevel, Index) and hlevel.equals(level):
954 lens = [len(idx) for idx in indexes]
955 codes_list.append(np.repeat(np.arange(len(hlevel)), lens))
956 else:
957 for key, index in zip(hlevel, indexes, strict=True):
958 # Find matching codes, include matching nan values as equal.
959 mask = (isna(level) & isna(key)) | (level == key)
960 if not mask.any():
961 raise ValueError(f"Key {key} not in level {level}")
962 i = np.nonzero(mask)[0][0]
964 to_concat.append(np.repeat(i, len(index)))
965 codes_list.append(np.concatenate(to_concat))
967 concat_index = _concat_indexes(indexes)
969 # these go at the end
970 if isinstance(concat_index, MultiIndex):
971 levels.extend(concat_index.levels)
972 codes_list.extend(concat_index.codes)
973 else:
974 codes, categories = factorize_from_iterable(concat_index)
975 levels.append(categories)
976 codes_list.append(codes)
978 if len(names) == len(levels):
979 names = list(names)
980 else:
981 # make sure that all of the passed indices have the same nlevels
982 nlevels_set = {idx.nlevels for idx in indexes}
983 if len(nlevels_set) != 1:
984 # GH#25413
985 raise ValueError(
986 "Cannot concat indices that do not have the same number "
987 f"of levels: got levels {sorted(nlevels_set)}. When passing "
988 "keys=, all objects must have the same index nlevels."
989 )
991 # also copies
992 names = list(names) + list(get_unanimous_names(*indexes))
994 return MultiIndex(
995 levels=levels, codes=codes_list, names=names, verify_integrity=False
996 )
998 new_index = indexes[0]
999 n = len(new_index)
1000 kpieces = len(indexes)
1002 # also copies
1003 new_names = list(names)
1004 new_levels = list(levels)
1006 # construct codes
1007 new_codes = []
1009 # do something a bit more speedy
1011 for hlevel, level in zip(zipped, levels, strict=True):
1012 hlevel_index = ensure_index(hlevel)
1013 # GH#64825 get_indexer_for (not get_indexer) so that an overlapping
1014 # IntervalIndex level, which is not unique-as-index, still resolves.
1015 mapped = level.get_indexer_for(hlevel_index)
1017 mask = mapped == -1
1018 if mask.any():
1019 raise ValueError(
1020 f"Values not found in passed level: {hlevel_index[mask]!s}"
1021 )
1023 new_codes.append(np.repeat(mapped, n))
1025 if isinstance(new_index, MultiIndex):
1026 new_levels.extend(new_index.levels)
1027 new_codes.extend(np.tile(lab, kpieces) for lab in new_index.codes)
1028 else:
1029 unique_index = new_index.unique()
1030 new_levels.append(unique_index)
1031 # GH#64825 get_indexer_for so an overlapping IntervalIndex resolves
1032 single_codes = unique_index.get_indexer_for(new_index)
1033 new_codes.append(np.tile(single_codes, kpieces))
1035 if len(new_names) < len(new_levels):
1036 new_names.extend(new_index.names)
1038 return MultiIndex(
1039 levels=new_levels, codes=new_codes, names=new_names, verify_integrity=False
1040 )