Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/reshape/concat.py: 14%

1""" 

2Concat routines. 

3""" 

4 

5from __future__ import annotations 

6 

7from collections import abc 

8from itertools import pairwise 

9import types 

10from typing import ( 

11 TYPE_CHECKING, 

12 Literal, 

13 cast, 

14 overload, 

15) 

16import warnings 

17 

18import numpy as np 

19 

20from pandas._libs import lib 

21from pandas.errors import Pandas4Warning 

22from pandas.util._decorators import set_module 

23from pandas.util._exceptions import find_stack_level 

24 

25from pandas.core.dtypes.common import ( 

26 is_bool, 

27 is_scalar, 

28) 

29from pandas.core.dtypes.concat import concat_compat 

30from pandas.core.dtypes.generic import ( 

31 ABCDataFrame, 

32 ABCSeries, 

33) 

34from pandas.core.dtypes.missing import isna 

35 

36from pandas.core.arrays.categorical import ( 

37 factorize_from_iterable, 

38 factorize_from_iterables, 

39) 

40import pandas.core.common as com 

41from pandas.core.indexes.api import ( 

42 Index, 

43 MultiIndex, 

44 all_indexes_same, 

45 default_index, 

46 ensure_index, 

47 get_objs_combined_axis, 

48 get_unanimous_names, 

49 union_indexes, 

50) 

51from pandas.core.indexes.datetimes import DatetimeIndex 

52from pandas.core.internals import concatenate_managers 

53 

54if TYPE_CHECKING: 

55 from collections.abc import ( 

56 Callable, 

57 Hashable, 

58 Iterable, 

59 Mapping, 

60 ) 

61 

62 from pandas._typing import ( 

63 Axis, 

64 AxisInt, 

65 HashableT, 

66 ) 

67 

68 from pandas import ( 

69 DataFrame, 

70 Series, 

71 ) 

72 

73# --------------------------------------------------------------------- 

74# Concatenate DataFrame objects 

75 

76 

77@overload 

78def concat( 

79 objs: Iterable[DataFrame] | Mapping[HashableT, DataFrame], 

80 *, 

81 axis: Literal[0, "index"] = ..., 

82 join: str = ..., 

83 ignore_index: bool = ..., 

84 keys: Iterable[Hashable] | None = ..., 

85 levels=..., 

86 names: list[HashableT] | None = ..., 

87 verify_integrity: bool = ..., 

88 sort: bool = ..., 

89 copy: bool | lib.NoDefault = ..., 

90) -> DataFrame: ... 

91 

92 

93@overload 

94def concat( 

95 objs: Iterable[Series] | Mapping[HashableT, Series], 

96 *, 

97 axis: Literal[0, "index"] = ..., 

98 join: str = ..., 

99 ignore_index: bool = ..., 

100 keys: Iterable[Hashable] | None = ..., 

101 levels=..., 

102 names: list[HashableT] | None = ..., 

103 verify_integrity: bool = ..., 

104 sort: bool = ..., 

105 copy: bool | lib.NoDefault = ..., 

106) -> Series: ... 

107 

108 

109@overload 

110def concat( 

111 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame], 

112 *, 

113 axis: Literal[0, "index"] = ..., 

114 join: str = ..., 

115 ignore_index: bool = ..., 

116 keys: Iterable[Hashable] | None = ..., 

117 levels=..., 

118 names: list[HashableT] | None = ..., 

119 verify_integrity: bool = ..., 

120 sort: bool = ..., 

121 copy: bool | lib.NoDefault = ..., 

122) -> DataFrame | Series: ... 

123 

124 

125@overload 

126def concat( 

127 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame], 

128 *, 

129 axis: Literal[1, "columns"], 

130 join: str = ..., 

131 ignore_index: bool = ..., 

132 keys: Iterable[Hashable] | None = ..., 

133 levels=..., 

134 names: list[HashableT] | None = ..., 

135 verify_integrity: bool = ..., 

136 sort: bool = ..., 

137 copy: bool | lib.NoDefault = ..., 

138) -> DataFrame: ... 

139 

140 

141@overload 

142def concat( 

143 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame], 

144 *, 

145 axis: Axis = ..., 

146 join: str = ..., 

147 ignore_index: bool = ..., 

148 keys: Iterable[Hashable] | None = ..., 

149 levels=..., 

150 names: list[HashableT] | None = ..., 

151 verify_integrity: bool = ..., 

152 sort: bool = ..., 

153 copy: bool | lib.NoDefault = ..., 

154) -> DataFrame | Series: ... 

155 

156 

157@set_module("pandas") 

158def concat( 

159 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame], 

160 *, 

161 axis: Axis = 0, 

162 join: str = "outer", 

163 ignore_index: bool = False, 

164 keys: Iterable[Hashable] | None = None, 

165 levels=None, 

166 names: list[HashableT] | None = None, 

167 verify_integrity: bool = False, 

168 sort: bool | lib.NoDefault = lib.no_default, 

169 copy: bool | lib.NoDefault = lib.no_default, 

170) -> DataFrame | Series: 

171 """ 

172 Concatenate pandas objects along a particular axis. 

173 

174 Allows optional set logic along the other axes. 

175 

176 Can also add a layer of hierarchical indexing on the concatenation axis, 

177 which may be useful if the labels are the same (or overlapping) on 

178 the passed axis number. 

179 

180 Parameters 

181 ---------- 

182 objs : an iterable or mapping of Series or DataFrame objects 

183 If a mapping is passed, the keys will be used as the `keys` 

184 argument, unless it is passed, in which case the values will be 

185 selected (see below). Any None objects will be dropped silently unless 

186 they are all None in which case a ValueError will be raised. 

187 axis : {0/'index', 1/'columns'}, default 0 

188 The axis to concatenate along. 

189 join : {'inner', 'outer'}, default 'outer' 

190 How to handle indexes on other axis (or axes). 

191 ignore_index : bool, default False 

192 If True, do not use the index values along the concatenation axis. The 

193 resulting axis will be labeled 0, ..., n - 1. This is useful if you are 

194 concatenating objects where the concatenation axis does not have 

195 meaningful indexing information. Note the index values on the other 

196 axes are still respected in the join. 

197 keys : sequence, default None 

198 If multiple levels passed, should contain tuples. Construct 

199 hierarchical index using the passed keys as the outermost level. 

200 levels : list of sequences, default None 

201 Specific levels (unique values) to use for constructing a 

202 MultiIndex. Otherwise they will be inferred from the keys. 

203 names : list, default None 

204 Names for the levels in the resulting hierarchical index. 

205 verify_integrity : bool, default False 

206 Check whether the new concatenated axis contains duplicates. This can 

207 be very expensive relative to the actual data concatenation. 

208 sort : bool, default False 

209 Sort non-concatenation axis. One exception to this is when the 

210 non-concatenation axis is a DatetimeIndex and join='outer' and the axis is 

211 not already aligned. In that case, the non-concatenation axis is always 

212 sorted lexicographically. 

213 copy : bool, default False 

214 This keyword is now ignored; changing its value will have no 

215 impact on the method. 

216 

217 .. deprecated:: 3.0.0 

218 

219 This keyword is ignored and will be removed in pandas 4.0. Since 

220 pandas 3.0, this method always returns a new object using a lazy 

221 copy mechanism that defers copies until necessary 

222 (Copy-on-Write). See the `user guide on Copy-on-Write 

223 <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__ 

224 for more details. 

225 

226 Returns 

227 ------- 

228 object, type of objs 

229 When concatenating all ``Series`` along the index (axis=0), a 

230 ``Series`` is returned. When ``objs`` contains at least one 

231 ``DataFrame``, a ``DataFrame`` is returned. When concatenating along 

232 the columns (axis=1), a ``DataFrame`` is returned. 

233 

234 See Also 

235 -------- 

236 DataFrame.join : Join DataFrames using indexes. 

237 DataFrame.merge : Merge DataFrames by indexes or columns. 

238 

239 Notes 

240 ----- 

241 The keys, levels, and names arguments are all optional. 

242 

243 A walkthrough of how this method fits in with other tools for combining 

244 pandas objects can be found `here 

245 <https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html>`__. 

246 

247 It is not recommended to build DataFrames by adding single rows in a 

248 for loop. Build a list of rows and make a DataFrame in a single concat. 

249 

250 Examples 

251 -------- 

252 Combine two ``Series``. 

253 

254 >>> s1 = pd.Series(["a", "b"]) 

255 >>> s2 = pd.Series(["c", "d"]) 

256 >>> pd.concat([s1, s2]) 

257 0 a 

258 1 b 

259 0 c 

260 1 d 

261 dtype: str 

262 

263 Clear the existing index and reset it in the result 

264 by setting the ``ignore_index`` option to ``True``. 

265 

266 >>> pd.concat([s1, s2], ignore_index=True) 

267 0 a 

268 1 b 

269 2 c 

270 3 d 

271 dtype: str 

272 

273 Add a hierarchical index at the outermost level of 

274 the data with the ``keys`` option. 

275 

276 >>> pd.concat([s1, s2], keys=["s1", "s2"]) 

277 s1 0 a 

278 1 b 

279 s2 0 c 

280 1 d 

281 dtype: str 

282 

283 Label the index keys you create with the ``names`` option. 

284 

285 >>> pd.concat([s1, s2], keys=["s1", "s2"], names=["Series name", "Row ID"]) 

286 Series name Row ID 

287 s1 0 a 

288 1 b 

289 s2 0 c 

290 1 d 

291 dtype: str 

292 

293 Combine two ``DataFrame`` objects with identical columns. 

294 

295 >>> df1 = pd.DataFrame([["a", 1], ["b", 2]], columns=["letter", "number"]) 

296 >>> df1 

297 letter number 

298 0 a 1 

299 1 b 2 

300 >>> df2 = pd.DataFrame([["c", 3], ["d", 4]], columns=["letter", "number"]) 

301 >>> df2 

302 letter number 

303 0 c 3 

304 1 d 4 

305 >>> pd.concat([df1, df2]) 

306 letter number 

307 0 a 1 

308 1 b 2 

309 0 c 3 

310 1 d 4 

311 

312 Combine ``DataFrame`` objects with overlapping columns 

313 and return everything. Columns outside the intersection will 

314 be filled with ``NaN`` values. 

315 

316 >>> df3 = pd.DataFrame( 

317 ... [["c", 3, "cat"], ["d", 4, "dog"]], columns=["letter", "number", "animal"] 

318 ... ) 

319 >>> df3 

320 letter number animal 

321 0 c 3 cat 

322 1 d 4 dog 

323 >>> pd.concat([df1, df3], sort=False) 

324 letter number animal 

325 0 a 1 NaN 

326 1 b 2 NaN 

327 0 c 3 cat 

328 1 d 4 dog 

329 

330 Combine ``DataFrame`` objects with overlapping columns 

331 and return only those that are shared by passing ``inner`` to 

332 the ``join`` keyword argument. 

333 

334 >>> pd.concat([df1, df3], join="inner") 

335 letter number 

336 0 a 1 

337 1 b 2 

338 0 c 3 

339 1 d 4 

340 

341 Combine ``DataFrame`` objects horizontally along the x axis by 

342 passing in ``axis=1``. 

343 

344 >>> df4 = pd.DataFrame( 

345 ... [["bird", "polly"], ["monkey", "george"]], columns=["animal", "name"] 

346 ... ) 

347 >>> pd.concat([df1, df4], axis=1) 

348 letter number animal name 

349 0 a 1 bird polly 

350 1 b 2 monkey george 

351 

352 Prevent the result from including duplicate index values with the 

353 ``verify_integrity`` option. 

354 

355 >>> df5 = pd.DataFrame([1], index=["a"]) 

356 >>> df5 

357 0 

358 a 1 

359 >>> df6 = pd.DataFrame([2], index=["a"]) 

360 >>> df6 

361 0 

362 a 2 

363 >>> pd.concat([df5, df6], verify_integrity=True) 

364 Traceback (most recent call last): 

365 ... 

366 ValueError: Indexes have overlapping values: ['a'] 

367 

368 Append a single row to the end of a ``DataFrame`` object. 

369 

370 >>> df7 = pd.DataFrame({"a": 1, "b": 2}, index=[0]) 

371 >>> df7 

372 a b 

373 0 1 2 

374 >>> new_row = pd.Series({"a": 3, "b": 4}) 

375 >>> new_row 

376 a 3 

377 b 4 

378 dtype: int64 

379 >>> pd.concat([df7, new_row.to_frame().T], ignore_index=True) 

380 a b 

381 0 1 2 

382 1 3 4 

383 """ 

384 if ignore_index and keys is not None: 

385 raise ValueError( 

386 f"Cannot set {ignore_index=} and specify keys. Either should be used." 

387 ) 

388 

389 if copy is not lib.no_default: 

390 warnings.warn( 

391 "The copy keyword is deprecated and will be removed in a future " 

392 "version. Copy-on-Write is active in pandas since 3.0 which utilizes " 

393 "a lazy copy mechanism that defers copies until necessary. Use " 

394 ".copy() to make an eager copy if necessary.", 

395 Pandas4Warning, 

396 stacklevel=find_stack_level(), 

397 ) 

398 if join == "outer": 

399 intersect = False 

400 elif join == "inner": 

401 intersect = True 

402 else: # pragma: no cover 

403 raise ValueError( 

404 "Only can inner (intersect) or outer (union) join the other axis" 

405 ) 

406 

407 objs, keys, ndims = _clean_keys_and_objs(objs, keys) 

408 

409 if sort is lib.no_default: 

410 if axis == 0: 

411 non_concat_axis = [ 

412 obj.columns if isinstance(obj, ABCDataFrame) else Index([obj.name]) 

413 for obj in objs 

414 ] 

415 else: 

416 non_concat_axis = [obj.index for obj in objs] 

417 

418 if ( 

419 intersect 

420 or any(not isinstance(index, DatetimeIndex) for index in non_concat_axis) 

421 or all(prev is curr for prev, curr in pairwise(non_concat_axis)) 

422 or ( 

423 all( 

424 prev[-1] <= curr[0] and prev.is_monotonic_increasing 

425 for prev, curr in pairwise(non_concat_axis) 

426 if not prev.empty and not curr.empty 

427 ) 

428 and non_concat_axis[-1].is_monotonic_increasing 

429 ) 

430 ): 

431 # Sorting or not will not impact the result. 

432 sort = False 

433 elif not is_bool(sort): 

434 raise ValueError( 

435 f"The 'sort' keyword only accepts boolean values; {sort} was passed." 

436 ) 

437 else: 

438 sort = bool(sort) 

439 

440 # select an object to be our result reference 

441 sample, objs = _get_sample_object(objs, ndims, keys, names, levels, intersect) 

442 

443 # Standardize axis parameter to int 

444 if sample.ndim == 1: 

445 from pandas import DataFrame 

446 

447 bm_axis = DataFrame._get_axis_number(axis) 

448 is_frame = False 

449 is_series = True 

450 else: 

451 bm_axis = sample._get_axis_number(axis) 

452 is_frame = True 

453 is_series = False 

454 

455 # Need to flip BlockManager axis in the DataFrame special case 

456 bm_axis = sample._get_block_manager_axis(bm_axis) 

457 

458 # if we have mixed ndims, then convert to highest ndim 

459 # creating column numbers as needed 

460 if len(ndims) > 1: 

461 objs = _sanitize_mixed_ndim(objs, sample, ignore_index, bm_axis) 

462 

463 orig_axis = axis 

464 axis = 1 - bm_axis if is_frame else 0 

465 names = names or getattr(keys, "names", None) 

466 result = _get_result( 

467 objs, 

468 is_series, 

469 bm_axis, 

470 ignore_index, 

471 intersect, 

472 sort, 

473 keys, 

474 levels, 

475 verify_integrity, 

476 names, 

477 axis, 

478 ) 

479 

480 if sort is lib.no_default: 

481 if orig_axis == 0: 

482 non_concat_axis = [ 

483 obj.columns if isinstance(obj, ABCDataFrame) else Index([obj.name]) 

484 for obj in objs 

485 ] 

486 else: 

487 non_concat_axis = [obj.index for obj in objs] 

488 no_sort_result_index = union_indexes(non_concat_axis, sort=False) 

489 orig = result.index if orig_axis == 1 else result.columns 

490 if not no_sort_result_index.equals(orig): 

491 msg = ( 

492 "Sorting by default when concatenating all DatetimeIndex is " 

493 "deprecated. In the future, pandas will respect the default " 

494 "of `sort=False`. Specify `sort=True` or `sort=False` to " 

495 "silence this message. If you see this warnings when not " 

496 "directly calling concat, report a bug to pandas." 

497 ) 

498 warnings.warn(msg, Pandas4Warning, stacklevel=find_stack_level()) 

499 

500 return result 

501 

502 

503def _sanitize_mixed_ndim( 

504 objs: list[Series | DataFrame], 

505 sample: Series | DataFrame, 

506 ignore_index: bool, 

507 axis: AxisInt, 

508) -> list[Series | DataFrame]: 

509 # if we have mixed ndims, then convert to highest ndim 

510 # creating column numbers as needed 

511 

512 new_objs = [] 

513 

514 current_column = 0 

515 max_ndim = sample.ndim 

516 for obj in objs: 

517 ndim = obj.ndim 

518 if ndim == max_ndim: 

519 pass 

520 

521 elif ndim != max_ndim - 1: 

522 raise ValueError( 

523 "cannot concatenate unaligned mixed dimensional NDFrame objects" 

524 ) 

525 

526 else: 

527 name = getattr(obj, "name", None) 

528 rename_columns = False 

529 if ignore_index or name is None: 

530 if axis == 1: 

531 # doing a row-wise concatenation so need everything 

532 # to line up 

533 if name is None: 

534 name = 0 

535 rename_columns = True 

536 # doing a column-wise concatenation so need series 

537 # to have unique names 

538 elif name is None: 

539 rename_columns = True 

540 name = current_column 

541 current_column += 1 

542 obj = sample._constructor(obj, copy=False) 

543 if isinstance(obj, ABCDataFrame) and rename_columns: 

544 obj.columns = range(name, name + 1, 1) 

545 else: 

546 obj = sample._constructor({name: obj}, copy=False) 

547 

548 new_objs.append(obj) 

549 

550 return new_objs 

551 

552 

553def _get_result( 

554 objs: list[Series | DataFrame], 

555 is_series: bool, 

556 bm_axis: AxisInt, 

557 ignore_index: bool, 

558 intersect: bool, 

559 sort: bool | lib.NoDefault, 

560 keys: Iterable[Hashable] | None, 

561 levels, 

562 verify_integrity: bool, 

563 names: list[HashableT] | None, 

564 axis: AxisInt, 

565): 

566 cons: Callable[..., DataFrame | Series] 

567 sample: DataFrame | Series 

568 

569 # series only 

570 if is_series: 

571 sample = cast("Series", objs[0]) 

572 

573 # stack blocks 

574 if bm_axis == 0: 

575 name = com.consensus_name_attr(objs) 

576 cons = sample._constructor 

577 

578 arrs = [ser._values for ser in objs] 

579 

580 res = concat_compat(arrs, axis=0) 

581 

582 if ignore_index: 

583 new_index: Index = default_index(len(res)) 

584 else: 

585 new_index = _get_concat_axis_series( 

586 objs, 

587 ignore_index, 

588 bm_axis, 

589 keys, 

590 levels, 

591 verify_integrity, 

592 names, 

593 ) 

594 

595 mgr = type(sample._mgr).from_array(res, index=new_index) 

596 

597 result = sample._constructor_from_mgr(mgr, axes=mgr.axes) 

598 result._name = name 

599 return result.__finalize__( 

600 types.SimpleNamespace(input_objs=objs, objs=objs), method="concat" 

601 ) 

602 

603 # combine as columns in a frame 

604 else: 

605 data = dict(enumerate(objs)) 

606 

607 # GH28330 Preserves subclassed objects through concat 

608 cons = sample._constructor_expanddim 

609 

610 index = get_objs_combined_axis( 

611 objs, 

612 axis=objs[0]._get_block_manager_axis(0), 

613 intersect=intersect, 

614 sort=sort, 

615 ) 

616 columns = _get_concat_axis_series( 

617 objs, ignore_index, bm_axis, keys, levels, verify_integrity, names 

618 ) 

619 df = cons(data, index=index, copy=False) 

620 df.columns = columns 

621 return df.__finalize__( 

622 types.SimpleNamespace(input_objs=objs, objs=objs), method="concat" 

623 ) 

624 

625 # combine block managers 

626 else: 

627 sample = cast("DataFrame", objs[0]) 

628 

629 mgrs_indexers = [] 

630 result_axes = new_axes( 

631 objs, 

632 bm_axis, 

633 intersect, 

634 sort, 

635 keys, 

636 names, 

637 axis, 

638 levels, 

639 verify_integrity, 

640 ignore_index, 

641 ) 

642 for obj in objs: 

643 indexers = {} 

644 for ax, new_labels in enumerate(result_axes): 

645 # ::-1 to convert BlockManager ax to DataFrame ax 

646 if ax == bm_axis: 

647 # Suppress reindexing on concat axis 

648 continue 

649 

650 # 1-ax to convert BlockManager axis to DataFrame axis 

651 obj_labels = obj.axes[1 - ax] 

652 if not new_labels.equals(obj_labels): 

653 indexers[ax] = obj_labels.get_indexer(new_labels) 

654 

655 mgrs_indexers.append((obj._mgr, indexers)) 

656 

657 new_data = concatenate_managers( 

658 mgrs_indexers, result_axes, concat_axis=bm_axis, copy=False 

659 ) 

660 

661 out = sample._constructor_from_mgr(new_data, axes=new_data.axes) 

662 return out.__finalize__( 

663 types.SimpleNamespace(input_objs=objs, objs=objs), method="concat" 

664 ) 

665 

666 

667def new_axes( 

668 objs: list[Series | DataFrame], 

669 bm_axis: AxisInt, 

670 intersect: bool, 

671 sort: bool | lib.NoDefault, 

672 keys: Iterable[Hashable] | None, 

673 names: list[HashableT] | None, 

674 axis: AxisInt, 

675 levels, 

676 verify_integrity: bool, 

677 ignore_index: bool, 

678) -> list[Index]: 

679 """Return the new [index, column] result for concat.""" 

680 return [ 

681 _get_concat_axis_dataframe( 

682 objs, 

683 axis, 

684 ignore_index, 

685 keys, 

686 names, 

687 levels, 

688 verify_integrity, 

689 ) 

690 if i == bm_axis 

691 else get_objs_combined_axis( 

692 objs, 

693 axis=objs[0]._get_block_manager_axis(i), 

694 intersect=intersect, 

695 sort=sort, 

696 ) 

697 for i in range(2) 

698 ] 

699 

700 

701def _get_concat_axis_series( 

702 objs: list[Series | DataFrame], 

703 ignore_index: bool, 

704 bm_axis: AxisInt, 

705 keys: Iterable[Hashable] | None, 

706 levels, 

707 verify_integrity: bool, 

708 names: list[HashableT] | None, 

709) -> Index: 

710 """Return result concat axis when concatenating Series objects.""" 

711 if ignore_index: 

712 return default_index(len(objs)) 

713 elif bm_axis == 0: 

714 indexes = [x.index for x in objs] 

715 if keys is None: 

716 if levels is not None: 

717 raise ValueError("levels supported only when keys is not None") 

718 concat_axis = _concat_indexes(indexes) 

719 else: 

720 concat_axis = _make_concat_multiindex(indexes, keys, levels, names) 

721 if verify_integrity and not concat_axis.is_unique: 

722 overlap = concat_axis[concat_axis.duplicated()].unique() 

723 raise ValueError(f"Indexes have overlapping values: {overlap}") 

724 return concat_axis 

725 elif keys is None: 

726 result_names: list[Hashable] = [None] * len(objs) 

727 num = 0 

728 has_names = False 

729 for i, x in enumerate(objs): 

730 if x.ndim != 1: 

731 raise TypeError( 

732 f"Cannot concatenate type 'Series' with " 

733 f"object of type '{type(x).__name__}'" 

734 ) 

735 if x.name is not None: 

736 result_names[i] = x.name 

737 has_names = True 

738 else: 

739 result_names[i] = num 

740 num += 1 

741 if has_names: 

742 return Index(result_names) 

743 else: 

744 return default_index(len(objs)) 

745 else: 

746 return ensure_index(keys).set_names(names) # type: ignore[arg-type] 

747 

748 

749def _get_concat_axis_dataframe( 

750 objs: list[Series | DataFrame], 

751 axis: AxisInt, 

752 ignore_index: bool, 

753 keys: Iterable[Hashable] | None, 

754 names: list[HashableT] | None, 

755 levels, 

756 verify_integrity: bool, 

757) -> Index: 

758 """Return result concat axis when concatenating DataFrame objects.""" 

759 indexes_gen = (x.axes[axis] for x in objs) 

760 

761 if ignore_index: 

762 return default_index(sum(len(i) for i in indexes_gen)) 

763 else: 

764 indexes = list(indexes_gen) 

765 

766 if keys is None: 

767 if levels is not None: 

768 raise ValueError("levels supported only when keys is not None") 

769 concat_axis = _concat_indexes(indexes) 

770 else: 

771 concat_axis = _make_concat_multiindex(indexes, keys, levels, names) 

772 

773 if verify_integrity and not concat_axis.is_unique: 

774 overlap = concat_axis[concat_axis.duplicated()].unique() 

775 raise ValueError(f"Indexes have overlapping values: {overlap}") 

776 

777 return concat_axis 

778 

779 

780def _clean_keys_and_objs( 

781 objs: Iterable[Series | DataFrame] | Mapping[HashableT, Series | DataFrame], 

782 keys, 

783) -> tuple[list[Series | DataFrame], Index | None, set[int]]: 

784 """ 

785 Returns 

786 ------- 

787 clean_objs : list[Series | DataFrame] 

788 List of DataFrame and Series with Nones removed. 

789 keys : Index | None 

790 None if keys was None 

791 Index if objs was a Mapping or keys was not None. Filtered where objs was None. 

792 ndim : set[int] 

793 Unique .ndim attribute of obj encountered. 

794 """ 

795 if isinstance(objs, abc.Mapping): 

796 if keys is None: 

797 keys = objs.keys() 

798 objs = [objs[k] for k in keys] 

799 elif isinstance(objs, (ABCSeries, ABCDataFrame)) or is_scalar(objs): 

800 raise TypeError( 

801 "first argument must be an iterable of pandas " 

802 f'objects, you passed an object of type "{type(objs).__name__}"' 

803 ) 

804 elif not isinstance(objs, abc.Sized): 

805 objs = list(objs) 

806 

807 if len(objs) == 0: 

808 raise ValueError("No objects to concatenate") 

809 

810 if keys is not None: 

811 if not isinstance(keys, Index): 

812 keys = Index(keys) 

813 if len(keys) != len(objs): 

814 # GH#43485 

815 raise ValueError( 

816 f"The length of the keys ({len(keys)}) must match " 

817 f"the length of the objects to concatenate ({len(objs)})" 

818 ) 

819 

820 # GH#1649 

821 key_indices = [] 

822 clean_objs = [] 

823 ndims = set() 

824 for i, obj in enumerate(objs): 

825 if obj is None: 

826 continue 

827 elif isinstance(obj, (ABCSeries, ABCDataFrame)): 

828 key_indices.append(i) 

829 clean_objs.append(obj) 

830 ndims.add(obj.ndim) 

831 else: 

832 msg = ( 

833 f"cannot concatenate object of type '{type(obj)}'; " 

834 "only Series and DataFrame objs are valid" 

835 ) 

836 raise TypeError(msg) 

837 

838 if keys is not None and len(key_indices) < len(keys): 

839 keys = keys.take(key_indices) 

840 

841 if len(clean_objs) == 0: 

842 raise ValueError("All objects passed were None") 

843 

844 return clean_objs, keys, ndims 

845 

846 

847def _get_sample_object( 

848 objs: list[Series | DataFrame], 

849 ndims: set[int], 

850 keys, 

851 names, 

852 levels, 

853 intersect: bool, 

854) -> tuple[Series | DataFrame, list[Series | DataFrame]]: 

855 # get the sample 

856 # want the highest ndim that we have, and must be non-empty 

857 # unless all objs are empty 

858 if len(ndims) > 1: 

859 max_ndim = max(ndims) 

860 for obj in objs: 

861 if obj.ndim == max_ndim and sum(obj.shape): # type: ignore[arg-type] 

862 return obj, objs 

863 elif keys is None and names is None and levels is None and not intersect: 

864 # filter out the empties if we have not multi-index possibilities 

865 # note to keep empty Series as it affect to result columns / name 

866 if ndims.pop() == 2: 

867 non_empties = [obj for obj in objs if sum(obj.shape)] 

868 else: 

869 non_empties = objs 

870 

871 if len(non_empties): 

872 return non_empties[0], non_empties 

873 

874 return objs[0], objs 

875 

876 

877def _concat_indexes(indexes) -> Index: 

878 return indexes[0].append(indexes[1:]) 

879 

880 

881def validate_unique_levels(levels: list[Index]) -> None: 

882 for level in levels: 

883 if not level.is_unique: 

884 raise ValueError(f"Level values not unique: {level.tolist()}") 

885 

886 

887def _make_concat_multiindex(indexes, keys, levels=None, names=None) -> MultiIndex: 

888 if (levels is None and isinstance(keys[0], tuple)) or ( 

889 levels is not None and len(levels) > 1 

890 ): 

891 zipped = list(zip(*keys, strict=True)) 

892 if names is None: 

893 names = [None] * len(zipped) 

894 

895 if levels is None: 

896 _, levels = factorize_from_iterables(zipped) 

897 else: 

898 levels = [ensure_index(x) for x in levels] 

899 validate_unique_levels(levels) 

900 else: 

901 zipped = [keys] 

902 if names is None: 

903 names = [None] 

904 

905 if levels is None: 

906 levels = [ensure_index(keys).unique()] 

907 else: 

908 levels = [ensure_index(x) for x in levels] 

909 validate_unique_levels(levels) 

910 

911 if not all_indexes_same(indexes): 

912 codes_list = [] 

913 

914 # things are potentially different sizes, so compute the exact codes 

915 # for each level and pass those to MultiIndex.from_arrays 

916 

917 for hlevel, level in zip(zipped, levels, strict=True): 

918 to_concat = []