Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/_testing/__init_

1from __future__ import annotations

3from concurrent.futures import ThreadPoolExecutor

4from decimal import Decimal

5import operator

6import os

7from sys import byteorder

8import threading

9from typing import (

10 TYPE_CHECKING,

11 Any,

12)

14import numpy as np

16from pandas._config import using_string_dtype

17from pandas._config.localization import (

18 can_set_locale,

19 get_locales,

20 set_locale,

21)

23from pandas.compat import HAS_PYARROW

25import pandas as pd

26from pandas import (

27 ArrowDtype,

28 DataFrame,

29 Index,

30 MultiIndex,

31 RangeIndex,

32 Series,

33)

34from pandas._testing._io import (

35 round_trip_pathlib,

36 round_trip_pickle,

37 write_to_compressed,

38)

39from pandas._testing._warnings import (

40 assert_produces_warning,

41 maybe_produces_warning,

42)

43from pandas._testing.asserters import (

44 assert_almost_equal,

45 assert_attr_equal,

46 assert_categorical_equal,

47 assert_class_equal,

48 assert_contains_all,

49 assert_copy,

50 assert_datetime_array_equal,

51 assert_dict_equal,

52 assert_equal,

53 assert_extension_array_equal,

54 assert_frame_equal,

55 assert_index_equal,

56 assert_indexing_slices_equivalent,

57 assert_interval_array_equal,

58 assert_is_sorted,

59 assert_metadata_equivalent,

60 assert_numpy_array_equal,

61 assert_period_array_equal,

62 assert_series_equal,

63 assert_sp_array_equal,

64 assert_timedelta_array_equal,

65 raise_assert_detail,

66)

67from pandas._testing.compat import (

68 get_dtype,

69 get_obj,

70)

71from pandas._testing.contexts import (

72 decompress_file,

73 raises_chained_assignment_error,

74 set_timezone,

75 with_csv_dialect,

76)

77from pandas.core.arrays import (

78 ArrowExtensionArray,

79 BaseMaskedArray,

80 NumpyExtensionArray,

81)

82from pandas.core.arrays._mixins import NDArrayBackedExtensionArray

83from pandas.core.construction import extract_array

85if TYPE_CHECKING:

86 from collections.abc import (

87 Callable,

88 Iterable,

89 )

90 from contextlib import AbstractContextManager

92 from pandas._typing import (

93 Dtype,

94 NpDtype,

95 )

97 from pandas.core.arrays import ExtensionArray

100UNSIGNED_INT_NUMPY_DTYPES: list[NpDtype] = ["uint8", "uint16", "uint32", "uint64"]

101UNSIGNED_INT_EA_DTYPES: list[Dtype] = ["UInt8", "UInt16", "UInt32", "UInt64"]

102SIGNED_INT_NUMPY_DTYPES: list[NpDtype] = [int, "int8", "int16", "int32", "int64"]

103SIGNED_INT_EA_DTYPES: list[Dtype] = ["Int8", "Int16", "Int32", "Int64"]

104ALL_INT_NUMPY_DTYPES = UNSIGNED_INT_NUMPY_DTYPES + SIGNED_INT_NUMPY_DTYPES

105ALL_INT_EA_DTYPES = UNSIGNED_INT_EA_DTYPES + SIGNED_INT_EA_DTYPES

106ALL_INT_DTYPES: list[Dtype] = [*ALL_INT_NUMPY_DTYPES, *ALL_INT_EA_DTYPES]

107

108FLOAT_NUMPY_DTYPES: list[NpDtype] = [float, "float32", "float64"]

109FLOAT_EA_DTYPES: list[Dtype] = ["Float32", "Float64"]

110ALL_FLOAT_DTYPES: list[Dtype] = [*FLOAT_NUMPY_DTYPES, *FLOAT_EA_DTYPES]

111

112COMPLEX_DTYPES: list[Dtype] = [complex, "complex64", "complex128"]

113if using_string_dtype():

114 STRING_DTYPES: list[Dtype] = ["U"]

115else:

116 STRING_DTYPES: list[Dtype] = [str, "str", "U"] # type: ignore[no-redef]

117COMPLEX_FLOAT_DTYPES: list[Dtype] = [*COMPLEX_DTYPES, *FLOAT_NUMPY_DTYPES]

118

119DATETIME64_DTYPES: list[Dtype] = ["datetime64[ns]", "M8[ns]"]

120TIMEDELTA64_DTYPES: list[Dtype] = ["timedelta64[ns]", "m8[ns]"]

121

122BOOL_DTYPES: list[Dtype] = [bool, "bool"]

123BYTES_DTYPES: list[Dtype] = [bytes, "bytes"]

124OBJECT_DTYPES: list[Dtype] = [object, "object"]

125

126ALL_REAL_NUMPY_DTYPES = FLOAT_NUMPY_DTYPES + ALL_INT_NUMPY_DTYPES

127ALL_REAL_EXTENSION_DTYPES = FLOAT_EA_DTYPES + ALL_INT_EA_DTYPES

128ALL_REAL_DTYPES: list[Dtype] = [*ALL_REAL_NUMPY_DTYPES, *ALL_REAL_EXTENSION_DTYPES]

129ALL_NUMERIC_DTYPES: list[Dtype] = [*ALL_REAL_DTYPES, *COMPLEX_DTYPES]

130

131ALL_NUMPY_DTYPES = (

132 ALL_REAL_NUMPY_DTYPES

133 + COMPLEX_DTYPES

134 + STRING_DTYPES

135 + DATETIME64_DTYPES

136 + TIMEDELTA64_DTYPES

137 + BOOL_DTYPES

138 + OBJECT_DTYPES

139 + BYTES_DTYPES

140)

141

142NARROW_NP_DTYPES = [

143 np.float16,

144 np.float32,

145 np.int8,

146 np.int16,

147 np.int32,

148 np.uint8,

149 np.uint16,

150 np.uint32,

151]

152

153PYTHON_DATA_TYPES = [

154 str,

155 int,

156 float,

157 complex,

158 list,

159 tuple,

160 range,

161 dict,

162 set,

163 frozenset,

164 bool,

165 bytes,

166 bytearray,

167 memoryview,

168]

169

170ENDIAN = {"little": "<", "big": ">"}[byteorder]

171

172NULL_OBJECTS = [None, np.nan, pd.NaT, float("nan"), pd.NA, Decimal("NaN")]

173NP_NAT_OBJECTS = [

174 cls("NaT", unit)

175 for cls in [np.datetime64, np.timedelta64]

176 for unit in [

177 "Y",

178 "M",

179 "W",

180 "D",

181 "h",

182 "m",

183 "s",

184 "ms",

185 "us",

186 "ns",

187 "ps",

188 "fs",

189 "as",

190 ]

191]

192

193if HAS_PYARROW:

194 import pyarrow as pa

195

196 UNSIGNED_INT_PYARROW_DTYPES = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()]

197 SIGNED_INT_PYARROW_DTYPES = [pa.int8(), pa.int16(), pa.int32(), pa.int64()]

198 ALL_INT_PYARROW_DTYPES = UNSIGNED_INT_PYARROW_DTYPES + SIGNED_INT_PYARROW_DTYPES

199 ALL_INT_PYARROW_DTYPES_STR_REPR = [

200 str(ArrowDtype(typ)) for typ in ALL_INT_PYARROW_DTYPES

201 ]

202

203 # pa.float16 doesn't seem supported

204 # https://github.com/apache/arrow/blob/master/python/pyarrow/src/arrow/python/helpers.cc#L86

205 FLOAT_PYARROW_DTYPES = [pa.float32(), pa.float64()]

206 FLOAT_PYARROW_DTYPES_STR_REPR = [

207 str(ArrowDtype(typ)) for typ in FLOAT_PYARROW_DTYPES

208 ]

209 DECIMAL_PYARROW_DTYPES = [pa.decimal128(7, 3)]

210 STRING_PYARROW_DTYPES = [pa.string()]

211 BINARY_PYARROW_DTYPES = [pa.binary()]

212

213 TIME_PYARROW_DTYPES = [

214 pa.time32("s"),

215 pa.time32("ms"),

216 pa.time64("us"),

217 pa.time64("ns"),

218 ]

219 DATE_PYARROW_DTYPES = [pa.date32(), pa.date64()]

220 DATETIME_PYARROW_DTYPES = [

221 pa.timestamp(unit=unit, tz=tz)

222 for unit in ["s", "ms", "us", "ns"]

223 for tz in [None, "UTC", "US/Pacific", "US/Eastern"]

224 ]

225 TIMEDELTA_PYARROW_DTYPES = [pa.duration(unit) for unit in ["s", "ms", "us", "ns"]]

226

227 BOOL_PYARROW_DTYPES = [pa.bool_()]

228

229 # TODO: Add container like pyarrow types:

230 # https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions

231 ALL_PYARROW_DTYPES = (

232 ALL_INT_PYARROW_DTYPES

233 + FLOAT_PYARROW_DTYPES

234 + DECIMAL_PYARROW_DTYPES

235 + STRING_PYARROW_DTYPES

236 + BINARY_PYARROW_DTYPES

237 + TIME_PYARROW_DTYPES

238 + DATE_PYARROW_DTYPES

239 + DATETIME_PYARROW_DTYPES

240 + TIMEDELTA_PYARROW_DTYPES

241 + BOOL_PYARROW_DTYPES

242 )

243 ALL_REAL_PYARROW_DTYPES_STR_REPR = (

244 ALL_INT_PYARROW_DTYPES_STR_REPR + FLOAT_PYARROW_DTYPES_STR_REPR

245 )

246else:

247 FLOAT_PYARROW_DTYPES_STR_REPR = []

248 ALL_INT_PYARROW_DTYPES_STR_REPR = []

249 ALL_PYARROW_DTYPES = []

250 ALL_REAL_PYARROW_DTYPES_STR_REPR = []

251

252ALL_REAL_NULLABLE_DTYPES = (

253 FLOAT_NUMPY_DTYPES + ALL_REAL_EXTENSION_DTYPES + ALL_REAL_PYARROW_DTYPES_STR_REPR

254)

255

256arithmetic_dunder_methods = [

257 "__add__",

258 "__radd__",

259 "__sub__",

260 "__rsub__",

261 "__mul__",

262 "__rmul__",

263 "__floordiv__",

264 "__rfloordiv__",

265 "__truediv__",

266 "__rtruediv__",

267 "__pow__",

268 "__rpow__",

269 "__mod__",

270 "__rmod__",

271]

272

273comparison_dunder_methods = ["__eq__", "__ne__", "__le__", "__lt__", "__ge__", "__gt__"]

274

275comparison_ops = [

276 operator.eq,

277 operator.ne,

278 operator.gt,

279 operator.ge,

280 operator.lt,

281 operator.le,

282]

283

284numeric_reductions = [

285 "count",

286 "sum",

287 "max",

288 "min",

289 "mean",

290 "prod",

291 "std",

292 "var",

293 "median",

294 "kurt",

295 "skew",

296 "sem",

297]

298

299boolean_reductions = ["all", "any"]

300

301all_reductions = numeric_reductions + boolean_reductions

302

303numeric_accumulations = ["cumsum", "cumprod", "cummin", "cummax"]

304

305

306# -----------------------------------------------------------------------------

307# Comparators

308

309

310def box_expected(expected: Any, box_cls: Any, transpose: bool = True) -> Any:

311 """

312 Helper function to wrap the expected output of a test in a given box_class.

313

314 Parameters

315 ----------

316 expected : np.ndarray, Index, Series

317 box_cls : {Index, Series, DataFrame}

318

319 Returns

320 -------

321 subclass of box_cls

322 """

323 if box_cls is pd.array:

324 if isinstance(expected, RangeIndex):

325 # pd.array would return an IntegerArray

326 expected = NumpyExtensionArray(np.asarray(expected._values))

327 else:

328 expected = pd.array(expected, copy=False)

329 elif box_cls is Index:

330 expected = Index(expected, copy=False)

331 elif box_cls is Series:

332 expected = Series(expected)

333 elif box_cls is DataFrame:

334 expected = Series(expected).to_frame()

335 if transpose:

336 # for vector operations, we need a DataFrame to be a single-row,

337 # not a single-column, in order to operate against non-DataFrame

338 # vectors of the same length. But convert to two rows to avoid

339 # single-row special cases in datetime arithmetic

340 expected = expected.T

341 expected = pd.concat([expected] * 2, ignore_index=True)

342 elif box_cls is np.ndarray or box_cls is np.array:

343 expected = np.array(expected)

344 elif box_cls is to_array:

345 expected = to_array(expected)

346 else:

347 raise NotImplementedError(box_cls)

348 return expected

349

350

351def to_array(obj: Any) -> ExtensionArray | np.ndarray:

352 """

353 Similar to pd.array, but does not cast numpy dtypes to nullable dtypes.

354 """

355 # temporary implementation until we get pd.array in place

356 dtype = getattr(obj, "dtype", None)

357

358 if dtype is None:

359 return np.asarray(obj)

360

361 return extract_array(obj, extract_numpy=True)

362

363

364class SubclassedSeries(Series):

365 _metadata = ["testattr", "name"]

366

367 @property

368 def _constructor(self) -> Callable: # type: ignore[override]

369 # For testing, those properties return a generic callable, and not

370 # the actual class. In this case that is equivalent, but it is to

371 # ensure we don't rely on the property returning a class

372 # See https://github.com/pandas-dev/pandas/pull/46018 and

373 # https://github.com/pandas-dev/pandas/issues/32638 and linked issues

374 return lambda *args, **kwargs: SubclassedSeries(*args, **kwargs)

375

376 @property

377 def _constructor_expanddim(self) -> Callable:

378 return lambda *args, **kwargs: SubclassedDataFrame(*args, **kwargs)

379

380

381class SubclassedDataFrame(DataFrame):

382 _metadata = ["testattr"]

383

384 @property

385 def _constructor(self) -> Callable: # type: ignore[override]

386 return lambda *args, **kwargs: SubclassedDataFrame(*args, **kwargs)

387

388 # error: Cannot override writeable attribute with read-only property

389 @property

390 def _constructor_sliced(self) -> Callable: # type: ignore[override]

391 return lambda *args, **kwargs: SubclassedSeries(*args, **kwargs)

392

393

394def convert_rows_list_to_csv_str(rows_list: list[str]) -> str:

395 """

396 Convert list of CSV rows to single CSV-formatted string for current OS.

397

398 This method is used for creating expected value of to_csv() method.

399

400 Parameters

401 ----------

402 rows_list : List[str]

403 Each element represents the row of csv.

404

405 Returns

406 -------

407 str

408 Expected output of to_csv() in current OS.

409 """

410 sep = os.linesep

411 return sep.join(rows_list) + sep

412

413

414def external_error_raised(

415 expected_exception: type[Exception],

416) -> AbstractContextManager:

417 """

418 Helper function to mark pytest.raises that have an external error message.

419

420 Parameters

421 ----------

422 expected_exception : Exception

423 Expected error to raise.

424

425 Returns

426 -------

427 Callable

428 Regular `pytest.raises` function with `match` equal to `None`.

429 """

430 import pytest

431

432 return pytest.raises(expected_exception, match=None)

433

434

435def get_op_from_name(op_name: str) -> Callable:

436 """

437 The operator function for a given op name.

438

439 Parameters

440 ----------

441 op_name : str

442 The op name, in form of "add" or "__add__".

443

444 Returns

445 -------

446 function

447 A function performing the operation.

448 """

449 short_opname = op_name.strip("_")

450 try:

451 op = getattr(operator, short_opname)

452 except AttributeError:

453 # Assume it is the reverse operator

454 rop = getattr(operator, short_opname[1:])

455 op = lambda x, y: rop(y, x)

456

457 return op

458

459

460# -----------------------------------------------------------------------------

461# Indexing test helpers

462

463

464def getitem(x: Any) -> Any:

465 return x

466

467

468def setitem(x: Any) -> Any:

469 return x

470

471

472def loc(x: Any) -> Any:

473 return x.loc

474

475

476def iloc(x: Any) -> Any:

477 return x.iloc

478

479

480def at(x: Any) -> Any:

481 return x.at

482

483

484def iat(x: Any) -> Any:

485 return x.iat

486

487

488# -----------------------------------------------------------------------------

489

490_UNITS = ["s", "ms", "us", "ns"]

491

492

493def get_finest_unit(left: str, right: str) -> str:

494 """

495 Find the higher of two datetime64 units.

496 """

497 if _UNITS.index(left) >= _UNITS.index(right):

498 return left

499 return right

500

501

502def _pa_buffer_addresses(pa_data: pa.ChunkedArray) -> set[int]:

503 """

504 Addresses of the non-empty buffers backing a pyarrow ChunkedArray.

505

506 Zero-length buffers are excluded because distinct empty arrays can be

507 handed out the same address, which would look like sharing.

508 """

509 return {

510 buf.address

511 for chunk in pa_data.iterchunks()

512 for buf in chunk.buffers()

513 if buf is not None and buf.size > 0

514 }

515

516

517def shares_memory(left: Any, right: Any) -> bool:

518 """

519 Pandas-compat for np.shares_memory.

520 """

521 if isinstance(left, np.ndarray) and isinstance(right, np.ndarray):

522 return np.shares_memory(left, right)

523 elif isinstance(left, np.ndarray):

524 # Call with reversed args to get to unpacking logic below.

525 return shares_memory(right, left)

526

527 if isinstance(left, RangeIndex):

528 return False

529 if isinstance(left, MultiIndex):

530 return any(shares_memory(codes, right) for codes in left._codes)

531 if isinstance(left, (Index, Series)):

532 if isinstance(right, (Index, Series)):

533 return shares_memory(left._values, right._values)

534 return shares_memory(left._values, right)

535

536 if isinstance(left, NDArrayBackedExtensionArray):

537 return shares_memory(left._ndarray, right)

538 if isinstance(left, pd.core.arrays.SparseArray):

539 return shares_memory(left.sp_values, right)

540 if isinstance(left, pd.core.arrays.IntervalArray):

541 return shares_memory(left._left, right) or shares_memory(left._right, right)

542

543 if isinstance(left, ArrowExtensionArray):

544 if isinstance(right, ArrowExtensionArray):

545 # https://github.com/pandas-dev/pandas/pull/43930#discussion_r736862669

546 left_addrs = _pa_buffer_addresses(left._pa_array)

547 right_addrs = _pa_buffer_addresses(right._pa_array)

548 return not left_addrs.isdisjoint(right_addrs)

549 else:

550 # if we have one ArrowExtensionArray and one other array, assume

551 # they can only share memory if they share the same numpy buffer

552 return np.shares_memory(left, right)

553

554 if isinstance(left, BaseMaskedArray):

555 # By convention, we'll say these share memory if they share *either*

556 # the _data or the _mask

557 if isinstance(right, BaseMaskedArray):

558 return shares_memory(left._data, right._data) or shares_memory(

559 left._mask, right._mask

560 )

561 return shares_memory(left._data, right) or shares_memory(left._mask, right)

562

563 if isinstance(left, DataFrame) and len(left._mgr.blocks) == 1:

564 arr = left._mgr.blocks[0].values

565 return shares_memory(arr, right)

566

567 raise NotImplementedError(type(left), type(right))

568

569

570def run_multithreaded(

571 closure: Callable,

572 max_workers: int,

573 arguments: Iterable | None = None,

574 pass_barrier: bool = False,

575) -> None:

576 with ThreadPoolExecutor(max_workers=max_workers) as tpe:

577 if arguments is None:

578 arguments = []

579 else:

580 arguments = list(arguments)

581

582 if pass_barrier:

583 barrier = threading.Barrier(max_workers)

584 arguments.append(barrier)

585

586 try:

587 futures = []

588 for _ in range(max_workers):

589 futures.append(tpe.submit(closure, *arguments)) # noqa: PERF401

590 except RuntimeError as e:

591 import pytest

592

593 pytest.skip(

594 f"Spawning {max_workers} threads failed with "

595 f"error {e!r} (likely due to resource limits on the "

596 "system running the tests)"

597 )

598 finally:

599 if len(futures) < max_workers and pass_barrier:

600 barrier.abort()

601 for f in futures:

602 f.result()

603

604

605__all__ = [

606 "ALL_INT_EA_DTYPES",

607 "ALL_INT_NUMPY_DTYPES",

608 "ALL_NUMPY_DTYPES",

609 "ALL_REAL_NUMPY_DTYPES",

610 "BOOL_DTYPES",

611 "BYTES_DTYPES",

612 "COMPLEX_DTYPES",

613 "DATETIME64_DTYPES",

614 "ENDIAN",

615 "FLOAT_EA_DTYPES",

616 "FLOAT_NUMPY_DTYPES",

617 "NARROW_NP_DTYPES",

618 "NP_NAT_OBJECTS",

619 "NULL_OBJECTS",

620 "OBJECT_DTYPES",

621 "SIGNED_INT_EA_DTYPES",

622 "SIGNED_INT_NUMPY_DTYPES",

623 "STRING_DTYPES",

624 "TIMEDELTA64_DTYPES",

625 "UNSIGNED_INT_EA_DTYPES",

626 "UNSIGNED_INT_NUMPY_DTYPES",

627 "SubclassedDataFrame",

628 "SubclassedSeries",

629 "assert_almost_equal",

630 "assert_attr_equal",

631 "assert_categorical_equal",

632 "assert_class_equal",

633 "assert_contains_all",

634 "assert_copy",

635 "assert_datetime_array_equal",

636 "assert_dict_equal",

637 "assert_equal",

638 "assert_extension_array_equal",

639 "assert_frame_equal",

640 "assert_index_equal",

641 "assert_indexing_slices_equivalent",

642 "assert_interval_array_equal",

643 "assert_is_sorted",

644 "assert_metadata_equivalent",

645 "assert_numpy_array_equal",

646 "assert_period_array_equal",

647 "assert_produces_warning",

648 "assert_series_equal",

649 "assert_sp_array_equal",

650 "assert_timedelta_array_equal",

651 "at",

652 "box_expected",

653 "can_set_locale",

654 "convert_rows_list_to_csv_str",

655 "decompress_file",

656 "external_error_raised",

657 "get_dtype",

658 "get_finest_unit",

659 "get_locales",

660 "get_obj",

661 "get_op_from_name",

662 "getitem",

663 "iat",

664 "iloc",

665 "loc",

666 "maybe_produces_warning",

667 "raise_assert_detail",

668 "raises_chained_assignment_error",

669 "round_trip_pathlib",

670 "round_trip_pickle",

671 "run_multithreaded",

672 "set_locale",

673 "set_timezone",

674 "setitem",

675 "shares_memory",

676 "to_array",

677 "with_csv_dialect",

678 "write_to_compressed",

679]

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/_testing/init.py: 44%

214 statements