Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/_testing/__init_

1from __future__ import annotations

3from concurrent.futures import ThreadPoolExecutor

4from decimal import Decimal

5import operator

6import os

7from sys import byteorder

8import threading

9from typing import (

10 TYPE_CHECKING,

11 ContextManager,

12)

14import numpy as np

16from pandas._config import using_string_dtype

17from pandas._config.localization import (

18 can_set_locale,

19 get_locales,

20 set_locale,

21)

23from pandas.compat import HAS_PYARROW

25import pandas as pd

26from pandas import (

27 ArrowDtype,

28 DataFrame,

29 Index,

30 MultiIndex,

31 RangeIndex,

32 Series,

33)

34from pandas._testing._io import (

35 round_trip_pathlib,

36 round_trip_pickle,

37 write_to_compressed,

38)

39from pandas._testing._warnings import (

40 assert_produces_warning,

41 maybe_produces_warning,

42)

43from pandas._testing.asserters import (

44 assert_almost_equal,

45 assert_attr_equal,

46 assert_categorical_equal,

47 assert_class_equal,

48 assert_contains_all,

49 assert_copy,

50 assert_datetime_array_equal,

51 assert_dict_equal,

52 assert_equal,

53 assert_extension_array_equal,

54 assert_frame_equal,

55 assert_index_equal,

56 assert_indexing_slices_equivalent,

57 assert_interval_array_equal,

58 assert_is_sorted,

59 assert_metadata_equivalent,

60 assert_numpy_array_equal,

61 assert_period_array_equal,

62 assert_series_equal,

63 assert_sp_array_equal,

64 assert_timedelta_array_equal,

65 raise_assert_detail,

66)

67from pandas._testing.compat import (

68 get_dtype,

69 get_obj,

70)

71from pandas._testing.contexts import (

72 decompress_file,

73 raises_chained_assignment_error,

74 set_timezone,

75 with_csv_dialect,

76)

77from pandas.core.arrays import (

78 ArrowExtensionArray,

79 BaseMaskedArray,

80 NumpyExtensionArray,

81)

82from pandas.core.arrays._mixins import NDArrayBackedExtensionArray

83from pandas.core.construction import extract_array

85if TYPE_CHECKING:

86 from collections.abc import Callable

88 from pandas._typing import (

89 Dtype,

90 NpDtype,

91 )

94UNSIGNED_INT_NUMPY_DTYPES: list[NpDtype] = ["uint8", "uint16", "uint32", "uint64"]

95UNSIGNED_INT_EA_DTYPES: list[Dtype] = ["UInt8", "UInt16", "UInt32", "UInt64"]

96SIGNED_INT_NUMPY_DTYPES: list[NpDtype] = [int, "int8", "int16", "int32", "int64"]

97SIGNED_INT_EA_DTYPES: list[Dtype] = ["Int8", "Int16", "Int32", "Int64"]

98ALL_INT_NUMPY_DTYPES = UNSIGNED_INT_NUMPY_DTYPES + SIGNED_INT_NUMPY_DTYPES

99ALL_INT_EA_DTYPES = UNSIGNED_INT_EA_DTYPES + SIGNED_INT_EA_DTYPES

100ALL_INT_DTYPES: list[Dtype] = [*ALL_INT_NUMPY_DTYPES, *ALL_INT_EA_DTYPES]

101

102FLOAT_NUMPY_DTYPES: list[NpDtype] = [float, "float32", "float64"]

103FLOAT_EA_DTYPES: list[Dtype] = ["Float32", "Float64"]

104ALL_FLOAT_DTYPES: list[Dtype] = [*FLOAT_NUMPY_DTYPES, *FLOAT_EA_DTYPES]

105

106COMPLEX_DTYPES: list[Dtype] = [complex, "complex64", "complex128"]

107if using_string_dtype():

108 STRING_DTYPES: list[Dtype] = ["U"]

109else:

110 STRING_DTYPES: list[Dtype] = [str, "str", "U"] # type: ignore[no-redef]

111COMPLEX_FLOAT_DTYPES: list[Dtype] = [*COMPLEX_DTYPES, *FLOAT_NUMPY_DTYPES]

112

113DATETIME64_DTYPES: list[Dtype] = ["datetime64[ns]", "M8[ns]"]

114TIMEDELTA64_DTYPES: list[Dtype] = ["timedelta64[ns]", "m8[ns]"]

115

116BOOL_DTYPES: list[Dtype] = [bool, "bool"]

117BYTES_DTYPES: list[Dtype] = [bytes, "bytes"]

118OBJECT_DTYPES: list[Dtype] = [object, "object"]

119

120ALL_REAL_NUMPY_DTYPES = FLOAT_NUMPY_DTYPES + ALL_INT_NUMPY_DTYPES

121ALL_REAL_EXTENSION_DTYPES = FLOAT_EA_DTYPES + ALL_INT_EA_DTYPES

122ALL_REAL_DTYPES: list[Dtype] = [*ALL_REAL_NUMPY_DTYPES, *ALL_REAL_EXTENSION_DTYPES]

123ALL_NUMERIC_DTYPES: list[Dtype] = [*ALL_REAL_DTYPES, *COMPLEX_DTYPES]

124

125ALL_NUMPY_DTYPES = (

126 ALL_REAL_NUMPY_DTYPES

127 + COMPLEX_DTYPES

128 + STRING_DTYPES

129 + DATETIME64_DTYPES

130 + TIMEDELTA64_DTYPES

131 + BOOL_DTYPES

132 + OBJECT_DTYPES

133 + BYTES_DTYPES

134)

135

136NARROW_NP_DTYPES = [

137 np.float16,

138 np.float32,

139 np.int8,

140 np.int16,

141 np.int32,

142 np.uint8,

143 np.uint16,

144 np.uint32,

145]

146

147PYTHON_DATA_TYPES = [

148 str,

149 int,

150 float,

151 complex,

152 list,

153 tuple,

154 range,

155 dict,

156 set,

157 frozenset,

158 bool,

159 bytes,

160 bytearray,

161 memoryview,

162]

163

164ENDIAN = {"little": "<", "big": ">"}[byteorder]

165

166NULL_OBJECTS = [None, np.nan, pd.NaT, float("nan"), pd.NA, Decimal("NaN")]

167NP_NAT_OBJECTS = [

168 cls("NaT", unit)

169 for cls in [np.datetime64, np.timedelta64]

170 for unit in [

171 "Y",

172 "M",

173 "W",

174 "D",

175 "h",

176 "m",

177 "s",

178 "ms",

179 "us",

180 "ns",

181 "ps",

182 "fs",

183 "as",

184 ]

185]

186

187if HAS_PYARROW:

188 import pyarrow as pa

189

190 UNSIGNED_INT_PYARROW_DTYPES = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()]

191 SIGNED_INT_PYARROW_DTYPES = [pa.int8(), pa.int16(), pa.int32(), pa.int64()]

192 ALL_INT_PYARROW_DTYPES = UNSIGNED_INT_PYARROW_DTYPES + SIGNED_INT_PYARROW_DTYPES

193 ALL_INT_PYARROW_DTYPES_STR_REPR = [

194 str(ArrowDtype(typ)) for typ in ALL_INT_PYARROW_DTYPES

195 ]

196

197 # pa.float16 doesn't seem supported

198 # https://github.com/apache/arrow/blob/master/python/pyarrow/src/arrow/python/helpers.cc#L86

199 FLOAT_PYARROW_DTYPES = [pa.float32(), pa.float64()]

200 FLOAT_PYARROW_DTYPES_STR_REPR = [

201 str(ArrowDtype(typ)) for typ in FLOAT_PYARROW_DTYPES

202 ]

203 DECIMAL_PYARROW_DTYPES = [pa.decimal128(7, 3)]

204 STRING_PYARROW_DTYPES = [pa.string()]

205 BINARY_PYARROW_DTYPES = [pa.binary()]

206

207 TIME_PYARROW_DTYPES = [

208 pa.time32("s"),

209 pa.time32("ms"),

210 pa.time64("us"),

211 pa.time64("ns"),

212 ]

213 DATE_PYARROW_DTYPES = [pa.date32(), pa.date64()]

214 DATETIME_PYARROW_DTYPES = [

215 pa.timestamp(unit=unit, tz=tz)

216 for unit in ["s", "ms", "us", "ns"]

217 for tz in [None, "UTC", "US/Pacific", "US/Eastern"]

218 ]

219 TIMEDELTA_PYARROW_DTYPES = [pa.duration(unit) for unit in ["s", "ms", "us", "ns"]]

220

221 BOOL_PYARROW_DTYPES = [pa.bool_()]

222

223 # TODO: Add container like pyarrow types:

224 # https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions

225 ALL_PYARROW_DTYPES = (

226 ALL_INT_PYARROW_DTYPES

227 + FLOAT_PYARROW_DTYPES

228 + DECIMAL_PYARROW_DTYPES

229 + STRING_PYARROW_DTYPES

230 + BINARY_PYARROW_DTYPES

231 + TIME_PYARROW_DTYPES

232 + DATE_PYARROW_DTYPES

233 + DATETIME_PYARROW_DTYPES

234 + TIMEDELTA_PYARROW_DTYPES

235 + BOOL_PYARROW_DTYPES

236 )

237 ALL_REAL_PYARROW_DTYPES_STR_REPR = (

238 ALL_INT_PYARROW_DTYPES_STR_REPR + FLOAT_PYARROW_DTYPES_STR_REPR

239 )

240else:

241 FLOAT_PYARROW_DTYPES_STR_REPR = []

242 ALL_INT_PYARROW_DTYPES_STR_REPR = []

243 ALL_PYARROW_DTYPES = []

244 ALL_REAL_PYARROW_DTYPES_STR_REPR = []

245

246ALL_REAL_NULLABLE_DTYPES = (

247 FLOAT_NUMPY_DTYPES + ALL_REAL_EXTENSION_DTYPES + ALL_REAL_PYARROW_DTYPES_STR_REPR

248)

249

250arithmetic_dunder_methods = [

251 "__add__",

252 "__radd__",

253 "__sub__",

254 "__rsub__",

255 "__mul__",

256 "__rmul__",

257 "__floordiv__",

258 "__rfloordiv__",

259 "__truediv__",

260 "__rtruediv__",

261 "__pow__",

262 "__rpow__",

263 "__mod__",

264 "__rmod__",

265]

266

267comparison_dunder_methods = ["__eq__", "__ne__", "__le__", "__lt__", "__ge__", "__gt__"]

268

269

270# -----------------------------------------------------------------------------

271# Comparators

272

273

274def box_expected(expected, box_cls, transpose: bool = True):

275 """

276 Helper function to wrap the expected output of a test in a given box_class.

277

278 Parameters

279 ----------

280 expected : np.ndarray, Index, Series

281 box_cls : {Index, Series, DataFrame}

282

283 Returns

284 -------

285 subclass of box_cls

286 """

287 if box_cls is pd.array:

288 if isinstance(expected, RangeIndex):

289 # pd.array would return an IntegerArray

290 expected = NumpyExtensionArray(np.asarray(expected._values))

291 else:

292 expected = pd.array(expected, copy=False)

293 elif box_cls is Index:

294 expected = Index(expected, copy=False)

295 elif box_cls is Series:

296 expected = Series(expected)

297 elif box_cls is DataFrame:

298 expected = Series(expected).to_frame()

299 if transpose:

300 # for vector operations, we need a DataFrame to be a single-row,

301 # not a single-column, in order to operate against non-DataFrame

302 # vectors of the same length. But convert to two rows to avoid

303 # single-row special cases in datetime arithmetic

304 expected = expected.T

305 expected = pd.concat([expected] * 2, ignore_index=True)

306 elif box_cls is np.ndarray or box_cls is np.array:

307 expected = np.array(expected)

308 elif box_cls is to_array:

309 expected = to_array(expected)

310 else:

311 raise NotImplementedError(box_cls)

312 return expected

313

314

315def to_array(obj):

316 """

317 Similar to pd.array, but does not cast numpy dtypes to nullable dtypes.

318 """

319 # temporary implementation until we get pd.array in place

320 dtype = getattr(obj, "dtype", None)

321

322 if dtype is None:

323 return np.asarray(obj)

324

325 return extract_array(obj, extract_numpy=True)

326

327

328class SubclassedSeries(Series):

329 _metadata = ["testattr", "name"]

330

331 @property

332 def _constructor(self):

333 # For testing, those properties return a generic callable, and not

334 # the actual class. In this case that is equivalent, but it is to

335 # ensure we don't rely on the property returning a class

336 # See https://github.com/pandas-dev/pandas/pull/46018 and

337 # https://github.com/pandas-dev/pandas/issues/32638 and linked issues

338 return lambda *args, **kwargs: SubclassedSeries(*args, **kwargs)

339

340 @property

341 def _constructor_expanddim(self):

342 return lambda *args, **kwargs: SubclassedDataFrame(*args, **kwargs)

343

344

345class SubclassedDataFrame(DataFrame):

346 _metadata = ["testattr"]

347

348 @property

349 def _constructor(self):

350 return lambda *args, **kwargs: SubclassedDataFrame(*args, **kwargs)

351

352 # error: Cannot override writeable attribute with read-only property

353 @property

354 def _constructor_sliced(self): # type: ignore[override]

355 return lambda *args, **kwargs: SubclassedSeries(*args, **kwargs)

356

357

358def convert_rows_list_to_csv_str(rows_list: list[str]) -> str:

359 """

360 Convert list of CSV rows to single CSV-formatted string for current OS.

361

362 This method is used for creating expected value of to_csv() method.

363

364 Parameters

365 ----------

366 rows_list : List[str]

367 Each element represents the row of csv.

368

369 Returns

370 -------

371 str

372 Expected output of to_csv() in current OS.

373 """

374 sep = os.linesep

375 return sep.join(rows_list) + sep

376

377

378def external_error_raised(expected_exception: type[Exception]) -> ContextManager:

379 """

380 Helper function to mark pytest.raises that have an external error message.

381

382 Parameters

383 ----------

384 expected_exception : Exception

385 Expected error to raise.

386

387 Returns

388 -------

389 Callable

390 Regular `pytest.raises` function with `match` equal to `None`.

391 """

392 import pytest

393

394 return pytest.raises(expected_exception, match=None)

395

396

397def get_cython_table_params(ndframe, func_names_and_expected):

398 """

399 Combine frame, functions from com._cython_table

400 keys and expected result.

401

402 Parameters

403 ----------

404 ndframe : DataFrame or Series

405 func_names_and_expected : Sequence of two items

406 The first item is a name of an NDFrame method ('sum', 'prod') etc.

407 The second item is the expected return value.

408

409 Returns

410 -------

411 list

412 List of three items (DataFrame, function, expected result)

413 """

414 results = []

415 for func_name, expected in func_names_and_expected:

416 results.append((ndframe, func_name, expected))

417 return results

418

419

420def get_op_from_name(op_name: str) -> Callable:

421 """

422 The operator function for a given op name.

423

424 Parameters

425 ----------

426 op_name : str

427 The op name, in form of "add" or "__add__".

428

429 Returns

430 -------

431 function

432 A function performing the operation.

433 """

434 short_opname = op_name.strip("_")

435 try:

436 op = getattr(operator, short_opname)

437 except AttributeError:

438 # Assume it is the reverse operator

439 rop = getattr(operator, short_opname[1:])

440 op = lambda x, y: rop(y, x)

441

442 return op

443

444

445# -----------------------------------------------------------------------------

446# Indexing test helpers

447

448

449def getitem(x):

450 return x

451

452

453def setitem(x):

454 return x

455

456

457def loc(x):

458 return x.loc

459

460

461def iloc(x):

462 return x.iloc

463

464

465def at(x):

466 return x.at

467

468

469def iat(x):

470 return x.iat

471

472

473# -----------------------------------------------------------------------------

474

475_UNITS = ["s", "ms", "us", "ns"]

476

477

478def get_finest_unit(left: str, right: str) -> str:

479 """

480 Find the higher of two datetime64 units.

481 """

482 if _UNITS.index(left) >= _UNITS.index(right):

483 return left

484 return right

485

486

487def shares_memory(left, right) -> bool:

488 """

489 Pandas-compat for np.shares_memory.

490 """

491 if isinstance(left, np.ndarray) and isinstance(right, np.ndarray):

492 return np.shares_memory(left, right)

493 elif isinstance(left, np.ndarray):

494 # Call with reversed args to get to unpacking logic below.

495 return shares_memory(right, left)

496

497 if isinstance(left, RangeIndex):

498 return False

499 if isinstance(left, MultiIndex):

500 return shares_memory(left._codes, right)

501 if isinstance(left, (Index, Series)):

502 if isinstance(right, (Index, Series)):

503 return shares_memory(left._values, right._values)

504 return shares_memory(left._values, right)

505

506 if isinstance(left, NDArrayBackedExtensionArray):

507 return shares_memory(left._ndarray, right)

508 if isinstance(left, pd.core.arrays.SparseArray):

509 return shares_memory(left.sp_values, right)

510 if isinstance(left, pd.core.arrays.IntervalArray):

511 return shares_memory(left._left, right) or shares_memory(left._right, right)

512

513 if isinstance(left, ArrowExtensionArray):

514 if isinstance(right, ArrowExtensionArray):

515 # https://github.com/pandas-dev/pandas/pull/43930#discussion_r736862669

516 left_pa_data = left._pa_array

517 right_pa_data = right._pa_array

518 left_buf1 = left_pa_data.chunk(0).buffers()[1]

519 right_buf1 = right_pa_data.chunk(0).buffers()[1]

520 return left_buf1.address == right_buf1.address

521 else:

522 # if we have one one ArrowExtensionArray and one other array, assume

523 # they can only share memory if they share the same numpy buffer

524 return np.shares_memory(left, right)

525

526 if isinstance(left, BaseMaskedArray) and isinstance(right, BaseMaskedArray):

527 # By convention, we'll say these share memory if they share *either*

528 # the _data or the _mask

529 return np.shares_memory(left._data, right._data) or np.shares_memory(

530 left._mask, right._mask

531 )

532

533 if isinstance(left, DataFrame) and len(left._mgr.blocks) == 1:

534 arr = left._mgr.blocks[0].values

535 return shares_memory(arr, right)

536

537 raise NotImplementedError(type(left), type(right))

538

539

540def run_multithreaded(closure, max_workers, arguments=None, pass_barrier=False):

541 with ThreadPoolExecutor(max_workers=max_workers) as tpe:

542 if arguments is None:

543 arguments = []

544 else:

545 arguments = list(arguments)

546

547 if pass_barrier:

548 barrier = threading.Barrier(max_workers)

549 arguments.append(barrier)

550

551 try:

552 futures = []

553 for _ in range(max_workers):

554 futures.append(tpe.submit(closure, *arguments)) # noqa: PERF401

555 except RuntimeError as e:

556 import pytest

557

558 pytest.skip(

559 f"Spawning {max_workers} threads failed with "

560 f"error {e!r} (likely due to resource limits on the "

561 "system running the tests)"

562 )

563 finally:

564 if len(futures) < max_workers and pass_barrier:

565 barrier.abort()

566 for f in futures:

567 f.result()

568

569

570__all__ = [

571 "ALL_INT_EA_DTYPES",

572 "ALL_INT_NUMPY_DTYPES",

573 "ALL_NUMPY_DTYPES",

574 "ALL_REAL_NUMPY_DTYPES",

575 "BOOL_DTYPES",

576 "BYTES_DTYPES",

577 "COMPLEX_DTYPES",

578 "DATETIME64_DTYPES",

579 "ENDIAN",

580 "FLOAT_EA_DTYPES",

581 "FLOAT_NUMPY_DTYPES",

582 "NARROW_NP_DTYPES",

583 "NP_NAT_OBJECTS",

584 "NULL_OBJECTS",

585 "OBJECT_DTYPES",

586 "SIGNED_INT_EA_DTYPES",

587 "SIGNED_INT_NUMPY_DTYPES",

588 "STRING_DTYPES",

589 "TIMEDELTA64_DTYPES",

590 "UNSIGNED_INT_EA_DTYPES",

591 "UNSIGNED_INT_NUMPY_DTYPES",

592 "SubclassedDataFrame",

593 "SubclassedSeries",

594 "assert_almost_equal",

595 "assert_attr_equal",

596 "assert_categorical_equal",

597 "assert_class_equal",

598 "assert_contains_all",

599 "assert_copy",

600 "assert_datetime_array_equal",

601 "assert_dict_equal",

602 "assert_equal",

603 "assert_extension_array_equal",

604 "assert_frame_equal",

605 "assert_index_equal",

606 "assert_indexing_slices_equivalent",

607 "assert_interval_array_equal",

608 "assert_is_sorted",

609 "assert_metadata_equivalent",

610 "assert_numpy_array_equal",

611 "assert_period_array_equal",

612 "assert_produces_warning",

613 "assert_series_equal",

614 "assert_sp_array_equal",

615 "assert_timedelta_array_equal",

616 "at",

617 "box_expected",

618 "can_set_locale",

619 "convert_rows_list_to_csv_str",

620 "decompress_file",

621 "external_error_raised",

622 "get_cython_table_params",

623 "get_dtype",

624 "get_finest_unit",

625 "get_locales",

626 "get_obj",

627 "get_op_from_name",

628 "getitem",

629 "iat",

630 "iloc",

631 "loc",

632 "maybe_produces_warning",

633 "raise_assert_detail",

634 "raises_chained_assignment_error",

635 "round_trip_pathlib",

636 "round_trip_pickle",

637 "run_multithreaded",

638 "set_locale",

639 "set_timezone",

640 "setitem",

641 "shares_memory",

642 "to_array",

643 "with_csv_dialect",

644 "write_to_compressed",

645]

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/_testing/init.py: 42%

210 statements