Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/strings/object_array.py: 22%

1from __future__ import annotations 

2 

3import functools 

4import re 

5import textwrap 

6from typing import ( 

7 TYPE_CHECKING, 

8 Literal, 

9 cast, 

10) 

11import unicodedata 

12 

13import numpy as np 

14 

15from pandas._libs import lib 

16import pandas._libs.missing as libmissing 

17import pandas._libs.ops as libops 

18from pandas.util._validators import validate_na_arg 

19 

20from pandas.core.dtypes.common import pandas_dtype 

21from pandas.core.dtypes.missing import isna 

22 

23if TYPE_CHECKING: 

24 from collections.abc import ( 

25 Callable, 

26 Sequence, 

27 ) 

28 

29 from pandas._typing import ( 

30 NpDtype, 

31 Scalar, 

32 ) 

33 

34 

35class ObjectStringArrayMixin: 

36 """ 

37 String Methods operating on object-dtype ndarrays. 

38 """ 

39 

40 def __len__(self) -> int: 

41 # For typing, _str_map relies on the object being sized. 

42 raise NotImplementedError 

43 

44 def _str_getitem(self, key): 

45 if isinstance(key, slice): 

46 return self._str_slice(start=key.start, stop=key.stop, step=key.step) 

47 else: 

48 return self._str_get(key) 

49 

50 def _str_map( 

51 self, 

52 f, 

53 na_value=lib.no_default, 

54 dtype: NpDtype | None = None, 

55 convert: bool = True, 

56 ): 

57 """ 

58 Map a callable over valid elements of the array. 

59 

60 Parameters 

61 ---------- 

62 f : Callable 

63 A function to call on each non-NA element. 

64 na_value : Scalar, optional 

65 The value to set for NA values. Might also be used for the 

66 fill value if the callable `f` raises an exception. 

67 This defaults to ``self.dtype.na_value`` which is ``np.nan`` 

68 for object-dtype and Categorical and ``pd.NA`` for StringArray. 

69 dtype : Dtype, optional 

70 The dtype of the result array. 

71 convert : bool, default True 

72 Whether to call `maybe_convert_objects` on the resulting ndarray 

73 """ 

74 if dtype is None: 

75 dtype = np.dtype("object") 

76 if na_value is lib.no_default: 

77 na_value = self.dtype.na_value # type: ignore[attr-defined] 

78 

79 if not len(self): 

80 return np.array([], dtype=dtype) 

81 

82 arr = np.asarray(self, dtype=object) 

83 mask = isna(arr) 

84 map_convert = convert and not np.all(mask) 

85 try: 

86 result = lib.map_infer_mask( 

87 arr, f, mask.view(np.uint8), convert=map_convert 

88 ) 

89 except (TypeError, AttributeError) as err: 

90 # Reraise the exception if callable `f` got wrong number of args. 

91 # The user may want to be warned by this, instead of getting NaN 

92 p_err = ( 

93 r"((takes)|(missing)) (?(2)from \d+ to )?\d+ " 

94 r"(?(3)required )positional arguments?" 

95 ) 

96 

97 if len(err.args) >= 1 and re.search(p_err, err.args[0]): 

98 # NOTE: matches CPython's TypeError when a user-supplied 

99 # callable (e.g. `repl` in str.replace) is called with the 

100 # wrong number of positional arguments. 

101 raise err 

102 

103 def g(x): 

104 # This type of fallback behavior can be removed once 

105 # we remove object-dtype .str accessor. 

106 try: 

107 return f(x) 

108 except (TypeError, AttributeError): 

109 return na_value 

110 

111 return self._str_map(g, na_value=na_value, dtype=dtype) 

112 if not isinstance(result, np.ndarray): 

113 return result 

114 if na_value is not np.nan: 

115 np.putmask(result, mask, na_value) 

116 if convert and result.dtype == object: 

117 result = lib.maybe_convert_objects(result) 

118 return result 

119 

120 def _str_count(self, pat, flags: int = 0): 

121 regex = re.compile(pat, flags=flags) 

122 f = lambda x: len(regex.findall(x)) 

123 return self._str_map(f, dtype="int64") 

124 

125 def _str_pad( 

126 self, 

127 width: int, 

128 side: Literal["left", "right", "both"] = "left", 

129 fillchar: str = " ", 

130 ): 

131 if side == "left": 

132 f = lambda x: x.rjust(width, fillchar) 

133 elif side == "right": 

134 f = lambda x: x.ljust(width, fillchar) 

135 elif side == "both": 

136 f = lambda x: x.center(width, fillchar) 

137 else: # pragma: no cover 

138 raise ValueError("Invalid side") 

139 return self._str_map(f) 

140 

141 def _str_contains( 

142 self, 

143 pat, 

144 case: bool = True, 

145 flags: int = 0, 

146 na=lib.no_default, 

147 regex: bool = True, 

148 ): 

149 validate_na_arg(na, name="na") 

150 if regex: 

151 if not case: 

152 flags |= re.IGNORECASE 

153 

154 pat = re.compile(pat, flags=flags) 

155 

156 f = lambda x: pat.search(x) is not None 

157 elif case: 

158 f = lambda x: pat in x 

159 else: 

160 upper_pat = pat.upper() 

161 f = lambda x: upper_pat in x.upper() 

162 return self._str_map(f, na, dtype=np.dtype("bool")) 

163 

164 def _str_startswith(self, pat, na=lib.no_default): 

165 validate_na_arg(na, name="na") 

166 f = lambda x: x.startswith(pat) 

167 return self._str_map(f, na_value=na, dtype=np.dtype(bool)) 

168 

169 def _str_endswith(self, pat, na=lib.no_default): 

170 validate_na_arg(na, name="na") 

171 f = lambda x: x.endswith(pat) 

172 return self._str_map(f, na_value=na, dtype=np.dtype(bool)) 

173 

174 def _str_replace( 

175 self, 

176 pat: str | re.Pattern, 

177 repl: str | Callable, 

178 n: int = -1, 

179 case: bool = True, 

180 flags: int = 0, 

181 regex: bool = True, 

182 ): 

183 if case is False: 

184 # add case flag, if provided 

185 flags |= re.IGNORECASE 

186 

187 if regex or flags or callable(repl): 

188 if not isinstance(pat, re.Pattern): 

189 if regex is False: 

190 pat = re.escape(pat) 

191 pat = re.compile(pat, flags=flags) 

192 

193 n = n if n >= 0 else 0 

194 f = lambda x: pat.sub(repl=repl, string=x, count=n) 

195 else: 

196 f = lambda x: x.replace(pat, repl, n) 

197 

198 return self._str_map(f, dtype=str) 

199 

200 def _str_repeat(self, repeats: int | Sequence[int]): 

201 if lib.is_integer(repeats): 

202 rint = cast("int", repeats) 

203 

204 def scalar_rep(x): 

205 try: 

206 return bytes.__mul__(x, rint) 

207 except TypeError: 

208 return str.__mul__(x, rint) 

209 

210 return self._str_map(scalar_rep, dtype=str) 

211 else: 

212 from pandas.core.arrays.string_ import BaseStringArray 

213 

214 def rep(x, r): 

215 if x is libmissing.NA: 

216 return x 

217 try: 

218 return bytes.__mul__(x, r) 

219 except TypeError: 

220 return str.__mul__(x, r) 

221 

222 result = libops.vec_binop( 

223 np.asarray(self), 

224 np.asarray(repeats, dtype=object), 

225 rep, 

226 ) 

227 if not isinstance(self, BaseStringArray): 

228 return result 

229 # Not going through map, so we have to do this here. 

230 return type(self)._from_sequence(result, dtype=self.dtype) 

231 

232 def _str_match( 

233 self, 

234 pat: str | re.Pattern, 

235 case: bool = True, 

236 flags: int = 0, 

237 na: Scalar | lib.NoDefault = lib.no_default, 

238 ): 

239 if not case: 

240 flags |= re.IGNORECASE 

241 

242 if isinstance(pat, re.Pattern): 

243 # pat already carries its own flags. IGNORECASE is the only one we 

244 # can contribute (via `case`), so it is the only bit we compare, 

245 # and only in the direction that would *add* it: a pat that is 

246 # already case-insensitive, e.g. via an inline "(?i)", is honored 

247 # as written. Comparing the full flag set would reject a pat 

248 # compiled with e.g. re.MULTILINE (GH#63108). 

249 if flags & ~re.IGNORECASE: 

250 raise ValueError( 

251 "Cannot pass flags in addition to an already-compiled pat" 

252 ) 

253 if flags & re.IGNORECASE and not pat.flags & re.IGNORECASE: 

254 raise ValueError("Cannot pass flags that do not match pat.flags") 

255 regex = pat 

256 else: 

257 regex = re.compile(pat, flags=flags) 

258 

259 f = lambda x: regex.match(x) is not None 

260 return self._str_map(f, na_value=na, dtype=np.dtype(bool)) 

261 

262 def _str_fullmatch( 

263 self, 

264 pat: str | re.Pattern, 

265 case: bool = True, 

266 flags: int = 0, 

267 na: Scalar | lib.NoDefault = lib.no_default, 

268 ): 

269 if not case: 

270 flags |= re.IGNORECASE 

271 

272 regex = re.compile(pat, flags=flags) 

273 

274 f = lambda x: regex.fullmatch(x) is not None 

275 return self._str_map(f, na_value=na, dtype=np.dtype(bool)) 

276 

277 def _str_encode(self, encoding, errors: str = "strict"): 

278 f = lambda x: x.encode(encoding, errors=errors) 

279 return self._str_map(f, dtype=object) 

280 

281 def _str_find(self, sub, start: int = 0, end=None): 

282 return self._str_find_(sub, start, end, side="left") 

283 

284 def _str_rfind(self, sub, start: int = 0, end=None): 

285 return self._str_find_(sub, start, end, side="right") 

286 

287 def _str_find_(self, sub, start, end, side): 

288 if side == "left": 

289 method = "find" 

290 elif side == "right": 

291 method = "rfind" 

292 else: # pragma: no cover 

293 raise ValueError("Invalid side") 

294 

295 if end is None: 

296 f = lambda x: getattr(x, method)(sub, start) 

297 else: 

298 f = lambda x: getattr(x, method)(sub, start, end) 

299 return self._str_map(f, dtype="int64") 

300 

301 def _str_findall(self, pat, flags: int = 0): 

302 regex = re.compile(pat, flags=flags) 

303 return self._str_map(regex.findall, dtype="object") 

304 

305 def _str_get(self, i): 

306 def f(x): 

307 if isinstance(x, dict): 

308 return x.get(i) 

309 elif len(x) > i >= -len(x): 

310 return x[i] 

311 return self.dtype.na_value # type: ignore[attr-defined] 

312 

313 return self._str_map(f) 

314 

315 def _str_index(self, sub, start: int = 0, end=None): 

316 f = lambda x: x.index(sub, start, end) 

317 return self._str_map(f, dtype="int64") 

318 

319 def _str_rindex(self, sub, start: int = 0, end=None): 

320 f = lambda x: x.rindex(sub, start, end) 

321 return self._str_map(f, dtype="int64") 

322 

323 def _str_join(self, sep: str): 

324 return self._str_map(sep.join) 

325 

326 def _str_partition(self, sep: str, expand): 

327 result = self._str_map(lambda x: x.partition(sep), dtype="object") 

328 return result 

329 

330 def _str_rpartition(self, sep: str, expand): 

331 return self._str_map(lambda x: x.rpartition(sep), dtype="object") 

332 

333 def _str_len(self): 

334 return self._str_map(len, dtype="int64") 

335 

336 def _str_slice(self, start=None, stop=None, step=None): 

337 obj = slice(start, stop, step) 

338 return self._str_map(lambda x: x[obj]) 

339 

340 def _str_slice_replace(self, start=None, stop=None, repl=None): 

341 if repl is None: 

342 repl = "" 

343 

344 def f(x): 

345 if x[start:stop] == "": 

346 local_stop = start 

347 else: 

348 local_stop = stop 

349 y = "" 

350 if start is not None: 

351 y += x[:start] 

352 y += repl 

353 if stop is not None: 

354 y += x[local_stop:] 

355 return y 

356 

357 return self._str_map(f) 

358 

359 def _str_split( 

360 self, 

361 pat: str | re.Pattern | None = None, 

362 n=-1, 

363 expand: bool = False, 

364 regex: bool | None = None, 

365 ): 

366 if pat is None: 

367 if n is None or n == 0: 

368 n = -1 

369 f = lambda x: x.split(pat, n) 

370 else: 

371 new_pat: str | re.Pattern 

372 if regex is True or isinstance(pat, re.Pattern): 

373 new_pat = re.compile(pat) 

374 elif regex is False: 

375 new_pat = pat 

376 # regex is None so link to old behavior #43563 

377 elif len(pat) == 1: 

378 new_pat = pat 

379 else: 

380 new_pat = re.compile(pat) 

381 

382 if isinstance(new_pat, re.Pattern): 

383 if n is None or n == -1: 

384 n = 0 

385 f = lambda x: new_pat.split(x, maxsplit=n) 

386 else: 

387 if n is None or n == 0: 

388 n = -1 

389 f = lambda x: x.split(pat, n) 

390 return self._str_map(f, dtype=object) 

391 

392 def _str_rsplit(self, pat=None, n=-1): 

393 if pat is not None and not isinstance(pat, str): 

394 msg = f"expected a string object, not {type(pat).__name__}" 

395 raise TypeError(msg) 

396 if n is None or n == 0: 

397 n = -1 

398 f = lambda x: x.rsplit(pat, n) 

399 return self._str_map(f, dtype="object") 

400 

401 def _str_translate(self, table): 

402 return self._str_map(lambda x: x.translate(table)) 

403 

404 def _str_wrap(self, width: int, **kwargs): 

405 kwargs["width"] = width 

406 tw = textwrap.TextWrapper(**kwargs) 

407 return self._str_map(lambda s: "\n".join(tw.wrap(s))) 

408 

409 def _str_get_dummies(self, sep: str = "|", dtype: NpDtype | None = None): 

410 from pandas import Series 

411 

412 if dtype is None: 

413 dtype = np.int64 

414 arr = Series(self).fillna("") 

415 try: 

416 arr = sep + arr + sep 

417 except (TypeError, NotImplementedError): 

418 arr = sep + arr.astype(str) + sep 

419 

420 tags: set[str] = set() 

421 for ts in Series(arr, copy=False).str.split(sep): 

422 tags.update(ts) 

423 tags2 = sorted(tags - {""}) 

424 

425 _dtype = pandas_dtype(dtype) 

426 dummies_dtype: NpDtype 

427 if isinstance(_dtype, np.dtype): 

428 dummies_dtype = _dtype 

429 else: 

430 dummies_dtype = np.bool_ 

431 dummies = np.empty((len(arr), len(tags2)), dtype=dummies_dtype, order="F") 

432 

433 def _isin(test_elements: str, element: str) -> bool: 

434 return element in test_elements 

435 

436 for i, t in enumerate(tags2): 

437 pat = sep + t + sep 

438 dummies[:, i] = lib.map_infer( 

439 arr.to_numpy(), functools.partial(_isin, element=pat) 

440 ) 

441 return dummies, tags2 

442 

443 def _str_upper(self): 

444 return self._str_map(lambda x: x.upper()) 

445 

446 def _str_isalnum(self): 

447 return self._str_map(str.isalnum, dtype="bool") 

448 

449 def _str_isalpha(self): 

450 return self._str_map(str.isalpha, dtype="bool") 

451 

452 def _str_isascii(self): 

453 return self._str_map(str.isascii, dtype="bool") 

454 

455 def _str_isdecimal(self): 

456 return self._str_map(str.isdecimal, dtype="bool") 

457 

458 def _str_isdigit(self): 

459 return self._str_map(str.isdigit, dtype="bool") 

460 

461 def _str_islower(self): 

462 return self._str_map(str.islower, dtype="bool") 

463 

464 def _str_isnumeric(self): 

465 return self._str_map(str.isnumeric, dtype="bool") 

466 

467 def _str_isspace(self): 

468 return self._str_map(str.isspace, dtype="bool") 

469 

470 def _str_istitle(self): 

471 return self._str_map(str.istitle, dtype="bool") 

472 

473 def _str_isupper(self): 

474 return self._str_map(str.isupper, dtype="bool") 

475 

476 def _str_capitalize(self): 

477 return self._str_map(str.capitalize) 

478 

479 def _str_casefold(self): 

480 return self._str_map(str.casefold) 

481 

482 def _str_title(self): 

483 return self._str_map(str.title) 

484 

485 def _str_swapcase(self): 

486 return self._str_map(str.swapcase) 

487 

488 def _str_lower(self): 

489 return self._str_map(str.lower) 

490 

491 def _str_normalize(self, form): 

492 f = lambda x: unicodedata.normalize(form, x) 

493 return self._str_map(f) 

494 

495 def _str_strip(self, to_strip=None): 

496 return self._str_map(lambda x: x.strip(to_strip)) 

497 

498 def _str_lstrip(self, to_strip=None): 

499 return self._str_map(lambda x: x.lstrip(to_strip)) 

500 

501 def _str_rstrip(self, to_strip=None): 

502 return self._str_map(lambda x: x.rstrip(to_strip)) 

503 

504 def _str_removeprefix(self, prefix: str): 

505 return self._str_map(lambda x: x.removeprefix(prefix)) 

506 

507 def _str_removesuffix(self, suffix: str): 

508 return self._str_map(lambda x: x.removesuffix(suffix)) 

509 

510 def _str_extract(self, pat: str | re.Pattern, flags: int = 0, expand: bool = True): 

511 regex = re.compile(pat, flags=flags) 

512 na_value = self.dtype.na_value # type: ignore[attr-defined] 

513 

514 if not expand: 

515 

516 def g(x): 

517 m = regex.search(x) 

518 return m.groups()[0] if m else na_value 

519 

520 return self._str_map(g, convert=False) 

521 

522 empty_row = [na_value] * regex.groups 

523 

524 def f(x): 

525 if not isinstance(x, str): 

526 return empty_row 

527 m = regex.search(x) 

528 if m: 

529 return [na_value if item is None else item for item in m.groups()] 

530 else: 

531 return empty_row 

532 

533 return [f(val) for val in np.asarray(self)] 

534 

535 def _str_zfill(self, width: int): 

536 return self._str_map(lambda x: x.zfill(width))