Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/core/arrays/numeric.py: 23%

1from __future__ import annotations 

2 

3import numbers 

4from typing import ( 

5 TYPE_CHECKING, 

6 Any, 

7 Self, 

8) 

9 

10import numpy as np 

11 

12from pandas._config import is_nan_na 

13 

14from pandas._libs import ( 

15 lib, 

16 missing as libmissing, 

17) 

18from pandas.errors import AbstractMethodError 

19from pandas.util._decorators import cache_readonly 

20 

21from pandas.core.dtypes.common import ( 

22 is_integer_dtype, 

23 is_string_dtype, 

24 pandas_dtype, 

25) 

26 

27from pandas.core.arrays.masked import ( 

28 BaseMaskedArray, 

29 BaseMaskedDtype, 

30) 

31 

32if TYPE_CHECKING: 

33 from collections.abc import ( 

34 Callable, 

35 Mapping, 

36 ) 

37 

38 import pyarrow 

39 

40 from pandas._typing import ( 

41 DtypeObj, 

42 npt, 

43 ) 

44 

45 from pandas.core.dtypes.dtypes import ExtensionDtype 

46 

47 

48class NumericDtype(BaseMaskedDtype): 

49 _default_np_dtype: np.dtype 

50 _checker: Callable[[Any], bool] # is_foo_dtype 

51 

52 def __repr__(self) -> str: 

53 return f"{self.name}Dtype()" 

54 

55 @cache_readonly 

56 def is_signed_integer(self) -> bool: 

57 return self.kind == "i" 

58 

59 @cache_readonly 

60 def is_unsigned_integer(self) -> bool: 

61 return self.kind == "u" 

62 

63 @property 

64 def _is_numeric(self) -> bool: 

65 return True 

66 

67 def __from_arrow__( 

68 self, array: pyarrow.Array | pyarrow.ChunkedArray 

69 ) -> BaseMaskedArray: 

70 """ 

71 Construct IntegerArray/FloatingArray from pyarrow Array/ChunkedArray. 

72 """ 

73 import pyarrow 

74 

75 from pandas.core.arrays.arrow._arrow_utils import ( 

76 pyarrow_array_to_numpy_and_mask, 

77 ) 

78 

79 array_class = self.construct_array_type() 

80 

81 pyarrow_type = pyarrow.from_numpy_dtype(self.type) 

82 if not array.type.equals(pyarrow_type) and not pyarrow.types.is_null( 

83 array.type 

84 ): 

85 # test_from_arrow_type_error raise for string, but allow 

86 # through itemsize conversion GH#31896 

87 rt_dtype = pandas_dtype(array.type.to_pandas_dtype()) 

88 if rt_dtype.kind not in "iuf": 

89 # Could allow "c" or potentially disallow float<->int conversion, 

90 # but at the moment we specifically test that uint<->int works 

91 raise TypeError( 

92 f"Expected array of {self} type, got {array.type} instead" 

93 ) 

94 

95 array = array.cast(pyarrow_type) 

96 

97 if isinstance(array, pyarrow.ChunkedArray): 

98 array = array.combine_chunks() 

99 

100 data, mask = pyarrow_array_to_numpy_and_mask(array, dtype=self.numpy_dtype) 

101 if data.dtype.kind == "f" and is_nan_na(): 

102 mask[np.isnan(data)] = False 

103 return array_class(data.copy(), ~mask, copy=False) 

104 

105 @classmethod 

106 def _get_dtype_mapping(cls) -> Mapping[np.dtype, NumericDtype]: 

107 raise AbstractMethodError(cls) 

108 

109 @classmethod 

110 def _standardize_dtype(cls, dtype: NumericDtype | str | np.dtype) -> NumericDtype: 

111 """ 

112 Convert a string representation or a numpy dtype to NumericDtype. 

113 """ 

114 if isinstance(dtype, str) and (dtype.startswith(("Int", "UInt", "Float"))): 

115 # Avoid DeprecationWarning from NumPy about np.dtype("Int64") 

116 # https://github.com/numpy/numpy/pull/7476 

117 dtype = dtype.lower() 

118 

119 if not isinstance(dtype, NumericDtype): 

120 mapping = cls._get_dtype_mapping() 

121 try: 

122 dtype = mapping[np.dtype(dtype)] 

123 except KeyError as err: 

124 raise ValueError(f"invalid dtype specified {dtype}") from err 

125 return dtype 

126 

127 @classmethod 

128 def _safe_cast(cls, values: np.ndarray, dtype: np.dtype, copy: bool) -> np.ndarray: 

129 """ 

130 Safely cast the values to the given dtype. 

131 

132 "safe" in this context means the casting is lossless. 

133 """ 

134 raise AbstractMethodError(cls) 

135 

136 

137def _coerce_to_data_and_mask(values, dtype, copy: bool, dtype_cls: type[NumericDtype]): 

138 checker = dtype_cls._checker 

139 default_dtype = dtype_cls._default_np_dtype 

140 

141 mask = None 

142 inferred_type = None 

143 

144 if dtype is None and hasattr(values, "dtype"): 

145 if checker(values.dtype): 

146 dtype = values.dtype 

147 

148 if dtype is not None: 

149 dtype = dtype_cls._standardize_dtype(dtype) 

150 

151 cls = dtype_cls().construct_array_type() 

152 if isinstance(values, cls): 

153 values, mask = values._data, values._mask 

154 if dtype is not None and values.dtype != dtype.numpy_dtype: 

155 if mask.any(): 

156 # values behind the mask are meaningless, so exclude them from 

157 # the lossiness check below 

158 values = values.copy() 

159 values[mask] = dtype_cls._internal_fill_value 

160 # GH#55232 raise instead of silently wrapping around, matching 

161 # the behavior when constructing from a list or ndarray 

162 values = dtype_cls._safe_cast(values, dtype.numpy_dtype, copy=False) 

163 

164 if copy: 

165 values = values.copy() 

166 mask = mask.copy() 

167 return values, mask 

168 

169 original = values 

170 if not copy: 

171 values = np.asarray(values) 

172 else: 

173 values = np.array(values, copy=copy) 

174 inferred_type = None 

175 if values.dtype == object or is_string_dtype(values.dtype): 

176 inferred_type = lib.infer_dtype(values, skipna=True) 

177 if inferred_type == "boolean" and dtype is None: 

178 # object dtype array of bools 

179 name = dtype_cls.__name__.strip("_") 

180 raise TypeError(f"{values.dtype} cannot be converted to {name}") 

181 

182 elif values.dtype.kind == "b" and checker(dtype): 

183 # fastpath 

184 mask = np.zeros(len(values), dtype=np.bool_) 

185 if not copy: 

186 values = np.asarray(values, dtype=default_dtype) 

187 else: 

188 values = np.array(values, dtype=default_dtype, copy=copy) 

189 

190 elif values.dtype.kind not in "iuf": 

191 name = dtype_cls.__name__.strip("_") 

192 raise TypeError(f"{values.dtype} cannot be converted to {name}") 

193 

194 if values.ndim != 1: 

195 raise TypeError("values must be a 1D list-like") 

196 

197 if mask is None: 

198 if values.dtype.kind in "iu": 

199 # fastpath 

200 mask = np.zeros(len(values), dtype=np.bool_) 

201 elif values.dtype.kind == "f": 

202 # np.isnan is faster than is_numeric_na() for floats 

203 # github issue: #60066 

204 if is_nan_na(): 

205 mask = np.isnan(values) 

206 else: 

207 mask = np.zeros(len(values), dtype=np.bool_) 

208 if dtype_cls.__name__.strip("_").startswith(("I", "U")): 

209 wrong = np.isnan(values) 

210 if wrong.any(): 

211 raise ValueError("Cannot cast NaN value to Integer dtype.") 

212 elif is_nan_na(): 

213 mask = libmissing.is_numeric_na(values) 

214 else: 

215 # is_numeric_na will raise on non-numeric NAs 

216 libmissing.is_numeric_na(values) 

217 mask = libmissing.is_pdna_or_none(values) 

218 else: 

219 assert len(mask) == len(values) 

220 

221 if mask.ndim != 1: 

222 raise TypeError("mask must be a 1D list-like") 

223 

224 # infer dtype if needed 

225 if dtype is None: 

226 dtype = default_dtype 

227 else: 

228 dtype = dtype.numpy_dtype 

229 

230 if is_integer_dtype(dtype) and values.dtype.kind == "f" and len(values) > 0: 

231 if mask.all(): 

232 values = np.ones(values.shape, dtype=dtype) 

233 else: 

234 idx = np.nanargmax(values) 

235 if int(values[idx]) != original[idx]: 

236 # We have ints that lost precision during the cast. 

237 inferred_type = lib.infer_dtype(original, skipna=True) 

238 if ( 

239 inferred_type not in ["floating", "mixed-integer-float"] 

240 and not mask.any() 

241 ): 

242 values = np.asarray(original, dtype=dtype) 

243 else: 

244 values = np.asarray(original, dtype="object") 

245 

246 # we copy as need to coerce here 

247 if mask.any(): 

248 values = values.copy() 

249 values[mask] = dtype_cls._internal_fill_value 

250 if inferred_type in ("string", "unicode"): 

251 # casts from str are always safe since they raise 

252 # a ValueError if the str cannot be parsed into a float 

253 values = values.astype(dtype, copy=copy) 

254 else: 

255 values = dtype_cls._safe_cast(values, dtype, copy=False) 

256 return values, mask 

257 

258 

259class NumericArray(BaseMaskedArray): 

260 """ 

261 Base class for IntegerArray and FloatingArray. 

262 """ 

263 

264 _dtype_cls: type[NumericDtype] 

265 

266 def __init__( 

267 self, values: np.ndarray, mask: npt.NDArray[np.bool_], copy: bool = False 

268 ) -> None: 

269 checker = self._dtype_cls._checker 

270 if not (isinstance(values, np.ndarray) and checker(values.dtype)): 

271 descr = ( 

272 "floating" 

273 if self._dtype_cls.kind == "f" # type: ignore[comparison-overlap] 

274 else "integer" 

275 ) 

276 raise TypeError( 

277 f"values should be {descr} numpy array. Use " 

278 "the 'pd.array' function instead" 

279 ) 

280 if values.dtype == np.float16: 

281 # If we don't raise here, then accessing self.dtype would raise 

282 raise TypeError("FloatingArray does not support np.float16 dtype.") 

283 

284 # NB: if is_nan_na() is True 

285 # then caller is responsible for ensuring 

286 # assert mask[np.isnan(values)].all() 

287 

288 super().__init__(values, mask, copy=copy) 

289 

290 @cache_readonly 

291 def dtype(self) -> NumericDtype: 

292 mapping = self._dtype_cls._get_dtype_mapping() 

293 return mapping[self._data.dtype] 

294 

295 @classmethod 

296 def _coerce_to_array( 

297 cls, value, *, dtype: DtypeObj, copy: bool = False 

298 ) -> tuple[np.ndarray, np.ndarray]: 

299 dtype_cls = cls._dtype_cls 

300 values, mask = _coerce_to_data_and_mask(value, dtype, copy, dtype_cls) 

301 return values, mask 

302 

303 @classmethod 

304 def _from_sequence_of_strings( 

305 cls, strings, *, dtype: ExtensionDtype, copy: bool = False 

306 ) -> Self: 

307 from pandas.core.tools.numeric import to_numeric 

308 

309 scalars = to_numeric(strings, errors="raise", dtype_backend="numpy_nullable") 

310 return cls._from_sequence(scalars, dtype=dtype, copy=copy) 

311 

312 _HANDLED_TYPES = (np.ndarray, numbers.Number)