Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from typing import (
4 TYPE_CHECKING,
5 Literal,
6)
8import numpy as np
10from pandas._libs import (
11 lib,
12 missing as libmissing,
13)
14from pandas._libs.tslibs import (
15 Timedelta,
16 Timestamp,
17)
18from pandas.util._decorators import set_module
19from pandas.util._validators import check_dtype_backend
21from pandas.core.dtypes.cast import maybe_downcast_numeric
22from pandas.core.dtypes.common import (
23 ensure_object,
24 is_bool_dtype,
25 is_decimal,
26 is_integer_dtype,
27 is_number,
28 is_numeric_dtype,
29 is_scalar,
30 is_string_dtype,
31 needs_i8_conversion,
32)
33from pandas.core.dtypes.dtypes import ArrowDtype
34from pandas.core.dtypes.generic import (
35 ABCIndex,
36 ABCSeries,
37)
39from pandas.core.arrays import BaseMaskedArray
40from pandas.core.arrays.string_ import StringDtype
42if TYPE_CHECKING:
43 from pandas._typing import (
44 DateTimeErrorChoices,
45 DtypeBackend,
46 npt,
47 )
50@set_module("pandas")
51def to_numeric(
52 arg,
53 errors: DateTimeErrorChoices = "raise",
54 downcast: Literal["integer", "signed", "unsigned", "float"] | None = None,
55 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
56):
57 """
58 Convert argument to a numeric type.
60 If the input is already of a numeric dtype, the dtype will be preserved.
61 For non-numeric inputs, the default return dtype is `float64` or `int64`
62 depending on the data supplied. Use the `downcast` parameter
63 to obtain other dtypes.
65 Please note that precision loss may occur if really large numbers
66 are passed in. Due to the internal limitations of `ndarray`, if
67 numbers smaller than `-9223372036854775808` (np.iinfo(np.int64).min)
68 or larger than `18446744073709551615` (np.iinfo(np.uint64).max) are
69 passed in, it is very likely they will be converted to float so that
70 they can be stored in an `ndarray`. These warnings apply similarly to
71 `Series` since it internally leverages `ndarray`.
73 Parameters
74 ----------
75 arg : scalar, list, tuple, 1-d array, or Series
76 Argument to be converted.
78 errors : {'raise', 'coerce'}, default 'raise'
79 - If 'raise', then invalid parsing will raise an exception.
80 - If 'coerce', then invalid parsing will be set as NaN.
82 downcast : str, default None
83 Can be 'integer', 'signed', 'unsigned', or 'float'.
84 If not None, and if the data has been successfully cast to a
85 numerical dtype (or if the data was numeric to begin with),
86 downcast that resulting data to the smallest numerical dtype
87 possible according to the following rules:
89 - 'integer' or 'signed': smallest signed int dtype (min.: np.int8)
90 - 'unsigned': smallest unsigned int dtype (min.: np.uint8)
91 - 'float': smallest float dtype (min.: np.float32)
93 As this behaviour is separate from the core conversion to
94 numeric values, any errors raised during the downcasting
95 will be surfaced regardless of the value of the 'errors' input.
97 In addition, downcasting will only occur if the size
98 of the resulting data's dtype is strictly larger than
99 the dtype it is to be cast to, so if none of the dtypes
100 checked satisfy that specification, no downcasting will be
101 performed on the data.
103 dtype_backend : {'numpy_nullable', 'pyarrow'}
104 Back-end data type applied to the resultant :class:`DataFrame`
105 (still experimental). If not specified, the default behavior
106 is to not use nullable data types. If specified, the behavior
107 is as follows:
109 * ``"numpy_nullable"``: returns nullable-dtype-backed object
110 * ``"pyarrow"``: returns with pyarrow-backed nullable object
112 .. versionadded:: 2.0
114 Returns
115 -------
116 ret
117 Numeric if parsing succeeded.
118 Return type depends on input. Series if Series, otherwise ndarray.
120 Raises
121 ------
122 ValueError
123 If the input contains non-numeric values and `errors='raise'`.
124 TypeError
125 If the input is not list-like, 1D, or scalar convertible to numeric,
126 such as nested lists or unsupported input types (e.g., dict).
128 See Also
129 --------
130 DataFrame.astype : Cast argument to a specified dtype.
131 to_datetime : Convert argument to datetime.
132 to_timedelta : Convert argument to timedelta.
133 numpy.ndarray.astype : Cast a numpy array to a specified type.
134 DataFrame.convert_dtypes : Convert dtypes.
136 Examples
137 --------
138 Take separate series and convert to numeric, coercing when told to
140 >>> s = pd.Series(["1.0", "2", -3])
141 >>> pd.to_numeric(s)
142 0 1.0
143 1 2.0
144 2 -3.0
145 dtype: float64
146 >>> pd.to_numeric(s, downcast="float")
147 0 1.0
148 1 2.0
149 2 -3.0
150 dtype: float32
151 >>> pd.to_numeric(s, downcast="signed")
152 0 1
153 1 2
154 2 -3
155 dtype: int8
156 >>> s = pd.Series(["apple", "1.0", "2", -3])
157 >>> pd.to_numeric(s, errors="coerce")
158 0 NaN
159 1 1.0
160 2 2.0
161 3 -3.0
162 dtype: float64
164 Downcasting of nullable integer and floating dtypes is supported:
166 >>> s = pd.Series([1, 2, 3], dtype="Int64")
167 >>> pd.to_numeric(s, downcast="integer")
168 0 1
169 1 2
170 2 3
171 dtype: Int8
172 >>> s = pd.Series([1.0, 2.1, 3.0], dtype="Float64")
173 >>> pd.to_numeric(s, downcast="float")
174 0 1.0
175 1 2.1
176 2 3.0
177 dtype: Float32
178 """
179 if downcast not in (None, "integer", "signed", "unsigned", "float"):
180 raise ValueError("invalid downcasting method provided")
182 if errors not in ("raise", "coerce"):
183 raise ValueError("invalid error value specified")
185 check_dtype_backend(dtype_backend)
187 is_series = False
188 is_index = False
189 is_scalars = False
191 if isinstance(arg, ABCSeries):
192 is_series = True
193 values = arg.values
194 elif isinstance(arg, ABCIndex):
195 is_index = True
196 if needs_i8_conversion(arg.dtype):
197 values = arg.view("i8")
198 else:
199 values = arg.values
200 elif isinstance(arg, (list, tuple)):
201 values = np.array(arg, dtype="O")
202 elif is_scalar(arg):
203 if is_decimal(arg):
204 return float(arg)
205 if is_number(arg):
206 return arg
207 if isinstance(arg, (Timedelta, Timestamp)):
208 return arg._value
209 is_scalars = True
210 values = np.array([arg], dtype="O")
211 elif getattr(arg, "ndim", 1) > 1:
212 raise TypeError("arg must be a list, tuple, 1-d array, or Series")
213 else:
214 values = arg
216 # GH33013: for IntegerArray & FloatingArray extract non-null values for casting
217 # save mask to reconstruct the full array after casting
218 mask: npt.NDArray[np.bool_] | None = None
219 if isinstance(values, BaseMaskedArray):
220 mask = values._mask
221 values = values._data[~mask]
223 values_dtype = getattr(values, "dtype", None)
224 if isinstance(values_dtype, ArrowDtype):
225 mask = values.isna()
226 values = values.dropna().to_numpy()
227 new_mask: np.ndarray | None = None
228 if is_numeric_dtype(values_dtype):
229 pass
230 elif lib.is_np_dtype(values_dtype, "mM"):
231 values = values.view(np.int64)
232 else:
233 values = ensure_object(values)
234 coerce_numeric = errors != "raise"
235 values, new_mask = lib.maybe_convert_numeric( # type: ignore[call-overload]
236 values,
237 set(),
238 coerce_numeric=coerce_numeric,
239 convert_to_masked_nullable=dtype_backend is not lib.no_default
240 or (
241 isinstance(values_dtype, StringDtype)
242 and values_dtype.na_value is libmissing.NA
243 ),
244 )
246 if new_mask is not None:
247 # Remove unnecessary values, is expected later anyway and enables
248 # downcasting
249 values = values[~new_mask]
250 elif (dtype_backend is not lib.no_default and new_mask is None) or (
251 isinstance(values_dtype, StringDtype) and values_dtype.na_value is libmissing.NA
252 ):
253 new_mask = np.zeros(values.shape, dtype=np.bool_)
255 # attempt downcast only if the data has been successfully converted
256 # to a numerical dtype and if a downcast method has been specified
257 if downcast is not None and is_numeric_dtype(values.dtype):
258 typecodes: str | None = None
260 if downcast in ("integer", "signed"):
261 typecodes = np.typecodes["Integer"]
262 elif downcast == "unsigned" and (not len(values) or np.min(values) >= 0):
263 typecodes = np.typecodes["UnsignedInteger"]
264 elif downcast == "float":
265 typecodes = np.typecodes["Float"]
267 # pandas support goes only to np.float32,
268 # as float dtypes smaller than that are
269 # extremely rare and not well supported
270 float_32_char = np.dtype(np.float32).char
271 float_32_ind = typecodes.index(float_32_char)
272 typecodes = typecodes[float_32_ind:]
274 if typecodes is not None:
275 # from smallest to largest
276 for typecode in typecodes:
277 dtype = np.dtype(typecode)
278 if dtype.itemsize <= values.dtype.itemsize:
279 values = maybe_downcast_numeric(values, dtype)
281 # successful conversion
282 if values.dtype == dtype:
283 break
285 # GH33013: for IntegerArray, BooleanArray & FloatingArray need to reconstruct
286 # masked array
287 if (mask is not None or new_mask is not None) and not is_string_dtype(values.dtype):
288 if mask is None or (new_mask is not None and new_mask.shape == mask.shape):
289 # GH 52588
290 mask = new_mask
291 else:
292 mask = mask.copy()
293 assert isinstance(mask, np.ndarray)
294 data = np.zeros(mask.shape, dtype=values.dtype)
295 data[~mask] = values
297 from pandas.core.arrays import (
298 ArrowExtensionArray,
299 BooleanArray,
300 FloatingArray,
301 IntegerArray,
302 )
304 klass: type[IntegerArray | BooleanArray | FloatingArray]
305 if is_integer_dtype(data.dtype):
306 klass = IntegerArray
307 elif is_bool_dtype(data.dtype):
308 klass = BooleanArray
309 else:
310 klass = FloatingArray
311 values = klass(data, mask)
313 if dtype_backend == "pyarrow" or isinstance(values_dtype, ArrowDtype):
314 values = ArrowExtensionArray(values.__arrow_array__())
316 if is_series:
317 return arg._constructor(values, index=arg.index, name=arg.name)
318 elif is_index:
319 # because we want to coerce to numeric if possible,
320 # do not use _shallow_copy
321 from pandas import Index
323 return Index(values, name=arg.name)
324 elif is_scalars:
325 return values[0]
326 else:
327 return values