Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from typing import (
4 TYPE_CHECKING,
5 Literal,
6)
8import numpy as np
10from pandas._libs import (
11 lib,
12 missing as libmissing,
13)
14from pandas._libs.tslibs import (
15 Timedelta,
16 Timestamp,
17)
18from pandas.util._decorators import set_module
19from pandas.util._validators import check_dtype_backend
21from pandas.core.dtypes.cast import maybe_downcast_numeric
22from pandas.core.dtypes.common import (
23 ensure_object,
24 is_bool_dtype,
25 is_decimal,
26 is_integer_dtype,
27 is_number,
28 is_numeric_dtype,
29 is_scalar,
30 is_string_dtype,
31 needs_i8_conversion,
32)
33from pandas.core.dtypes.dtypes import ArrowDtype
34from pandas.core.dtypes.generic import (
35 ABCIndex,
36 ABCSeries,
37)
39from pandas.core.arrays import BaseMaskedArray
40from pandas.core.arrays.string_ import StringDtype
42if TYPE_CHECKING:
43 from pandas._typing import (
44 DateTimeErrorChoices,
45 DtypeBackend,
46 npt,
47 )
50@set_module("pandas")
51def to_numeric(
52 arg,
53 errors: DateTimeErrorChoices = "raise",
54 downcast: Literal["integer", "signed", "unsigned", "float"] | None = None,
55 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
56):
57 """
58 Convert argument to a numeric type.
60 If the input is already of a numeric dtype, the dtype will be preserved.
61 For non-numeric inputs, the default return dtype is `float64` or `int64`
62 depending on the data supplied. Use the `downcast` parameter
63 to obtain other dtypes.
65 Please note that precision loss may occur if really large numbers
66 are passed in. Due to the internal limitations of `ndarray`, if
67 numbers smaller than `-9223372036854775808` (np.iinfo(np.int64).min)
68 or larger than `18446744073709551615` (np.iinfo(np.uint64).max) are
69 passed in, it is very likely they will be converted to float so that
70 they can be stored in an `ndarray`. These warnings apply similarly to
71 `Series` since it internally leverages `ndarray`.
73 Parameters
74 ----------
75 arg : scalar, list, tuple, 1-d array, Index, or Series
76 Argument to be converted.
78 errors : {'raise', 'coerce'}, default 'raise'
79 - If 'raise', then invalid parsing will raise an exception.
80 - If 'coerce', then invalid parsing will be set as NaN.
82 downcast : str, default None
83 Can be 'integer', 'signed', 'unsigned', or 'float'.
84 If not None, and if the data has been successfully cast to a
85 numerical dtype (or if the data was numeric to begin with),
86 downcast that resulting data to the smallest numerical dtype
87 possible according to the following rules:
89 - 'integer' or 'signed': smallest signed int dtype (min.: np.int8)
90 - 'unsigned': smallest unsigned int dtype (min.: np.uint8)
91 - 'float': smallest float dtype (min.: np.float32)
93 As this behaviour is separate from the core conversion to
94 numeric values, any errors raised during the downcasting
95 will be surfaced regardless of the value of the 'errors' input.
97 In addition, downcasting will only occur if the size
98 of the resulting data's dtype is strictly larger than
99 the dtype it is to be cast to, so if none of the dtypes
100 checked satisfy that specification, no downcasting will be
101 performed on the data.
103 dtype_backend : {'numpy_nullable', 'pyarrow'}
104 Back-end data type applied to the result
105 (still experimental). If not specified, the default behavior
106 is to not use nullable data types. If specified, the behavior
107 is as follows:
109 * ``"numpy_nullable"``: returns nullable-dtype-backed object
110 * ``"pyarrow"``: returns with pyarrow-backed nullable object
112 .. versionadded:: 2.0
114 Returns
115 -------
116 scalar, Series, Index, numpy.ndarray, or ExtensionArray
117 Numeric scalars are returned unchanged. Other scalars are returned
118 as a NumPy scalar: ``np.int64`` if possible (``np.uint64`` for
119 positive values above the ``int64`` maximum), otherwise ``np.float64``,
120 before any ``downcast`` is applied. With ``dtype_backend="pyarrow"``,
121 a Python scalar is returned instead.
122 For 1-d: :class:`Series` if Series, :class:`Index` if Index,
123 otherwise an array as specified by ``dtype_backend``.
125 If ``errors='coerce'``, un-parseable values become ``NaN``
126 (or ``pd.NA`` if ``dtype_backend`` is not the default).
127 If ``errors='raise'`` (default),
128 a :exc:`ValueError` is raised for invalid values.
130 Raises
131 ------
132 ValueError
133 If the input contains non-numeric values and `errors='raise'`.
134 TypeError
135 If the input is not list-like, 1D, or scalar convertible to numeric,
136 such as nested lists or unsupported input types (e.g., dict).
138 See Also
139 --------
140 DataFrame.astype : Cast argument to a specified dtype.
141 to_datetime : Convert argument to datetime.
142 to_timedelta : Convert argument to timedelta.
143 numpy.ndarray.astype : Cast a numpy array to a specified type.
144 DataFrame.convert_dtypes : Convert dtypes.
146 Examples
147 --------
148 Take separate series and convert to numeric, coercing when told to
150 >>> s = pd.Series(["1.0", "2", -3])
151 >>> pd.to_numeric(s)
152 0 1.0
153 1 2.0
154 2 -3.0
155 dtype: float64
156 >>> pd.to_numeric(s, downcast="float")
157 0 1.0
158 1 2.0
159 2 -3.0
160 dtype: float32
161 >>> pd.to_numeric(s, downcast="signed")
162 0 1
163 1 2
164 2 -3
165 dtype: int8
166 >>> s = pd.Series(["apple", "1.0", "2", -3])
167 >>> pd.to_numeric(s, errors="coerce")
168 0 NaN
169 1 1.0
170 2 2.0
171 3 -3.0
172 dtype: float64
174 Downcasting of nullable integer and floating dtypes is supported:
176 >>> s = pd.Series([1, 2, 3], dtype="Int64")
177 >>> pd.to_numeric(s, downcast="integer")
178 0 1
179 1 2
180 2 3
181 dtype: Int8
182 >>> s = pd.Series([1.0, 2.1, 3.0], dtype="Float64")
183 >>> pd.to_numeric(s, downcast="float")
184 0 1.0
185 1 2.1
186 2 3.0
187 dtype: Float32
189 Scalar input returns a scalar:
191 >>> pd.to_numeric("42")
192 np.int64(42)
193 >>> pd.to_numeric("3.14")
194 np.float64(3.14)
196 Index input returns an Index:
198 >>> pd.to_numeric(pd.Index(["1", "2", "3"]))
199 Index([1, 2, 3], dtype='int64')
200 """
201 if downcast not in (None, "integer", "signed", "unsigned", "float"):
202 raise ValueError("invalid downcasting method provided")
204 if errors not in ("raise", "coerce"):
205 raise ValueError("invalid error value specified")
207 check_dtype_backend(dtype_backend)
209 is_series = False
210 is_index = False
211 is_scalars = False
213 if isinstance(arg, ABCSeries):
214 is_series = True
215 values = arg._values
216 if needs_i8_conversion(arg.dtype):
217 values = values.view("i8")
218 elif isinstance(arg, ABCIndex):
219 is_index = True
220 if needs_i8_conversion(arg.dtype):
221 values = arg.view("i8")
222 else:
223 values = arg.values
224 elif isinstance(arg, (list, tuple)):
225 values = np.array(arg, dtype="O")
226 elif is_scalar(arg):
227 if is_decimal(arg):
228 return float(arg)
229 if is_number(arg):
230 return arg
231 if isinstance(arg, (Timedelta, Timestamp)):
232 return arg._value
233 is_scalars = True
234 values = np.array([arg], dtype="O")
235 elif getattr(arg, "ndim", 1) > 1:
236 raise TypeError("arg must be a list, tuple, 1-d array, or Series")
237 else:
238 values = arg
240 # GH33013: for IntegerArray & FloatingArray extract non-null values for casting
241 # save mask to reconstruct the full array after casting
242 mask: npt.NDArray[np.bool_] | None = None
243 if isinstance(values, BaseMaskedArray):
244 mask = values._mask
245 values = values._data[~mask]
247 values_dtype = getattr(values, "dtype", None)
248 if isinstance(values_dtype, ArrowDtype):
249 mask = values.isna()
250 values = values.dropna().to_numpy()
251 new_mask: np.ndarray | None = None
252 if is_numeric_dtype(values_dtype):
253 pass
254 elif lib.is_np_dtype(values_dtype, "mM"):
255 values = values.view(np.int64)
256 else:
257 values = ensure_object(values)
258 coerce_numeric = errors != "raise"
259 values, new_mask = lib.maybe_convert_numeric( # type: ignore[call-overload]
260 values,
261 set(),
262 coerce_numeric=coerce_numeric,
263 convert_to_masked_nullable=dtype_backend is not lib.no_default
264 or (
265 isinstance(values_dtype, StringDtype)
266 and values_dtype.na_value is libmissing.NA
267 ),
268 )
270 if new_mask is not None:
271 # Remove unnecessary values, is expected later anyway and enables
272 # downcasting
273 values = values[~new_mask]
274 elif (dtype_backend is not lib.no_default and new_mask is None) or (
275 isinstance(values_dtype, StringDtype) and values_dtype.na_value is libmissing.NA
276 ):
277 new_mask = np.zeros(values.shape, dtype=np.bool_)
279 # attempt downcast only if the data has been successfully converted
280 # to a numerical dtype and if a downcast method has been specified
281 if downcast is not None and is_numeric_dtype(values.dtype):
282 typecodes: str | None = None
284 if downcast in ("integer", "signed"):
285 typecodes = np.typecodes["Integer"]
286 elif downcast == "unsigned" and (not len(values) or np.min(values) >= 0):
287 typecodes = np.typecodes["UnsignedInteger"]
288 elif downcast == "float":
289 typecodes = np.typecodes["Float"]
291 # pandas support goes only to np.float32,
292 # as float dtypes smaller than that are
293 # extremely rare and not well supported
294 float_32_char = np.dtype(np.float32).char
295 float_32_ind = typecodes.index(float_32_char)
296 typecodes = typecodes[float_32_ind:]
298 if typecodes is not None:
299 # from smallest to largest
300 for typecode in typecodes:
301 dtype = np.dtype(typecode)
302 if dtype.itemsize <= values.dtype.itemsize:
303 values = maybe_downcast_numeric(values, dtype)
305 # successful conversion
306 if values.dtype == dtype:
307 break
309 # GH33013: for IntegerArray, BooleanArray & FloatingArray need to reconstruct
310 # masked array
311 if (mask is not None or new_mask is not None) and not is_string_dtype(values.dtype):
312 if mask is None or (new_mask is not None and new_mask.shape == mask.shape):
313 # GH 52588
314 mask = new_mask
315 else:
316 mask = mask.copy()
317 assert isinstance(mask, np.ndarray)
318 data = np.zeros(mask.shape, dtype=values.dtype)
319 data[~mask] = values
321 from pandas.core.arrays import (
322 ArrowExtensionArray,
323 BooleanArray,
324 FloatingArray,
325 IntegerArray,
326 )
328 klass: type[IntegerArray | BooleanArray | FloatingArray]
329 if is_integer_dtype(data.dtype):
330 klass = IntegerArray
331 elif is_bool_dtype(data.dtype):
332 klass = BooleanArray
333 else:
334 klass = FloatingArray
335 values = klass(data, mask)
337 if dtype_backend == "pyarrow" or isinstance(values_dtype, ArrowDtype):
338 values = ArrowExtensionArray(values.__arrow_array__())
340 if is_series:
341 return arg._constructor(values, index=arg.index, name=arg.name)
342 elif is_index:
343 # because we want to coerce to numeric if possible,
344 # do not use _shallow_copy
345 from pandas import Index
347 return Index(values, name=arg.name)
348 elif is_scalars:
349 return values[0]
350 else:
351 return values