Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Top level ``eval`` module.
3"""
5from __future__ import annotations
7import tokenize
8from typing import (
9 TYPE_CHECKING,
10 Any,
11)
12import warnings
14from pandas.util._decorators import set_module
15from pandas.util._exceptions import find_stack_level
16from pandas.util._validators import validate_bool_kwarg
18from pandas.core.dtypes.common import (
19 is_extension_array_dtype,
20 is_string_dtype,
21)
23from pandas.core.computation.engines import ENGINES
24from pandas.core.computation.expr import (
25 PARSERS,
26 Expr,
27)
28from pandas.core.computation.parsing import tokenize_string
29from pandas.core.computation.scope import ensure_scope
30from pandas.core.generic import NDFrame
32from pandas.io.formats.printing import pprint_thing
34if TYPE_CHECKING:
35 from pandas.core.computation.ops import BinOp
38def _check_engine(engine: str | None) -> str:
39 """
40 Make sure a valid engine is passed.
42 Parameters
43 ----------
44 engine : str
45 String to validate.
47 Raises
48 ------
49 KeyError
50 * If an invalid engine is passed.
51 ImportError
52 * If numexpr was requested but doesn't exist.
54 Returns
55 -------
56 str
57 Engine name.
58 """
59 from pandas.core.computation.check import NUMEXPR_INSTALLED
60 from pandas.core.computation.expressions import USE_NUMEXPR
62 if engine is None:
63 engine = "numexpr" if USE_NUMEXPR else "python"
65 if engine not in ENGINES:
66 valid_engines = list(ENGINES.keys())
67 raise KeyError(
68 f"Invalid engine '{engine}' passed, valid engines are {valid_engines}"
69 )
71 # TODO: validate this in a more general way (thinking of future engines
72 # that won't necessarily be import-able)
73 # Could potentially be done on engine instantiation
74 if engine == "numexpr" and not NUMEXPR_INSTALLED:
75 raise ImportError(
76 "'numexpr' is not installed or an unsupported version. Cannot use "
77 "engine='numexpr' for query/eval if 'numexpr' is not installed"
78 )
80 return engine
83def _check_parser(parser: str) -> None:
84 """
85 Make sure a valid parser is passed.
87 Parameters
88 ----------
89 parser : str
91 Raises
92 ------
93 KeyError
94 * If an invalid parser is passed
95 """
96 if parser not in PARSERS:
97 raise KeyError(
98 f"Invalid parser '{parser}' passed, valid parsers are {PARSERS.keys()}"
99 )
102def _check_resolvers(resolvers) -> None:
103 if resolvers is not None:
104 for resolver in resolvers:
105 if not hasattr(resolver, "__getitem__"):
106 name = type(resolver).__name__
107 raise TypeError(
108 f"Resolver of type '{name}' does not "
109 "implement the __getitem__ method"
110 )
113def _check_expression(expr) -> None:
114 """
115 Make sure an expression is not an empty string
117 Parameters
118 ----------
119 expr : object
120 An object that can be converted to a string
122 Raises
123 ------
124 ValueError
125 * If expr is an empty string
126 """
127 if not expr:
128 raise ValueError("expr cannot be an empty string")
131def _convert_expression(expr) -> str:
132 """
133 Convert an object to an expression.
135 This function converts an object to an expression (a unicode string) and
136 checks to make sure it isn't empty after conversion. This is used to
137 convert operators to their string representation for recursive calls to
138 :func:`~pandas.eval`.
140 Parameters
141 ----------
142 expr : object
143 The object to be converted to a string.
145 Returns
146 -------
147 str
148 The string representation of an object.
150 Raises
151 ------
152 ValueError
153 * If the expression is empty.
154 """
155 s = pprint_thing(expr)
156 _check_expression(s)
157 return s
160def _check_for_locals(expr: str, stack_level: int, parser: str) -> None:
161 at_top_of_stack = stack_level == 0
162 not_pandas_parser = parser != "pandas"
164 if not_pandas_parser:
165 msg = "The '@' prefix is only supported by the pandas parser"
166 elif at_top_of_stack:
167 msg = (
168 "The '@' prefix is not allowed in top-level eval calls.\n"
169 "please refer to your variables by name without the '@' prefix."
170 )
172 if at_top_of_stack or not_pandas_parser:
173 for toknum, tokval in tokenize_string(expr):
174 if toknum == tokenize.OP and tokval == "@":
175 raise SyntaxError(msg)
178@set_module("pandas")
179def eval(
180 expr: str | BinOp, # we leave BinOp out of the docstr bc it isn't for users
181 parser: str = "pandas",
182 engine: str | None = None,
183 local_dict=None,
184 global_dict=None,
185 resolvers=(),
186 level: int = 0,
187 target=None,
188 inplace: bool = False,
189) -> Any:
190 """
191 Evaluate a Python expression as a string using various backends.
193 .. warning::
195 This function can run arbitrary code which can make you vulnerable to code
196 injection if you pass user input to this function.
198 Parameters
199 ----------
200 expr : str
201 The expression to evaluate. This string cannot contain any Python
202 `statements
203 <https://docs.python.org/3/reference/simple_stmts.html#simple-statements>`__,
204 only Python `expressions
205 <https://docs.python.org/3/reference/simple_stmts.html#expression-statements>`__.
207 By default, with the numexpr engine, the following operations are supported:
209 - Arithmetic operations: ``+``, ``-``, ``*``, ``/``, ``**``, ``%``
210 - Boolean operations: ``|`` (or), ``&`` (and), and ``~`` (not)
211 - Comparison operators: ``<``, ``<=``, ``==``, ``!=``, ``>=``, ``>``
213 Furthermore, the following mathematical functions are supported:
215 - Trigonometric: ``sin``, ``cos``, ``tan``, ``arcsin``, ``arccos``, \
216 ``arctan``, ``arctan2``, ``sinh``, ``cosh``, ``tanh``, ``arcsinh``, \
217 ``arccosh`` and ``arctanh``
218 - Logarithms: ``log`` natural, ``log10`` base 10, ``log1p`` log(1+x)
219 - Absolute Value ``abs``
220 - Square root ``sqrt``
221 - Exponential ``exp`` and Exponential minus one ``expm1``
223 See the numexpr engine `documentation
224 <https://numexpr.readthedocs.io/en/latest/user_guide.html#supported-functions>`__
225 for further function support details.
227 Using the ``'python'`` engine allows the use of native Python operators
228 such as floor division ``//``, in addition to built-in and user-defined
229 Python functions.
231 Additionally, the ``'pandas'`` parser allows the use of :keyword:`and`,
232 :keyword:`or`, and :keyword:`not` with the same semantics as the
233 corresponding bitwise operators.
234 parser : {'pandas', 'python'}, default 'pandas'
235 The parser to use to construct the syntax tree from the expression. The
236 default of ``'pandas'`` parses code slightly different than standard
237 Python. Alternatively, you can parse an expression using the
238 ``'python'`` parser to retain strict Python semantics. See the
239 :ref:`enhancing performance <enhancingperf.eval>` documentation for
240 more details.
241 engine : {'python', 'numexpr'}, optional, default None
243 The engine used to evaluate the expression. Supported engines are
245 - None : tries to use ``numexpr``, falls back to ``python``
246 - ``'numexpr'`` : This is the default engine when ``numexpr`` is installed.
247 Evaluates pandas objects using numexpr for large speed ups in complex
248 expressions with large frames.
249 - ``'python'`` : Performs operations as if you had ``eval``'d in top
250 level python. This engine is generally not that useful.
252 More backends may be available in the future.
253 local_dict : dict or None, optional
254 A dictionary of local variables, taken from locals() by default.
255 global_dict : dict or None, optional
256 A dictionary of global variables, taken from globals() by default.
257 resolvers : list of dict-like or None, optional
258 A list of objects implementing the ``__getitem__`` special method that
259 you can use to inject an additional collection of namespaces to use for
260 variable lookup. For example, this is used in the
261 :meth:`~DataFrame.query` method to inject the
262 ``DataFrame.index`` and ``DataFrame.columns``
263 variables that refer to their respective :class:`~pandas.DataFrame`
264 instance attributes.
265 level : int, optional
266 The number of prior stack frames to traverse and add to the current
267 scope. Most users will **not** need to change this parameter.
268 target : object, optional, default None
269 This is the target object for assignment. It is used when there is
270 variable assignment in the expression. If so, then `target` must
271 support item assignment with string keys, and if a copy is being
272 returned, it must also support `.copy()`.
273 inplace : bool, default False
274 If `target` is provided, and the expression mutates `target`, whether
275 to modify `target` inplace. Otherwise, return a copy of `target` with
276 the mutation.
278 Returns
279 -------
280 ndarray, numeric scalar, DataFrame, Series, or None
281 The completion value of evaluating the given code or None if ``inplace=True``.
283 Raises
284 ------
285 ValueError
286 There are many instances where such an error can be raised:
288 - `target=None`, but the expression is multiline.
289 - The expression is multiline, but not all them have item assignment.
290 An example of such an arrangement is this:
292 a = b + 1
293 a + 2
295 Here, there are expressions on different lines, making it multiline,
296 but the last line has no variable assigned to the output of `a + 2`.
297 - `inplace=True`, but the expression is missing item assignment.
298 - Item assignment is provided, but the `target` does not support
299 string item assignment.
300 - Item assignment is provided and `inplace=False`, but the `target`
301 does not support the `.copy()` method
303 See Also
304 --------
305 DataFrame.query : Evaluates a boolean expression to query the columns
306 of a frame.
307 DataFrame.eval : Evaluate a string describing operations on
308 DataFrame columns.
310 Notes
311 -----
312 The ``dtype`` of any objects involved in an arithmetic ``%`` operation are
313 recursively cast to ``float64``.
315 See the :ref:`enhancing performance <enhancingperf.eval>` documentation for
316 more details.
318 Examples
319 --------
320 >>> df = pd.DataFrame({"animal": ["dog", "pig"], "age": [10, 20]})
321 >>> df
322 animal age
323 0 dog 10
324 1 pig 20
326 We can add a new column using ``pd.eval``:
328 >>> pd.eval("double_age = df.age * 2", target=df)
329 animal age double_age
330 0 dog 10 20
331 1 pig 20 40
332 """
333 inplace = validate_bool_kwarg(inplace, "inplace")
335 exprs: list[str | BinOp]
336 if isinstance(expr, str):
337 _check_expression(expr)
338 exprs = [e.strip() for e in expr.splitlines() if e.strip() != ""]
339 else:
340 # ops.BinOp; for internal compat, not intended to be passed by users
341 exprs = [expr]
342 multi_line = len(exprs) > 1
344 if multi_line and target is None:
345 raise ValueError(
346 "multi-line expressions are only valid in the "
347 "context of data, use DataFrame.eval"
348 )
349 engine = _check_engine(engine)
350 _check_parser(parser)
351 _check_resolvers(resolvers)
353 ret = None
354 first_expr = True
355 target_modified = False
357 for expr in exprs:
358 expr = _convert_expression(expr)
359 _check_for_locals(expr, level, parser)
361 # get our (possibly passed-in) scope
362 env = ensure_scope(
363 level + 1,
364 global_dict=global_dict,
365 local_dict=local_dict,
366 resolvers=resolvers,
367 target=target,
368 )
370 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
372 if engine == "numexpr" and (
373 (
374 is_extension_array_dtype(parsed_expr.terms.return_type)
375 and not is_string_dtype(parsed_expr.terms.return_type)
376 )
377 or (
378 getattr(parsed_expr.terms, "operand_types", None) is not None
379 and any(
380 (is_extension_array_dtype(elem) and not is_string_dtype(elem))
381 for elem in parsed_expr.terms.operand_types
382 )
383 )
384 ):
385 warnings.warn(
386 "Engine has switched to 'python' because numexpr does not support "
387 "extension array dtypes. Please set your engine to python manually.",
388 RuntimeWarning,
389 stacklevel=find_stack_level(),
390 )
391 engine = "python"
393 # construct the engine and evaluate the parsed expression
394 eng = ENGINES[engine]
395 eng_inst = eng(parsed_expr)
396 ret = eng_inst.evaluate()
398 if parsed_expr.assigner is None:
399 if multi_line:
400 raise ValueError(
401 "Multi-line expressions are only valid "
402 "if all expressions contain an assignment"
403 )
404 if inplace:
405 raise ValueError("Cannot operate inplace if there is no assignment")
407 # assign if needed
408 assigner = parsed_expr.assigner
409 if env.target is not None and assigner is not None:
410 target_modified = True
412 # if returning a copy, copy only on the first assignment
413 if not inplace and first_expr:
414 try:
415 target = env.target
416 if isinstance(target, NDFrame):
417 target = target.copy(deep=False)
418 else:
419 target = target.copy()
420 except AttributeError as err:
421 raise ValueError("Cannot return a copy of the target") from err
422 else:
423 target = env.target
425 # TypeError is most commonly raised (e.g. int, list), but you
426 # get IndexError if you try to do this assignment on np.ndarray.
427 # we will ignore numpy warnings here; e.g. if trying
428 # to use a non-numeric indexer
429 try:
430 if inplace and isinstance(target, NDFrame):
431 target.loc[:, assigner] = ret
432 else:
433 target[assigner] = ret # pyright: ignore[reportIndexIssue]
434 except (TypeError, IndexError) as err:
435 raise ValueError("Cannot assign expression output to target") from err
437 if not resolvers:
438 resolvers = ({assigner: ret},)
439 else:
440 # existing resolver needs updated to handle
441 # case of mutating existing column in copy
442 for resolver in resolvers:
443 if assigner in resolver:
444 resolver[assigner] = ret
445 break
446 else:
447 resolvers += ({assigner: ret},)
449 ret = None
450 first_expr = False
452 # We want to exclude `inplace=None` as being False.
453 if inplace is False:
454 return target if target_modified else ret