Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Top level ``eval`` module.
3"""
5from __future__ import annotations
7import tokenize
8from typing import (
9 TYPE_CHECKING,
10 Any,
11)
12import warnings
14from pandas._config.config import _global_config as config
16from pandas._libs import lib
17from pandas.errors import Pandas4Warning
18from pandas.util._decorators import set_module
19from pandas.util._exceptions import find_stack_level
20from pandas.util._validators import validate_bool_kwarg
22from pandas.core.dtypes.common import (
23 is_extension_array_dtype,
24 is_string_dtype,
25)
27from pandas.core.computation.engines import ENGINES
28from pandas.core.computation.expr import (
29 PARSERS,
30 Expr,
31)
32from pandas.core.computation.parsing import tokenize_string
33from pandas.core.computation.scope import ensure_scope
34from pandas.core.generic import NDFrame
36from pandas.io.formats.printing import pprint_thing
38if TYPE_CHECKING:
39 from pandas.core.computation.ops import BinOp
42def _check_engine(engine: str | None) -> str:
43 """
44 Make sure a valid engine is passed.
46 Parameters
47 ----------
48 engine : str
49 String to validate.
51 Raises
52 ------
53 KeyError
54 * If an invalid engine is passed.
55 ImportError
56 * If numexpr was requested but doesn't exist.
58 Returns
59 -------
60 str
61 Engine name.
62 """
63 from pandas.core.computation.check import (
64 NUMEXPR_BLOCKED_VERSION,
65 NUMEXPR_INSTALLED,
66 warn_numexpr_blocked,
67 )
68 from pandas.core.computation.expressions import USE_NUMEXPR
70 if engine is None and not NUMEXPR_INSTALLED and config["compute"]["use_numexpr"]:
71 # report an unusable numexpr where we would have used it, GH#66956. An
72 # explicit engine="numexpr" raises below, which needs no warning.
73 warn_numexpr_blocked()
75 if engine is None:
76 engine = "numexpr" if USE_NUMEXPR else "python"
78 if engine not in ENGINES:
79 valid_engines = list(ENGINES.keys())
80 raise KeyError(
81 f"Invalid engine '{engine}' passed, valid engines are {valid_engines}"
82 )
84 # TODO: validate this in a more general way (thinking of future engines
85 # that won't necessarily be import-able)
86 # Could potentially be done on engine instantiation
87 if engine == "numexpr" and not NUMEXPR_INSTALLED:
88 if NUMEXPR_BLOCKED_VERSION is not None:
89 # the deferred warning does not fire here, so the raise names the version
90 msg = (
91 f"numexpr {NUMEXPR_BLOCKED_VERSION} is installed, but can silently "
92 "return incorrect results, so pandas does not use it. Install "
93 "numexpr 2.14.2 or newer to use engine='numexpr'."
94 )
95 else:
96 msg = (
97 "'numexpr' is not installed or an unsupported version. Cannot use "
98 "engine='numexpr' for query/eval if 'numexpr' is not installed"
99 )
100 raise ImportError(msg)
102 return engine
105def _check_parser(parser: str) -> None:
106 """
107 Make sure a valid parser is passed.
109 Parameters
110 ----------
111 parser : str
113 Raises
114 ------
115 KeyError
116 * If an invalid parser is passed
117 """
118 if parser not in PARSERS:
119 raise KeyError(
120 f"Invalid parser '{parser}' passed, valid parsers are {PARSERS.keys()}"
121 )
124def _check_resolvers(resolvers) -> None:
125 if resolvers is not None:
126 for resolver in resolvers:
127 if not hasattr(resolver, "__getitem__"):
128 name = type(resolver).__name__
129 raise TypeError(
130 f"Resolver of type '{name}' does not "
131 "implement the __getitem__ method"
132 )
135def _check_expression(expr) -> None:
136 """
137 Make sure an expression is not an empty string
139 Parameters
140 ----------
141 expr : object
142 An object that can be converted to a string
144 Raises
145 ------
146 ValueError
147 * If expr is an empty string
148 """
149 if not expr:
150 raise ValueError("expr cannot be an empty string")
153def _convert_expression(expr) -> str:
154 """
155 Convert an object to an expression.
157 This function converts an object to an expression (a unicode string) and
158 checks to make sure it isn't empty after conversion. This is used to
159 convert operators to their string representation for recursive calls to
160 :func:`~pandas.eval`.
162 Parameters
163 ----------
164 expr : object
165 The object to be converted to a string.
167 Returns
168 -------
169 str
170 The string representation of an object.
172 Raises
173 ------
174 ValueError
175 * If the expression is empty.
176 """
177 s = pprint_thing(expr)
178 _check_expression(s)
179 return s
182def _check_for_locals(expr: str, stack_level: int, parser: str) -> None:
183 at_top_of_stack = stack_level == 0
184 not_pandas_parser = parser != "pandas"
186 if not_pandas_parser:
187 msg = "The '@' prefix is only supported by the pandas parser"
188 elif at_top_of_stack:
189 msg = (
190 "The '@' prefix is not allowed in top-level eval calls.\n"
191 "please refer to your variables by name without the '@' prefix."
192 )
194 if at_top_of_stack or not_pandas_parser:
195 for toknum, tokval in tokenize_string(expr):
196 if toknum == tokenize.OP and tokval == "@":
197 raise SyntaxError(msg)
200@set_module("pandas")
201def eval(
202 expr: str | BinOp, # we leave BinOp out of the docstr bc it isn't for users
203 parser: str = "pandas",
204 engine: str | None = None,
205 local_dict=None,
206 global_dict=None,
207 resolvers=(),
208 level: int = 0,
209 target=None,
210 inplace: bool | lib.NoDefault = lib.no_default,
211) -> Any:
212 """
213 Evaluate a Python expression as a string using various backends.
215 .. warning::
217 This function can run arbitrary code which can make you vulnerable to code
218 injection if you pass user input to this function.
220 Parameters
221 ----------
222 expr : str
223 The expression to evaluate. This string cannot contain any Python
224 `statements
225 <https://docs.python.org/3/reference/simple_stmts.html#simple-statements>`__,
226 only Python `expressions
227 <https://docs.python.org/3/reference/simple_stmts.html#expression-statements>`__.
229 By default, with the numexpr engine, the following operations are supported:
231 - Arithmetic operations: ``+``, ``-``, ``*``, ``/``, ``**``, ``%``
232 - Boolean operations: ``|`` (or), ``&`` (and), and ``~`` (not)
233 - Comparison operators: ``<``, ``<=``, ``==``, ``!=``, ``>=``, ``>``
235 Furthermore, the following mathematical functions are supported:
237 - Trigonometric: ``sin``, ``cos``, ``tan``, ``arcsin``, ``arccos``, \
238 ``arctan``, ``arctan2``, ``sinh``, ``cosh``, ``tanh``, ``arcsinh``, \
239 ``arccosh`` and ``arctanh``
240 - Logarithms: ``log`` natural, ``log10`` base 10, ``log1p`` log(1+x)
241 - Absolute Value ``abs``
242 - Square root ``sqrt``
243 - Exponential ``exp`` and Exponential minus one ``expm1``
245 See the numexpr engine `documentation
246 <https://numexpr.readthedocs.io/en/latest/user_guide.html#supported-functions>`__
247 for further function support details.
249 Using the ``'python'`` engine allows the use of native Python operators
250 such as floor division ``//``, in addition to built-in and user-defined
251 Python functions.
253 Additionally, the ``'pandas'`` parser allows the use of :keyword:`and`,
254 :keyword:`or`, and :keyword:`not` with the same semantics as the
255 corresponding bitwise operators.
256 parser : {'pandas', 'python'}, default 'pandas'
257 The parser to use to construct the syntax tree from the expression. The
258 default of ``'pandas'`` parses code slightly different than standard
259 Python. Alternatively, you can parse an expression using the
260 ``'python'`` parser to retain strict Python semantics. See the
261 :ref:`enhancing performance <enhancingperf.eval>` documentation for
262 more details.
263 engine : {'python', 'numexpr'}, optional, default None
265 The engine used to evaluate the expression. Supported engines are
267 - None : tries to use ``numexpr``, falls back to ``python``
268 - ``'numexpr'`` : This is the default engine when ``numexpr`` is installed.
269 Evaluates pandas objects using numexpr for large speed ups in complex
270 expressions with large frames.
271 - ``'python'`` : Performs operations as if you had ``eval``'d in top
272 level python. This engine is generally not that useful.
274 More backends may be available in the future.
275 local_dict : dict or None, optional
276 A dictionary of local variables, taken from locals() by default.
277 global_dict : dict or None, optional
278 A dictionary of global variables, taken from globals() by default.
279 resolvers : list of dict-like or None, optional
280 A list of objects implementing the ``__getitem__`` special method that
281 you can use to inject an additional collection of namespaces to use for
282 variable lookup. For example, this is used in the
283 :meth:`~DataFrame.query` method to inject the
284 ``DataFrame.index`` and ``DataFrame.columns``
285 variables that refer to their respective :class:`~pandas.DataFrame`
286 instance attributes.
287 level : int, optional
288 The number of prior stack frames to traverse and add to the current
289 scope. Most users will **not** need to change this parameter.
290 target : object, optional, default None
291 This is the target object for assignment. It is used when there is
292 variable assignment in the expression. If so, then `target` must
293 support item assignment with string keys, and if a copy is being
294 returned, it must also support `.copy()`.
295 inplace : bool, default False
296 If `target` is provided, and the expression mutates `target`, whether
297 to modify `target` inplace. Otherwise, return a copy of `target` with
298 the mutation.
300 .. deprecated:: 3.1.0
302 This keyword is deprecated and will be removed in pandas 4.0.
303 See the `whatsnew note on PDEP-8
304 <https://pandas.pydata.org/docs/dev/whatsnew/v3.1.0.html#deprecation-inplace>`__
305 for more details.
307 Returns
308 -------
309 ndarray, numeric scalar, DataFrame, Series, or None
310 The completion value of evaluating the given code or None if ``inplace=True``.
312 Raises
313 ------
314 ValueError
315 There are many instances where such an error can be raised:
317 - `target=None`, but the expression is multiline.
318 - The expression is multiline, but not all them have item assignment.
319 An example of such an arrangement is this:
321 a = b + 1
322 a + 2
324 Here, there are expressions on different lines, making it multiline,
325 but the last line has no variable assigned to the output of `a + 2`.
326 - `inplace=True`, but the expression is missing item assignment.
327 - Item assignment is provided, but the `target` does not support
328 string item assignment.
329 - Item assignment is provided and `inplace=False`, but the `target`
330 does not support the `.copy()` method
332 See Also
333 --------
334 DataFrame.query : Evaluates a boolean expression to query the columns
335 of a frame.
336 DataFrame.eval : Evaluate a string describing operations on
337 DataFrame columns.
339 Notes
340 -----
341 The ``dtype`` of any objects involved in an arithmetic ``%`` operation are
342 recursively cast to ``float64``.
344 See the :ref:`enhancing performance <enhancingperf.eval>` documentation for
345 more details.
347 Examples
348 --------
349 >>> df = pd.DataFrame({"animal": ["dog", "pig"], "age": [10, 20]})
350 >>> df
351 animal age
352 0 dog 10
353 1 pig 20
355 We can add a new column using ``pd.eval``:
357 >>> pd.eval("double_age = df.age * 2", target=df)
358 animal age double_age
359 0 dog 10 20
360 1 pig 20 40
361 """
362 if inplace is not lib.no_default:
363 # GH#63207
364 warnings.warn(
365 "The inplace keyword in eval is deprecated and will be removed "
366 "in a future version (PDEP-8).\nSee "
367 "https://pandas.pydata.org/docs/dev/whatsnew/v3.1.0.html#deprecation-inplace"
368 " for more details.",
369 Pandas4Warning,
370 stacklevel=find_stack_level(),
371 )
372 else:
373 inplace = False
375 inplace = validate_bool_kwarg(inplace, "inplace")
377 exprs: list[str | BinOp]
378 if isinstance(expr, str):
379 _check_expression(expr)
380 exprs = [e.strip() for e in expr.splitlines() if e.strip() != ""]
381 elif isinstance(expr, NDFrame):
382 # GH#16289 a Series/DataFrame would otherwise be converted to its
383 # (possibly truncated) repr and parsed, producing a confusing error
384 raise ValueError(f"expr must be a string to be evaluated, {type(expr)} given")
385 else:
386 # ops.BinOp; for internal compat, not intended to be passed by users
387 exprs = [expr]
388 multi_line = len(exprs) > 1
390 if multi_line and target is None:
391 raise ValueError(
392 "multi-line expressions are only valid in the "
393 "context of data, use DataFrame.eval"
394 )
395 engine = _check_engine(engine)
396 _check_parser(parser)
397 _check_resolvers(resolvers)
399 ret = None
400 first_expr = True
401 target_modified = False
403 for expr in exprs:
404 expr = _convert_expression(expr)
405 _check_for_locals(expr, level, parser)
407 # get our (possibly passed-in) scope
408 env = ensure_scope(
409 level + 1,
410 global_dict=global_dict,
411 local_dict=local_dict,
412 resolvers=resolvers,
413 target=target,
414 )
416 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
418 if engine == "numexpr" and (
419 (
420 is_extension_array_dtype(parsed_expr.terms.return_type)
421 and not is_string_dtype(parsed_expr.terms.return_type)
422 )
423 or (
424 getattr(parsed_expr.terms, "operand_types", None) is not None
425 and any(
426 (is_extension_array_dtype(elem) and not is_string_dtype(elem))
427 for elem in parsed_expr.terms.operand_types
428 )
429 )
430 ):
431 warnings.warn(
432 "Engine has switched to 'python' because numexpr does not support "
433 "extension array dtypes. Please set your engine to python manually.",
434 RuntimeWarning,
435 stacklevel=find_stack_level(),
436 )
437 engine = "python"
439 # construct the engine and evaluate the parsed expression
440 eng = ENGINES[engine]
441 eng_inst = eng(parsed_expr)
442 ret = eng_inst.evaluate()
444 if parsed_expr.assigner is None:
445 if multi_line:
446 raise ValueError(
447 "Multi-line expressions are only valid "
448 "if all expressions contain an assignment"
449 )
450 if inplace:
451 raise ValueError("Cannot operate inplace if there is no assignment")
453 # assign if needed
454 assigner = parsed_expr.assigner
455 if env.target is not None and assigner is not None:
456 target_modified = True
458 # if returning a copy, copy only on the first assignment
459 if not inplace and first_expr:
460 try:
461 target = env.target
462 if isinstance(target, NDFrame):
463 target = target.copy(deep=False)
464 else:
465 target = target.copy()
466 except AttributeError as err:
467 raise ValueError("Cannot return a copy of the target") from err
468 else:
469 target = env.target
471 # TypeError is most commonly raised (e.g. int, list), but you
472 # get IndexError if you try to do this assignment on np.ndarray.
473 # we will ignore numpy warnings here; e.g. if trying
474 # to use a non-numeric indexer
475 try:
476 if inplace and isinstance(target, NDFrame):
477 target.loc[:, assigner] = ret
478 else:
479 target[assigner] = ret # pyright: ignore[reportIndexIssue]
480 except (TypeError, IndexError) as err:
481 raise ValueError("Cannot assign expression output to target") from err
483 if not resolvers:
484 resolvers = ({assigner: ret},)
485 else:
486 # existing resolver needs updated to handle
487 # case of mutating existing column in copy
488 for resolver in resolvers:
489 if assigner in resolver:
490 resolver[assigner] = ret
491 break
492 else:
493 resolvers += ({assigner: ret},)
495 ret = None
496 first_expr = False
498 # We want to exclude `inplace=None` as being False.
499 if inplace is False:
500 return target if target_modified else ret