Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2:func:`~pandas.eval` source string parsing functions
3"""
5from __future__ import annotations
7from enum import Enum
8from io import StringIO
9from keyword import iskeyword
10import token
11import tokenize
12from typing import TYPE_CHECKING
14if TYPE_CHECKING:
15 from collections.abc import (
16 Hashable,
17 Iterator,
18 )
20# A token value Python's tokenizer probably will never use.
21BACKTICK_QUOTED_STRING = 100
24def create_valid_python_identifier(name: str) -> str:
25 """
26 Create valid Python identifiers from any string.
28 Check if name contains any special characters. If it contains any
29 special characters, the special characters will be replaced by
30 a special string and a prefix is added.
32 Raises
33 ------
34 SyntaxError
35 If the returned name is not a Python valid identifier, raise an exception.
36 """
37 if name.isidentifier() and not iskeyword(name):
38 return name
40 # Escape characters that fall outside the ASCII range (U+0001..U+007F).
41 # GH 49633
42 gen = (
43 (c, "".join(chr(b) for b in c.encode("ascii", "backslashreplace")))
44 for c in name
45 )
46 name = "".join(
47 c_escaped.replace("\\", "_UNICODE_" if c != c_escaped else "_BACKSLASH_")
48 for c, c_escaped in gen
49 )
51 # Create a dict with the special characters and their replacement string.
52 # EXACT_TOKEN_TYPES contains these special characters
53 # token.tok_name contains a readable description of the replacement string.
54 special_characters_replacements = {
55 char: f"_{token.tok_name[tokval]}_"
56 for char, tokval in (tokenize.EXACT_TOKEN_TYPES.items())
57 }
58 special_characters_replacements.update(
59 {
60 " ": "_",
61 "?": "_QUESTIONMARK_",
62 "!": "_EXCLAMATIONMARK_",
63 "$": "_DOLLARSIGN_",
64 "€": "_EUROSIGN_",
65 "°": "_DEGREESIGN_",
66 "'": "_SINGLEQUOTE_",
67 '"': "_DOUBLEQUOTE_",
68 "#": "_HASH_",
69 "`": "_BACKTICK_",
70 }
71 )
73 name = "".join([special_characters_replacements.get(char, char) for char in name])
74 name = f"BACKTICK_QUOTED_STRING_{name}"
76 if not name.isidentifier():
77 raise SyntaxError(f"Could not convert '{name}' to a valid Python identifier.")
79 return name
82def clean_backtick_quoted_toks(tok: tuple[int, str]) -> tuple[int, str]:
83 """
84 Clean up a column name if surrounded by backticks.
86 Backtick quoted string are indicated by a certain tokval value. If a string
87 is a backtick quoted token it will processed by
88 :func:`_create_valid_python_identifier` so that the parser can find this
89 string when the query is executed.
90 In this case the tok will get the NAME tokval.
92 Parameters
93 ----------
94 tok : tuple of int, str
95 ints correspond to the all caps constants in the tokenize module
97 Returns
98 -------
99 tok : Tuple[int, str]
100 Either the input or token or the replacement values
101 """
102 toknum, tokval = tok
103 if toknum == BACKTICK_QUOTED_STRING:
104 return tokenize.NAME, create_valid_python_identifier(tokval)
105 return toknum, tokval
108def clean_column_name(name: Hashable) -> Hashable:
109 """
110 Function to emulate the cleaning of a backtick quoted name.
112 The purpose for this function is to see what happens to the name of
113 identifier if it goes to the process of being parsed a Python code
114 inside a backtick quoted string and than being cleaned
115 (removed of any special characters).
117 Parameters
118 ----------
119 name : hashable
120 Name to be cleaned.
122 Returns
123 -------
124 name : hashable
125 Returns the name after tokenizing and cleaning.
126 """
127 try:
128 # Escape backticks
129 name = name.replace("`", "``") if isinstance(name, str) else name
131 tokenized = tokenize_string(f"`{name}`")
132 tokval = next(tokenized)[1]
133 return create_valid_python_identifier(tokval)
134 except SyntaxError:
135 return name
138class ParseState(Enum):
139 DEFAULT = 0
140 IN_BACKTICK = 1
141 IN_SINGLE_QUOTE = 2
142 IN_DOUBLE_QUOTE = 3
145def _split_by_backtick(s: str) -> list[tuple[bool, str]]:
146 """
147 Splits a str into substrings along backtick characters (`).
149 Disregards backticks inside quotes.
151 Parameters
152 ----------
153 s : str
154 The Python source code string.
156 Returns
157 -------
158 substrings: list[tuple[bool, str]]
159 List of tuples, where each tuple has two elements:
160 The first is a boolean indicating if the substring is backtick-quoted.
161 The second is the actual substring.
162 """
163 substrings = []
164 substr: list[str] = [] # Will join into a string before adding to `substrings`
165 i = 0
166 parse_state = ParseState.DEFAULT
167 while i < len(s):
168 char = s[i]
170 match char:
171 case "`":
172 # start of a backtick-quoted string
173 if parse_state == ParseState.DEFAULT:
174 if substr:
175 substrings.append((False, "".join(substr)))
177 substr = [char]
178 i += 1
179 parse_state = ParseState.IN_BACKTICK
180 continue
182 elif parse_state == ParseState.IN_BACKTICK:
183 # escaped backtick inside a backtick-quoted string
184 next_char = s[i + 1] if (i != len(s) - 1) else None
185 if next_char == "`":
186 substr.append(char)
187 substr.append(next_char)
188 i += 2
189 continue
191 # end of the backtick-quoted string
192 else:
193 substr.append(char)
194 substrings.append((True, "".join(substr)))
196 substr = []
197 i += 1
198 parse_state = ParseState.DEFAULT
199 continue
200 case "'":
201 # start of a single-quoted string
202 if parse_state == ParseState.DEFAULT:
203 parse_state = ParseState.IN_SINGLE_QUOTE
204 # end of a single-quoted string
205 elif (parse_state == ParseState.IN_SINGLE_QUOTE) and (s[i - 1] != "\\"):
206 parse_state = ParseState.DEFAULT
207 case '"':
208 # start of a double-quoted string
209 if parse_state == ParseState.DEFAULT:
210 parse_state = ParseState.IN_DOUBLE_QUOTE
211 # end of a double-quoted string
212 elif (parse_state == ParseState.IN_DOUBLE_QUOTE) and (s[i - 1] != "\\"):
213 parse_state = ParseState.DEFAULT
214 substr.append(char)
215 i += 1
217 if substr:
218 substrings.append((False, "".join(substr)))
220 return substrings
223def tokenize_string(source: str) -> Iterator[tuple[int, str]]:
224 """
225 Tokenize a Python source code string.
227 Parameters
228 ----------
229 source : str
230 The Python source code string.
232 Returns
233 -------
234 tok_generator : Iterator[Tuple[int, str]]
235 An iterator yielding all tokens with only toknum and tokval (Tuple[ing, str]).
236 """
237 # GH 59285
238 # Escape characters, including backticks
239 source = "".join(
240 (
241 create_valid_python_identifier(substring[1:-1])
242 if is_backtick_quoted
243 else substring
244 )
245 for is_backtick_quoted, substring in _split_by_backtick(source)
246 )
248 line_reader = StringIO(source).readline
249 token_generator = tokenize.generate_tokens(line_reader)
251 for toknum, tokval, _, _, _ in token_generator:
252 yield toknum, tokval