Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This copy of shlex.py from Python 3.6 is distributed with argcomplete.
2# It contains only the shlex class, with modifications as noted.
4"""A lexical analyzer class for simple shell-like syntaxes."""
6# Module and documentation by Eric S. Raymond, 21 Dec 1998
7# Input stacking and error message cleanup added by ESR, March 2000
8# push_source() and pop_source() made explicit by ESR, January 2001.
9# Posix compliance, split(), string arguments, and
10# iterator interface by Gustavo Niemeyer, April 2003.
11# changes to tokenize more like Posix shells by Vinay Sajip, July 2016.
13from __future__ import annotations
15import os
16import sys
17from collections import deque
18from io import StringIO
21class shlex:
22 "A lexical analyzer class for simple shell-like syntaxes."
24 def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False):
25 # Modified by argcomplete: 2/3 compatibility
26 if isinstance(instream, str):
27 instream = StringIO(instream)
28 if instream is not None:
29 self.instream = instream
30 self.infile = infile
31 else:
32 self.instream = sys.stdin
33 self.infile = None
34 self.posix = posix
35 if posix:
36 self.eof = None
37 else:
38 self.eof = ''
39 self.commenters = '#'
40 self.wordchars = 'abcdfeghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
41 # Modified by argcomplete: 2/3 compatibility
42 # if self.posix:
43 # self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
44 # 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
45 self.whitespace = ' \t\r\n'
46 self.whitespace_split = False
47 self.quotes = '\'"'
48 self.escape = '\\'
49 self.escapedquotes = '"'
50 self.state: str | None = ' '
51 self.pushback: deque = deque()
52 self.lineno = 1
53 self.debug = 0
54 self.token = ''
55 self.filestack: deque = deque()
56 self.source = None
57 if not punctuation_chars:
58 punctuation_chars = ''
59 elif punctuation_chars is True:
60 punctuation_chars = '();<>|&'
61 self.punctuation_chars = punctuation_chars
62 if punctuation_chars:
63 # _pushback_chars is a push back queue used by lookahead logic
64 self._pushback_chars: deque = deque()
65 # these chars added because allowed in file names, args, wildcards
66 self.wordchars += '~-./*?='
67 # remove any punctuation chars from wordchars
68 t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
69 self.wordchars = self.wordchars.translate(t)
71 # Modified by argcomplete: Record last wordbreak position
72 self.last_wordbreak_pos = None
73 self.wordbreaks = ''
75 def push_token(self, tok):
76 "Push a token onto the stack popped by the get_token method"
77 if self.debug >= 1:
78 print("shlex: pushing token " + repr(tok))
79 self.pushback.appendleft(tok)
81 def push_source(self, newstream, newfile=None):
82 "Push an input source onto the lexer's input source stack."
83 # Modified by argcomplete: 2/3 compatibility
84 if isinstance(newstream, str):
85 newstream = StringIO(newstream)
86 self.filestack.appendleft((self.infile, self.instream, self.lineno))
87 self.infile = newfile
88 self.instream = newstream
89 self.lineno = 1
90 if self.debug:
91 if newfile is not None:
92 print(f'shlex: pushing to file {self.infile}')
93 else:
94 print(f'shlex: pushing to stream {self.instream}')
96 def pop_source(self):
97 "Pop the input source stack."
98 self.instream.close()
99 (self.infile, self.instream, self.lineno) = self.filestack.popleft()
100 if self.debug:
101 print('shlex: popping to %s, line %d' % (self.instream, self.lineno))
102 self.state = ' '
104 def get_token(self):
105 "Get a token from the input stream (or from stack if it's nonempty)"
106 if self.pushback:
107 tok = self.pushback.popleft()
108 if self.debug >= 1:
109 print("shlex: popping token " + repr(tok))
110 return tok
111 # No pushback. Get a token.
112 raw = self.read_token()
113 # Handle inclusions
114 if self.source is not None:
115 while raw == self.source:
116 spec = self.sourcehook(self.read_token())
117 if spec:
118 (newfile, newstream) = spec
119 self.push_source(newstream, newfile)
120 raw = self.get_token()
121 # Maybe we got EOF instead?
122 while raw == self.eof:
123 if not self.filestack:
124 return self.eof
125 else:
126 self.pop_source()
127 raw = self.get_token()
128 # Neither inclusion nor EOF
129 if self.debug >= 1:
130 if raw != self.eof:
131 print("shlex: token=" + repr(raw))
132 else:
133 print("shlex: token=EOF")
134 return raw
136 def read_token(self):
137 quoted = False
138 escapedstate = ' '
139 while True:
140 if self.punctuation_chars and self._pushback_chars:
141 nextchar = self._pushback_chars.pop()
142 else:
143 nextchar = self.instream.read(1)
144 if nextchar == '\n':
145 self.lineno += 1
146 if self.debug >= 3:
147 print(f"shlex: in state {self.state!r} I see character: {nextchar!r}")
148 if self.state is None:
149 self.token = '' # past end of file
150 break
151 elif self.state == ' ':
152 if not nextchar:
153 self.state = None # end of file
154 break
155 elif nextchar in self.whitespace:
156 if self.debug >= 2:
157 print("shlex: I see whitespace in whitespace state")
158 if self.token or (self.posix and quoted):
159 break # emit current token
160 else:
161 continue
162 elif nextchar in self.commenters:
163 self.instream.readline()
164 self.lineno += 1
165 elif self.posix and nextchar in self.escape:
166 escapedstate = 'a'
167 self.state = nextchar
168 elif nextchar in self.wordchars:
169 self.token = nextchar
170 self.state = 'a'
171 elif nextchar in self.punctuation_chars:
172 self.token = nextchar
173 self.state = 'c'
174 elif nextchar in self.quotes:
175 if not self.posix:
176 self.token = nextchar
177 self.state = nextchar
178 elif self.whitespace_split:
179 self.token = nextchar
180 self.state = 'a'
181 # Modified by argcomplete: Record last wordbreak position
182 if nextchar in self.wordbreaks:
183 self.last_wordbreak_pos = len(self.token) - 1
184 else:
185 self.token = nextchar
186 if self.token or (self.posix and quoted):
187 break # emit current token
188 else:
189 continue
190 elif self.state in self.quotes:
191 quoted = True
192 if not nextchar: # end of file
193 if self.debug >= 2:
194 print("shlex: I see EOF in quotes state")
195 # XXX what error should be raised here?
196 raise ValueError("No closing quotation")
197 if nextchar == self.state:
198 if not self.posix:
199 self.token += nextchar
200 self.state = ' '
201 break
202 else:
203 self.state = 'a'
204 elif self.posix and nextchar in self.escape and self.state in self.escapedquotes:
205 escapedstate = self.state
206 self.state = nextchar
207 else:
208 self.token += nextchar
209 elif self.state in self.escape:
210 if not nextchar: # end of file
211 if self.debug >= 2:
212 print("shlex: I see EOF in escape state")
213 # XXX what error should be raised here?
214 raise ValueError("No escaped character")
215 # In posix shells, only the quote itself or the escape
216 # character may be escaped within quotes.
217 if escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate:
218 self.token += self.state
219 self.token += nextchar
220 self.state = escapedstate
221 elif self.state in ('a', 'c'):
222 if not nextchar:
223 self.state = None # end of file
224 break
225 elif nextchar in self.whitespace:
226 if self.debug >= 2:
227 print("shlex: I see whitespace in word state")
228 self.state = ' '
229 if self.token or (self.posix and quoted):
230 break # emit current token
231 else:
232 continue
233 elif nextchar in self.commenters:
234 self.instream.readline()
235 self.lineno += 1
236 if self.posix:
237 self.state = ' '
238 if self.token or (self.posix and quoted):
239 break # emit current token
240 else:
241 continue
242 elif self.posix and nextchar in self.quotes:
243 self.state = nextchar
244 elif self.posix and nextchar in self.escape:
245 escapedstate = 'a'
246 self.state = nextchar
247 elif self.state == 'c':
248 if nextchar in self.punctuation_chars:
249 self.token += nextchar
250 else:
251 if nextchar not in self.whitespace:
252 self._pushback_chars.append(nextchar)
253 self.state = ' '
254 break
255 elif nextchar in self.wordchars or nextchar in self.quotes or self.whitespace_split:
256 self.token += nextchar
257 # Modified by argcomplete: Record last wordbreak position
258 if nextchar in self.wordbreaks:
259 self.last_wordbreak_pos = len(self.token) - 1
260 else:
261 if self.punctuation_chars:
262 self._pushback_chars.append(nextchar)
263 else:
264 self.pushback.appendleft(nextchar)
265 if self.debug >= 2:
266 print("shlex: I see punctuation in word state")
267 self.state = ' '
268 if self.token or (self.posix and quoted):
269 break # emit current token
270 else:
271 continue
272 result: str | None = self.token
273 self.token = ''
274 if self.posix and not quoted and result == '':
275 result = None
276 if self.debug > 1:
277 if result:
278 print("shlex: raw token=" + repr(result))
279 else:
280 print("shlex: raw token=EOF")
281 # Modified by argcomplete: Record last wordbreak position
282 if self.state == ' ':
283 self.last_wordbreak_pos = None
284 return result
286 def sourcehook(self, newfile):
287 "Hook called on a filename to be sourced."
288 if newfile[0] == '"':
289 newfile = newfile[1:-1]
290 # This implements cpp-like semantics for relative-path inclusion.
291 # Modified by argcomplete: 2/3 compatibility
292 if isinstance(self.infile, str) and not os.path.isabs(newfile):
293 newfile = os.path.join(os.path.dirname(self.infile), newfile)
294 return (newfile, open(newfile, "r"))
296 def error_leader(self, infile=None, lineno=None):
297 "Emit a C-compiler-like, Emacs-friendly error-message leader."
298 if infile is None:
299 infile = self.infile
300 if lineno is None:
301 lineno = self.lineno
302 return "\"%s\", line %d: " % (infile, lineno)
304 def __iter__(self):
305 return self
307 def __next__(self):
308 token = self.get_token()
309 if token == self.eof:
310 raise StopIteration
311 return token
313 # Modified by argcomplete: 2/3 compatibility
314 next = __next__