Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/argcomplete/packages/_shlex.py: 61%
236 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:31 +0000
1# This copy of shlex.py from Python 3.6 is distributed with argcomplete.
2# It contains only the shlex class, with modifications as noted.
4"""A lexical analyzer class for simple shell-like syntaxes."""
6# Module and documentation by Eric S. Raymond, 21 Dec 1998
7# Input stacking and error message cleanup added by ESR, March 2000
8# push_source() and pop_source() made explicit by ESR, January 2001.
9# Posix compliance, split(), string arguments, and
10# iterator interface by Gustavo Niemeyer, April 2003.
11# changes to tokenize more like Posix shells by Vinay Sajip, July 2016.
13import os
14import sys
15from collections import deque
16from io import StringIO
17from typing import Optional
20class shlex:
21 "A lexical analyzer class for simple shell-like syntaxes."
23 def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False):
24 # Modified by argcomplete: 2/3 compatibility
25 if isinstance(instream, str):
26 instream = StringIO(instream)
27 if instream is not None:
28 self.instream = instream
29 self.infile = infile
30 else:
31 self.instream = sys.stdin
32 self.infile = None
33 self.posix = posix
34 if posix:
35 self.eof = None
36 else:
37 self.eof = ''
38 self.commenters = '#'
39 self.wordchars = 'abcdfeghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
40 # Modified by argcomplete: 2/3 compatibility
41 # if self.posix:
42 # self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
43 # 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
44 self.whitespace = ' \t\r\n'
45 self.whitespace_split = False
46 self.quotes = '\'"'
47 self.escape = '\\'
48 self.escapedquotes = '"'
49 self.state: Optional[str] = ' '
50 self.pushback: deque = deque()
51 self.lineno = 1
52 self.debug = 0
53 self.token = ''
54 self.filestack: deque = deque()
55 self.source = None
56 if not punctuation_chars:
57 punctuation_chars = ''
58 elif punctuation_chars is True:
59 punctuation_chars = '();<>|&'
60 self.punctuation_chars = punctuation_chars
61 if punctuation_chars:
62 # _pushback_chars is a push back queue used by lookahead logic
63 self._pushback_chars: deque = deque()
64 # these chars added because allowed in file names, args, wildcards
65 self.wordchars += '~-./*?='
66 # remove any punctuation chars from wordchars
67 t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
68 self.wordchars = self.wordchars.translate(t)
70 # Modified by argcomplete: Record last wordbreak position
71 self.last_wordbreak_pos = None
72 self.wordbreaks = ''
74 def push_token(self, tok):
75 "Push a token onto the stack popped by the get_token method"
76 if self.debug >= 1:
77 print("shlex: pushing token " + repr(tok))
78 self.pushback.appendleft(tok)
80 def push_source(self, newstream, newfile=None):
81 "Push an input source onto the lexer's input source stack."
82 # Modified by argcomplete: 2/3 compatibility
83 if isinstance(newstream, str):
84 newstream = StringIO(newstream)
85 self.filestack.appendleft((self.infile, self.instream, self.lineno))
86 self.infile = newfile
87 self.instream = newstream
88 self.lineno = 1
89 if self.debug:
90 if newfile is not None:
91 print('shlex: pushing to file %s' % (self.infile,))
92 else:
93 print('shlex: pushing to stream %s' % (self.instream,))
95 def pop_source(self):
96 "Pop the input source stack."
97 self.instream.close()
98 (self.infile, self.instream, self.lineno) = self.filestack.popleft()
99 if self.debug:
100 print('shlex: popping to %s, line %d' % (self.instream, self.lineno))
101 self.state = ' '
103 def get_token(self):
104 "Get a token from the input stream (or from stack if it's nonempty)"
105 if self.pushback:
106 tok = self.pushback.popleft()
107 if self.debug >= 1:
108 print("shlex: popping token " + repr(tok))
109 return tok
110 # No pushback. Get a token.
111 raw = self.read_token()
112 # Handle inclusions
113 if self.source is not None:
114 while raw == self.source:
115 spec = self.sourcehook(self.read_token())
116 if spec:
117 (newfile, newstream) = spec
118 self.push_source(newstream, newfile)
119 raw = self.get_token()
120 # Maybe we got EOF instead?
121 while raw == self.eof:
122 if not self.filestack:
123 return self.eof
124 else:
125 self.pop_source()
126 raw = self.get_token()
127 # Neither inclusion nor EOF
128 if self.debug >= 1:
129 if raw != self.eof:
130 print("shlex: token=" + repr(raw))
131 else:
132 print("shlex: token=EOF")
133 return raw
135 def read_token(self):
136 quoted = False
137 escapedstate = ' '
138 while True:
139 if self.punctuation_chars and self._pushback_chars:
140 nextchar = self._pushback_chars.pop()
141 else:
142 nextchar = self.instream.read(1)
143 if nextchar == '\n':
144 self.lineno += 1
145 if self.debug >= 3:
146 print("shlex: in state %r I see character: %r" % (self.state, nextchar))
147 if self.state is None:
148 self.token = '' # past end of file
149 break
150 elif self.state == ' ':
151 if not nextchar:
152 self.state = None # end of file
153 break
154 elif nextchar in self.whitespace:
155 if self.debug >= 2:
156 print("shlex: I see whitespace in whitespace state")
157 if self.token or (self.posix and quoted):
158 break # emit current token
159 else:
160 continue
161 elif nextchar in self.commenters:
162 self.instream.readline()
163 self.lineno += 1
164 elif self.posix and nextchar in self.escape:
165 escapedstate = 'a'
166 self.state = nextchar
167 elif nextchar in self.wordchars:
168 self.token = nextchar
169 self.state = 'a'
170 elif nextchar in self.punctuation_chars:
171 self.token = nextchar
172 self.state = 'c'
173 elif nextchar in self.quotes:
174 if not self.posix:
175 self.token = nextchar
176 self.state = nextchar
177 elif self.whitespace_split:
178 self.token = nextchar
179 self.state = 'a'
180 else:
181 self.token = nextchar
182 if self.token or (self.posix and quoted):
183 break # emit current token
184 else:
185 continue
186 elif self.state in self.quotes:
187 quoted = True
188 if not nextchar: # end of file
189 if self.debug >= 2:
190 print("shlex: I see EOF in quotes state")
191 # XXX what error should be raised here?
192 raise ValueError("No closing quotation")
193 if nextchar == self.state:
194 if not self.posix:
195 self.token += nextchar
196 self.state = ' '
197 break
198 else:
199 self.state = 'a'
200 elif self.posix and nextchar in self.escape and self.state in self.escapedquotes:
201 escapedstate = self.state
202 self.state = nextchar
203 else:
204 self.token += nextchar
205 elif self.state in self.escape:
206 if not nextchar: # end of file
207 if self.debug >= 2:
208 print("shlex: I see EOF in escape state")
209 # XXX what error should be raised here?
210 raise ValueError("No escaped character")
211 # In posix shells, only the quote itself or the escape
212 # character may be escaped within quotes.
213 if escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate:
214 self.token += self.state
215 self.token += nextchar
216 self.state = escapedstate
217 elif self.state in ('a', 'c'):
218 if not nextchar:
219 self.state = None # end of file
220 break
221 elif nextchar in self.whitespace:
222 if self.debug >= 2:
223 print("shlex: I see whitespace in word state")
224 self.state = ' '
225 if self.token or (self.posix and quoted):
226 break # emit current token
227 else:
228 continue
229 elif nextchar in self.commenters:
230 self.instream.readline()
231 self.lineno += 1
232 if self.posix:
233 self.state = ' '
234 if self.token or (self.posix and quoted):
235 break # emit current token
236 else:
237 continue
238 elif self.posix and nextchar in self.quotes:
239 self.state = nextchar
240 elif self.posix and nextchar in self.escape:
241 escapedstate = 'a'
242 self.state = nextchar
243 elif self.state == 'c':
244 if nextchar in self.punctuation_chars:
245 self.token += nextchar
246 else:
247 if nextchar not in self.whitespace:
248 self._pushback_chars.append(nextchar)
249 self.state = ' '
250 break
251 elif nextchar in self.wordchars or nextchar in self.quotes or self.whitespace_split:
252 self.token += nextchar
253 # Modified by argcomplete: Record last wordbreak position
254 if nextchar in self.wordbreaks:
255 self.last_wordbreak_pos = len(self.token) - 1
256 else:
257 if self.punctuation_chars:
258 self._pushback_chars.append(nextchar)
259 else:
260 self.pushback.appendleft(nextchar)
261 if self.debug >= 2:
262 print("shlex: I see punctuation in word state")
263 self.state = ' '
264 if self.token or (self.posix and quoted):
265 break # emit current token
266 else:
267 continue
268 result: Optional[str] = self.token
269 self.token = ''
270 if self.posix and not quoted and result == '':
271 result = None
272 if self.debug > 1:
273 if result:
274 print("shlex: raw token=" + repr(result))
275 else:
276 print("shlex: raw token=EOF")
277 # Modified by argcomplete: Record last wordbreak position
278 if self.state == ' ':
279 self.last_wordbreak_pos = None
280 return result
282 def sourcehook(self, newfile):
283 "Hook called on a filename to be sourced."
284 if newfile[0] == '"':
285 newfile = newfile[1:-1]
286 # This implements cpp-like semantics for relative-path inclusion.
287 # Modified by argcomplete: 2/3 compatibility
288 if isinstance(self.infile, str) and not os.path.isabs(newfile):
289 newfile = os.path.join(os.path.dirname(self.infile), newfile)
290 return (newfile, open(newfile, "r"))
292 def error_leader(self, infile=None, lineno=None):
293 "Emit a C-compiler-like, Emacs-friendly error-message leader."
294 if infile is None:
295 infile = self.infile
296 if lineno is None:
297 lineno = self.lineno
298 return "\"%s\", line %d: " % (infile, lineno)
300 def __iter__(self):
301 return self
303 def __next__(self):
304 token = self.get_token()
305 if token == self.eof:
306 raise StopIteration
307 return token
309 # Modified by argcomplete: 2/3 compatibility
310 next = __next__