Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pygments/lexers/robotframework.py: 55%

1""" 

2 pygments.lexers.robotframework 

3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

4 

5 Lexer for Robot Framework. 

6 

7 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS. 

8 :license: BSD, see LICENSE for details. 

9""" 

10 

11# Copyright 2012 Nokia Siemens Networks Oyj 

12# 

13# Licensed under the Apache License, Version 2.0 (the "License"); 

14# you may not use this file except in compliance with the License. 

15# You may obtain a copy of the License at 

16# 

17# http://www.apache.org/licenses/LICENSE-2.0 

18# 

19# Unless required by applicable law or agreed to in writing, software 

20# distributed under the License is distributed on an "AS IS" BASIS, 

21# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

22# See the License for the specific language governing permissions and 

23# limitations under the License. 

24 

25import re 

26 

27from pygments.lexer import Lexer 

28from pygments.token import Token 

29 

30__all__ = ['RobotFrameworkLexer'] 

31 

32 

33HEADING = Token.Generic.Heading 

34SETTING = Token.Keyword.Namespace 

35IMPORT = Token.Name.Namespace 

36TC_KW_NAME = Token.Generic.Subheading 

37KEYWORD = Token.Name.Function 

38ARGUMENT = Token.String 

39VARIABLE = Token.Name.Variable 

40COMMENT = Token.Comment 

41SEPARATOR = Token.Punctuation 

42SYNTAX = Token.Punctuation 

43GHERKIN = Token.Generic.Emph 

44ERROR = Token.Error 

45 

46 

47def normalize(string, remove=''): 

48 string = string.lower() 

49 for char in remove + ' ': 

50 if char in string: 

51 string = string.replace(char, '') 

52 return string 

53 

54 

55class RobotFrameworkLexer(Lexer): 

56 """ 

57 For Robot Framework test data. 

58 

59 Supports both space and pipe separated plain text formats. 

60 

61 .. versionadded:: 1.6 

62 """ 

63 name = 'RobotFramework' 

64 url = 'http://robotframework.org' 

65 aliases = ['robotframework'] 

66 filenames = ['*.robot', '*.resource'] 

67 mimetypes = ['text/x-robotframework'] 

68 

69 def __init__(self, **options): 

70 options['tabsize'] = 2 

71 options['encoding'] = 'UTF-8' 

72 Lexer.__init__(self, **options) 

73 

74 def get_tokens_unprocessed(self, text): 

75 row_tokenizer = RowTokenizer() 

76 var_tokenizer = VariableTokenizer() 

77 index = 0 

78 for row in text.splitlines(): 

79 for value, token in row_tokenizer.tokenize(row): 

80 for value, token in var_tokenizer.tokenize(value, token): 

81 if value: 

82 yield index, token, str(value) 

83 index += len(value) 

84 

85 

86class VariableTokenizer: 

87 

88 def tokenize(self, string, token): 

89 var = VariableSplitter(string, identifiers='$@%&') 

90 if var.start < 0 or token in (COMMENT, ERROR): 

91 yield string, token 

92 return 

93 for value, token in self._tokenize(var, string, token): 

94 if value: 

95 yield value, token 

96 

97 def _tokenize(self, var, string, orig_token): 

98 before = string[:var.start] 

99 yield before, orig_token 

100 yield var.identifier + '{', SYNTAX 

101 yield from self.tokenize(var.base, VARIABLE) 

102 yield '}', SYNTAX 

103 if var.index is not None: 

104 yield '[', SYNTAX 

105 yield from self.tokenize(var.index, VARIABLE) 

106 yield ']', SYNTAX 

107 yield from self.tokenize(string[var.end:], orig_token) 

108 

109 

110class RowTokenizer: 

111 

112 def __init__(self): 

113 self._table = UnknownTable() 

114 self._splitter = RowSplitter() 

115 testcases = TestCaseTable() 

116 settings = SettingTable(testcases.set_default_template) 

117 variables = VariableTable() 

118 keywords = KeywordTable() 

119 self._tables = {'settings': settings, 'setting': settings, 

120 'metadata': settings, 

121 'variables': variables, 'variable': variables, 

122 'testcases': testcases, 'testcase': testcases, 

123 'tasks': testcases, 'task': testcases, 

124 'keywords': keywords, 'keyword': keywords, 

125 'userkeywords': keywords, 'userkeyword': keywords} 

126 

127 def tokenize(self, row): 

128 commented = False 

129 heading = False 

130 for index, value in enumerate(self._splitter.split(row)): 

131 # First value, and every second after that, is a separator. 

132 index, separator = divmod(index-1, 2) 

133 if value.startswith('#'): 

134 commented = True 

135 elif index == 0 and value.startswith('*'): 

136 self._table = self._start_table(value) 

137 heading = True 

138 yield from self._tokenize(value, index, commented, 

139 separator, heading) 

140 self._table.end_row() 

141 

142 def _start_table(self, header): 

143 name = normalize(header, remove='*') 

144 return self._tables.get(name, UnknownTable()) 

145 

146 def _tokenize(self, value, index, commented, separator, heading): 

147 if commented: 

148 yield value, COMMENT 

149 elif separator: 

150 yield value, SEPARATOR 

151 elif heading: 

152 yield value, HEADING 

153 else: 

154 yield from self._table.tokenize(value, index) 

155 

156 

157class RowSplitter: 

158 _space_splitter = re.compile('( {2,})') 

159 _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))') 

160 

161 def split(self, row): 

162 splitter = (row.startswith('| ') and self._split_from_pipes 

163 or self._split_from_spaces) 

164 yield from splitter(row) 

165 yield '\n' 

166 

167 def _split_from_spaces(self, row): 

168 yield '' # Start with (pseudo)separator similarly as with pipes 

169 yield from self._space_splitter.split(row) 

170 

171 def _split_from_pipes(self, row): 

172 _, separator, rest = self._pipe_splitter.split(row, 1) 

173 yield separator 

174 while self._pipe_splitter.search(rest): 

175 cell, separator, rest = self._pipe_splitter.split(rest, 1) 

176 yield cell 

177 yield separator 

178 yield rest 

179 

180 

181class Tokenizer: 

182 _tokens = None 

183 

184 def __init__(self): 

185 self._index = 0 

186 

187 def tokenize(self, value): 

188 values_and_tokens = self._tokenize(value, self._index) 

189 self._index += 1 

190 if isinstance(values_and_tokens, type(Token)): 

191 values_and_tokens = [(value, values_and_tokens)] 

192 return values_and_tokens 

193 

194 def _tokenize(self, value, index): 

195 index = min(index, len(self._tokens) - 1) 

196 return self._tokens[index] 

197 

198 def _is_assign(self, value): 

199 if value.endswith('='): 

200 value = value[:-1].strip() 

201 var = VariableSplitter(value, identifiers='$@&') 

202 return var.start == 0 and var.end == len(value) 

203 

204 

205class Comment(Tokenizer): 

206 _tokens = (COMMENT,) 

207 

208 

209class Setting(Tokenizer): 

210 _tokens = (SETTING, ARGUMENT) 

211 _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown', 

212 'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition', 

213 'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate') 

214 _import_settings = ('library', 'resource', 'variables') 

215 _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags', 

216 'testtimeout','tasktimeout') 

217 _custom_tokenizer = None 

218 

219 def __init__(self, template_setter=None): 

220 Tokenizer.__init__(self) 

221 self._template_setter = template_setter 

222 

223 def _tokenize(self, value, index): 

224 if index == 1 and self._template_setter: 

225 self._template_setter(value) 

226 if index == 0: 

227 normalized = normalize(value) 

228 if normalized in self._keyword_settings: 

229 self._custom_tokenizer = KeywordCall(support_assign=False) 

230 elif normalized in self._import_settings: 

231 self._custom_tokenizer = ImportSetting() 

232 elif normalized not in self._other_settings: 

233 return ERROR 

234 elif self._custom_tokenizer: 

235 return self._custom_tokenizer.tokenize(value) 

236 return Tokenizer._tokenize(self, value, index) 

237 

238 

239class ImportSetting(Tokenizer): 

240 _tokens = (IMPORT, ARGUMENT) 

241 

242 

243class TestCaseSetting(Setting): 

244 _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition', 

245 'template') 

246 _import_settings = () 

247 _other_settings = ('documentation', 'tags', 'timeout') 

248 

249 def _tokenize(self, value, index): 

250 if index == 0: 

251 type = Setting._tokenize(self, value[1:-1], index) 

252 return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)] 

253 return Setting._tokenize(self, value, index) 

254 

255 

256class KeywordSetting(TestCaseSetting): 

257 _keyword_settings = ('teardown',) 

258 _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags') 

259 

260 

261class Variable(Tokenizer): 

262 _tokens = (SYNTAX, ARGUMENT) 

263 

264 def _tokenize(self, value, index): 

265 if index == 0 and not self._is_assign(value): 

266 return ERROR 

267 return Tokenizer._tokenize(self, value, index) 

268 

269 

270class KeywordCall(Tokenizer): 

271 _tokens = (KEYWORD, ARGUMENT) 

272 

273 def __init__(self, support_assign=True): 

274 Tokenizer.__init__(self) 

275 self._keyword_found = not support_assign 

276 self._assigns = 0 

277 

278 def _tokenize(self, value, index): 

279 if not self._keyword_found and self._is_assign(value): 

280 self._assigns += 1 

281 return SYNTAX # VariableTokenizer tokenizes this later. 

282 if self._keyword_found: 

283 return Tokenizer._tokenize(self, value, index - self._assigns) 

284 self._keyword_found = True 

285 return GherkinTokenizer().tokenize(value, KEYWORD) 

286 

287 

288class GherkinTokenizer: 

289 _gherkin_prefix = re.compile('^(Given|When|Then|And|But) ', re.IGNORECASE) 

290 

291 def tokenize(self, value, token): 

292 match = self._gherkin_prefix.match(value) 

293 if not match: 

294 return [(value, token)] 

295 end = match.end() 

296 return [(value[:end], GHERKIN), (value[end:], token)] 

297 

298 

299class TemplatedKeywordCall(Tokenizer): 

300 _tokens = (ARGUMENT,) 

301 

302 

303class ForLoop(Tokenizer): 

304 

305 def __init__(self): 

306 Tokenizer.__init__(self) 

307 self._in_arguments = False 

308 

309 def _tokenize(self, value, index): 

310 token = self._in_arguments and ARGUMENT or SYNTAX 

311 if value.upper() in ('IN', 'IN RANGE'): 

312 self._in_arguments = True 

313 return token 

314 

315 

316class _Table: 

317 _tokenizer_class = None 

318 

319 def __init__(self, prev_tokenizer=None): 

320 self._tokenizer = self._tokenizer_class() 

321 self._prev_tokenizer = prev_tokenizer 

322 self._prev_values_on_row = [] 

323 

324 def tokenize(self, value, index): 

325 if self._continues(value, index): 

326 self._tokenizer = self._prev_tokenizer 

327 yield value, SYNTAX 

328 else: 

329 yield from self._tokenize(value, index) 

330 self._prev_values_on_row.append(value) 

331 

332 def _continues(self, value, index): 

333 return value == '...' and all(self._is_empty(t) 

334 for t in self._prev_values_on_row) 

335 

336 def _is_empty(self, value): 

337 return value in ('', '\\') 

338 

339 def _tokenize(self, value, index): 

340 return self._tokenizer.tokenize(value) 

341 

342 def end_row(self): 

343 self.__init__(prev_tokenizer=self._tokenizer) 

344 

345 

346class UnknownTable(_Table): 

347 _tokenizer_class = Comment 

348 

349 def _continues(self, value, index): 

350 return False 

351 

352 

353class VariableTable(_Table): 

354 _tokenizer_class = Variable 

355 

356 

357class SettingTable(_Table): 

358 _tokenizer_class = Setting 

359 

360 def __init__(self, template_setter, prev_tokenizer=None): 

361 _Table.__init__(self, prev_tokenizer) 

362 self._template_setter = template_setter 

363 

364 def _tokenize(self, value, index): 

365 if index == 0 and normalize(value) == 'testtemplate': 

366 self._tokenizer = Setting(self._template_setter) 

367 return _Table._tokenize(self, value, index) 

368 

369 def end_row(self): 

370 self.__init__(self._template_setter, prev_tokenizer=self._tokenizer) 

371 

372 

373class TestCaseTable(_Table): 

374 _setting_class = TestCaseSetting 

375 _test_template = None 

376 _default_template = None 

377 

378 @property 

379 def _tokenizer_class(self): 

380 if self._test_template or (self._default_template and 

381 self._test_template is not False): 

382 return TemplatedKeywordCall 

383 return KeywordCall 

384 

385 def _continues(self, value, index): 

386 return index > 0 and _Table._continues(self, value, index) 

387 

388 def _tokenize(self, value, index): 

389 if index == 0: 

390 if value: 

391 self._test_template = None 

392 return GherkinTokenizer().tokenize(value, TC_KW_NAME) 

393 if index == 1 and self._is_setting(value): 

394 if self._is_template(value): 

395 self._test_template = False 

396 self._tokenizer = self._setting_class(self.set_test_template) 

397 else: 

398 self._tokenizer = self._setting_class() 

399 if index == 1 and self._is_for_loop(value): 

400 self._tokenizer = ForLoop() 

401 if index == 1 and self._is_empty(value): 

402 return [(value, SYNTAX)] 

403 return _Table._tokenize(self, value, index) 

404 

405 def _is_setting(self, value): 

406 return value.startswith('[') and value.endswith(']') 

407 

408 def _is_template(self, value): 

409 return normalize(value) == '[template]' 

410 

411 def _is_for_loop(self, value): 

412 return value.startswith(':') and normalize(value, remove=':') == 'for' 

413 

414 def set_test_template(self, template): 

415 self._test_template = self._is_template_set(template) 

416 

417 def set_default_template(self, template): 

418 self._default_template = self._is_template_set(template) 

419 

420 def _is_template_set(self, template): 

421 return normalize(template) not in ('', '\\', 'none', '${empty}') 

422 

423 

424class KeywordTable(TestCaseTable): 

425 _tokenizer_class = KeywordCall 

426 _setting_class = KeywordSetting 

427 

428 def _is_template(self, value): 

429 return False 

430 

431 

432# Following code copied directly from Robot Framework 2.7.5. 

433 

434class VariableSplitter: 

435 

436 def __init__(self, string, identifiers): 

437 self.identifier = None 

438 self.base = None 

439 self.index = None 

440 self.start = -1 

441 self.end = -1 

442 self._identifiers = identifiers 

443 self._may_have_internal_variables = False 

444 try: 

445 self._split(string) 

446 except ValueError: 

447 pass 

448 else: 

449 self._finalize() 

450 

451 def get_replaced_base(self, variables): 

452 if self._may_have_internal_variables: 

453 return variables.replace_string(self.base) 

454 return self.base 

455 

456 def _finalize(self): 

457 self.identifier = self._variable_chars[0] 

458 self.base = ''.join(self._variable_chars[2:-1]) 

459 self.end = self.start + len(self._variable_chars) 

460 if self._has_list_or_dict_variable_index(): 

461 self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1]) 

462 self.end += len(self._list_and_dict_variable_index_chars) 

463 

464 def _has_list_or_dict_variable_index(self): 

465 return self._list_and_dict_variable_index_chars\ 

466 and self._list_and_dict_variable_index_chars[-1] == ']' 

467 

468 def _split(self, string): 

469 start_index, max_index = self._find_variable(string) 

470 self.start = start_index 

471 self._open_curly = 1 

472 self._state = self._variable_state 

473 self._variable_chars = [string[start_index], '{'] 

474 self._list_and_dict_variable_index_chars = [] 

475 self._string = string 

476 start_index += 2 

477 for index, char in enumerate(string[start_index:]): 

478 index += start_index # Giving start to enumerate only in Py 2.6+ 

479 try: 

480 self._state(char, index) 

481 except StopIteration: 

482 return 

483 if index == max_index and not self._scanning_list_variable_index(): 

484 return 

485 

486 def _scanning_list_variable_index(self): 

487 return self._state in [self._waiting_list_variable_index_state, 

488 self._list_variable_index_state] 

489 

490 def _find_variable(self, string): 

491 max_end_index = string.rfind('}') 

492 if max_end_index == -1: 

493 raise ValueError('No variable end found') 

494 if self._is_escaped(string, max_end_index): 

495 return self._find_variable(string[:max_end_index]) 

496 start_index = self._find_start_index(string, 1, max_end_index) 

497 if start_index == -1: 

498 raise ValueError('No variable start found') 

499 return start_index, max_end_index 

500 

501 def _find_start_index(self, string, start, end): 

502 index = string.find('{', start, end) - 1 

503 if index < 0: 

504 return -1 

505 if self._start_index_is_ok(string, index): 

506 return index 

507 return self._find_start_index(string, index+2, end) 

508 

509 def _start_index_is_ok(self, string, index): 

510 return string[index] in self._identifiers\ 

511 and not self._is_escaped(string, index) 

512 

513 def _is_escaped(self, string, index): 

514 escaped = False 

515 while index > 0 and string[index-1] == '\\': 

516 index -= 1 

517 escaped = not escaped 

518 return escaped 

519 

520 def _variable_state(self, char, index): 

521 self._variable_chars.append(char) 

522 if char == '}' and not self._is_escaped(self._string, index): 

523 self._open_curly -= 1 

524 if self._open_curly == 0: 

525 if not self._is_list_or_dict_variable(): 

526 raise StopIteration 

527 self._state = self._waiting_list_variable_index_state 

528 elif char in self._identifiers: 

529 self._state = self._internal_variable_start_state 

530 

531 def _is_list_or_dict_variable(self): 

532 return self._variable_chars[0] in ('@','&') 

533 

534 def _internal_variable_start_state(self, char, index): 

535 self._state = self._variable_state 

536 if char == '{': 

537 self._variable_chars.append(char) 

538 self._open_curly += 1 

539 self._may_have_internal_variables = True 

540 else: 

541 self._variable_state(char, index) 

542 

543 def _waiting_list_variable_index_state(self, char, index): 

544 if char != '[': 

545 raise StopIteration 

546 self._list_and_dict_variable_index_chars.append(char) 

547 self._state = self._list_variable_index_state 

548 

549 def _list_variable_index_state(self, char, index): 

550 self._list_and_dict_variable_index_chars.append(char) 

551 if char == ']': 

552 raise StopIteration