Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pygments/lexers/crystal.py: 47%
60 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-01 06:54 +0000
1"""
2 pygments.lexers.crystal
3 ~~~~~~~~~~~~~~~~~~~~~~~
5 Lexer for Crystal.
7 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
13from pygments.lexer import ExtendedRegexLexer, include, bygroups, default, \
14 words, line_re
15from pygments.token import Comment, Operator, Keyword, Name, String, Number, \
16 Punctuation, Error, Whitespace
18__all__ = ['CrystalLexer']
21CRYSTAL_OPERATORS = [
22 '!=', '!~', '!', '%', '&&', '&', '**', '*', '+', '-', '/', '<=>', '<<', '<=', '<',
23 '===', '==', '=~', '=', '>=', '>>', '>', '[]=', '[]?', '[]', '^', '||', '|', '~'
24]
27class CrystalLexer(ExtendedRegexLexer):
28 """
29 For Crystal source code.
31 .. versionadded:: 2.2
32 """
34 name = 'Crystal'
35 url = 'http://crystal-lang.org'
36 aliases = ['cr', 'crystal']
37 filenames = ['*.cr']
38 mimetypes = ['text/x-crystal']
40 flags = re.DOTALL | re.MULTILINE
42 def heredoc_callback(self, match, ctx):
43 # okay, this is the hardest part of parsing Crystal...
44 # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line
46 start = match.start(1)
47 yield start, Operator, match.group(1) # <<-?
48 yield match.start(2), String.Heredoc, match.group(2) # quote ", ', `
49 yield match.start(3), String.Delimiter, match.group(3) # heredoc name
50 yield match.start(4), String.Heredoc, match.group(4) # quote again
52 heredocstack = ctx.__dict__.setdefault('heredocstack', [])
53 outermost = not bool(heredocstack)
54 heredocstack.append((match.group(1) == '<<-', match.group(3)))
56 ctx.pos = match.start(5)
57 ctx.end = match.end(5)
58 # this may find other heredocs, so limit the recursion depth
59 if len(heredocstack) < 100:
60 yield from self.get_tokens_unprocessed(context=ctx)
61 else:
62 yield ctx.pos, String.Heredoc, match.group(5)
63 ctx.pos = match.end()
65 if outermost:
66 # this is the outer heredoc again, now we can process them all
67 for tolerant, hdname in heredocstack:
68 lines = []
69 for match in line_re.finditer(ctx.text, ctx.pos):
70 if tolerant:
71 check = match.group().strip()
72 else:
73 check = match.group().rstrip()
74 if check == hdname:
75 for amatch in lines:
76 yield amatch.start(), String.Heredoc, amatch.group()
77 yield match.start(), String.Delimiter, match.group()
78 ctx.pos = match.end()
79 break
80 else:
81 lines.append(match)
82 else:
83 # end of heredoc not found -- error!
84 for amatch in lines:
85 yield amatch.start(), Error, amatch.group()
86 ctx.end = len(ctx.text)
87 del heredocstack[:]
89 def gen_crystalstrings_rules():
90 states = {}
91 states['strings'] = [
92 (r'\:\w+[!?]?', String.Symbol),
93 (words(CRYSTAL_OPERATORS, prefix=r'\:'), String.Symbol),
94 (r":'(\\\\|\\[^\\]|[^'\\])*'", String.Symbol),
95 # This allows arbitrary text after '\ for simplicity
96 (r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char),
97 (r':"', String.Symbol, 'simple-sym'),
98 # Crystal doesn't have "symbol:"s but this simplifies function args
99 (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)),
100 (r'"', String.Double, 'simple-string'),
101 (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
102 ]
104 # double-quoted string and symbol
105 for name, ttype, end in ('string', String.Double, '"'), \
106 ('sym', String.Symbol, '"'), \
107 ('backtick', String.Backtick, '`'):
108 states['simple-'+name] = [
109 include('string-escaped' if name == 'sym' else 'string-intp-escaped'),
110 (r'[^\\%s#]+' % end, ttype),
111 (r'[\\#]', ttype),
112 (end, ttype, '#pop'),
113 ]
115 # https://crystal-lang.org/docs/syntax_and_semantics/literals/string.html#percent-string-literals
116 for lbrace, rbrace, bracecc, name in \
117 ('\\{', '\\}', '{}', 'cb'), \
118 ('\\[', '\\]', '\\[\\]', 'sb'), \
119 ('\\(', '\\)', '()', 'pa'), \
120 ('<', '>', '<>', 'ab'), \
121 ('\\|', '\\|', '\\|', 'pi'):
122 states[name+'-intp-string'] = [
123 (r'\\' + lbrace, String.Other),
124 ] + (lbrace != rbrace) * [
125 (lbrace, String.Other, '#push'),
126 ] + [
127 (rbrace, String.Other, '#pop'),
128 include('string-intp-escaped'),
129 (r'[\\#' + bracecc + ']', String.Other),
130 (r'[^\\#' + bracecc + ']+', String.Other),
131 ]
132 states['strings'].append((r'%Q?' + lbrace, String.Other,
133 name+'-intp-string'))
134 states[name+'-string'] = [
135 (r'\\[\\' + bracecc + ']', String.Other),
136 ] + (lbrace != rbrace) * [
137 (lbrace, String.Other, '#push'),
138 ] + [
139 (rbrace, String.Other, '#pop'),
140 (r'[\\#' + bracecc + ']', String.Other),
141 (r'[^\\#' + bracecc + ']+', String.Other),
142 ]
143 # https://crystal-lang.org/docs/syntax_and_semantics/literals/array.html#percent-array-literals
144 states['strings'].append((r'%[qwi]' + lbrace, String.Other,
145 name+'-string'))
146 states[name+'-regex'] = [
147 (r'\\[\\' + bracecc + ']', String.Regex),
148 ] + (lbrace != rbrace) * [
149 (lbrace, String.Regex, '#push'),
150 ] + [
151 (rbrace + '[imsx]*', String.Regex, '#pop'),
152 include('string-intp'),
153 (r'[\\#' + bracecc + ']', String.Regex),
154 (r'[^\\#' + bracecc + ']+', String.Regex),
155 ]
156 states['strings'].append((r'%r' + lbrace, String.Regex,
157 name+'-regex'))
159 return states
161 tokens = {
162 'root': [
163 (r'#.*?$', Comment.Single),
164 # keywords
165 (words('''
166 abstract asm begin break case do else elsif end ensure extend if in
167 include next of private protected require rescue return select self super
168 then unless until when while with yield
169 '''.split(), suffix=r'\b'), Keyword),
170 (words('''
171 previous_def forall out uninitialized __DIR__ __FILE__ __LINE__
172 __END_LINE__
173 '''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Keyword.Pseudo),
174 # https://crystal-lang.org/docs/syntax_and_semantics/is_a.html
175 (r'\.(is_a\?|nil\?|responds_to\?|as\?|as\b)', Keyword.Pseudo),
176 (words(['true', 'false', 'nil'], suffix=r'\b'), Keyword.Constant),
177 # start of function, class and module names
178 (r'(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
179 bygroups(Keyword, Whitespace, Name.Namespace)),
180 (r'(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)',
181 bygroups(Keyword, Whitespace, Name.Namespace), 'funcname'),
182 (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
183 (r'(annotation|class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)',
184 bygroups(Keyword, Whitespace, Name.Namespace), 'classname'),
185 # https://crystal-lang.org/api/toplevel.html
186 (words('''
187 instance_sizeof offsetof pointerof sizeof typeof
188 '''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Keyword.Pseudo),
189 # macros
190 (r'(?<!\.)(debugger\b|p!|pp!|record\b|spawn\b)', Name.Builtin.Pseudo),
191 # builtins
192 (words('''
193 abort at_exit caller exit gets loop main p pp print printf puts
194 raise rand read_line sleep spawn sprintf system
195 '''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin),
196 # https://crystal-lang.org/api/Object.html#macro-summary
197 (r'(?<!\.)(((class_)?((getter|property)\b[!?]?|setter\b))|'
198 r'(def_(clone|equals|equals_and_hash|hash)|delegate|forward_missing_to)\b)',
199 Name.Builtin.Pseudo),
200 # normal heredocs
201 (r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)',
202 heredoc_callback),
203 # empty string heredocs
204 (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback),
205 (r'__END__', Comment.Preproc, 'end-part'),
206 # multiline regex (after keywords or assignments)
207 (r'(?:^|(?<=[=<>~!:])|'
208 r'(?<=(?:\s|;)when\s)|'
209 r'(?<=(?:\s|;)or\s)|'
210 r'(?<=(?:\s|;)and\s)|'
211 r'(?<=\.index\s)|'
212 r'(?<=\.scan\s)|'
213 r'(?<=\.sub\s)|'
214 r'(?<=\.sub!\s)|'
215 r'(?<=\.gsub\s)|'
216 r'(?<=\.gsub!\s)|'
217 r'(?<=\.match\s)|'
218 r'(?<=(?:\s|;)if\s)|'
219 r'(?<=(?:\s|;)elsif\s)|'
220 r'(?<=^when\s)|'
221 r'(?<=^index\s)|'
222 r'(?<=^scan\s)|'
223 r'(?<=^sub\s)|'
224 r'(?<=^gsub\s)|'
225 r'(?<=^sub!\s)|'
226 r'(?<=^gsub!\s)|'
227 r'(?<=^match\s)|'
228 r'(?<=^if\s)|'
229 r'(?<=^elsif\s)'
230 r')(\s*)(/)', bygroups(Whitespace, String.Regex), 'multiline-regex'),
231 # multiline regex (in method calls or subscripts)
232 (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
233 # multiline regex (this time the funny no whitespace rule)
234 (r'(\s+)(/)(?![\s=])', bygroups(Whitespace, String.Regex),
235 'multiline-regex'),
236 # lex numbers and ignore following regular expressions which
237 # are division operators in fact (grrrr. i hate that. any
238 # better ideas?)
239 # since pygments 0.7 we also eat a "?" operator after numbers
240 # so that the char operator does not work. Chars are not allowed
241 # there so that you can use the ternary operator.
242 # stupid example:
243 # x>=0?n[x]:""
244 (r'(0o[0-7]+(?:_[0-7]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
245 bygroups(Number.Oct, Whitespace, Operator)),
246 (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
247 bygroups(Number.Hex, Whitespace, Operator)),
248 (r'(0b[01]+(?:_[01]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
249 bygroups(Number.Bin, Whitespace, Operator)),
250 # 3 separate expressions for floats because any of the 3 optional
251 # parts makes it a float
252 (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)(?:e[+-]?[0-9]+)?'
253 r'(?:_?f[0-9]+)?)(\s*)([/?])?',
254 bygroups(Number.Float, Whitespace, Operator)),
255 (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)'
256 r'(?:_?f[0-9]+)?)(\s*)([/?])?',
257 bygroups(Number.Float, Whitespace, Operator)),
258 (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)?'
259 r'(?:_?f[0-9]+))(\s*)([/?])?',
260 bygroups(Number.Float, Whitespace, Operator)),
261 (r'(0\b|[1-9][\d]*(?:_\d+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
262 bygroups(Number.Integer, Whitespace, Operator)),
263 # Names
264 (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
265 (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
266 (r'\$\w+', Name.Variable.Global),
267 (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
268 (r'\$-[0adFiIlpvw]', Name.Variable.Global),
269 (r'::', Operator),
270 include('strings'),
271 # https://crystal-lang.org/reference/syntax_and_semantics/literals/char.html
272 (r'\?(\\[MC]-)*' # modifiers
273 r'(\\([\\abefnrtv#"\']|[0-7]{1,3}|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|u\{[a-fA-F0-9 ]+\})|\S)'
274 r'(?!\w)',
275 String.Char),
276 (r'[A-Z][A-Z_]+\b(?!::|\.)', Name.Constant),
277 # macro expansion
278 (r'\{%', String.Interpol, 'in-macro-control'),
279 (r'\{\{', String.Interpol, 'in-macro-expr'),
280 # annotations
281 (r'(@\[)(\s*)([A-Z]\w*(::[A-Z]\w*)*)',
282 bygroups(Operator, Whitespace, Name.Decorator), 'in-annot'),
283 # this is needed because Crystal attributes can look
284 # like keywords (class) or like this: ` ?!?
285 (words(CRYSTAL_OPERATORS, prefix=r'(\.|::)'),
286 bygroups(Operator, Name.Operator)),
287 (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
288 bygroups(Operator, Name)),
289 # Names can end with [!?] unless it's "!="
290 (r'[a-zA-Z_]\w*(?:[!?](?!=))?', Name),
291 (r'(\[|\]\??|\*\*|<=>?|>=|<<?|>>?|=~|===|'
292 r'!~|&&?|\|\||\.{1,3})', Operator),
293 (r'[-+/*%=<>&!^|~]=?', Operator),
294 (r'[(){};,/?:\\]', Punctuation),
295 (r'\s+', Whitespace)
296 ],
297 'funcname': [
298 (r'(?:([a-zA-Z_]\w*)(\.))?'
299 r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|'
300 r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)',
301 bygroups(Name.Class, Operator, Name.Function), '#pop'),
302 default('#pop')
303 ],
304 'classname': [
305 (r'[A-Z_]\w*', Name.Class),
306 (r'(\()(\s*)([A-Z_]\w*)(\s*)(\))',
307 bygroups(Punctuation, Whitespace, Name.Class, Whitespace, Punctuation)),
308 default('#pop')
309 ],
310 'in-intp': [
311 (r'\{', String.Interpol, '#push'),
312 (r'\}', String.Interpol, '#pop'),
313 include('root'),
314 ],
315 'string-intp': [
316 (r'#\{', String.Interpol, 'in-intp'),
317 ],
318 'string-escaped': [
319 # https://crystal-lang.org/reference/syntax_and_semantics/literals/string.html
320 (r'\\([\\abefnrtv#"\']|[0-7]{1,3}|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|u\{[a-fA-F0-9 ]+\})',
321 String.Escape)
322 ],
323 'string-intp-escaped': [
324 include('string-intp'),
325 include('string-escaped'),
326 ],
327 'interpolated-regex': [
328 include('string-intp'),
329 (r'[\\#]', String.Regex),
330 (r'[^\\#]+', String.Regex),
331 ],
332 'interpolated-string': [
333 include('string-intp'),
334 (r'[\\#]', String.Other),
335 (r'[^\\#]+', String.Other),
336 ],
337 'multiline-regex': [
338 include('string-intp'),
339 (r'\\\\', String.Regex),
340 (r'\\/', String.Regex),
341 (r'[\\#]', String.Regex),
342 (r'[^\\/#]+', String.Regex),
343 (r'/[imsx]*', String.Regex, '#pop'),
344 ],
345 'end-part': [
346 (r'.+', Comment.Preproc, '#pop')
347 ],
348 'in-macro-control': [
349 (r'\{%', String.Interpol, '#push'),
350 (r'%\}', String.Interpol, '#pop'),
351 (r'(for|verbatim)\b', Keyword),
352 include('root'),
353 ],
354 'in-macro-expr': [
355 (r'\{\{', String.Interpol, '#push'),
356 (r'\}\}', String.Interpol, '#pop'),
357 include('root'),
358 ],
359 'in-annot': [
360 (r'\[', Operator, '#push'),
361 (r'\]', Operator, '#pop'),
362 include('root'),
363 ],
364 }
365 tokens.update(gen_crystalstrings_rules())