Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pygments/lexers/julia.py: 37%
52 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-01 06:54 +0000
1"""
2 pygments.lexers.julia
3 ~~~~~~~~~~~~~~~~~~~~~
5 Lexers for the Julia language.
7 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \
12 words, include
13from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
14 Number, Punctuation, Generic, Whitespace
15from pygments.util import shebang_matches
16from pygments.lexers._julia_builtins import OPERATORS_LIST, DOTTED_OPERATORS_LIST, \
17 KEYWORD_LIST, BUILTIN_LIST, LITERAL_LIST
19__all__ = ['JuliaLexer', 'JuliaConsoleLexer']
21# see https://docs.julialang.org/en/v1/manual/variables/#Allowed-Variable-Names
22allowed_variable = \
23 '(?:[a-zA-Z_\u00A1-\U0010ffff][a-zA-Z_0-9!\u00A1-\U0010ffff]*)'
24# see https://github.com/JuliaLang/julia/blob/master/src/flisp/julia_opsuffs.h
25operator_suffixes = r'[²³¹ʰʲʳʷʸˡˢˣᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵇᵈᵉᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₕₖₗₘₙₚₛₜⱼⱽ]*'
27class JuliaLexer(RegexLexer):
28 """
29 For Julia source code.
31 .. versionadded:: 1.6
32 """
34 name = 'Julia'
35 url = 'https://julialang.org/'
36 aliases = ['julia', 'jl']
37 filenames = ['*.jl']
38 mimetypes = ['text/x-julia', 'application/x-julia']
40 tokens = {
41 'root': [
42 (r'\n', Whitespace),
43 (r'[^\S\n]+', Whitespace),
44 (r'#=', Comment.Multiline, "blockcomment"),
45 (r'#.*$', Comment),
46 (r'[\[\](),;]', Punctuation),
48 # symbols
49 # intercept range expressions first
50 (r'(' + allowed_variable + r')(\s*)(:)(' + allowed_variable + ')',
51 bygroups(Name, Whitespace, Operator, Name)),
52 # then match :name which does not follow closing brackets, digits, or the
53 # ::, <:, and :> operators
54 (r'(?<![\]):<>\d.])(:' + allowed_variable + ')', String.Symbol),
56 # type assertions - excludes expressions like ::typeof(sin) and ::avec[1]
57 (r'(?<=::)(\s*)(' + allowed_variable + r')\b(?![(\[])',
58 bygroups(Whitespace, Keyword.Type)),
59 # type comparisons
60 # - MyType <: A or MyType >: A
61 ('(' + allowed_variable + r')(\s*)([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])',
62 bygroups(Keyword.Type, Whitespace, Operator, Whitespace, Keyword.Type)),
63 # - <: B or >: B
64 (r'([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])',
65 bygroups(Operator, Whitespace, Keyword.Type)),
66 # - A <: or A >:
67 (r'\b(' + allowed_variable + r')(\s*)([<>]:)',
68 bygroups(Keyword.Type, Whitespace, Operator)),
70 # operators
71 # Suffixes aren't actually allowed on all operators, but we'll ignore that
72 # since those cases are invalid Julia code.
73 (words([*OPERATORS_LIST, *DOTTED_OPERATORS_LIST],
74 suffix=operator_suffixes), Operator),
75 (words(['.' + o for o in DOTTED_OPERATORS_LIST],
76 suffix=operator_suffixes), Operator),
77 (words(['...', '..']), Operator),
79 # NOTE
80 # Patterns below work only for definition sites and thus hardly reliable.
81 #
82 # functions
83 # (r'(function)(\s+)(' + allowed_variable + ')',
84 # bygroups(Keyword, Text, Name.Function)),
86 # chars
87 (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|"
88 r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char),
90 # try to match trailing transpose
91 (r'(?<=[.\w)\]])(\'' + operator_suffixes + ')+', Operator),
93 # raw strings
94 (r'(raw)(""")', bygroups(String.Affix, String), 'tqrawstring'),
95 (r'(raw)(")', bygroups(String.Affix, String), 'rawstring'),
96 # regular expressions
97 (r'(r)(""")', bygroups(String.Affix, String.Regex), 'tqregex'),
98 (r'(r)(")', bygroups(String.Affix, String.Regex), 'regex'),
99 # other strings
100 (r'(' + allowed_variable + ')?(""")',
101 bygroups(String.Affix, String), 'tqstring'),
102 (r'(' + allowed_variable + ')?(")',
103 bygroups(String.Affix, String), 'string'),
105 # backticks
106 (r'(' + allowed_variable + ')?(```)',
107 bygroups(String.Affix, String.Backtick), 'tqcommand'),
108 (r'(' + allowed_variable + ')?(`)',
109 bygroups(String.Affix, String.Backtick), 'command'),
111 # type names
112 # - names that begin a curly expression
113 ('(' + allowed_variable + r')(\{)',
114 bygroups(Keyword.Type, Punctuation), 'curly'),
115 # - names as part of bare 'where'
116 (r'(where)(\s+)(' + allowed_variable + ')',
117 bygroups(Keyword, Whitespace, Keyword.Type)),
118 # - curly expressions in general
119 (r'(\{)', Punctuation, 'curly'),
120 # - names as part of type declaration
121 (r'(abstract|primitive)([ \t]+)(type\b)([\s()]+)(' +
122 allowed_variable + r')',
123 bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)),
124 (r'(mutable(?=[ \t]))?([ \t]+)?(struct\b)([\s()]+)(' +
125 allowed_variable + r')',
126 bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)),
128 # macros
129 (r'@' + allowed_variable, Name.Decorator),
130 (words([*OPERATORS_LIST, '..', '.', *DOTTED_OPERATORS_LIST],
131 prefix='@', suffix=operator_suffixes), Name.Decorator),
133 # keywords
134 (words(KEYWORD_LIST, suffix=r'\b'), Keyword),
135 # builtin types
136 (words(BUILTIN_LIST, suffix=r'\b'), Keyword.Type),
137 # builtin literals
138 (words(LITERAL_LIST, suffix=r'\b'), Name.Builtin),
140 # names
141 (allowed_variable, Name),
143 # numbers
144 (r'(\d+((_\d+)+)?\.(?!\.)(\d+((_\d+)+)?)?|\.\d+((_\d+)+)?)([eEf][+-]?[0-9]+)?', Number.Float),
145 (r'\d+((_\d+)+)?[eEf][+-]?[0-9]+', Number.Float),
146 (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?(\.([a-fA-F0-9]+((_[a-fA-F0-9]+)+)?)?)?p[+-]?\d+', Number.Float),
147 (r'0b[01]+((_[01]+)+)?', Number.Bin),
148 (r'0o[0-7]+((_[0-7]+)+)?', Number.Oct),
149 (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?', Number.Hex),
150 (r'\d+((_\d+)+)?', Number.Integer),
152 # single dot operator matched last to permit e.g. ".1" as a float
153 (words(['.']), Operator),
154 ],
156 "blockcomment": [
157 (r'[^=#]', Comment.Multiline),
158 (r'#=', Comment.Multiline, '#push'),
159 (r'=#', Comment.Multiline, '#pop'),
160 (r'[=#]', Comment.Multiline),
161 ],
163 'curly': [
164 (r'\{', Punctuation, '#push'),
165 (r'\}', Punctuation, '#pop'),
166 (allowed_variable, Keyword.Type),
167 include('root'),
168 ],
170 'tqrawstring': [
171 (r'"""', String, '#pop'),
172 (r'([^"]|"[^"][^"])+', String),
173 ],
174 'rawstring': [
175 (r'"', String, '#pop'),
176 (r'\\"', String.Escape),
177 (r'([^"\\]|\\[^"])+', String),
178 ],
180 # Interpolation is defined as "$" followed by the shortest full
181 # expression, which is something we can't parse. Include the most
182 # common cases here: $word, and $(paren'd expr).
183 'interp': [
184 (r'\$' + allowed_variable, String.Interpol),
185 (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
186 ],
187 'in-intp': [
188 (r'\(', Punctuation, '#push'),
189 (r'\)', Punctuation, '#pop'),
190 include('root'),
191 ],
193 'string': [
194 (r'(")(' + allowed_variable + r'|\d+)?',
195 bygroups(String, String.Affix), '#pop'),
196 # FIXME: This escape pattern is not perfect.
197 (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
198 include('interp'),
199 # @printf and @sprintf formats
200 (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]',
201 String.Interpol),
202 (r'[^"$%\\]+', String),
203 (r'.', String),
204 ],
205 'tqstring': [
206 (r'(""")(' + allowed_variable + r'|\d+)?',
207 bygroups(String, String.Affix), '#pop'),
208 (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
209 include('interp'),
210 (r'[^"$%\\]+', String),
211 (r'.', String),
212 ],
214 'regex': [
215 (r'(")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'),
216 (r'\\"', String.Regex),
217 (r'[^\\"]+', String.Regex),
218 ],
220 'tqregex': [
221 (r'(""")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'),
222 (r'[^"]+', String.Regex),
223 ],
225 'command': [
226 (r'(`)(' + allowed_variable + r'|\d+)?',
227 bygroups(String.Backtick, String.Affix), '#pop'),
228 (r'\\[`$]', String.Escape),
229 include('interp'),
230 (r'[^\\`$]+', String.Backtick),
231 (r'.', String.Backtick),
232 ],
233 'tqcommand': [
234 (r'(```)(' + allowed_variable + r'|\d+)?',
235 bygroups(String.Backtick, String.Affix), '#pop'),
236 (r'\\\$', String.Escape),
237 include('interp'),
238 (r'[^\\`$]+', String.Backtick),
239 (r'.', String.Backtick),
240 ],
241 }
243 def analyse_text(text):
244 return shebang_matches(text, r'julia')
247class JuliaConsoleLexer(Lexer):
248 """
249 For Julia console sessions. Modeled after MatlabSessionLexer.
251 .. versionadded:: 1.6
252 """
253 name = 'Julia console'
254 aliases = ['jlcon', 'julia-repl']
256 def get_tokens_unprocessed(self, text):
257 jllexer = JuliaLexer(**self.options)
258 start = 0
259 curcode = ''
260 insertions = []
261 output = False
262 error = False
264 for line in text.splitlines(keepends=True):
265 if line.startswith('julia>'):
266 insertions.append((len(curcode), [(0, Generic.Prompt, line[:6])]))
267 curcode += line[6:]
268 output = False
269 error = False
270 elif line.startswith('help?>') or line.startswith('shell>'):
271 yield start, Generic.Prompt, line[:6]
272 yield start + 6, Text, line[6:]
273 output = False
274 error = False
275 elif line.startswith(' ') and not output:
276 insertions.append((len(curcode), [(0, Whitespace, line[:6])]))
277 curcode += line[6:]
278 else:
279 if curcode:
280 yield from do_insertions(
281 insertions, jllexer.get_tokens_unprocessed(curcode))
282 curcode = ''
283 insertions = []
284 if line.startswith('ERROR: ') or error:
285 yield start, Generic.Error, line
286 error = True
287 else:
288 yield start, Generic.Output, line
289 output = True
290 start += len(line)
292 if curcode:
293 yield from do_insertions(
294 insertions, jllexer.get_tokens_unprocessed(curcode))