Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2 pygments.lexers
3 ~~~~~~~~~~~~~~~
5 Pygments lexers.
7 :copyright: Copyright 2006-2025 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
12import sys
13import types
14import fnmatch
15from os.path import basename
17from pygments.lexers._mapping import LEXERS
18from pygments.modeline import get_filetype_from_buffer
19from pygments.plugin import find_plugin_lexers
20from pygments.util import ClassNotFound, guess_decode
22COMPAT = {
23 'Python3Lexer': 'PythonLexer',
24 'Python3TracebackLexer': 'PythonTracebackLexer',
25 'LeanLexer': 'Lean3Lexer',
26}
28__all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
29 'guess_lexer', 'load_lexer_from_file'] + list(LEXERS) + list(COMPAT)
31_lexer_cache = {}
32_pattern_cache = {}
35def _fn_matches(fn, glob):
36 """Return whether the supplied file name fn matches pattern filename."""
37 if glob not in _pattern_cache:
38 pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
39 return pattern.match(fn)
40 return _pattern_cache[glob].match(fn)
43def _load_lexers(module_name):
44 """Load a lexer (and all others in the module too)."""
45 mod = __import__(module_name, None, None, ['__all__'])
46 for lexer_name in mod.__all__:
47 cls = getattr(mod, lexer_name)
48 _lexer_cache[cls.name] = cls
51def get_all_lexers(plugins=True):
52 """Return a generator of tuples in the form ``(name, aliases,
53 filenames, mimetypes)`` of all know lexers.
55 If *plugins* is true (the default), plugin lexers supplied by entrypoints
56 are also returned. Otherwise, only builtin ones are considered.
57 """
58 for item in LEXERS.values():
59 yield item[1:]
60 if plugins:
61 for lexer in find_plugin_lexers():
62 yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes
65def find_lexer_class(name):
66 """
67 Return the `Lexer` subclass that with the *name* attribute as given by
68 the *name* argument.
69 """
70 if name in _lexer_cache:
71 return _lexer_cache[name]
72 # lookup builtin lexers
73 for module_name, lname, aliases, _, _ in LEXERS.values():
74 if name == lname:
75 _load_lexers(module_name)
76 return _lexer_cache[name]
77 # continue with lexers from setuptools entrypoints
78 for cls in find_plugin_lexers():
79 if cls.name == name:
80 return cls
83def find_lexer_class_by_name(_alias):
84 """
85 Return the `Lexer` subclass that has `alias` in its aliases list, without
86 instantiating it.
88 Like `get_lexer_by_name`, but does not instantiate the class.
90 Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is
91 found.
93 .. versionadded:: 2.2
94 """
95 if not _alias:
96 raise ClassNotFound(f'no lexer for alias {_alias!r} found')
97 # lookup builtin lexers
98 for module_name, name, aliases, _, _ in LEXERS.values():
99 if _alias.lower() in aliases:
100 if name not in _lexer_cache:
101 _load_lexers(module_name)
102 return _lexer_cache[name]
103 # continue with lexers from setuptools entrypoints
104 for cls in find_plugin_lexers():
105 if _alias.lower() in cls.aliases:
106 return cls
107 raise ClassNotFound(f'no lexer for alias {_alias!r} found')
110def get_lexer_by_name(_alias, **options):
111 """
112 Return an instance of a `Lexer` subclass that has `alias` in its
113 aliases list. The lexer is given the `options` at its
114 instantiation.
116 Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is
117 found.
118 """
119 if not _alias:
120 raise ClassNotFound(f'no lexer for alias {_alias!r} found')
122 # lookup builtin lexers
123 for module_name, name, aliases, _, _ in LEXERS.values():
124 if _alias.lower() in aliases:
125 if name not in _lexer_cache:
126 _load_lexers(module_name)
127 return _lexer_cache[name](**options)
128 # continue with lexers from setuptools entrypoints
129 for cls in find_plugin_lexers():
130 if _alias.lower() in cls.aliases:
131 return cls(**options)
132 raise ClassNotFound(f'no lexer for alias {_alias!r} found')
135def load_lexer_from_file(filename, lexername="CustomLexer", **options):
136 """Load a lexer from a file.
138 This method expects a file located relative to the current working
139 directory, which contains a Lexer class. By default, it expects the
140 Lexer to be name CustomLexer; you can specify your own class name
141 as the second argument to this function.
143 Users should be very careful with the input, because this method
144 is equivalent to running eval on the input file.
146 Raises ClassNotFound if there are any problems importing the Lexer.
148 .. versionadded:: 2.2
149 """
150 try:
151 # This empty dict will contain the namespace for the exec'd file
152 custom_namespace = {}
153 with open(filename, 'rb') as f:
154 exec(f.read(), custom_namespace)
155 # Retrieve the class `lexername` from that namespace
156 if lexername not in custom_namespace:
157 raise ClassNotFound(f'no valid {lexername} class found in {filename}')
158 lexer_class = custom_namespace[lexername]
159 # And finally instantiate it with the options
160 return lexer_class(**options)
161 except OSError as err:
162 raise ClassNotFound(f'cannot read {filename}: {err}')
163 except ClassNotFound:
164 raise
165 except Exception as err:
166 raise ClassNotFound(f'error when loading custom lexer: {err}')
169def find_lexer_class_for_filename(_fn, code=None):
170 """Get a lexer for a filename.
172 If multiple lexers match the filename pattern, use ``analyse_text()`` to
173 figure out which one is more appropriate.
175 Returns None if not found.
176 """
177 matches = []
178 fn = basename(_fn)
179 for modname, name, _, filenames, _ in LEXERS.values():
180 for filename in filenames:
181 if _fn_matches(fn, filename):
182 if name not in _lexer_cache:
183 _load_lexers(modname)
184 matches.append((_lexer_cache[name], filename))
185 for cls in find_plugin_lexers():
186 for filename in cls.filenames:
187 if _fn_matches(fn, filename):
188 matches.append((cls, filename))
190 if isinstance(code, bytes):
191 # decode it, since all analyse_text functions expect unicode
192 code = guess_decode(code)
194 def get_rating(info):
195 cls, filename = info
196 # explicit patterns get a bonus
197 bonus = '*' not in filename and 0.5 or 0
198 # The class _always_ defines analyse_text because it's included in
199 # the Lexer class. The default implementation returns None which
200 # gets turned into 0.0. Run scripts/detect_missing_analyse_text.py
201 # to find lexers which need it overridden.
202 if code:
203 return cls.analyse_text(code) + bonus, cls.__name__
204 return cls.priority + bonus, cls.__name__
206 if matches:
207 matches.sort(key=get_rating)
208 # print "Possible lexers, after sort:", matches
209 return matches[-1][0]
212def get_lexer_for_filename(_fn, code=None, **options):
213 """Get a lexer for a filename.
215 Return a `Lexer` subclass instance that has a filename pattern
216 matching `fn`. The lexer is given the `options` at its
217 instantiation.
219 Raise :exc:`pygments.util.ClassNotFound` if no lexer for that filename
220 is found.
222 If multiple lexers match the filename pattern, use their ``analyse_text()``
223 methods to figure out which one is more appropriate.
224 """
225 res = find_lexer_class_for_filename(_fn, code)
226 if not res:
227 raise ClassNotFound(f'no lexer for filename {_fn!r} found')
228 return res(**options)
231def get_lexer_for_mimetype(_mime, **options):
232 """
233 Return a `Lexer` subclass instance that has `mime` in its mimetype
234 list. The lexer is given the `options` at its instantiation.
236 Will raise :exc:`pygments.util.ClassNotFound` if not lexer for that mimetype
237 is found.
238 """
239 for modname, name, _, _, mimetypes in LEXERS.values():
240 if _mime in mimetypes:
241 if name not in _lexer_cache:
242 _load_lexers(modname)
243 return _lexer_cache[name](**options)
244 for cls in find_plugin_lexers():
245 if _mime in cls.mimetypes:
246 return cls(**options)
247 raise ClassNotFound(f'no lexer for mimetype {_mime!r} found')
250def _iter_lexerclasses(plugins=True):
251 """Return an iterator over all lexer classes."""
252 for key in sorted(LEXERS):
253 module_name, name = LEXERS[key][:2]
254 if name not in _lexer_cache:
255 _load_lexers(module_name)
256 yield _lexer_cache[name]
257 if plugins:
258 yield from find_plugin_lexers()
261def guess_lexer_for_filename(_fn, _text, **options):
262 """
263 As :func:`guess_lexer()`, but only lexers which have a pattern in `filenames`
264 or `alias_filenames` that matches `filename` are taken into consideration.
266 :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can
267 handle the content.
268 """
269 fn = basename(_fn)
270 primary = {}
271 matching_lexers = set()
272 for lexer in _iter_lexerclasses():
273 for filename in lexer.filenames:
274 if _fn_matches(fn, filename):
275 matching_lexers.add(lexer)
276 primary[lexer] = True
277 for filename in lexer.alias_filenames:
278 if _fn_matches(fn, filename):
279 matching_lexers.add(lexer)
280 primary[lexer] = False
281 if not matching_lexers:
282 raise ClassNotFound(f'no lexer for filename {fn!r} found')
283 if len(matching_lexers) == 1:
284 return matching_lexers.pop()(**options)
285 result = []
286 for lexer in matching_lexers:
287 rv = lexer.analyse_text(_text)
288 if rv == 1.0:
289 return lexer(**options)
290 result.append((rv, lexer))
292 def type_sort(t):
293 # sort by:
294 # - analyse score
295 # - is primary filename pattern?
296 # - priority
297 # - last resort: class name
298 return (t[0], primary[t[1]], t[1].priority, t[1].__name__)
299 result.sort(key=type_sort)
301 return result[-1][1](**options)
304def guess_lexer(_text, **options):
305 """
306 Return a `Lexer` subclass instance that's guessed from the text in
307 `text`. For that, the :meth:`.analyse_text()` method of every known lexer
308 class is called with the text as argument, and the lexer which returned the
309 highest value will be instantiated and returned.
311 :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can
312 handle the content.
313 """
315 if not isinstance(_text, str):
316 inencoding = options.get('inencoding', options.get('encoding'))
317 if inencoding:
318 _text = _text.decode(inencoding or 'utf8')
319 else:
320 _text, _ = guess_decode(_text)
322 # try to get a vim modeline first
323 ft = get_filetype_from_buffer(_text)
325 if ft is not None:
326 try:
327 return get_lexer_by_name(ft, **options)
328 except ClassNotFound:
329 pass
331 best_lexer = [0.0, None]
332 for lexer in _iter_lexerclasses():
333 rv = lexer.analyse_text(_text)
334 if rv == 1.0:
335 return lexer(**options)
336 if rv > best_lexer[0]:
337 best_lexer[:] = (rv, lexer)
338 if not best_lexer[0] or best_lexer[1] is None:
339 raise ClassNotFound('no lexer matching the text found')
340 return best_lexer[1](**options)
343class _automodule(types.ModuleType):
344 """Automatically import lexers."""
346 def __getattr__(self, name):
347 info = LEXERS.get(name)
348 if info:
349 _load_lexers(info[0])
350 cls = _lexer_cache[info[1]]
351 setattr(self, name, cls)
352 return cls
353 if name in COMPAT:
354 return getattr(self, COMPAT[name])
355 raise AttributeError(name)
358oldmod = sys.modules[__name__]
359newmod = _automodule(__name__)
360newmod.__dict__.update(oldmod.__dict__)
361sys.modules[__name__] = newmod
362del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types