Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2This module provides the :module:`hyperscan` backend for :class:`~pathspec.gitignore.GitIgnoreSpec`.
4WARNING: The *pathspec._backends.hyperscan* package is not part of the public
5API. Its contents and structure are likely to change.
6"""
7from __future__ import annotations
9from collections.abc import (
10 Sequence)
11from typing import (
12 Any,
13 Callable, # Replaced by `collections.abc.Callable` in 3.9.2.
14 Optional, # Replaced by `X | None` in 3.10.
15 Union) # Replaced by `X | Y` in 3.10.
17try:
18 import hyperscan
19except ModuleNotFoundError:
20 hyperscan = None # type: ignore[assignment]
22from pathspec.pattern import (
23 RegexPattern)
24from pathspec.patterns.gitignore.spec import (
25 GitIgnoreSpecPattern,
26 _BYTES_ENCODING,
27 _DIR_MARK_CG,
28 _DIR_MARK_OPT)
29from pathspec._typing import (
30 override) # Added in 3.12.
32from ._base import (
33 HS_FLAGS,
34 HyperscanExprDat,
35 HyperscanExprDebug)
36from .pathspec import (
37 HyperscanPsBackend)
40class HyperscanGiBackend(HyperscanPsBackend):
41 """
42 The :class:`HyperscanGiBackend` class is the :module:`hyperscan`
43 implementation used by :class:`~pathspec.gitignore.GitIgnoreSpec`. The
44 Hyperscan database uses block mode for matching files.
45 """
47 # Change type hint.
48 _out: tuple[Optional[bool], int, Optional[bool], int] # type: ignore[assignment]
50 def __init__(
51 self,
52 patterns: Sequence[RegexPattern],
53 *,
54 _debug_exprs: Optional[bool] = None,
55 _test_sort: Optional[Callable[[list], None]] = None,
56 ) -> None:
57 """
58 Initialize the :class:`HyperscanMatcher` instance.
60 *patterns* (:class:`Sequence` of :class:`.RegexPattern`) contains the
61 compiled patterns.
62 """
63 super().__init__(patterns, _debug_exprs=_debug_exprs, _test_sort=_test_sort)
65 self._out = (None, -1, None, -1)
66 """
67 *_out* (:class:`tuple`) stores the current match:
69 - *0* (:class:`bool` or :data:`None`) is the directory match include.
71 - *1* (:class:`int`) is the directory match index.
73 - *2* (:class:`bool` or :data:`None`) is the file match include.
75 - *3* (:class:`int`) is the file match index.
76 """
78 @override
79 @staticmethod
80 def _init_db(
81 db: hyperscan.Database, # type: ignore
82 debug: bool,
83 patterns: list[tuple[int, RegexPattern]],
84 sort_ids: Optional[Callable[[list[int]], None]],
85 ) -> list[HyperscanExprDat]:
86 """
87 Create the Hyperscan database from the given patterns.
89 *db* (:class:`hyperscan.Hyperscan`) is the Hyperscan database.
91 *debug* (:class:`bool`) is whether to include additional debugging
92 information for the expressions.
94 *patterns* (:class:`~collections.abc.Sequence` of :class:`.RegexPattern`)
95 contains the patterns.
97 *sort_ids* (:class:`callable` or :data:`None`) is a function used to sort
98 the compiled expression ids. This is used during testing to ensure the order
99 of expressions is not accidentally relied on.
101 Returns a :class:`list` indexed by expression id (:class:`int`) to its data
102 (:class:`HyperscanExprDat`).
103 """
104 # WARNING: Hyperscan raises a `hyperscan.error` exception when compiled with
105 # zero elements.
106 assert patterns, patterns
108 # Prepare patterns.
109 expr_data: list[HyperscanExprDat] = []
110 exprs: list[bytes] = []
111 for pattern_index, pattern in patterns:
112 assert pattern.include is not None, (pattern_index, pattern)
113 assert pattern.regex is not None, (pattern_index, pattern)
115 # Encode regex.
116 assert isinstance(pattern, RegexPattern), pattern
117 regex = pattern.regex.pattern
119 use_regexes: list[tuple[Union[str, bytes], bool]] = []
120 if isinstance(pattern, GitIgnoreSpecPattern):
121 # GitIgnoreSpecPattern uses capture groups for its directory marker but
122 # Hyperscan does not support capture groups. Handle this scenario.
123 regex_str: str
124 if isinstance(regex, str):
125 regex_str = regex
126 else:
127 assert isinstance(regex, bytes), regex
128 regex_str = regex.decode(_BYTES_ENCODING)
130 if _DIR_MARK_CG in regex_str:
131 # Found directory marker.
132 if regex_str.endswith(_DIR_MARK_OPT):
133 # Regex has optional directory marker. Split regex into directory
134 # and file variants.
135 base_regex = regex_str[:-len(_DIR_MARK_OPT)]
136 use_regexes.append((f'{base_regex}/', True))
137 use_regexes.append((f'{base_regex}$', False))
138 else:
139 # Remove capture group.
140 base_regex = regex_str.replace(_DIR_MARK_CG, '/')
141 use_regexes.append((base_regex, True))
143 if not use_regexes:
144 # No special case for regex.
145 use_regexes.append((regex, False))
147 for regex, is_dir_pattern in use_regexes:
148 if isinstance(regex, bytes):
149 regex_bytes = regex
150 else:
151 assert isinstance(regex, str), regex
152 regex_bytes = regex.encode('utf8')
154 if debug:
155 expr_data.append(HyperscanExprDebug(
156 include=pattern.include,
157 index=pattern_index,
158 is_dir_pattern=is_dir_pattern,
159 regex=regex,
160 ))
161 else:
162 expr_data.append(HyperscanExprDat(
163 include=pattern.include,
164 index=pattern_index,
165 is_dir_pattern=is_dir_pattern,
166 ))
168 exprs.append(regex_bytes)
170 # Sort expressions.
171 ids = list(range(len(exprs)))
172 if sort_ids is not None:
173 sort_ids(ids)
174 exprs = [exprs[__id] for __id in ids]
176 # Compile patterns.
177 db.compile(
178 expressions=exprs,
179 ids=ids,
180 elements=len(exprs),
181 flags=HS_FLAGS,
182 )
183 return expr_data
185 @override
186 def match_file(self, file: str) -> tuple[Optional[bool], Optional[int]]:
187 """
188 Check the file against the patterns.
190 *file* (:class:`str`) is the normalized file path to check.
192 Returns a :class:`tuple` containing whether to include *file* (:class:`bool`
193 or :data:`None`), and the index of the last matched pattern (:class:`int` or
194 :data:`None`).
195 """
196 # NOTICE: According to benchmarking, a method callback is 13% faster than
197 # using a closure here.
198 db = self._db
199 if db is None:
200 # Database was not initialized because there were no patterns. Return no
201 # match.
202 return (None, None)
204 self._out = (None, -1, None, -1)
205 db.scan(file.encode('utf8'), match_event_handler=self.__on_match)
207 dir_include, dir_index, file_include, file_index = self._out
208 if dir_include:
209 out_include, out_index = dir_include, dir_index
210 elif file_include is not None:
211 out_include, out_index = file_include, file_index
212 else:
213 out_include, out_index = dir_include, dir_index
215 return (out_include, out_index if out_index != -1 else None)
217 @override
218 def __on_match(
219 self,
220 expr_id: int,
221 _from: int,
222 _to: int,
223 _flags: int,
224 _context: Any,
225 ) -> Optional[bool]:
226 """
227 Called on each match.
229 *expr_id* (:class:`int`) is the expression id (index) of the matched
230 pattern.
231 """
232 expr_dat = self._expr_data[expr_id]
234 # WARNING: Hyperscan does not guarantee matches will be produced in order!
235 # Resolve the ancestor directory and the file separately: a file negation
236 # only applies while no ancestor directory is excluded.
237 include = expr_dat.include
238 index = expr_dat.index
239 dir_include, dir_index, file_include, file_index = self._out
240 if expr_dat.is_dir_pattern:
241 # Pattern matched by a directory pattern.
242 if index > dir_index:
243 self._out = (include, index, file_include, file_index) # type: ignore
244 elif index > file_index:
245 # Pattern matched by a file pattern.
246 self._out = (dir_include, dir_index, include, index) # type: ignore
248 return None