Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2This module provides common classes for the gitignore patterns.
3"""
4from __future__ import annotations
6import re
7from typing import (
8 Literal,
9 Optional, # Replaced by `X | None` in 3.10.
10 Union) # Replaced by `X | Y` in 3.10.
12from pathspec.pattern import (
13 RegexPattern)
14from pathspec._typing import (
15 AnyStr) # Removed in 3.18.
17_BRACKET_TOKEN_REGEX = re.compile((
18 # Range, including escaped endpoints, but not the start of a POSIX class.
19 r'(\\.|[^\\])-(?!\[:)(\\.|[^\\])'
20 # Escaped character.
21 r'|\\(.)'
22 # POSIX class.
23 r'|\[:(\^?)([^:\]]*):\]'
24 # Literal characters reserved for future regex set operations.
25 r'|([-\[&~|])'
26), re.DOTALL)
27"""
28Matches tokens inside a bracket expression:
30- A character range (groups 1 and 2).
32- A backslash escaped character (group 3).
34- Matches a POSIX bracket character class token such as ``[:alpha:]`` inside a
35 bracket expression. Group 4 captures a leading caret (unsupported by git);
36 group 5 captures the class name.
38- A literal character requiring regex escaping (group 6).
39"""
41_BYTES_ENCODING = 'latin1'
42"""
43The encoding to use when parsing a byte string pattern.
44"""
46_POSIX_CLASS_TO_REGEX: dict[str, str] = {
47 # Git's wildmatch implements POSIX bracket character classes using its own
48 # ASCII (locale-independent) ``is*`` functions, so each class maps to an
49 # explicit ASCII set. These are NOT the Unicode-aware equivalents (``\w``,
50 # ``\d``, ``\s``); using those would over-match non-ASCII characters that git
51 # never matches.
52 'alnum': '0-9A-Za-z',
53 'alpha': 'A-Za-z',
54 'blank': '\\t ',
55 'cntrl': '\\x00-\\x1f\\x7f',
56 'digit': '0-9',
57 'graph': '\\x21-\\x2e\\x30-\\x7e',
58 'lower': 'a-z',
59 'print': '\\x20-\\x2e\\x30-\\x7e',
60 'punct': '!-.:-@\\[-`{-~',
61 'space': '\\t\\n\\r ',
62 'upper': 'A-Z',
63 'xdigit': '0-9A-Fa-f',
64}
65"""
66Maps each POSIX bracket character class name to ASCII regex ranges, excluding
67the directory separator as required by git's wildmatch behavior.
68"""
71def _collapse_star_runs(segments: list[str]) -> list[str]:
72 """
73 Replace each pattern segment that consists only of three or more asterisks
74 (e.g., ``***``) with a double-asterisk (``**``). Git's wildmatch treats a run
75 of asterisks as a single globstar when it spans a whole path segment.
77 *segments* (:class:`list` of :class:`str`) are the pattern segments.
79 Returns the modified segments (:class:`list` of :class:`str`).
80 """
81 return [
82 '**' if len(seg) > 2 and seg.count('*') == len(seg) else seg
83 for seg in segments
84 ]
87def _strip_trailing_ws(pattern: str) -> str:
88 """
89 Strip trailing spaces and line endings from the pattern while considering
90 that they can be escaped. Other whitespace is part of the filename.
92 *pattern* (:class:`str`) is the pattern.
94 Returns the modified pattern (:class:`str`).
95 """
96 i = len(pattern) - 1
97 if i == -1 or pattern[i] not in ' \r\n':
98 # Fast path: pattern does not end with a space or line ending.
99 return pattern
101 # Scan past spaces and line endings.
102 i -= 1
103 while i >= 0 and pattern[i] in ' \r\n':
104 i -= 1
106 last_ws = i + 1
108 # Count backslashes.
109 while i >= 0 and pattern[i] == '\\':
110 i -= 1
112 bs_count = last_ws - i - 1
113 if bs_count % 2 == 1:
114 # Odd count, first whitespace character is escaped, strip the rest.
115 last_ws += 1
117 # Strip trailing whitespace.
118 return pattern[:last_ws]
121def _sub_bracket_token(match: re.Match[str]) -> str:
122 """
123 Translate a single token matched by :data:`_BRACKET_TOKEN_REGEX`. A token can
124 be a range (groups 1 and 2), a backslash escaped character (group 3), or a
125 POSIX bracket character class (groups 4 and 5), or a literal regex set
126 operator character (group 6).
128 *match* (:class:`re.Match`) is the match object.
130 Returns the replacement string (:class:`str`).
131 """
132 range_start, range_end, escape_char, class_negated, class_name, literal_char = match.groups()
134 # Character range. A bracket expression cannot match a directory separator.
135 if range_start is not None:
136 range_start, range_end = range_start[-1], range_end[-1]
137 if range_start < '/' < range_end:
138 return re.escape(range_start) + '-.0-' + re.escape(range_end)
139 return re.escape(range_start) + '-' + re.escape(range_end)
141 # Backslash escaped character.
142 if escape_char:
143 return re.escape(escape_char)
144 elif literal_char:
145 return re.escape(literal_char)
147 # POSIX bracket character class.
148 class_regex = _POSIX_CLASS_TO_REGEX.get(class_name)
149 if class_regex is not None and not class_negated:
150 return class_regex
152 raise _PosixClassError((
153 f"Invalid character class={match.group(0)!r} found in pattern="
154 f"{match.string!r}."
155 )) # _PosixClassError
158class _GitIgnoreBasePattern(RegexPattern):
159 """
160 .. warning:: This class is not part of the public API. It is subject to
161 change.
163 The :class:`_GitIgnoreBasePattern` class is the base implementation for a
164 compiled gitignore pattern.
165 """
167 # Keep the dict-less class hierarchy.
168 __slots__ = ()
170 def __init__(
171 self,
172 pattern: Union[AnyStr, re.Pattern, None],
173 include: Optional[bool] = None,
174 *,
175 errors: Optional[Literal['literal', 'null', 'raise']] = None,
176 ) -> None:
177 """
178 Initializes the :class:`_GitIgnoreBasePattern` instance.
180 *pattern* (:class:`str`, :class:`bytes`, :class:`re.Pattern`, or
181 :data:`None`) is the pattern to compile into a regular expression.
183 *include* (:class:`bool` or :data:`None`) must be :data:`None` unless
184 *pattern* is a precompiled regular expression (:class:`re.Pattern`) in which
185 case it is whether matched files should be included (:data:`True`), excluded
186 (:data:`False`), or is a null operation (:data:`None`).
188 *errors* (:class:`str` :data:`None`) is how to handle invalid notation in
189 the pattern. Default is :data:`None` for ``'null'`` because that is the
190 behavior of Git.
192 - ``'literal'``: Most invalid notation will be treated as a literal string.
193 In cases where the gitignore documentation clearly states the pattern
194 should never match, a null-operation will be returned.
196 - ``'null'``: Invalid notation will result in a null-operation.
198 - ``'raise'``: Invalid notation will raise a :exc:`GitIgnorePatternError`.
199 """
200 super().__init__(pattern, include, errors=errors)
202 @staticmethod
203 def escape(s: AnyStr) -> AnyStr:
204 """
205 Escape special characters in the given string.
207 *s* (:class:`str` or :class:`bytes`) a filename or a string that you want to
208 escape, usually before adding it to a ".gitignore".
210 Returns the escaped string (:class:`str` or :class:`bytes`).
211 """
212 if isinstance(s, str):
213 return_type = str
214 string = s
215 elif isinstance(s, bytes):
216 return_type = bytes
217 string = s.decode(_BYTES_ENCODING)
218 else:
219 raise TypeError(f"s:{s!r} is not a unicode or byte string.")
221 # Reference: https://git-scm.com/docs/gitignore#_pattern_format
222 out_string = ''.join((f"\\{x}" if x in '\\[]!*#?' else x) for x in string)
224 # EDGE CASE: Git strips trailing spaces from a pattern unless they are
225 # escaped with a backslash. Escape them so an escaped filename that ends
226 # with a space still matches that file.
227 stripped = out_string.rstrip(' ')
228 trailing = len(out_string) - len(stripped)
229 if trailing:
230 out_string = stripped + '\\ ' * trailing
232 if return_type is bytes:
233 out_bytes = out_string.encode(_BYTES_ENCODING)
234 return out_bytes # type: ignore[return-value]
235 else:
236 return out_string # type: ignore[return-value]
238 @staticmethod
239 def _translate_segment_glob(
240 pattern: str,
241 errors: Literal['literal', 'raise'],
242 ) -> str:
243 """
244 Translates the glob pattern to a regular expression. This is used in the
245 constructor to translate a path segment glob pattern to its corresponding
246 regular expression.
248 *pattern* (:class:`str`) is the glob pattern.
250 *errors* (:class:`str`) is how to handle invalid pattern notation in the
251 pattern:
253 - ``'literal'``: Invalid notation will be treated as a literal string.
255 - ``'raise'``: Invalid notation will raise an exception.
257 Raises :exc:`._PosixClassError` when an invalid POXIS class is found and
258 *errors* is ``'raise'``.
260 Raises :exc:`._RangeNotationError` when an invalid range notation is found
261 and *errors* is ``'raise'``.
263 Raises :exc:`._TrailingBackslashError` when a trailing backslash is found at
264 the end of the pattern, regardless of the value of *errors*.
266 Returns the regular expression (:class:`str`).
267 """
268 # NOTE: This is derived from `fnmatch.translate()` and is similar to the
269 # POSIX function `fnmatch()` with the `FNM_PATHNAME` flag set.
271 escape = False
272 regex = ''
273 i, end = 0, len(pattern)
274 while i < end:
275 # Get next character.
276 char = pattern[i]
277 i += 1
279 if escape:
280 # Escape the character.
281 escape = False
282 regex += re.escape(char)
284 elif char == '\\':
285 # Escape character, escape next character.
286 escape = True
288 elif char == '*':
289 # Multi-character wildcard. Match any string (except slashes), including
290 # an empty string.
291 while i < end and pattern[i] == '*':
292 i += 1
293 regex += '[^/]*'
295 elif char == '*':
296 # Multi-character wildcard. Match any string (except slashes), including
297 # an empty string. Consecutive asterisks within a segment are equivalent
298 # to one. Collapse them.
299 while i < end and pattern[i] == '*':
300 i += 1
302 regex += '[^/]*'
304 elif char == '?':
305 # Single-character wildcard. Match any single character (except a
306 # slash).
307 regex += '[^/]'
309 elif char == '[':
310 # Bracket expression (range notation) wildcard. Except for the beginning
311 # exclamation mark, the whole bracket expression can be used directly as
312 # regex, but we have to find where the expression ends.
313 # - "[][!]" matches ']', '[' and '!'.
314 # - "[]-]" matches ']' and '-'.
315 # - "[!]a-]" matches any character except ']', 'a' and '-'.
316 bracket_start = i - 1
317 j = i
319 # Pass bracket expression negation.
320 if j < end and (pattern[j] == '!' or pattern[j] == '^'):
321 j += 1
323 # Pass first closing bracket if it is at the beginning of the
324 # expression.
325 if j < end and pattern[j] == ']':
326 j += 1
328 # Find closing bracket. Stop once we reach the end or find it.
329 while j < end and pattern[j] != ']':
330 if pattern[j] == '\\':
331 # Skip over an escaped character so an escaped closing bracket
332 # ("\]") does not end the bracket expression.
333 j += 2
334 elif pattern[j:j+2] == '[:':
335 # Skip over a POSIX character class token ("[:name:]") so its
336 # internal closing bracket is not mistaken for the end of the whole
337 # bracket expression.
338 close = pattern.find(':]', j + 2)
339 if close == -1:
340 j = end
341 break
342 j = close + 2
343 else:
344 j += 1
346 if j < end:
347 # Found end of bracket expression. Increment j to be one past the
348 # closing bracket:
349 #
350 # [...]
351 # ^ ^
352 # i j
353 #
354 j += 1
355 expr = '['
356 is_expr_negated = False
358 if pattern[i] == '!':
359 # Bracket expression needs to be negated.
360 expr += '^'
361 is_expr_negated = True
362 i += 1
363 elif pattern[i] == '^':
364 # POSIX declares that the regex bracket expression negation "[^...]"
365 # is undefined in a glob pattern. Python's `fnmatch.translate()`
366 # escapes the caret ('^') as a literal. Git supports the using a
367 # caret for negation. Maintain consistency with Git because that is
368 # the expected behavior.
369 expr += '^'
370 is_expr_negated = True
371 i += 1
373 # Build regex bracket expression.
374 #
375 # A backslash escapes the next character (e.g. "[\]]" matches ']'),
376 # the same as outside of a bracket expression.
377 #
378 # Translate POSIX character classes (e.g. "[:alpha:]") into their
379 # ASCII regex equivalents. Git's wildmatch supports these, but
380 # Python's `re` does not, so passing them through verbatim builds a
381 # broken regex that silently mismatches (and warns about a nested
382 # set).
383 try:
384 body = _BRACKET_TOKEN_REGEX.sub(_sub_bracket_token, pattern[i:j - 1])
385 except _PosixClassError:
386 if errors == 'raise':
387 # EDGE CASE: Git discards patterns with an invalid range notation
388 # or an invalid POSIX class.
389 raise
390 else:
391 # Treat the whole bracket expression as a literal.
392 regex += re.escape(pattern[bracket_start:j])
393 i = j
394 continue
396 if is_expr_negated:
397 # Negated expressions must also exclude the directory separator.
398 # Preserve literal ']' and '-' when inserting '/' ahead of them.
399 if body.startswith((']', '-')):
400 body = '\\' + body
401 expr += '/'
403 expr += body + ']'
405 if errors == 'raise':
406 try:
407 re.compile(expr)
408 except re.error as e: # Renamed to `re.PatternError` in 3.13.
409 raise _RangeNotationError((
410 f"Invalid range notation={pattern[i:j]!r} found in "
411 f"pattern={pattern!r}."
412 )) from e
414 # Add regex bracket expression to regex result.
415 regex += expr
417 # Set i to one past the closing bracket.
418 i = j
420 else:
421 # Failed to find closing bracket.
422 if errors == 'raise':
423 # Treat invalid range notation as an error.
424 raise _RangeNotationError((
425 f"Invalid range notation={pattern[i:j]!r} found in {pattern=!r}."
426 ))
427 else:
428 # Treat opening bracket as a bracket literal instead of as an
429 # expression.
430 regex += '\\['
432 else:
433 # Regular character, escape it for regex.
434 regex += re.escape(char)
436 if escape:
437 # Trailing backslash found. According to the gitignore docs, this "is an
438 # invalid pattern that never matches".
439 raise _TrailingBackslashError()
441 return regex
444class GitIgnorePatternError(ValueError):
445 """
446 The :class:`GitIgnorePatternError` class indicates an invalid gitignore
447 pattern.
448 """
449 pass
452class _PosixClassError(GitIgnorePatternError):
453 """
454 Raised internally when a bracket expression contains an unknown or negated
455 POSIX character class name. Git treats such a pattern as malformed.
456 """
457 pass
460class _RangeNotationError(GitIgnorePatternError):
461 """
462 Raised internally when an invalid range notation was found in a gitignore
463 pattern.
464 """
465 pass
468class _TrailingBackslashError(GitIgnorePatternError):
469 """
470 Raised internally when a trailing backslash is found at the end of a gitignore
471 pattern.
472 """
473 pass