Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2This module provides :class:`GitIgnoreSpecPattern` which implements Git's
3`gitignore`_ patterns, and handles edge-cases where Git's behavior differs from
4what's documented. Git allows including files from excluded directories which
5appears to contradict the documentation. Git discards patterns with invalid
6range notation. This is used by :class:`~pathspec.gitignore.GitIgnoreSpec` to
7fully replicate Git's handling.
9.. _`gitignore`: https://git-scm.com/docs/gitignore
10"""
12from typing import (
13 Literal,
14 Optional) # Replaced by `X | None` in 3.10.
16from pathspec._typing import (
17 AnyStr, # Removed in 3.18.
18 assert_unreachable,
19 override) # Added in 3.12.
21from .base import (
22 GitIgnorePatternError,
23 _BYTES_ENCODING,
24 _GitIgnoreBasePattern,
25 _PosixClassError,
26 _RangeNotationError,
27 _TrailingBackslashError,
28 _collapse_star_runs,
29 _strip_trailing_ws)
31_DIR_MARK = 'ps_d'
32"""
33The regex group name for the directory marker. This is only used by
34:class:`GitIgnoreSpec`.
35"""
37_DIR_MARK_CG = f'(?P<{_DIR_MARK}>/)'
38"""
39This regular expression matches the directory marker.
40"""
42_DIR_MARK_OPT = rf'(?:{_DIR_MARK_CG}|\Z)'
43"""
44This regular expression matches the optional directory marker and sub-path.
45"""
47_MATCH_ALL = f'^(?s:.+/)?[^/]+{_DIR_MARK_OPT}'
48"""
49This regular expression matches every path. It is the expansion of the patterns
50"*" and "**" (i.e., "**/{any name}"), and it has to capture the directory marker
51like any other pattern so that :class:`.GitIgnoreSpec` can tell a directory
52match from a file match.
53"""
56class GitIgnoreSpecPattern(_GitIgnoreBasePattern):
57 """
58 The :class:`GitIgnoreSpecPattern` class represents a compiled gitignore
59 pattern with special handling for edge-cases to replicate Git's behavior.
61 This is registered under the deprecated name "gitwildmatch" for backward
62 compatibility with v0.12. The registered name will be removed in a future
63 version.
64 """
66 # Keep the dict-less class hierarchy.
67 __slots__ = ()
69 @staticmethod
70 def __normalize_segments(
71 is_dir_pattern: bool,
72 pattern_segs: list[str],
73 requires_dir_descendant: bool,
74 ) -> tuple[Optional[list[str]], Optional[str]]:
75 """
76 Normalize the pattern segments to make processing easier.
78 *is_dir_pattern* (:class:`bool`) is whether the pattern is a directory
79 pattern (i.e., ends with a slash '/').
81 *pattern_segs* (:class:`list` of :class:`str`) contains the pattern
82 segments. This may be modified in place.
84 *requires_dir_descendant* (:class:`bool`) indicates whether a terminal
85 directory globstar must consume a descendant segment before the directory
86 marker.
88 Raises :exc:`ValueError` if the pattern normalizes to nothing.
90 Returns a :class:`tuple` containing either:
92 - The normalized segments (:class:`list` of :class:`str`; or :data:`None`).
94 - The regular expression override (:class:`str` or :data:`None`).
95 """
96 if not pattern_segs[0]:
97 # A pattern beginning with a slash ('/') should match relative to the root
98 # directory. Remove the empty first segment to make the pattern relative
99 # to root.
100 del pattern_segs[0]
102 elif len(pattern_segs) == 1 or (len(pattern_segs) == 2 and not pattern_segs[1]):
103 # A single segment pattern with or without a trailing slash ('/') will
104 # match any descendant path. This is equivalent to "**/{pattern}". Prepend
105 # a double-asterisk segment to make the pattern relative to root.
106 if pattern_segs[0] != '**':
107 pattern_segs.insert(0, '**')
109 else:
110 # A pattern without a beginning slash ('/') but contains at least one
111 # prepended directory (e.g., "dir/{pattern}") should match relative to the
112 # root directory. No segment modification is needed.
113 pass
115 if not pattern_segs:
116 # After normalization, we end up with no pattern at all. This must be
117 # because the pattern is invalid.
118 raise ValueError("Pattern normalized to nothing.")
120 if not pattern_segs[-1]:
121 # A pattern ending with a slash ('/') will match all descendant paths if
122 # it is a directory but not if it is a regular file. This is equivalent to
123 # "{pattern}/**". Set the empty last segment to a double-asterisk to
124 # include all descendants.
125 pattern_segs[-1] = '**'
127 # EDGE CASE: Collapse duplicate double-asterisk sequences (i.e., '**/**').
128 # Iterate over the segments in reverse order and remove the duplicate double
129 # asterisks as we go.
130 for i in range(len(pattern_segs) - 1, 0, -1):
131 prev = pattern_segs[i-1]
132 seg = pattern_segs[i]
133 if prev == '**' and seg == '**':
134 del pattern_segs[i]
136 seg_count = len(pattern_segs)
137 if seg_count == 1 and pattern_segs[0] == '**':
138 if is_dir_pattern:
139 # The pattern "**/" will be normalized to "**", but it should match
140 # everything except for files in the root. Special case this pattern.
141 return (None, _DIR_MARK_CG)
142 else:
143 # The pattern "**" will match every path. Special case this pattern.
144 return (None, _MATCH_ALL)
146 elif (
147 seg_count == 2
148 and pattern_segs[0] == '**'
149 and pattern_segs[1] == '*'
150 ):
151 # The pattern "*" will be normalized to "**/*" and will match every
152 # path. Special case this pattern for efficiency.
153 return (None, _MATCH_ALL)
155 elif (
156 seg_count == 3
157 and pattern_segs[0] == '**'
158 and pattern_segs[1] == '*'
159 and pattern_segs[2] == '**'
160 and not requires_dir_descendant
161 ):
162 # The pattern "*/" will be normalized to "**/*/**" which will match every
163 # file not in the root directory. Special case this pattern for
164 # efficiency.
165 if is_dir_pattern:
166 return (None, _DIR_MARK_CG)
167 else:
168 return (None, '/')
170 # No regular expression override, return modified pattern segments.
171 return (pattern_segs, None)
173 @override
174 @classmethod
175 def pattern_to_regex(
176 cls,
177 pattern: AnyStr,
178 *,
179 errors: Optional[Literal['literal', 'null', 'raise']] = None,
180 ) -> tuple[Optional[AnyStr], Optional[bool]]:
181 """
182 Convert the pattern into a regular expression.
184 *pattern* (:class:`str` or :class:`bytes`) is the pattern to convert into a
185 regular expression.
187 *errors* (:class:`str` :data:`None`) is how to handle invalid notation in
188 the pattern. Default is :data:`None` for ``'null'`` because that is the
189 behavior of Git.
191 - ``'literal'``: Most invalid notation will be treated as a literal
192 string. In cases where the gitignore documentation clearly states the
193 pattern should never match, a null-operation will be returned.
195 - ``'null'``: Invalid notation will result in a null-operation.
197 - ``'raise'``: Invalid notation will raise a :exc:`.GitIgnorePatternError`.
199 Raises :exc:`.GitIgnorePatternError` if the pattern fails to process,
200 regardless of the value of *errors*.
202 Returns a :class:`tuple` containing:
204 - *pattern* (:class:`str`, :class:`bytes` or :data:`None`) is the uncompiled
205 regular expression.
207 - *include* (:class:`bool` or :data:`None`) is whether matched files should
208 be included (:data:`True`), excluded (:data:`False`), or is a
209 null-operation (:data:`None`).
210 """
211 if isinstance(pattern, str):
212 pattern_str = pattern
213 return_type = str
214 elif isinstance(pattern, bytes):
215 pattern_str = pattern.decode(_BYTES_ENCODING)
216 return_type = bytes
217 else:
218 raise TypeError(f"{pattern=!r} is not a unicode or byte string.")
220 original_pattern = pattern_str
221 del pattern
223 if errors is None:
224 errors = 'null'
225 elif errors not in ('literal', 'null', 'raise'):
226 raise ValueError(f"{errors=!r} is not a valid value.")
228 seg_errors: Literal['literal', 'raise']
229 if errors == 'null':
230 seg_errors = 'raise'
231 elif errors in ('literal', 'raise'):
232 seg_errors = errors
233 else:
234 assert_unreachable(f"Failed to map {errors=!r} to seg_errors.")
236 # Strip trailing whitespace.
237 pattern_str = _strip_trailing_ws(pattern_str)
239 regex: Optional[str]
240 include: Optional[bool]
242 if pattern_str.startswith('#'):
243 # A pattern starting with a hash ('#') serves as a comment (neither
244 # includes nor excludes files). Escape the hash with a backslash to match
245 # a literal hash (i.e., '\#').
246 return (None, None)
248 elif pattern_str == '/':
249 # EDGE CASE: According to `git check-ignore` (v2.4.1), a single '/' does
250 # not match any file.
251 return (None, None)
253 if pattern_str.startswith('!'):
254 # A pattern starting with an exclamation mark ('!') negates the pattern
255 # (exclude instead of include). Escape the exclamation mark with a
256 # backslash to match a literal exclamation mark (i.e., '\!').
257 include = False
258 # Remove leading exclamation mark.
259 pattern_str = pattern_str[1:]
260 else:
261 include = True
263 if not pattern_str:
264 # A blank pattern is a null-operation (neither includes nor excludes
265 # files).
266 return (None, None)
268 # Split pattern into segments.
269 orig_segs = _collapse_star_runs(pattern_str.split('/'))
271 # Check whether the pattern is specifically a directory pattern before
272 # normalization.
273 is_dir_pattern = not orig_segs[-1]
274 # A terminal directory globstar with a concrete prefix must consume a
275 # descendant segment before the directory marker. Keep this semantic bit
276 # separate from normalized segments, which collapse adjacent ``**``.
277 requires_dir_descendant = (
278 is_dir_pattern
279 and len(orig_segs) > 2
280 and orig_segs[-2] == '**'
281 and any(seg not in ('', '**') for seg in orig_segs[:-2])
282 )
284 # Normalize pattern to make processing easier.
285 try:
286 pattern_segs, override_regex = cls.__normalize_segments(
287 is_dir_pattern, orig_segs, requires_dir_descendant,
288 )
289 except ValueError as e:
290 raise GitIgnorePatternError((
291 f"Invalid git pattern: {original_pattern!r}"
292 )) from e # GitIgnorePatternError
294 if override_regex is not None:
295 # Use regex override.
296 regex = override_regex
298 elif pattern_segs is not None:
299 # Build regular expression from pattern.
300 try:
301 regex_parts = cls.__translate_segments(
302 seg_errors, is_dir_pattern, pattern_segs, requires_dir_descendant,
303 )
304 except (_PosixClassError, _RangeNotationError) as e:
305 if errors == 'raise':
306 raise GitIgnorePatternError((
307 f"Invalid git pattern: {original_pattern!r}"
308 )) from e # GitIgnorePatternError
309 else:
310 return (None, None)
311 except _TrailingBackslashError as e:
312 # EDGE CASE: The gitignore docs say a trailing backlash is invalid and
313 # never matches.
314 if errors == 'raise':
315 raise GitIgnorePatternError((
316 f"Invalid git pattern: {original_pattern!r}"
317 )) from e # GitIgnorePatternError
318 else:
319 return (None, None)
321 regex = ''.join(regex_parts)
323 else:
324 assert_unreachable((
325 f"{override_regex=} and {pattern_segs=} cannot both be null."
326 )) # assert_unreachable
328 # Encode regex if needed.
329 out_regex: AnyStr
330 if regex is not None and return_type is bytes:
331 regex_bytes = regex.encode(_BYTES_ENCODING)
332 out_regex = regex_bytes # type: ignore[assignment]
333 else:
334 out_regex = regex # type: ignore[assignment]
336 return (out_regex, include)
338 @classmethod
339 def __translate_segments(
340 cls,
341 errors: Literal['literal', 'raise'],
342 is_dir_pattern: bool,
343 pattern_segs: list[str],
344 requires_dir_descendant: bool,
345 ) -> list[str]:
346 """
347 Translate the pattern segments to regular expressions.
349 *errors* (:class:`str`) is how to handle invalid pattern notation in the
350 pattern:
352 - ``'literal'``: Invalid notation will be treated as a literal string.
354 - ``'raise'``: Invalid notation will raise an exception.
356 *is_dir_pattern* (:class:`bool`) is whether the pattern is a directory
357 pattern (i.e., ends with a slash '/').
359 *pattern_segs* (:class:`list` of :class:`str`) contains the pattern
360 segments.
362 *requires_dir_descendant* (:class:`bool`) indicates whether a terminal
363 directory globstar must consume a descendant segment before the directory
364 marker.
366 Raises :exc:`._PosixClassError` when an invalid POXIS class is found and
367 *errors* is ``'literal'``.
369 Raises :exc:`._RangeNotationError` when an invalid range notation is found
370 and *errors* is ``'literal'``.
372 Raises :exc:`._TrailingBackslashError` when a trailing backslash is found at
373 the end of the pattern, regardless of the value of *errors*.
375 Returns the regular expression parts (:class:`list` of :class:`str`).
376 """
377 # Build regular expression from pattern.
378 out_parts = []
379 need_slash = False
380 end = len(pattern_segs) - 1
381 for i, seg in enumerate(pattern_segs):
382 if seg == '**':
383 if i == 0:
384 # A normalized pattern beginning with double-asterisks ('**') will
385 # match any leading path segments.
386 out_parts.append('^(?s:.+/)?')
388 elif i < end:
389 # A pattern with inner double-asterisks ('**') will match multiple (or
390 # zero) inner path segments.
391 out_parts.append('(?s:/.+)?')
392 need_slash = True
394 else:
395 assert i == end, (i, end)
396 # A normalized pattern ending with double-asterisks ('**') will match
397 # nonempty trailing path segments, not the parent directory itself.
398 if is_dir_pattern:
399 if requires_dir_descendant:
400 out_parts.append('(?s:/.+)')
401 out_parts.append(_DIR_MARK_CG)
402 else:
403 out_parts.append('/[^/]')
405 else:
406 # Match path segment.
407 if i == 0:
408 # Anchor to root directory.
409 out_parts.append('^')
411 if need_slash:
412 out_parts.append('/')
414 if seg == '*':
415 # Match whole path segment.
416 out_parts.append('[^/]+')
418 else:
419 # Match segment glob pattern.
420 # - EDGE CASE: Git discards patterns with invalid range notation.
421 out_parts.append(cls._translate_segment_glob(seg, errors))
423 if i == end:
424 # A pattern ending without a slash ('/') will match a file or a
425 # directory (with paths underneath it). E.g., "foo" matches "foo",
426 # "foo/bar", "foo/bar/baz", etc.
427 out_parts.append(_DIR_MARK_OPT)
429 need_slash = True
431 return out_parts