Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of Hypothesis, which may be found at
2# https://github.com/HypothesisWorks/hypothesis/
3#
4# Copyright the Hypothesis Authors.
5# Individual contributors are listed in AUTHORS.rst and the git log.
6#
7# This Source Code Form is subject to the terms of the Mozilla Public License,
8# v. 2.0. If a copy of the MPL was not distributed with this file, You can
9# obtain one at https://mozilla.org/MPL/2.0/.
11import copy
12import re
13import warnings
14from collections.abc import Collection
15from functools import cache, lru_cache, partial
16from typing import Any, cast
18from hypothesis.errors import (
19 CannotInvert,
20 HypothesisWarning,
21 InvalidArgument,
22 NonRoundTrippableCharactersWarning,
23)
24from hypothesis.internal import charmap
25from hypothesis.internal.charmap import Categories
26from hypothesis.internal.conjecture.choice import ChoiceT
27from hypothesis.internal.conjecture.data import ConjectureData
28from hypothesis.internal.conjecture.providers import COLLECTION_DEFAULT_MAX_SIZE
29from hypothesis.internal.filtering import max_len, min_len
30from hypothesis.internal.intervalsets import IntervalSet
31from hypothesis.internal.reflection import get_pretty_function_description
32from hypothesis.strategies._internal.collections import ListStrategy
33from hypothesis.strategies._internal.lazy import unwrap_strategies
34from hypothesis.strategies._internal.strategies import (
35 OneOfStrategy,
36 SampledFromStrategy,
37 SearchStrategy,
38)
39from hypothesis.vendor.pretty import pretty
42# Cache size is limited by sys.maxunicode, but passing None makes it slightly faster.
43@cache
44# this is part of our forward-facing validation, so we do *not* tell mypyc that c
45# should be a str, because we don't want it to validate it before we can.
46def _check_is_single_character(c: object) -> str:
47 # In order to mitigate the performance cost of this check, we use a shared cache,
48 # even at the cost of showing the culprit strategy in the error message.
49 if not isinstance(c, str):
50 type_ = get_pretty_function_description(type(c))
51 raise InvalidArgument(f"Got non-string {c!r} (type {type_})")
52 if len(c) != 1:
53 raise InvalidArgument(f"Got {c!r} (length {len(c)} != 1)")
54 return c
57def _check_alphabet_elements(alphabet: Collection[str]) -> None:
58 non_string = [c for c in alphabet if not isinstance(c, str)]
59 if non_string:
60 raise InvalidArgument(
61 "The following elements in alphabet are not unicode "
62 f"strings: {non_string!r}"
63 )
64 not_one_char = [c for c in alphabet if len(c) != 1]
65 if not_one_char:
66 raise InvalidArgument(
67 "The following elements in alphabet are not of length one, "
68 f"which leads to violation of size constraints: {not_one_char!r}"
69 )
70 if alphabet in ["ascii", "utf-8"]:
71 warnings.warn(
72 f"alphabet={alphabet!r}: it seems like you are trying to use the "
73 f"codec {alphabet!r}, but this generates strings using the literal "
74 f"characters {list(alphabet)!r}. To specify the {alphabet} codec, "
75 f"use alphabet=st.characters(codec={alphabet!r}). If you intended "
76 "to use character literals, you can silence this warning by "
77 "reordering the characters.",
78 HypothesisWarning,
79 # this stacklevel is of course incorrect, but breaking out of the
80 # levels of LazyStrategy and validation isn't worthwhile.
81 stacklevel=1,
82 )
85class OneCharStringStrategy(SearchStrategy[str]):
86 """A strategy which generates single character strings of text type."""
88 def __init__(self, intervals: IntervalSet, force_repr: str | None = None) -> None:
89 super().__init__()
90 assert isinstance(intervals, IntervalSet)
91 self.intervals = intervals
92 self._force_repr = force_repr
94 @classmethod
95 def from_characters_args(
96 cls,
97 *,
98 codec: str | None = None,
99 min_codepoint: int | None = None,
100 max_codepoint: int | None = None,
101 categories: Categories | None = None,
102 exclude_characters: Collection[str] = "",
103 include_characters: Collection[str] = "",
104 ) -> "OneCharStringStrategy":
105 assert set(categories or ()).issubset(charmap.categories())
106 intervals = charmap.query(
107 min_codepoint=min_codepoint,
108 max_codepoint=max_codepoint,
109 categories=categories,
110 exclude_characters=exclude_characters,
111 )
112 include_intervals = IntervalSet.from_string("".join(include_characters))
113 if codec is not None:
114 encodable, non_roundtrip = charmap.intervals_from_codec(codec)
115 intervals &= encodable
116 if undecided := (intervals & non_roundtrip) - include_intervals:
117 chars = "".join(map(chr, undecided))
118 # also show the \u-escaped form, in case the raw repr doesn't
119 # display or copy-paste cleanly in the user's terminal
120 aka = "" if ascii(chars) == repr(chars) else f" (aka {chars!a})"
121 warnings.warn(
122 f"Characters {chars!r}{aka} can be encoded with "
123 f"codec={codec!r}, but do not decode back to the same "
124 "character, so strings containing them do not round-trip. "
125 "Pass each of them in either include_characters, to "
126 f"generate them without this warning, or "
127 f"exclude_characters={chars!r}, to generate only "
128 "characters which round-trip.",
129 NonRoundTrippableCharactersWarning,
130 # this stacklevel is of course incorrect, but breaking out
131 # of the levels of LazyStrategy and validation isn't
132 # worthwhile.
133 stacklevel=1,
134 )
135 # include_characters are generated even if excluded by other arguments,
136 # such as the passed categories or codepoint range. (overlap with
137 # exclude_characters raises an error in st.characters())
138 intervals |= include_intervals
140 _arg_repr = ", ".join(
141 f"{k}={v!r}"
142 for k, v in [
143 ("codec", codec),
144 ("min_codepoint", min_codepoint),
145 ("max_codepoint", max_codepoint),
146 ("categories", categories),
147 ("exclude_characters", exclude_characters),
148 ("include_characters", include_characters),
149 ]
150 if v not in (None, "")
151 and not (
152 k == "categories"
153 # v has to be `categories` here. Help mypy along to infer that.
154 and set(cast(Categories, v)) == set(charmap.categories()) - {"Cs"}
155 )
156 )
157 if not intervals:
158 raise InvalidArgument(
159 "No characters are allowed to be generated by this "
160 f"combination of arguments: {_arg_repr}"
161 )
162 return cls(intervals, force_repr=f"characters({_arg_repr})")
164 @classmethod
165 def from_alphabet(
166 cls, alphabet: Collection[str] | SearchStrategy[str]
167 ) -> "OneCharStringStrategy | None":
168 # Shared logic for the `alphabet=` parameter of st.text and st.from_regex.
169 # Returns None if `alphabet` cannot be statically resolved to a set of characters,
170 # since each caller may wish to handle this case differently.
171 if not isinstance(alphabet, SearchStrategy):
172 _check_alphabet_elements(alphabet)
173 return cls.from_characters_args(categories=(), include_characters=alphabet)
175 char_strategy = unwrap_strategies(alphabet)
176 if isinstance(char_strategy, cls):
177 return char_strategy
178 elif isinstance(char_strategy, SampledFromStrategy):
179 if char_strategy._transformations:
180 # resolving from .elements would ignore the .map/.filter calls
181 return None
182 _check_alphabet_elements(char_strategy.elements)
183 return cls.from_characters_args(
184 categories=(),
185 include_characters=char_strategy.elements,
186 )
187 elif isinstance(char_strategy, OneOfStrategy):
188 intervals = IntervalSet()
189 for s in char_strategy.element_strategies:
190 resolved = cls.from_alphabet(s)
191 if resolved is None:
192 return None
193 intervals = intervals.union(resolved.intervals)
194 return cls(intervals, force_repr=repr(alphabet))
195 return None
197 def __repr__(self) -> str:
198 return self._force_repr or f"OneCharStringStrategy({self.intervals!r})"
200 def do_draw(self, data: ConjectureData) -> str:
201 return data.draw_string(self.intervals, min_size=1, max_size=1)
203 def _invert(self, value: Any) -> tuple[ChoiceT, ...]:
204 if not isinstance(value, str) or len(value) != 1:
205 raise CannotInvert(f"{value!r} is not a single character")
206 if ord(value) not in self.intervals:
207 raise CannotInvert(f"{value!r} is not in {self.intervals!r}")
208 return (value,)
211_nonempty_names = (
212 "capitalize",
213 "expandtabs",
214 "join",
215 "lower",
216 "rsplit",
217 "split",
218 "splitlines",
219 "swapcase",
220 "title",
221 "upper",
222)
223_nonempty_and_content_names = (
224 "islower",
225 "isupper",
226 "isalnum",
227 "isalpha",
228 "isascii",
229 "isdigit",
230 "isspace",
231 "istitle",
232 "lstrip",
233 "rstrip",
234 "strip",
235)
238class TextStrategy(ListStrategy[str]):
239 def do_draw(self, data):
240 # if our element strategy is OneCharStringStrategy, we can skip the
241 # ListStrategy draw and jump right to data.draw_string.
242 # Doing so for user-provided element strategies is not correct in
243 # general, as they may define a different distribution than data.draw_string.
244 elems = unwrap_strategies(self.element_strategy)
245 if isinstance(elems, OneCharStringStrategy):
246 return data.draw_string(
247 elems.intervals,
248 min_size=self.min_size,
249 max_size=(
250 COLLECTION_DEFAULT_MAX_SIZE
251 if self.max_size == float("inf")
252 else self.max_size
253 ),
254 )
255 return "".join(super().do_draw(data))
257 def _invert(self, value: Any) -> tuple[ChoiceT, ...]:
258 if not isinstance(value, str):
259 raise CannotInvert(f"{value!r} is not a string")
260 elems = unwrap_strategies(self.element_strategy)
261 if not isinstance(elems, OneCharStringStrategy):
262 # a non-standard element strategy is drawn one character at a time
263 return ListStrategy._invert(self, list(value))
264 effective_max = (
265 COLLECTION_DEFAULT_MAX_SIZE
266 if self.max_size == float("inf")
267 else self.max_size
268 )
269 if not (self.min_size <= len(value) <= effective_max):
270 raise CannotInvert(
271 f"len({value!r})={len(value)} outside "
272 f"[{self.min_size}, {effective_max}]"
273 )
274 if any(ord(c) not in elems.intervals for c in value):
275 raise CannotInvert(f"{value!r} contains chars outside {elems!r}")
276 return (value,)
278 def __repr__(self) -> str:
279 args = []
280 if repr(self.element_strategy) != "characters()":
281 args.append(repr(self.element_strategy))
282 if self.min_size:
283 args.append(f"min_size={self.min_size}")
284 if self.max_size < float("inf"):
285 args.append(f"max_size={self.max_size}")
286 return f"text({', '.join(args)})"
288 # See https://docs.python.org/3/library/stdtypes.html#string-methods
289 # These methods always return Truthy values for any nonempty string.
290 _nonempty_filters = (
291 *ListStrategy._nonempty_filters,
292 str,
293 str.casefold,
294 str.encode,
295 *(getattr(str, n) for n in _nonempty_names),
296 )
297 _nonempty_and_content_filters = (
298 str.isdecimal,
299 str.isnumeric,
300 *(getattr(str, n) for n in _nonempty_and_content_names),
301 )
303 def filter(self, condition):
304 elems = unwrap_strategies(self.element_strategy)
305 if (
306 condition is str.isidentifier
307 and self.max_size >= 1
308 and isinstance(elems, OneCharStringStrategy)
309 ):
310 from hypothesis.strategies import builds, nothing
312 id_start, id_continue = _identifier_characters()
313 if not (elems.intervals & id_start):
314 return nothing()
315 return builds(
316 "{}{}".format,
317 OneCharStringStrategy(elems.intervals & id_start),
318 TextStrategy(
319 OneCharStringStrategy(elems.intervals & id_continue),
320 min_size=max(0, self.min_size - 1),
321 max_size=self.max_size - 1,
322 ),
323 # Filter to ensure that NFKC normalization keeps working in future
324 ).filter(str.isidentifier)
325 if (new := _string_filter_rewrite(self, str, condition)) is not None:
326 return new
327 return super().filter(condition)
330def _string_filter_rewrite(self, kind, condition):
331 if condition in (kind.lower, kind.title, kind.upper):
332 k = kind.__name__
333 warnings.warn(
334 f"You applied {k}.{condition.__name__} as a filter, but this allows "
335 f"all nonempty strings! Did you mean {k}.is{condition.__name__}?",
336 HypothesisWarning,
337 stacklevel=2,
338 )
340 if (
341 (
342 kind is bytes
343 or isinstance(
344 unwrap_strategies(self.element_strategy), OneCharStringStrategy
345 )
346 )
347 and isinstance(pattern := getattr(condition, "__self__", None), re.Pattern)
348 and isinstance(pattern.pattern, kind)
349 ):
350 from hypothesis.strategies._internal.regex import regex_strategy
352 if condition.__name__ == "match":
353 # Replace with an easier-to-handle equivalent condition
354 caret, close = ("^(?:", ")") if kind is str else (b"^(?:", b")")
355 pattern = re.compile(caret + pattern.pattern + close, flags=pattern.flags)
356 condition = pattern.search
358 if condition.__name__ in ("search", "findall", "fullmatch"):
359 s = regex_strategy(
360 pattern,
361 fullmatch=condition.__name__ == "fullmatch",
362 alphabet=self.element_strategy if kind is str else None,
363 )
364 if self.min_size > 0:
365 s = s.filter(partial(min_len, self.min_size))
366 if self.max_size < 1e999:
367 s = s.filter(partial(max_len, self.max_size))
368 return s
369 elif condition.__name__ in ("finditer", "scanner"):
370 # PyPy implements `finditer` as an alias to their `scanner` method
371 warnings.warn(
372 f"You applied {pretty(condition)} as a filter, but this allows "
373 f"any string at all! Did you mean .findall ?",
374 HypothesisWarning,
375 stacklevel=3,
376 )
377 return self
378 elif condition.__name__ == "split":
379 warnings.warn(
380 f"You applied {pretty(condition)} as a filter, but this allows "
381 f"any nonempty string! Did you mean .search ?",
382 HypothesisWarning,
383 stacklevel=3,
384 )
385 return self.filter(bool)
387 # We use ListStrategy filter logic for the conditions that *only* imply
388 # the string is nonempty. Here, we increment the min_size but still apply
389 # the filter for conditions that imply nonempty *and specific contents*.
390 if condition in self._nonempty_and_content_filters and self.max_size >= 1:
391 self = copy.copy(self)
392 self.min_size = max(1, self.min_size)
393 return ListStrategy.filter(self, condition)
395 return None
398# Excerpted from https://www.unicode.org/Public/15.0.0/ucd/PropList.txt
399# Python updates it's Unicode version between minor releases, but fortunately
400# these properties do not change between the Unicode versions in question.
401_PROPLIST = """
402# ================================================
4041885..1886 ; Other_ID_Start # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA
4052118 ; Other_ID_Start # Sm SCRIPT CAPITAL P
406212E ; Other_ID_Start # So ESTIMATED SYMBOL
407309B..309C ; Other_ID_Start # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
409# Total code points: 6
411# ================================================
41300B7 ; Other_ID_Continue # Po MIDDLE DOT
4140387 ; Other_ID_Continue # Po GREEK ANO TELEIA
4151369..1371 ; Other_ID_Continue # No [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE
41619DA ; Other_ID_Continue # No NEW TAI LUE THAM DIGIT ONE
418# Total code points: 12
419"""
422@lru_cache
423def _identifier_characters() -> tuple[IntervalSet, IntervalSet]:
424 """See https://docs.python.org/3/reference/lexical_analysis.html#identifiers"""
425 # Start by computing the set of special characters
426 chars = {"Other_ID_Start": "", "Other_ID_Continue": ""}
427 for line in _PROPLIST.splitlines():
428 if m := re.match(r"([0-9A-F.]+) +; (\w+) # ", line):
429 codes, prop = m.groups()
430 span = range(int(codes[:4], base=16), int(codes[-4:], base=16) + 1)
431 chars[prop] += "".join(chr(x) for x in span)
433 # Then get the basic set by Unicode category and known extras
434 id_start = charmap.query(
435 categories=("Lu", "Ll", "Lt", "Lm", "Lo", "Nl"),
436 include_characters="_" + chars["Other_ID_Start"],
437 )
438 id_start -= IntervalSet.from_string(
439 # Magic value: the characters which NFKC-normalize to be invalid identifiers.
440 # Conveniently they're all in `id_start`, so we only need to do this once.
441 "\u037a\u0e33\u0eb3\u2e2f\u309b\u309c\ufc5e\ufc5f\ufc60\ufc61\ufc62\ufc63"
442 "\ufdfa\ufdfb\ufe70\ufe72\ufe74\ufe76\ufe78\ufe7a\ufe7c\ufe7e\uff9e\uff9f"
443 )
444 id_continue = id_start | charmap.query(
445 categories=("Mn", "Mc", "Nd", "Pc"),
446 include_characters=chars["Other_ID_Continue"],
447 )
448 return id_start, id_continue
451class BytesStrategy(SearchStrategy):
452 def __init__(self, min_size: int, max_size: int | None):
453 super().__init__()
454 self.min_size = min_size
455 self.max_size = (
456 max_size if max_size is not None else COLLECTION_DEFAULT_MAX_SIZE
457 )
459 def do_draw(self, data: ConjectureData) -> bytes:
460 return data.draw_bytes(self.min_size, self.max_size)
462 def _invert(self, value: Any) -> tuple[ChoiceT, ...]:
463 if not isinstance(value, bytes):
464 raise CannotInvert(f"{value!r} is not bytes")
465 if not (self.min_size <= len(value) <= self.max_size):
466 raise CannotInvert(
467 f"len({value!r})={len(value)} outside "
468 f"[{self.min_size}, {self.max_size}]"
469 )
470 return (value,)
472 _nonempty_filters = (
473 *ListStrategy._nonempty_filters,
474 bytes,
475 *(getattr(bytes, n) for n in _nonempty_names),
476 )
477 _nonempty_and_content_filters = (
478 *(getattr(bytes, n) for n in _nonempty_and_content_names),
479 )
481 def filter(self, condition):
482 if (new := _string_filter_rewrite(self, bytes, condition)) is not None:
483 return new
484 return ListStrategy.filter(self, condition)