Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of Hypothesis, which may be found at
2# https://github.com/HypothesisWorks/hypothesis/
3#
4# Copyright the Hypothesis Authors.
5# Individual contributors are listed in AUTHORS.rst and the git log.
6#
7# This Source Code Form is subject to the terms of the Mozilla Public License,
8# v. 2.0. If a copy of the MPL was not distributed with this file, You can
9# obtain one at https://mozilla.org/MPL/2.0/.
11import copy
12import re
13import warnings
14from collections.abc import Collection
15from functools import cache, lru_cache, partial
16from typing import cast
18from hypothesis.errors import HypothesisWarning, InvalidArgument
19from hypothesis.internal import charmap
20from hypothesis.internal.charmap import Categories
21from hypothesis.internal.conjecture.data import ConjectureData
22from hypothesis.internal.conjecture.providers import COLLECTION_DEFAULT_MAX_SIZE
23from hypothesis.internal.filtering import max_len, min_len
24from hypothesis.internal.intervalsets import IntervalSet
25from hypothesis.internal.reflection import get_pretty_function_description
26from hypothesis.strategies._internal.collections import ListStrategy
27from hypothesis.strategies._internal.lazy import unwrap_strategies
28from hypothesis.strategies._internal.strategies import (
29 OneOfStrategy,
30 SampledFromStrategy,
31 SearchStrategy,
32)
33from hypothesis.vendor.pretty import pretty
36# Cache size is limited by sys.maxunicode, but passing None makes it slightly faster.
37@cache
38# this is part of our forward-facing validation, so we do *not* tell mypyc that c
39# should be a str, because we don't want it to validate it before we can.
40def _check_is_single_character(c: object) -> str:
41 # In order to mitigate the performance cost of this check, we use a shared cache,
42 # even at the cost of showing the culprit strategy in the error message.
43 if not isinstance(c, str):
44 type_ = get_pretty_function_description(type(c))
45 raise InvalidArgument(f"Got non-string {c!r} (type {type_})")
46 if len(c) != 1:
47 raise InvalidArgument(f"Got {c!r} (length {len(c)} != 1)")
48 return c
51def _check_alphabet_elements(alphabet: Collection[str]) -> None:
52 non_string = [c for c in alphabet if not isinstance(c, str)]
53 if non_string:
54 raise InvalidArgument(
55 "The following elements in alphabet are not unicode "
56 f"strings: {non_string!r}"
57 )
58 not_one_char = [c for c in alphabet if len(c) != 1]
59 if not_one_char:
60 raise InvalidArgument(
61 "The following elements in alphabet are not of length one, "
62 f"which leads to violation of size constraints: {not_one_char!r}"
63 )
64 if alphabet in ["ascii", "utf-8"]:
65 warnings.warn(
66 f"alphabet={alphabet!r}: it seems like you are trying to use the "
67 f"codec {alphabet!r}, but this generates strings using the literal "
68 f"characters {list(alphabet)!r}. To specify the {alphabet} codec, "
69 f"use alphabet=st.characters(codec={alphabet!r}). If you intended "
70 "to use character literals, you can silence this warning by "
71 "reordering the characters.",
72 HypothesisWarning,
73 # this stacklevel is of course incorrect, but breaking out of the
74 # levels of LazyStrategy and validation isn't worthwhile.
75 stacklevel=1,
76 )
79class OneCharStringStrategy(SearchStrategy[str]):
80 """A strategy which generates single character strings of text type."""
82 def __init__(self, intervals: IntervalSet, force_repr: str | None = None) -> None:
83 super().__init__()
84 assert isinstance(intervals, IntervalSet)
85 self.intervals = intervals
86 self._force_repr = force_repr
88 @classmethod
89 def from_characters_args(
90 cls,
91 *,
92 codec: str | None = None,
93 min_codepoint: int | None = None,
94 max_codepoint: int | None = None,
95 categories: Categories | None = None,
96 exclude_characters: Collection[str] = "",
97 include_characters: Collection[str] = "",
98 ) -> "OneCharStringStrategy":
99 assert set(categories or ()).issubset(charmap.categories())
100 intervals = charmap.query(
101 min_codepoint=min_codepoint,
102 max_codepoint=max_codepoint,
103 categories=categories,
104 exclude_characters=exclude_characters,
105 include_characters=include_characters,
106 )
107 if codec is not None:
108 intervals &= charmap.intervals_from_codec(codec)
110 _arg_repr = ", ".join(
111 f"{k}={v!r}"
112 for k, v in [
113 ("codec", codec),
114 ("min_codepoint", min_codepoint),
115 ("max_codepoint", max_codepoint),
116 ("categories", categories),
117 ("exclude_characters", exclude_characters),
118 ("include_characters", include_characters),
119 ]
120 if v not in (None, "")
121 and not (
122 k == "categories"
123 # v has to be `categories` here. Help mypy along to infer that.
124 and set(cast(Categories, v)) == set(charmap.categories()) - {"Cs"}
125 )
126 )
127 if not intervals:
128 raise InvalidArgument(
129 "No characters are allowed to be generated by this "
130 f"combination of arguments: {_arg_repr}"
131 )
132 return cls(intervals, force_repr=f"characters({_arg_repr})")
134 @classmethod
135 def from_alphabet(
136 cls, alphabet: Collection[str] | SearchStrategy[str]
137 ) -> "OneCharStringStrategy | None":
138 # Shared logic for the `alphabet=` parameter of st.text and st.from_regex.
139 # Returns None if `alphabet` cannot be statically resolved to a set of characters,
140 # since each caller may wish to handle this case differently.
141 if not isinstance(alphabet, SearchStrategy):
142 _check_alphabet_elements(alphabet)
143 return cls.from_characters_args(categories=(), include_characters=alphabet)
145 char_strategy = unwrap_strategies(alphabet)
146 if isinstance(char_strategy, cls):
147 return char_strategy
148 elif isinstance(char_strategy, SampledFromStrategy):
149 if char_strategy._transformations:
150 # resolving from .elements would ignore the .map/.filter calls
151 return None
152 _check_alphabet_elements(char_strategy.elements)
153 return cls.from_characters_args(
154 categories=(),
155 include_characters=char_strategy.elements,
156 )
157 elif isinstance(char_strategy, OneOfStrategy):
158 intervals = IntervalSet()
159 for s in char_strategy.element_strategies:
160 resolved = cls.from_alphabet(s)
161 if resolved is None:
162 return None
163 intervals = intervals.union(resolved.intervals)
164 return cls(intervals, force_repr=repr(alphabet))
165 return None
167 def __repr__(self) -> str:
168 return self._force_repr or f"OneCharStringStrategy({self.intervals!r})"
170 def do_draw(self, data: ConjectureData) -> str:
171 return data.draw_string(self.intervals, min_size=1, max_size=1)
174_nonempty_names = (
175 "capitalize",
176 "expandtabs",
177 "join",
178 "lower",
179 "rsplit",
180 "split",
181 "splitlines",
182 "swapcase",
183 "title",
184 "upper",
185)
186_nonempty_and_content_names = (
187 "islower",
188 "isupper",
189 "isalnum",
190 "isalpha",
191 "isascii",
192 "isdigit",
193 "isspace",
194 "istitle",
195 "lstrip",
196 "rstrip",
197 "strip",
198)
201class TextStrategy(ListStrategy[str]):
202 def do_draw(self, data):
203 # if our element strategy is OneCharStringStrategy, we can skip the
204 # ListStrategy draw and jump right to data.draw_string.
205 # Doing so for user-provided element strategies is not correct in
206 # general, as they may define a different distribution than data.draw_string.
207 elems = unwrap_strategies(self.element_strategy)
208 if isinstance(elems, OneCharStringStrategy):
209 return data.draw_string(
210 elems.intervals,
211 min_size=self.min_size,
212 max_size=(
213 COLLECTION_DEFAULT_MAX_SIZE
214 if self.max_size == float("inf")
215 else self.max_size
216 ),
217 )
218 return "".join(super().do_draw(data))
220 def __repr__(self) -> str:
221 args = []
222 if repr(self.element_strategy) != "characters()":
223 args.append(repr(self.element_strategy))
224 if self.min_size:
225 args.append(f"min_size={self.min_size}")
226 if self.max_size < float("inf"):
227 args.append(f"max_size={self.max_size}")
228 return f"text({', '.join(args)})"
230 # See https://docs.python.org/3/library/stdtypes.html#string-methods
231 # These methods always return Truthy values for any nonempty string.
232 _nonempty_filters = (
233 *ListStrategy._nonempty_filters,
234 str,
235 str.casefold,
236 str.encode,
237 *(getattr(str, n) for n in _nonempty_names),
238 )
239 _nonempty_and_content_filters = (
240 str.isdecimal,
241 str.isnumeric,
242 *(getattr(str, n) for n in _nonempty_and_content_names),
243 )
245 def filter(self, condition):
246 elems = unwrap_strategies(self.element_strategy)
247 if (
248 condition is str.isidentifier
249 and self.max_size >= 1
250 and isinstance(elems, OneCharStringStrategy)
251 ):
252 from hypothesis.strategies import builds, nothing
254 id_start, id_continue = _identifier_characters()
255 if not (elems.intervals & id_start):
256 return nothing()
257 return builds(
258 "{}{}".format,
259 OneCharStringStrategy(elems.intervals & id_start),
260 TextStrategy(
261 OneCharStringStrategy(elems.intervals & id_continue),
262 min_size=max(0, self.min_size - 1),
263 max_size=self.max_size - 1,
264 ),
265 # Filter to ensure that NFKC normalization keeps working in future
266 ).filter(str.isidentifier)
267 if (new := _string_filter_rewrite(self, str, condition)) is not None:
268 return new
269 return super().filter(condition)
272def _string_filter_rewrite(self, kind, condition):
273 if condition in (kind.lower, kind.title, kind.upper):
274 k = kind.__name__
275 warnings.warn(
276 f"You applied {k}.{condition.__name__} as a filter, but this allows "
277 f"all nonempty strings! Did you mean {k}.is{condition.__name__}?",
278 HypothesisWarning,
279 stacklevel=2,
280 )
282 if (
283 (
284 kind is bytes
285 or isinstance(
286 unwrap_strategies(self.element_strategy), OneCharStringStrategy
287 )
288 )
289 and isinstance(pattern := getattr(condition, "__self__", None), re.Pattern)
290 and isinstance(pattern.pattern, kind)
291 ):
292 from hypothesis.strategies._internal.regex import regex_strategy
294 if condition.__name__ == "match":
295 # Replace with an easier-to-handle equivalent condition
296 caret, close = ("^(?:", ")") if kind is str else (b"^(?:", b")")
297 pattern = re.compile(caret + pattern.pattern + close, flags=pattern.flags)
298 condition = pattern.search
300 if condition.__name__ in ("search", "findall", "fullmatch"):
301 s = regex_strategy(
302 pattern,
303 fullmatch=condition.__name__ == "fullmatch",
304 alphabet=self.element_strategy if kind is str else None,
305 )
306 if self.min_size > 0:
307 s = s.filter(partial(min_len, self.min_size))
308 if self.max_size < 1e999:
309 s = s.filter(partial(max_len, self.max_size))
310 return s
311 elif condition.__name__ in ("finditer", "scanner"):
312 # PyPy implements `finditer` as an alias to their `scanner` method
313 warnings.warn(
314 f"You applied {pretty(condition)} as a filter, but this allows "
315 f"any string at all! Did you mean .findall ?",
316 HypothesisWarning,
317 stacklevel=3,
318 )
319 return self
320 elif condition.__name__ == "split":
321 warnings.warn(
322 f"You applied {pretty(condition)} as a filter, but this allows "
323 f"any nonempty string! Did you mean .search ?",
324 HypothesisWarning,
325 stacklevel=3,
326 )
327 return self.filter(bool)
329 # We use ListStrategy filter logic for the conditions that *only* imply
330 # the string is nonempty. Here, we increment the min_size but still apply
331 # the filter for conditions that imply nonempty *and specific contents*.
332 if condition in self._nonempty_and_content_filters and self.max_size >= 1:
333 self = copy.copy(self)
334 self.min_size = max(1, self.min_size)
335 return ListStrategy.filter(self, condition)
337 return None
340# Excerpted from https://www.unicode.org/Public/15.0.0/ucd/PropList.txt
341# Python updates it's Unicode version between minor releases, but fortunately
342# these properties do not change between the Unicode versions in question.
343_PROPLIST = """
344# ================================================
3461885..1886 ; Other_ID_Start # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA
3472118 ; Other_ID_Start # Sm SCRIPT CAPITAL P
348212E ; Other_ID_Start # So ESTIMATED SYMBOL
349309B..309C ; Other_ID_Start # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
351# Total code points: 6
353# ================================================
35500B7 ; Other_ID_Continue # Po MIDDLE DOT
3560387 ; Other_ID_Continue # Po GREEK ANO TELEIA
3571369..1371 ; Other_ID_Continue # No [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE
35819DA ; Other_ID_Continue # No NEW TAI LUE THAM DIGIT ONE
360# Total code points: 12
361"""
364@lru_cache
365def _identifier_characters() -> tuple[IntervalSet, IntervalSet]:
366 """See https://docs.python.org/3/reference/lexical_analysis.html#identifiers"""
367 # Start by computing the set of special characters
368 chars = {"Other_ID_Start": "", "Other_ID_Continue": ""}
369 for line in _PROPLIST.splitlines():
370 if m := re.match(r"([0-9A-F.]+) +; (\w+) # ", line):
371 codes, prop = m.groups()
372 span = range(int(codes[:4], base=16), int(codes[-4:], base=16) + 1)
373 chars[prop] += "".join(chr(x) for x in span)
375 # Then get the basic set by Unicode category and known extras
376 id_start = charmap.query(
377 categories=("Lu", "Ll", "Lt", "Lm", "Lo", "Nl"),
378 include_characters="_" + chars["Other_ID_Start"],
379 )
380 id_start -= IntervalSet.from_string(
381 # Magic value: the characters which NFKC-normalize to be invalid identifiers.
382 # Conveniently they're all in `id_start`, so we only need to do this once.
383 "\u037a\u0e33\u0eb3\u2e2f\u309b\u309c\ufc5e\ufc5f\ufc60\ufc61\ufc62\ufc63"
384 "\ufdfa\ufdfb\ufe70\ufe72\ufe74\ufe76\ufe78\ufe7a\ufe7c\ufe7e\uff9e\uff9f"
385 )
386 id_continue = id_start | charmap.query(
387 categories=("Mn", "Mc", "Nd", "Pc"),
388 include_characters=chars["Other_ID_Continue"],
389 )
390 return id_start, id_continue
393class BytesStrategy(SearchStrategy):
394 def __init__(self, min_size: int, max_size: int | None):
395 super().__init__()
396 self.min_size = min_size
397 self.max_size = (
398 max_size if max_size is not None else COLLECTION_DEFAULT_MAX_SIZE
399 )
401 def do_draw(self, data: ConjectureData) -> bytes:
402 return data.draw_bytes(self.min_size, self.max_size)
404 _nonempty_filters = (
405 *ListStrategy._nonempty_filters,
406 bytes,
407 *(getattr(bytes, n) for n in _nonempty_names),
408 )
409 _nonempty_and_content_filters = (
410 *(getattr(bytes, n) for n in _nonempty_and_content_names),
411 )
413 def filter(self, condition):
414 if (new := _string_filter_rewrite(self, bytes, condition)) is not None:
415 return new
416 return ListStrategy.filter(self, condition)