Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of Hypothesis, which may be found at
2# https://github.com/HypothesisWorks/hypothesis/
3#
4# Copyright the Hypothesis Authors.
5# Individual contributors are listed in AUTHORS.rst and the git log.
6#
7# This Source Code Form is subject to the terms of the Mozilla Public License,
8# v. 2.0. If a copy of the MPL was not distributed with this file, You can
9# obtain one at https://mozilla.org/MPL/2.0/.
11import codecs
12import gzip
13import json
14import os
15import sys
16import tempfile
17import unicodedata
18from collections.abc import Collection, Iterable
19from functools import cache
20from pathlib import Path
21from typing import Literal, TypeAlias
23from hypothesis.configuration import storage_directory
24from hypothesis.control import _current_build_context
25from hypothesis.errors import InvalidArgument
26from hypothesis.internal.intervalsets import IntervalSet, IntervalsT
28# See https://en.wikipedia.org/wiki/Unicode_character_property#General_Category
29CategoryName: TypeAlias = Literal[
30 "L", # Letter
31 "Lu", # Letter, uppercase
32 "Ll", # Letter, lowercase
33 "Lt", # Letter, titlecase
34 "Lm", # Letter, modifier
35 "Lo", # Letter, other
36 "M", # Mark
37 "Mn", # Mark, nonspacing
38 "Mc", # Mark, spacing combining
39 "Me", # Mark, enclosing
40 "N", # Number
41 "Nd", # Number, decimal digit
42 "Nl", # Number, letter
43 "No", # Number, other
44 "P", # Punctuation
45 "Pc", # Punctuation, connector
46 "Pd", # Punctuation, dash
47 "Ps", # Punctuation, open
48 "Pe", # Punctuation, close
49 "Pi", # Punctuation, initial quote
50 "Pf", # Punctuation, final quote
51 "Po", # Punctuation, other
52 "S", # Symbol
53 "Sm", # Symbol, math
54 "Sc", # Symbol, currency
55 "Sk", # Symbol, modifier
56 "So", # Symbol, other
57 "Z", # Separator
58 "Zs", # Separator, space
59 "Zl", # Separator, line
60 "Zp", # Separator, paragraph
61 "C", # Other
62 "Cc", # Other, control
63 "Cf", # Other, format
64 "Cs", # Other, surrogate
65 "Co", # Other, private use
66 "Cn", # Other, not assigned
67]
68Categories: TypeAlias = Iterable[CategoryName]
69CategoriesTuple: TypeAlias = tuple[CategoryName, ...]
72def charmap_file(fname: str = "charmap") -> Path:
73 return storage_directory(
74 "unicode_data", unicodedata.unidata_version, f"{fname}.json.gz"
75 ).path
78_charmap: dict[CategoryName, IntervalsT] | None = None
81def charmap() -> dict[CategoryName, IntervalsT]:
82 """Return a dict that maps a Unicode category, to a tuple of 2-tuples
83 covering the codepoint intervals for characters in that category.
85 >>> charmap()['Co']
86 ((57344, 63743), (983040, 1048573), (1048576, 1114109))
87 """
88 global _charmap
89 # Best-effort caching in the face of missing files and/or unwritable
90 # filesystems is fairly simple: check if loaded, else try loading,
91 # else calculate and try writing the cache.
92 if _charmap is None:
93 f = charmap_file()
94 try:
95 with gzip.GzipFile(f, "rb") as d:
96 tmp_charmap = dict(json.load(d))
98 except Exception:
99 # This loop is reduced to using only local variables for performance;
100 # indexing and updating containers is a ~3x slowdown. This doesn't fix
101 # https://github.com/HypothesisWorks/hypothesis/issues/2108 but it helps.
102 category = unicodedata.category # Local variable -> ~20% speedup!
103 tmp_charmap = {}
104 last_cat = category(chr(0))
105 last_start = 0
106 for i in range(1, sys.maxunicode + 1):
107 cat = category(chr(i))
108 if cat != last_cat:
109 tmp_charmap.setdefault(last_cat, []).append((last_start, i - 1))
110 last_cat, last_start = cat, i
111 tmp_charmap.setdefault(last_cat, []).append((last_start, sys.maxunicode))
113 try:
114 # Write the Unicode table atomically
115 storage_dir = storage_directory("tmp")
116 storage_dir.create_if_missing()
117 fd, tmpfile = tempfile.mkstemp(dir=storage_dir.path)
118 os.close(fd)
119 # Explicitly set the mtime to get reproducible output
120 with gzip.GzipFile(tmpfile, "wb", mtime=1) as fp:
121 result = json.dumps(sorted(tmp_charmap.items()))
122 fp.write(result.encode())
124 os.renames(tmpfile, f)
125 except Exception:
126 pass
128 # convert between lists and tuples
129 _charmap = {
130 k: tuple(tuple(pair) for pair in pairs) for k, pairs in tmp_charmap.items()
131 }
132 # each value is a tuple of 2-tuples (that is, tuples of length 2)
133 # and both elements of that tuple are integers.
134 for vs in _charmap.values():
135 ints = list(sum(vs, ()))
136 assert all(isinstance(x, int) for x in ints)
137 assert ints == sorted(ints)
138 assert all(len(tup) == 2 for tup in vs)
140 assert _charmap is not None
141 return _charmap
144@cache
145def intervals_from_codec(
146 codec_name: str,
147) -> tuple[IntervalSet, IntervalSet]: # pragma: no cover
148 """Return IntervalSets of characters which can be encoded with this codec,
149 and the subset which encode successfully but do not decode back to the
150 same character."""
151 assert codec_name == codecs.lookup(codec_name).name
152 fname = charmap_file(f"codec-v2-{codec_name}")
153 try:
154 with gzip.GzipFile(fname) as gzf:
155 encodable_intervals, non_roundtrip_intervals = json.load(gzf)
157 except Exception:
158 # This loop is kinda slow, but hopefully we don't need to do it very often!
159 encodable_intervals = []
160 non_roundtrip_intervals = []
161 for i in range(sys.maxunicode + 1):
162 char = chr(i)
163 try:
164 encoded = char.encode(codec_name)
165 except Exception: # usually _but not always_ UnicodeEncodeError
166 continue
167 encodable_intervals.append((i, i))
168 try:
169 # A few legacy codecs have lossy fallback mappings - e.g. under
170 # shift_jis the yen sign encodes to the byte which decodes as a
171 # backslash - so we track the non-round-tripping subset too.
172 roundtrips = encoded.decode(codec_name) == char
173 except Exception:
174 roundtrips = False
175 if not roundtrips:
176 non_roundtrip_intervals.append((i, i))
178 res = IntervalSet(encodable_intervals)
179 res = res.union(res)
180 non_roundtrip = IntervalSet(non_roundtrip_intervals)
181 non_roundtrip = non_roundtrip.union(non_roundtrip)
182 try:
183 # Write the Unicode table atomically
184 storage_dir = storage_directory("tmp")
185 storage_dir.create_if_missing()
186 fd, tmpfile = tempfile.mkstemp(dir=storage_dir.path)
187 os.close(fd)
188 # Explicitly set the mtime to get reproducible output
189 with gzip.GzipFile(tmpfile, "wb", mtime=1) as f:
190 f.write(json.dumps([res.intervals, non_roundtrip.intervals]).encode())
191 os.renames(tmpfile, fname)
192 except Exception:
193 pass
194 return res, non_roundtrip
197_categories: Categories | None = None
200def categories() -> Categories:
201 """Return a tuple of Unicode categories in a normalised order.
203 >>> categories() # doctest: +ELLIPSIS
204 ('Zl', 'Zp', 'Co', 'Me', 'Pc', ..., 'Cc', 'Cs')
205 """
206 global _categories
207 if _categories is None:
208 cm = charmap()
209 categories = sorted(cm.keys(), key=lambda c: len(cm[c]))
210 categories.remove("Cc") # Other, Control
211 categories.remove("Cs") # Other, Surrogate
212 categories.append("Cc")
213 categories.append("Cs")
214 _categories = tuple(categories)
215 return _categories
218def as_general_categories(cats: Categories, name: str = "cats") -> CategoriesTuple:
219 """Return a tuple of Unicode categories in a normalised order.
221 This function expands one-letter designations of a major class to include
222 all subclasses:
224 >>> as_general_categories(['N'])
225 ('Nd', 'Nl', 'No')
227 See section 4.5 of the Unicode standard for more on classes:
228 https://www.unicode.org/versions/Unicode10.0.0/ch04.pdf
230 If the collection ``cats`` includes any elements that do not represent a
231 major class or a class with subclass, a deprecation warning is raised.
232 """
233 major_classes = ("L", "M", "N", "P", "S", "Z", "C")
234 cs = categories()
235 out = set(cats)
236 for c in cats:
237 if c in major_classes:
238 out.discard(c)
239 out.update(x for x in cs if x.startswith(c))
240 elif c not in cs:
241 raise InvalidArgument(
242 f"In {name}={cats!r}, {c!r} is not a valid Unicode category."
243 )
244 return tuple(c for c in cs if c in out)
247category_index_cache: dict[frozenset[CategoryName], IntervalsT] = {frozenset(): ()}
250def _category_key(cats: Iterable[str] | None) -> CategoriesTuple:
251 """Return a normalised tuple of all Unicode categories that are in
252 `include`, but not in `exclude`.
254 If include is None then default to including all categories.
255 Any item in include that is not a unicode character will be excluded.
257 >>> _category_key(exclude=['So'], include=['Lu', 'Me', 'Cs', 'So'])
258 ('Me', 'Lu', 'Cs')
259 """
260 cs = categories()
261 if cats is None:
262 cats = set(cs)
263 return tuple(c for c in cs if c in cats)
266def _query_for_key(key: Categories) -> IntervalsT:
267 """Return a tuple of codepoint intervals covering characters that match one
268 or more categories in the tuple of categories `key`.
270 >>> _query_for_key(categories())
271 ((0, 1114111),)
272 >>> _query_for_key(('Zl', 'Zp', 'Co'))
273 ((8232, 8233), (57344, 63743), (983040, 1048573), (1048576, 1114109))
274 """
275 key = tuple(key)
276 # ignore ordering on the cache key to increase potential cache hits.
277 cache_key = frozenset(key)
278 context = _current_build_context.value
279 if context is None or not context.data.provider.avoid_realization:
280 try:
281 return category_index_cache[cache_key]
282 except KeyError:
283 pass
284 elif not key: # pragma: no cover # only on alternative backends
285 return ()
286 assert key
287 if set(key) == set(categories()):
288 result = IntervalSet([(0, sys.maxunicode)])
289 else:
290 result = IntervalSet(_query_for_key(key[:-1])).union(
291 IntervalSet(charmap()[key[-1]])
292 )
293 assert isinstance(result, IntervalSet)
294 if context is None or not context.data.provider.avoid_realization:
295 category_index_cache[cache_key] = result.intervals
296 return result.intervals
299limited_category_index_cache: dict[
300 tuple[CategoriesTuple, int, int, IntervalsT, IntervalsT], IntervalSet
301] = {}
304def query(
305 *,
306 categories: Categories | None = None,
307 min_codepoint: int | None = None,
308 max_codepoint: int | None = None,
309 include_characters: Collection[str] = "",
310 exclude_characters: Collection[str] = "",
311) -> IntervalSet:
312 """Return a tuple of intervals covering the codepoints for all characters
313 that meet the criteria.
315 >>> query()
316 ((0, 1114111),)
317 >>> query(min_codepoint=0, max_codepoint=128)
318 ((0, 128),)
319 >>> query(min_codepoint=0, max_codepoint=128, categories=['Lu'])
320 ((65, 90),)
321 >>> query(min_codepoint=0, max_codepoint=128, categories=['Lu'],
322 ... include_characters='☃')
323 ((65, 90), (9731, 9731))
324 """
325 if min_codepoint is None:
326 min_codepoint = 0
327 if max_codepoint is None:
328 max_codepoint = sys.maxunicode
330 if min_codepoint > max_codepoint:
331 raise InvalidArgument(
332 f"min_codepoint={min_codepoint} is greater than max_codepoint={max_codepoint}"
333 )
335 catkey = _category_key(categories)
336 character_intervals = IntervalSet.from_string("".join(include_characters))
337 exclude_intervals = IntervalSet.from_string("".join(exclude_characters))
338 qkey = (
339 catkey,
340 min_codepoint,
341 max_codepoint,
342 character_intervals.intervals,
343 exclude_intervals.intervals,
344 )
345 context = _current_build_context.value
346 if context is None or not context.data.provider.avoid_realization:
347 try:
348 return limited_category_index_cache[qkey]
349 except KeyError:
350 pass
352 result = []
353 for u, v in _query_for_key(catkey):
354 if v >= min_codepoint and u <= max_codepoint:
355 result.append((max(u, min_codepoint), min(v, max_codepoint)))
357 result = (IntervalSet(result) | character_intervals) - exclude_intervals
358 if context is None or not context.data.provider.avoid_realization:
359 limited_category_index_cache[qkey] = result
361 return result