Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/slugify/slugify.py: 22%

1from __future__ import annotations 

2 

3import re 

4import unicodedata 

5from collections.abc import Iterable 

6from html.entities import name2codepoint 

7from importlib import import_module 

8from typing import Literal 

9 

10from ._legacy import slugify as _legacy_slugify, smart_truncate 

11 

12__all__ = ['slugify', 'smart_truncate', 'Backend', 'ReplacementStage', 'Algorithm'] 

13 

14 

15CHAR_ENTITY_PATTERN = re.compile(r'&(%s);' % '|'.join(name2codepoint)) 

16DECIMAL_PATTERN = re.compile(r'&#(\d+);') 

17MODERN_HEX_PATTERN = re.compile(r'&#[xX]([\da-fA-F]+);') 

18QUOTE_PATTERN = re.compile(r"[']+") 

19DISALLOWED_CHARS_PATTERN = re.compile(r'[^-a-zA-Z0-9]+') 

20DISALLOWED_UNICODE_CHARS_PATTERN = re.compile(r'[\W_]+') 

21DUPLICATE_DASH_PATTERN = re.compile(r'-{2,}') 

22NUMBERS_PATTERN = re.compile(r'(?<=\d),(?=\d)') 

23DEFAULT_SEPARATOR = '-' 

24 

25Backend = Literal['auto', 'text-unidecode', 'unidecode', 'anyascii'] 

26ReplacementStage = Literal['both', 'pre', 'post'] 

27Algorithm = Literal['legacy', 'modern'] 

28 

29 

30def _numeric_reference(match: re.Match[str], base: int) -> str: 

31 """Leave invalid references for ordinary filtering, independently of neighbors.""" 

32 try: 

33 value = int(match.group(1), base) 

34 if 0xD800 <= value <= 0xDFFF: 

35 return match.group(0) 

36 return chr(value) 

37 except (ValueError, OverflowError): 

38 return match.group(0) 

39 

40 

41def _decode_entities(text: str, entities: bool, decimal: bool, hexadecimal: bool) -> str: 

42 if entities: 

43 text = CHAR_ENTITY_PATTERN.sub(lambda m: chr(name2codepoint[m.group(1)]), text) 

44 if decimal: 

45 text = DECIMAL_PATTERN.sub(lambda m: _numeric_reference(m, 10), text) 

46 if hexadecimal: 

47 text = MODERN_HEX_PATTERN.sub(lambda m: _numeric_reference(m, 16), text) 

48 return text 

49 

50 

51def _transliterate(text: str, backend: Backend) -> str: 

52 # Every backend maps 7-bit ASCII to itself; skip importing one (text-unidecode's table is ~3 MB). 

53 if text.isascii(): 

54 return text 

55 if backend == 'auto': 

56 try: 

57 module = import_module('unidecode') 

58 except ModuleNotFoundError as error: 

59 if error.name != 'unidecode': 

60 raise 

61 module = import_module('text_unidecode') 

62 else: 

63 module = import_module(backend.replace('-', '_')) 

64 # These third-party modules share a string-to-string API; some are untyped. 

65 result: str = getattr(module, 'anyascii' if backend == 'anyascii' else 'unidecode')(text) 

66 return result 

67 

68 

69def _modern_truncate(text: str, max_length: int, word_boundary: bool, separator: str, save_order: bool) -> str: 

70 """Budget internal dash-separated tokens before mapping output delimiters. 

71 

72 Output delimiters may also occur in words; never strip or split emitted text. 

73 Post replacements retain the historical global dash-to-separator mapping. 

74 """ 

75 if max_length <= 0: 

76 return text.replace(DEFAULT_SEPARATOR, separator) 

77 output_length = len(text) + text.count(DEFAULT_SEPARATOR) * (len(separator) - 1) 

78 if output_length <= max_length: 

79 return text.replace(DEFAULT_SEPARATOR, separator) 

80 tokens = text.split(DEFAULT_SEPARATOR) 

81 if word_boundary: 

82 words: list[str] = [] 

83 length = 0 

84 previous_index = 0 

85 first_word = True 

86 for index, word in enumerate(tokens): 

87 if not word: 

88 continue 

89 # Keep the full delimiter run attached to each retained word. 

90 # Skipping an oversized first word must not add a leading delimiter. 

91 delimiter = separator * (index - previous_index) if words or first_word else '' 

92 previous_index = index 

93 first_word = False 

94 next_length = length + len(delimiter) + len(word) 

95 if next_length <= max_length: 

96 words.append(delimiter + word) 

97 length = next_length 

98 elif save_order: 

99 break 

100 if words: 

101 return ''.join(words) 

102 # A hard cut may shorten a word but must not emit a partial/trailing delimiter. 

103 parts: list[str] = [] 

104 length = 0 

105 previous_index = 0 

106 for index, word in enumerate(tokens): 

107 if not word: 

108 continue 

109 # Keep repeated delimiters with the next nonempty word until it fits. 

110 delimiter = separator * (index - previous_index) 

111 remaining = max_length - length - len(delimiter) 

112 if remaining <= 0: 

113 break 

114 parts.append(delimiter + word[:remaining]) 

115 length += len(delimiter) + min(len(word), remaining) 

116 previous_index = index 

117 if len(word) > remaining: 

118 break 

119 return ''.join(parts) 

120 

121 

122def _modern_slugify( 

123 text: str | bytes | bytearray, 

124 entities: bool, 

125 decimal: bool, 

126 hexadecimal: bool, 

127 max_length: int, 

128 word_boundary: bool, 

129 separator: str, 

130 save_order: bool, 

131 stopwords: Iterable[str], 

132 regex_pattern: re.Pattern[str] | str | None, 

133 lowercase: bool, 

134 replacements: Iterable[Iterable[str]], 

135 allow_unicode: bool, 

136 replacement_stage: ReplacementStage, 

137 backend: Backend, 

138) -> str: 

139 # bool is an int subclass: legacy silently treats max_length=True as 1. 

140 # Modern rejects it, and a non-str separator, up front. 

141 if isinstance(max_length, bool) or not isinstance(max_length, int): 

142 raise TypeError(f"max_length must be an int, not {type(max_length).__name__}") 

143 if not isinstance(separator, str): 

144 raise TypeError(f"separator must be str, not {type(separator).__name__}") 

145 if not isinstance(text, str): 

146 if not isinstance(text, (bytes, bytearray)): 

147 raise TypeError(f'text must be str, bytes or bytearray, not {type(text).__name__}') 

148 text = text.decode('utf-8', 'ignore') 

149 if replacement_stage not in ('both', 'pre', 'post'): 

150 raise ValueError("replacement_stage must be 'both', 'pre' or 'post'") 

151 if backend not in ('auto', 'text-unidecode', 'unidecode', 'anyascii'): 

152 raise ValueError("backend must be 'auto', 'text-unidecode', 'unidecode' or 'anyascii'") 

153 

154 # Materialize rules once: consumption of an iterator would affect output. 

155 rules = tuple((old, new) for old, new in replacements) if replacements else () 

156 if rules and replacement_stage in ('both', 'pre'): 

157 for old, new in rules: 

158 text = text.replace(old, new) 

159 

160 text = _decode_entities(text, entities, decimal, hexadecimal) 

161 text = QUOTE_PATTERN.sub(DEFAULT_SEPARATOR, text) 

162 if allow_unicode: 

163 text = unicodedata.normalize('NFKC', text) 

164 else: 

165 text = _transliterate(unicodedata.normalize('NFKD', text), backend) 

166 text = unicodedata.normalize('NFKC' if allow_unicode else 'NFKD', text) 

167 if lowercase: 

168 text = text.lower() 

169 text = QUOTE_PATTERN.sub('', text) 

170 text = NUMBERS_PATTERN.sub('', text) 

171 pattern = regex_pattern or (DISALLOWED_UNICODE_CHARS_PATTERN if allow_unicode else DISALLOWED_CHARS_PATTERN) 

172 text = re.sub(pattern, DEFAULT_SEPARATOR, text) 

173 text = DUPLICATE_DASH_PATTERN.sub(DEFAULT_SEPARATOR, text).strip(DEFAULT_SEPARATOR) 

174 

175 if stopwords: 

176 excluded = {word.lower() if lowercase else word for word in stopwords} 

177 text = DEFAULT_SEPARATOR.join(word for word in text.split(DEFAULT_SEPARATOR) if word not in excluded) 

178 if rules and replacement_stage in ('both', 'post'): 

179 for old, new in rules: 

180 text = text.replace(old, new) 

181 

182 return _modern_truncate(text, max_length, word_boundary, separator, save_order) 

183 

184 

185def slugify( 

186 text: str | bytes | bytearray, 

187 entities: bool = True, 

188 decimal: bool = True, 

189 hexadecimal: bool = True, 

190 max_length: int = 0, 

191 word_boundary: bool = False, 

192 separator: str = DEFAULT_SEPARATOR, 

193 save_order: bool = False, 

194 stopwords: Iterable[str] = (), 

195 regex_pattern: re.Pattern[str] | str | None = None, 

196 lowercase: bool = True, 

197 replacements: Iterable[Iterable[str]] = (), 

198 allow_unicode: bool = False, 

199 *, 

200 replacement_stage: ReplacementStage = 'both', 

201 backend: Backend = 'auto', 

202 algorithm: Algorithm = 'legacy', 

203) -> str: 

204 """Make a slug with the legacy output pipeline by default, permanently. 

205 

206 algorithm='legacy' (the default) dispatches to the frozen legacy pipeline in 

207 slugify._legacy, which must never change. algorithm='modern' opts into early 

208 entity decoding, reusable iterator rules, stable stopword membership, a final 

209 emitted-character length budget, and up-front argument type validation. 

210 Bytes and bytearray are decoded as UTF-8, ignoring invalid bytes. 

211 replacements are ordered literal rules, before and after cleanup by default; 

212 replacement_stage selects 'pre', 'post' or 'both'. Post rules are unfiltered. 

213 regex_pattern matches disallowed characters; stopwords match internal words. 

214 allow_unicode uses NFKC without transliteration. backend='auto' prefers 

215 installed Unidecode, falling back to text-unidecode; explicit choices do not 

216 fall back. All new controls are keyword-only. 

217 """ 

218 if algorithm not in ('legacy', 'modern'): 

219 raise ValueError("algorithm must be 'legacy' or 'modern'") 

220 if algorithm == 'legacy': 

221 return _legacy_slugify( 

222 text, entities, decimal, hexadecimal, max_length, word_boundary, 

223 separator, save_order, stopwords, regex_pattern, lowercase, 

224 replacements, allow_unicode, 

225 replacement_stage=replacement_stage, backend=backend) 

226 return _modern_slugify( 

227 text, entities, decimal, hexadecimal, max_length, word_boundary, 

228 separator, save_order, stopwords, regex_pattern, lowercase, 

229 replacements, allow_unicode, replacement_stage, backend)