Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/sacremoses/tokenize.py: 64%

1#!/usr/bin/env python3 

2# -*- coding: utf-8 -*- 

3 

4import re 

5 

6from sacremoses.corpus import Perluniprops 

7from sacremoses.corpus import NonbreakingPrefixes 

8from sacremoses.util import is_cjk 

9from sacremoses.indic import VIRAMAS, NUKTAS 

10 

11perluniprops = Perluniprops() 

12nonbreaking_prefixes = NonbreakingPrefixes() 

13 

14 

15class MosesTokenizer(object): 

16 """ 

17 This is a Python port of the Moses Tokenizer from 

18 https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl 

19 """ 

20 

21 # Perl Unicode Properties character sets. 

22 IsN = str("".join(perluniprops.chars("IsN"))) 

23 IsAlnum = str( 

24 "".join(perluniprops.chars("IsAlnum")) + "".join(VIRAMAS) + "".join(NUKTAS) 

25 ) 

26 IsSc = str("".join(perluniprops.chars("IsSc"))) 

27 IsSo = str("".join(perluniprops.chars("IsSo"))) 

28 IsAlpha = str( 

29 "".join(perluniprops.chars("IsAlpha")) + "".join(VIRAMAS) + "".join(NUKTAS) 

30 ) 

31 IsLower = str("".join(perluniprops.chars("IsLower"))) 

32 

33 # Remove ASCII junk. 

34 DEDUPLICATE_SPACE = re.compile(r"\s+"), r" " 

35 ASCII_JUNK = re.compile(r"[\000-\037]"), r"" 

36 

37 # Neurotic Perl heading space, multi-space and trailing space chomp. 

38 # These regexes are kept for reference purposes and shouldn't be used!! 

39 MID_STRIP = r" +", r" " # Use DEDUPLICATE_SPACE instead. 

40 LEFT_STRIP = r"^ ", r"" # Uses text.lstrip() instead. 

41 RIGHT_STRIP = r" $", r"" # Uses text.rstrip() instead. 

42 

43 # Pad all "other" special characters not in IsAlnum. 

44 PAD_NOT_ISALNUM = re.compile(r"([^{}\s\.'\`\,\-])".format(IsAlnum)), r" \1 " 

45 

46 # Splits all hyphens (regardless of circumstances), e.g. 

47 # 'foo-bar' -> 'foo @-@ bar' 

48 AGGRESSIVE_HYPHEN_SPLIT = ( 

49 re.compile(r"([{alphanum}])\-(?=[{alphanum}])".format(alphanum=IsAlnum)), 

50 r"\1 @-@ ", 

51 ) 

52 

53 # Make multi-dots stay together. 

54 REPLACE_DOT_WITH_LITERALSTRING_1 = re.compile(r"\.([\.]+)"), " DOTMULTI\1" 

55 REPLACE_DOT_WITH_LITERALSTRING_2 = re.compile(r"DOTMULTI\.([^\.])"), "DOTDOTMULTI \1" 

56 REPLACE_DOT_WITH_LITERALSTRING_3 = re.compile(r"DOTMULTI\."), "DOTDOTMULTI" 

57 

58 # Separate out "," except if within numbers (5,300) 

59 # e.g. A,B,C,D,E > A , B,C , D,E 

60 # First application uses up B so rule can't see B,C 

61 # two-step version here may create extra spaces but these are removed later 

62 # will also space digit,letter or letter,digit forms (redundant with next section) 

63 COMMA_SEPARATE_1 = re.compile(r"([^{}])[,]".format(IsN)), r"\1 , " 

64 COMMA_SEPARATE_2 = re.compile(r"[,]([^{}])".format(IsN)), r" , \1" 

65 COMMA_SEPARATE_3 = re.compile(r"([{}])[,]$".format(IsN)), r"\1 , " 

66 

67 # Attempt to get correct directional quotes. 

68 DIRECTIONAL_QUOTE_1 = re.compile(r"^``"), r"`` " 

69 DIRECTIONAL_QUOTE_2 = re.compile(r'^"'), r"`` " 

70 DIRECTIONAL_QUOTE_3 = re.compile(r"^`([^`])"), r"` \1" 

71 DIRECTIONAL_QUOTE_4 = re.compile(r"^'"), r"` " 

72 DIRECTIONAL_QUOTE_5 = re.compile(r'([ ([{<])"'), r"\1 `` " 

73 DIRECTIONAL_QUOTE_6 = re.compile(r"([ ([{<])``"), r"\1 `` " 

74 DIRECTIONAL_QUOTE_7 = re.compile(r"([ ([{<])`([^`])"), r"\1 ` \2" 

75 DIRECTIONAL_QUOTE_8 = re.compile(r"([ ([{<])'"), r"\1 ` " 

76 

77 # Replace ... with _ELLIPSIS_ 

78 REPLACE_ELLIPSIS = re.compile(r"\.\.\."), r" _ELLIPSIS_ " 

79 # Restore _ELLIPSIS_ with ... 

80 RESTORE_ELLIPSIS = re.compile(r"_ELLIPSIS_"), r"\.\.\." 

81 

82 # Pad , with tailing space except if within numbers, e.g. 5,300 

83 COMMA_1 = re.compile(r"([^{numbers}])[,]([^{numbers}])".format(numbers=IsN)), r"\1 , \2" 

84 COMMA_2 = re.compile(r"([{numbers}])[,]([^{numbers}])".format(numbers=IsN)), r"\1 , \2" 

85 COMMA_3 = re.compile(r"([^{numbers}])[,]([{numbers}])".format(numbers=IsN)), r"\1 , \2" 

86 

87 # Pad unicode symbols with spaces. 

88 SYMBOLS = re.compile(r"([;:@#\$%&{}{}])".format(IsSc, IsSo)), r" \1 " 

89 

90 # Separate out intra-token slashes. PTB tokenization doesn't do this, so 

91 # the tokens should be merged prior to parsing with a PTB-trained parser. 

92 # e.g. "and/or" -> "and @/@ or" 

93 INTRATOKEN_SLASHES = ( 

94 r"([{alphanum}])\/([{alphanum}])".format(alphanum=IsAlnum), 

95 r"$1 \@\/\@ $2", 

96 ) 

97 

98 # Splits final period at end of string. 

99 FINAL_PERIOD = re.compile(r"""([^.])([.])([\]\)}>"']*) ?$"""), r"\1 \2\3" 

100 # Pad all question marks and exclamation marks with spaces. 

101 PAD_QUESTION_EXCLAMATION_MARK = re.compile(r"([?!])"), r" \1 " 

102 

103 # Handles parentheses, brackets and converts them to PTB symbols. 

104 PAD_PARENTHESIS = re.compile(r"([\]\[\(\){}<>])"), r" \1 " 

105 CONVERT_PARENTHESIS_1 = re.compile(r"\("), "-LRB-" 

106 CONVERT_PARENTHESIS_2 = re.compile(r"\)"), "-RRB-" 

107 CONVERT_PARENTHESIS_3 = re.compile(r"\["), "-LSB-" 

108 CONVERT_PARENTHESIS_4 = re.compile(r"\]"), "-RSB-" 

109 CONVERT_PARENTHESIS_5 = re.compile(r"\{"), "-LCB-" 

110 CONVERT_PARENTHESIS_6 = re.compile(r"\}"), "-RCB-" 

111 

112 # Pads double dashes with spaces. 

113 PAD_DOUBLE_DASHES = re.compile(r"--"), " -- " 

114 

115 # Adds spaces to start and end of string to simplify further regexps. 

116 PAD_START_OF_STR = re.compile(r"^"), " " 

117 PAD_END_OF_STR = re.compile(r"$"), " " 

118 

119 # Converts double quotes to two single quotes and pad with spaces. 

120 CONVERT_DOUBLE_TO_SINGLE_QUOTES = re.compile(r'"'), " '' " 

121 # Handles single quote in possessives or close-single-quote. 

122 HANDLES_SINGLE_QUOTES = re.compile(r"([^'])' "), r"\1 ' " 

123 

124 # Pad apostrophe in possessive or close-single-quote. 

125 APOSTROPHE = re.compile(r"([^'])'"), r"\1 ' " 

126 

127 # Prepend space on contraction apostrophe. 

128 CONTRACTION_1 = re.compile(r"'([sSmMdD]) "), r" '\1 " 

129 CONTRACTION_2 = re.compile(r"'ll "), r" 'll " 

130 CONTRACTION_3 = re.compile(r"'re "), r" 're " 

131 CONTRACTION_4 = re.compile(r"'ve "), r" 've " 

132 CONTRACTION_5 = re.compile(r"n't "), r" n't " 

133 CONTRACTION_6 = re.compile(r"'LL "), r" 'LL " 

134 CONTRACTION_7 = re.compile(r"'RE "), r" 'RE " 

135 CONTRACTION_8 = re.compile(r"'VE "), r" 'VE " 

136 CONTRACTION_9 = re.compile(r"N'T "), r" N'T " 

137 

138 # Informal Contractions. 

139 CONTRACTION_10 = re.compile(r" ([Cc])annot "), r" \1an not " 

140 CONTRACTION_11 = re.compile(r" ([Dd])'ye "), r" \1' ye " 

141 CONTRACTION_12 = re.compile(r" ([Gg])imme "), r" \1im me " 

142 CONTRACTION_13 = re.compile(r" ([Gg])onna "), r" \1on na " 

143 CONTRACTION_14 = re.compile(r" ([Gg])otta "), r" \1ot ta " 

144 CONTRACTION_15 = re.compile(r" ([Ll])emme "), r" \1em me " 

145 CONTRACTION_16 = re.compile(r" ([Mm])ore'n "), r" \1ore 'n " 

146 CONTRACTION_17 = re.compile(r" '([Tt])is "), r" '\1 is " 

147 CONTRACTION_18 = re.compile(r" '([Tt])was "), r" '\1 was " 

148 CONTRACTION_19 = re.compile(r" ([Ww])anna "), r" \1an na " 

149 

150 # Clean out extra spaces 

151 CLEAN_EXTRA_SPACE_1 = re.compile(r" *"), r" " 

152 CLEAN_EXTRA_SPACE_2 = re.compile(r"^ *"), r"" 

153 CLEAN_EXTRA_SPACE_3 = re.compile(r" *$"), r"" 

154 

155 # Neurotic Perl regexes to escape special characters. 

156 ESCAPE_AMPERSAND = re.compile(r"&"), r"&amp;" 

157 ESCAPE_PIPE = re.compile(r"\|"), r"&#124;" 

158 ESCAPE_LEFT_ANGLE_BRACKET = re.compile(r"<"), r"&lt;" 

159 ESCAPE_RIGHT_ANGLE_BRACKET = re.compile(r">"), r"&gt;" 

160 ESCAPE_SINGLE_QUOTE = re.compile(r"\'"), r"&apos;" 

161 ESCAPE_DOUBLE_QUOTE = re.compile(r"\""), r"&quot;" 

162 ESCAPE_LEFT_SQUARE_BRACKET = re.compile(r"\["), r"&#91;" 

163 ESCAPE_RIGHT_SQUARE_BRACKET = re.compile(r"]"), r"&#93;" 

164 

165 EN_SPECIFIC_1 = re.compile(r"([^{alpha}])[']([^{alpha}])".format(alpha=IsAlpha)), r"\1 ' \2" 

166 EN_SPECIFIC_2 = ( 

167 re.compile(r"([^{alpha}{isn}])[']([{alpha}])".format(alpha=IsAlpha, isn=IsN)), 

168 r"\1 ' \2", 

169 ) 

170 EN_SPECIFIC_3 = re.compile(r"([{alpha}])[']([^{alpha}])".format(alpha=IsAlpha)), r"\1 ' \2" 

171 EN_SPECIFIC_4 = re.compile(r"([{alpha}])[']([{alpha}])".format(alpha=IsAlpha)), r"\1 '\2" 

172 EN_SPECIFIC_5 = re.compile(r"([{isn}])[']([s])".format(isn=IsN)), r"\1 '\2" 

173 

174 ENGLISH_SPECIFIC_APOSTROPHE = [ 

175 EN_SPECIFIC_1, 

176 EN_SPECIFIC_2, 

177 EN_SPECIFIC_3, 

178 EN_SPECIFIC_4, 

179 EN_SPECIFIC_5, 

180 ] 

181 

182 FR_IT_SPECIFIC_1 = re.compile(r"([^{alpha}])[']([^{alpha}])".format(alpha=IsAlpha)), r"\1 ' \2" 

183 FR_IT_SPECIFIC_2 = re.compile(r"([^{alpha}])[']([{alpha}])".format(alpha=IsAlpha)), r"\1 ' \2" 

184 FR_IT_SPECIFIC_3 = re.compile(r"([{alpha}])[']([^{alpha}])".format(alpha=IsAlpha)), r"\1 ' \2" 

185 FR_IT_SPECIFIC_4 = re.compile(r"([{alpha}])[']([{alpha}])".format(alpha=IsAlpha)), r"\1' \2" 

186 

187 FR_IT_SPECIFIC_APOSTROPHE = [ 

188 FR_IT_SPECIFIC_1, 

189 FR_IT_SPECIFIC_2, 

190 FR_IT_SPECIFIC_3, 

191 FR_IT_SPECIFIC_4, 

192 ] 

193 

194 NON_SPECIFIC_APOSTROPHE = re.compile(r"\'"), " ' " 

195 

196 TRAILING_DOT_APOSTROPHE = re.compile(r"\.' ?$"), " . ' " 

197 

198 BASIC_PROTECTED_PATTERN_1 = r"<\/?\S+\/?>" 

199 BASIC_PROTECTED_PATTERN_2 = r'<\S+( [a-zA-Z0-9]+\="?[^"]")+ ?\/?>' 

200 BASIC_PROTECTED_PATTERN_3 = r"<\S+( [a-zA-Z0-9]+\='?[^']')+ ?\/?>" 

201 BASIC_PROTECTED_PATTERN_4 = r"[\w\-\_\.]+\@([\w\-\_]+\.)+[a-zA-Z]{2,}" 

202 BASIC_PROTECTED_PATTERN_5 = r"(http[s]?|ftp):\/\/[^:\/\s]+(\/\w+)*\/[\w\-\.]+" 

203 

204 MOSES_PENN_REGEXES_1 = [ 

205 DEDUPLICATE_SPACE, 

206 ASCII_JUNK, 

207 DIRECTIONAL_QUOTE_1, 

208 DIRECTIONAL_QUOTE_2, 

209 DIRECTIONAL_QUOTE_3, 

210 DIRECTIONAL_QUOTE_4, 

211 DIRECTIONAL_QUOTE_5, 

212 DIRECTIONAL_QUOTE_6, 

213 DIRECTIONAL_QUOTE_7, 

214 DIRECTIONAL_QUOTE_8, 

215 REPLACE_ELLIPSIS, 

216 COMMA_1, 

217 COMMA_2, 

218 COMMA_3, 

219 SYMBOLS, 

220 INTRATOKEN_SLASHES, 

221 FINAL_PERIOD, 

222 PAD_QUESTION_EXCLAMATION_MARK, 

223 PAD_PARENTHESIS, 

224 CONVERT_PARENTHESIS_1, 

225 CONVERT_PARENTHESIS_2, 

226 CONVERT_PARENTHESIS_3, 

227 CONVERT_PARENTHESIS_4, 

228 CONVERT_PARENTHESIS_5, 

229 CONVERT_PARENTHESIS_6, 

230 PAD_DOUBLE_DASHES, 

231 PAD_START_OF_STR, 

232 PAD_END_OF_STR, 

233 CONVERT_DOUBLE_TO_SINGLE_QUOTES, 

234 HANDLES_SINGLE_QUOTES, 

235 APOSTROPHE, 

236 CONTRACTION_1, 

237 CONTRACTION_2, 

238 CONTRACTION_3, 

239 CONTRACTION_4, 

240 CONTRACTION_5, 

241 CONTRACTION_6, 

242 CONTRACTION_7, 

243 CONTRACTION_8, 

244 CONTRACTION_9, 

245 CONTRACTION_10, 

246 CONTRACTION_11, 

247 CONTRACTION_12, 

248 CONTRACTION_13, 

249 CONTRACTION_14, 

250 CONTRACTION_15, 

251 CONTRACTION_16, 

252 CONTRACTION_17, 

253 CONTRACTION_18, 

254 CONTRACTION_19, 

255 ] 

256 

257 MOSES_PENN_REGEXES_2 = [ 

258 RESTORE_ELLIPSIS, 

259 CLEAN_EXTRA_SPACE_1, 

260 CLEAN_EXTRA_SPACE_2, 

261 CLEAN_EXTRA_SPACE_3, 

262 ESCAPE_AMPERSAND, 

263 ESCAPE_PIPE, 

264 ESCAPE_LEFT_ANGLE_BRACKET, 

265 ESCAPE_RIGHT_ANGLE_BRACKET, 

266 ESCAPE_SINGLE_QUOTE, 

267 ESCAPE_DOUBLE_QUOTE, 

268 ] 

269 

270 MOSES_ESCAPE_XML_REGEXES = [ 

271 ESCAPE_AMPERSAND, 

272 ESCAPE_PIPE, 

273 ESCAPE_LEFT_ANGLE_BRACKET, 

274 ESCAPE_RIGHT_ANGLE_BRACKET, 

275 ESCAPE_SINGLE_QUOTE, 

276 ESCAPE_DOUBLE_QUOTE, 

277 ESCAPE_LEFT_SQUARE_BRACKET, 

278 ESCAPE_RIGHT_SQUARE_BRACKET, 

279 ] 

280 

281 BASIC_PROTECTED_PATTERNS = [ 

282 BASIC_PROTECTED_PATTERN_1, 

283 BASIC_PROTECTED_PATTERN_2, 

284 BASIC_PROTECTED_PATTERN_3, 

285 BASIC_PROTECTED_PATTERN_4, 

286 BASIC_PROTECTED_PATTERN_5, 

287 ] 

288 WEB_PROTECTED_PATTERNS = [ 

289 r"((https?|ftp|rsync)://|www\.)[^ ]*", # URLs 

290 r"[\w\-\_\.]+\@([\w\-\_]+\.)+[a-zA-Z]{2,}", # Emails user@host.domain 

291 r"@[a-zA-Z0-9_]+", # @handler such as twitter/github ID 

292 r"#[a-zA-Z0-9_]+", # @hashtag 

293 # TODO: emojis especially the multi codepoints 

294 ] 

295 

296 def __init__(self, lang="en", custom_nonbreaking_prefixes_file=None): 

297 # Initialize the object. 

298 super(MosesTokenizer, self).__init__() 

299 self.lang = lang 

300 

301 # Initialize the language specific nonbreaking prefixes. 

302 self.NONBREAKING_PREFIXES = [ 

303 _nbp.strip() for _nbp in nonbreaking_prefixes.words(lang) 

304 ] 

305 

306 # Load custom nonbreaking prefixes file. 

307 if custom_nonbreaking_prefixes_file: 

308 self.NONBREAKING_PREFIXES = [] 

309 # Explicit encoding: without it Python uses the locale default, 

310 # which is cp1252 on Windows. 33 of the 39 bundled prefix lists are 

311 # non-ASCII; el/cs/as/bn/ga/gu are not cp1252-decodable at all 

312 # (hard UnicodeDecodeError) and ca/de/es/et/fi/fr decode to silent 

313 # mojibake, which is worse. A caller's own prefix file is no 

314 # different, so pin UTF-8 here too. 

315 # `line not in self.NONBREAKING_PREFIXES` scanned a growing list on 

316 # every line, which is O(n**2) in the size of the caller's file: 8k 

317 # prefixes took ~1s and 64k would take a minute (CWE-407). Track 

318 # membership in a set instead. NONBREAKING_PREFIXES stays a list -- 

319 # it is public and its order is preserved -- and duplicates are 

320 # still dropped, keeping the first occurrence as before. 

321 seen = set() 

322 with open(custom_nonbreaking_prefixes_file, "r", encoding="utf-8") as fin: 

323 for line in fin: 

324 line = line.strip() 

325 if line and not line.startswith("#") and line not in seen: 

326 seen.add(line) 

327 self.NONBREAKING_PREFIXES.append(line) 

328 

329 self.NUMERIC_ONLY_PREFIXES = [ 

330 w.rpartition(" ")[0] 

331 for w in self.NONBREAKING_PREFIXES 

332 if self.has_numeric_only(w) 

333 ] 

334 # Add CJK characters to alpha and alnum. 

335 if self.lang in ["zh", "ja", "ko", "cjk"]: 

336 cjk_chars = "" 

337 if self.lang in ["ko", "cjk"]: 

338 cjk_chars += str("".join(perluniprops.chars("Hangul"))) 

339 if self.lang in ["zh", "cjk"]: 

340 cjk_chars += str("".join(perluniprops.chars("Han"))) 

341 if self.lang in ["ja", "cjk"]: 

342 cjk_chars += str("".join(perluniprops.chars("Hiragana"))) 

343 cjk_chars += str("".join(perluniprops.chars("Katakana"))) 

344 cjk_chars += str("".join(perluniprops.chars("Han"))) 

345 self.IsAlpha += cjk_chars 

346 self.IsAlnum += cjk_chars 

347 # Overwrite the alnum regexes. 

348 self.PAD_NOT_ISALNUM = re.compile(r"([^{}\s\.'\`\,\-])".format(self.IsAlnum)), r" \1 " 

349 self.AGGRESSIVE_HYPHEN_SPLIT = ( 

350 re.compile(r"([{alphanum}])\-(?=[{alphanum}])".format(alphanum=self.IsAlnum)), 

351 r"\1 @-@ ", 

352 ) 

353 self.INTRATOKEN_SLASHES = ( 

354 re.compile(r"([{alphanum}])\/([{alphanum}])".format(alphanum=self.IsAlnum)), 

355 r"$1 \@\/\@ $2", 

356 ) 

357 

358 #: A maximal run of two or more dots, and the marker it becomes. 

359 MULTIDOT_RUN = re.compile(r"\.{2,}") 

360 MULTIDOT_MARKER = re.compile(r"(?:DOT)+MULTI") 

361 

362 def replace_multidots(self, text): 

363 # A run of k dots becomes " " + "DOT" * k + "MULTI", plus a trailing 

364 # space when something follows it. This is done in a single pass: the 

365 # original peeled one dot per iteration and re-scanned the whole string 

366 # each time, which is O(n**2) in the length of a dot run and let a 

367 # ".....'-long input burn minutes of CPU (CWE-407). 

368 def to_marker(match): 

369 marker = " " + "DOT" * (match.end() - match.start()) + "MULTI" 

370 return marker + " " if match.end() < len(text) else marker 

371 

372 return self.MULTIDOT_RUN.sub(to_marker, text) 

373 

374 def restore_multidots(self, text): 

375 # Inverse of the above, also in a single pass. 

376 return self.MULTIDOT_MARKER.sub( 

377 lambda match: "." * ((match.end() - match.start() - 5) // 3), text 

378 ) 

379 

380 def islower(self, text): 

381 return not set(text).difference(set(self.IsLower)) 

382 

383 def isanyalpha(self, text): 

384 return any(set(text).intersection(set(self.IsAlpha))) 

385 

386 #: Upper bound on how many spans one ``protected_patterns`` call may 

387 #: protect. Each protected token costs a full ``str.replace`` pass over the 

388 #: text, so an unbounded count is quadratic as well as overflowing the 

389 #: fixed-width placeholder. 

390 MAX_PROTECTED_TOKENS = 1000 

391 

392 #: The placeholder stem, plus any run of the escape character after it. 

393 PROTECT_MARKER = "THISISPROTECTED" 

394 PROTECT_ESCAPE = "X" 

395 PROTECT_MARKER_RUN = re.compile(PROTECT_MARKER + PROTECT_ESCAPE + "*") 

396 

397 def unused_protect_marker(self, text): 

398 """Return a placeholder stem that does not already occur in ``text``. 

399 

400 The placeholder used to be the constant ``THISISPROTECTED000``, and the 

401 input was never checked for it. Because restoring is a blind 

402 ``str.replace``, text containing that literal had it rewritten into a 

403 protected span -- letting whoever supplies the text relocate or 

404 duplicate a URL, e-mail or handle anywhere in the output. Appending one 

405 more escape character than the longest run already present makes the 

406 stem unrepresentable in the input, in a single linear pass. 

407 """ 

408 runs = self.PROTECT_MARKER_RUN.findall(text) 

409 if not runs: 

410 return self.PROTECT_MARKER 

411 longest = max(len(run) for run in runs) - len(self.PROTECT_MARKER) 

412 return self.PROTECT_MARKER + self.PROTECT_ESCAPE * (longest + 1) 

413 

414 #: One ``\s``, not ``[\s]+``. For a boolean test the ``+`` is redundant -- 

415 #: if a *run* of whitespace precedes the marker then its last character 

416 #: does too -- but it made every starting position rescan the whole 

417 #: whitespace run, which is O(n**2) (CWE-407): 50k spaces took 96s. This 

418 #: is the same regex the 2021 ReDoS fix (8ad457e) trimmed a leading 

419 #: ``(.*)`` from; the quadratic tail was left behind. 

420 NUMERIC_ONLY_MARKER = re.compile(r"\s\#NUMERIC_ONLY\#") 

421 

422 def has_numeric_only(self, text): 

423 return bool(self.NUMERIC_ONLY_MARKER.search(text)) 

424 

425 def handles_nonbreaking_prefixes(self, text): 

426 # Splits the text into tokens to check for nonbreaking prefixes. 

427 tokens = text.split() 

428 num_tokens = len(tokens) 

429 for i, token in enumerate(tokens): 

430 # Checks if token ends with a fullstop. 

431 token_ends_with_period = re.search(r"^(\S+)\.$", token) 

432 if token_ends_with_period: 

433 prefix = token_ends_with_period.group(1) 

434 # Checks for 3 conditions if 

435 # i. the prefix contains a fullstop and 

436 # any char in the prefix is within the IsAlpha charset 

437 # ii. the prefix is in the list of nonbreaking prefixes and 

438 # does not contain #NUMERIC_ONLY# 

439 # iii. the token is not the last token and that the 

440 # next token contains all lowercase. 

441 if ( 

442 ("." in prefix and self.isanyalpha(prefix)) 

443 or ( 

444 prefix in self.NONBREAKING_PREFIXES 

445 and prefix not in self.NUMERIC_ONLY_PREFIXES 

446 ) 

447 or ( 

448 i != num_tokens - 1 

449 and tokens[i + 1] 

450 and self.islower(tokens[i + 1][0]) 

451 ) 

452 ): 

453 pass # No change to the token. 

454 # Checks if the prefix is in NUMERIC_ONLY_PREFIXES 

455 # and ensures that the next word is a digit. 

456 elif ( 

457 prefix in self.NUMERIC_ONLY_PREFIXES 

458 and (i + 1) < num_tokens 

459 and re.search(r"^[0-9]+", tokens[i + 1]) 

460 ): 

461 pass # No change to the token. 

462 else: # Otherwise, adds a space after the tokens before a dot. 

463 tokens[i] = prefix + " ." 

464 return " ".join(tokens) # Stitch the tokens back. 

465 

466 def escape_xml(self, text): 

467 for regexp, substitution in self.MOSES_ESCAPE_XML_REGEXES: 

468 text = regexp.sub(substitution, text) 

469 return text 

470 

471 def penn_tokenize(self, text, return_str=False): 

472 """ 

473 This is a Python port of the Penn treebank tokenizer adapted by the Moses 

474 machine translation community. 

475 """ 

476 # Converts input string into unicode. 

477 text = str(text) 

478 # Perform a chain of regex substituitions using MOSES_PENN_REGEXES_1 

479 for regexp, substitution in self.MOSES_PENN_REGEXES_1: 

480 text = regexp.sub(substitution, text) 

481 # Handles nonbreaking prefixes. 

482 text = self.handles_nonbreaking_prefixes(text) 

483 # Restore ellipsis, clean extra spaces, escape XML symbols. 

484 for regexp, substitution in self.MOSES_PENN_REGEXES_2: 

485 text = regexp.sub(substitution, text) 

486 return text if return_str else text.split() 

487 

488 def tokenize( 

489 self, 

490 text, 

491 aggressive_dash_splits=False, 

492 return_str=False, 

493 escape=True, 

494 protected_patterns=None, 

495 ): 

496 """ 

497 Python port of the Moses tokenizer. 

498 

499 :param tokens: A single string, i.e. sentence text. 

500 :type tokens: str 

501 :param aggressive_dash_splits: Option to trigger dash split rules . 

502 :type aggressive_dash_splits: bool 

503 """ 

504 # Converts input string into unicode. 

505 text = str(text) 

506 # De-duplicate spaces and clean ASCII junk 

507 for regexp, substitution in [self.DEDUPLICATE_SPACE, self.ASCII_JUNK]: 

508 text = regexp.sub(substitution, text) 

509 

510 if protected_patterns: 

511 protected_patterns = [re.compile(p, re.IGNORECASE) for p in protected_patterns] 

512 # Find the tokens that needs to be protected. 

513 protected_tokens = [ 

514 match.group() 

515 for protected_pattern in protected_patterns 

516 for match in protected_pattern.finditer(text) 

517 ] 

518 # A real check, not an ``assert``: ``python -O`` and 

519 # ``PYTHONOPTIMIZE=1`` (routine in slim container images) strip 

520 # asserts, and without this bound the zfill(3) placeholder space 

521 # wraps -- token 1000 becomes THISISPROTECTED1000, which the 

522 # restore loop matches with the i=100 placeholder first and 

523 # silently corrupts. The count comes straight from the input text, 

524 # so it is attacker-controlled. 

525 if len(protected_tokens) > self.MAX_PROTECTED_TOKENS: 

526 raise ValueError( 

527 "too many protected tokens: %d matches exceeds the limit " 

528 "of %d. Use narrower protected_patterns, or tokenize the " 

529 "text in smaller pieces." 

530 % (len(protected_tokens), self.MAX_PROTECTED_TOKENS) 

531 ) 

532 marker = self.unused_protect_marker(text) 

533 

534 # Apply the protected_patterns, longest match first. 

535 for i, token in sorted(enumerate(protected_tokens), key=lambda pair:len(pair[1]), reverse=True): 

536 substituition = marker + str(i).zfill(3) 

537 text = text.replace(token, substituition) 

538 

539 # Strips heading and trailing spaces. 

540 text = text.strip() 

541 # FIXME!!! 

542 """ 

543 # For Finnish and Swedish, seperate out all "other" special characters. 

544 if self.lang in ["fi", "sv"]: 

545 # In Finnish and Swedish, the colon can be used inside words 

546 # as an apostrophe-like character: 

547 # USA:n, 20:een, EU:ssa, USA:s, S:t 

548 regexp, substitution = self.FI_SV_COLON_APOSTROPHE 

549 text = regexp.sub(substitution, text) 

550 # If a colon is not immediately followed by lower-case characters, 

551 # separate it out anyway. 

552 regexp, substitution = self.FI_SV_COLON_NO_LOWER_FOLLOW 

553 text = regexp.sub(substitution, text) 

554 else: 

555 """ 

556 # Separate special characters outside of IsAlnum character set. 

557 regexp, substitution = self.PAD_NOT_ISALNUM 

558 text = regexp.sub(substitution, text) 

559 # Aggressively splits dashes 

560 if aggressive_dash_splits: 

561 regexp, substitution = self.AGGRESSIVE_HYPHEN_SPLIT 

562 text = regexp.sub(substitution, text) 

563 

564 # Replaces multidots with "DOTDOTMULTI" literal strings. 

565 text = self.replace_multidots(text) 

566 

567 # Separate out "," except if within numbers e.g. 5,300 

568 for regexp, substitution in [ 

569 self.COMMA_SEPARATE_1, 

570 self.COMMA_SEPARATE_2, 

571 self.COMMA_SEPARATE_3, 

572 ]: 

573 text = regexp.sub(substitution, text) 

574 

575 # (Language-specific) apostrophe tokenization. 

576 if self.lang == "en": 

577 for regexp, substitution in self.ENGLISH_SPECIFIC_APOSTROPHE: 

578 text = regexp.sub(substitution, text) 

579 elif self.lang in ["fr", "it"]: 

580 for regexp, substitution in self.FR_IT_SPECIFIC_APOSTROPHE: 

581 text = regexp.sub(substitution, text) 

582 # FIXME!!! 

583 ##elif self.lang == "so": 

584 ## for regexp, substitution in self.SO_SPECIFIC_APOSTROPHE: 

585 ## text = re.sub(regexp, substitution, text) 

586 else: 

587 regexp, substitution = self.NON_SPECIFIC_APOSTROPHE 

588 text = regexp.sub(substitution, text) 

589 

590 # Handles nonbreaking prefixes. 

591 text = self.handles_nonbreaking_prefixes(text) 

592 # Cleans up extraneous spaces. 

593 regexp, substitution = self.DEDUPLICATE_SPACE 

594 text = regexp.sub(substitution, text).strip() 

595 # Split trailing ".'". 

596 regexp, substituition = self.TRAILING_DOT_APOSTROPHE 

597 text = regexp.sub(substituition, text) 

598 

599 # Restore the protected tokens. 

600 if protected_patterns: 

601 for i, token in enumerate(protected_tokens): 

602 substituition = marker + str(i).zfill(3) 

603 text = text.replace(substituition, token) 

604 

605 # Restore multidots. 

606 text = self.restore_multidots(text) 

607 if escape: 

608 # Escape XML symbols. 

609 text = self.escape_xml(text) 

610 

611 return text if return_str else text.split() 

612 

613 

614class MosesDetokenizer(object): 

615 """ 

616 This is a Python port of the Moses Detokenizer from 

617 https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/detokenizer.perl 

618 

619 """ 

620 

621 # Currency Symbols. 

622 IsAlnum = str("".join(perluniprops.chars("IsAlnum"))) 

623 IsAlpha = str("".join(perluniprops.chars("IsAlpha"))) 

624 IsSc = str("".join(perluniprops.chars("IsSc"))) 

625 

626 AGGRESSIVE_HYPHEN_SPLIT = re.compile(r" \@\-\@ "), r"-" 

627 

628 # Merge multiple spaces. 

629 ONE_SPACE = re.compile(r" {2,}"), " " 

630 

631 # Unescape special characters. 

632 UNESCAPE_FACTOR_SEPARATOR = re.compile(r"&#124;"), r"|" 

633 UNESCAPE_LEFT_ANGLE_BRACKET = re.compile(r"&lt;"), r"<" 

634 UNESCAPE_RIGHT_ANGLE_BRACKET = re.compile(r"&gt;"), r">" 

635 UNESCAPE_DOUBLE_QUOTE = re.compile(r"&quot;"), r'"' 

636 UNESCAPE_SINGLE_QUOTE = re.compile(r"&apos;"), r"'" 

637 UNESCAPE_SYNTAX_NONTERMINAL_LEFT = re.compile(r"&#91;"), r"[" 

638 UNESCAPE_SYNTAX_NONTERMINAL_RIGHT = re.compile(r"&#93;"), r"]" 

639 UNESCAPE_AMPERSAND = re.compile(r"&amp;"), r"&" 

640 # The legacy regexes are used to support outputs from older Moses versions. 

641 UNESCAPE_FACTOR_SEPARATOR_LEGACY = re.compile(r"&bar;"), r"|" 

642 UNESCAPE_SYNTAX_NONTERMINAL_LEFT_LEGACY = re.compile(r"&bra;"), r"[" 

643 UNESCAPE_SYNTAX_NONTERMINAL_RIGHT_LEGACY = re.compile(r"&ket;"), r"]" 

644 

645 MOSES_UNESCAPE_XML_REGEXES = [ 

646 UNESCAPE_FACTOR_SEPARATOR_LEGACY, 

647 UNESCAPE_FACTOR_SEPARATOR, 

648 UNESCAPE_LEFT_ANGLE_BRACKET, 

649 UNESCAPE_RIGHT_ANGLE_BRACKET, 

650 UNESCAPE_SYNTAX_NONTERMINAL_LEFT_LEGACY, 

651 UNESCAPE_SYNTAX_NONTERMINAL_RIGHT_LEGACY, 

652 UNESCAPE_DOUBLE_QUOTE, 

653 UNESCAPE_SINGLE_QUOTE, 

654 UNESCAPE_SYNTAX_NONTERMINAL_LEFT, 

655 UNESCAPE_SYNTAX_NONTERMINAL_RIGHT, 

656 UNESCAPE_AMPERSAND, 

657 ] 

658 

659 FINNISH_MORPHSET_1 = [ 

660 "N", 

661 "n", 

662 "A", 

663 "a", 

664 "\xc4", 

665 "\xe4", 

666 "ssa", 

667 "Ssa", 

668 "ss\xe4", 

669 "Ss\xe4", 

670 "sta", 

671 "st\xe4", 

672 "Sta", 

673 "St\xe4", 

674 "hun", 

675 "Hun", 

676 "hyn", 

677 "Hyn", 

678 "han", 

679 "Han", 

680 "h\xe4n", 

681 "H\xe4n", 

682 "h\xf6n", 

683 "H\xf6n", 

684 "un", 

685 "Un", 

686 "yn", 

687 "Yn", 

688 "an", 

689 "An", 

690 "\xe4n", 

691 "\xc4n", 

692 "\xf6n", 

693 "\xd6n", 

694 "seen", 

695 "Seen", 

696 "lla", 

697 "Lla", 

698 "ll\xe4", 

699 "Ll\xe4", 

700 "lta", 

701 "Lta", 

702 "lt\xe4", 

703 "Lt\xe4", 

704 "lle", 

705 "Lle", 

706 "ksi", 

707 "Ksi", 

708 "kse", 

709 "Kse", 

710 "tta", 

711 "Tta", 

712 "ine", 

713 "Ine", 

714 ] 

715 

716 FINNISH_MORPHSET_2 = ["ni", "si", "mme", "nne", "nsa"] 

717 

718 FINNISH_MORPHSET_3 = [ 

719 "ko", 

720 "k\xf6", 

721 "han", 

722 "h\xe4n", 

723 "pa", 

724 "p\xe4", 

725 "kaan", 

726 "k\xe4\xe4n", 

727 "kin", 

728 ] 

729 

730 FINNISH_REGEX = re.compile(r"^({})({})?({})$".format( 

731 "|".join(FINNISH_MORPHSET_1), 

732 "|".join(FINNISH_MORPHSET_2), 

733 "|".join(FINNISH_MORPHSET_3), 

734 )) 

735 

736 IS_CURRENCY_SYMBOL = re.compile(r"^[{}\(\[\{{\¿\¡]+$".format(IsSc)) 

737 

738 IS_ENGLISH_CONTRACTION = re.compile(r"^['][{}]".format(IsAlpha)) 

739 

740 IS_FRENCH_CONRTACTION = re.compile(r"[{}][']$".format(IsAlpha)) 

741 

742 STARTS_WITH_ALPHA = re.compile(r"^[{}]".format(IsAlpha)) 

743 

744 IS_PUNCT = re.compile(r"^[\,\.\?\!\:\;\\\%\}\]\)]+$") 

745 

746 IS_OPEN_QUOTE = re.compile(r"""^[\'\"„“`]+$""") 

747 

748 def __init__(self, lang="en"): 

749 super(MosesDetokenizer, self).__init__() 

750 self.lang = lang 

751 

752 def unescape_xml(self, text): 

753 for regexp, substitution in self.MOSES_UNESCAPE_XML_REGEXES: 

754 text = regexp.sub(substitution, text) 

755 return text 

756 

757 def tokenize(self, tokens, return_str=True, unescape=True): 

758 """ 

759 Python port of the Moses detokenizer. 

760 :param tokens: A list of strings, i.e. tokenized text. 

761 :type tokens: list(str) 

762 :return: str 

763 """ 

764 # Convert the list of tokens into a string and pad it with spaces. 

765 text = r" {} ".format(" ".join(tokens)) 

766 # Converts input string into unicode. 

767 text = str(text) 

768 # Detokenize the agressive hyphen split. 

769 regexp, substitution = self.AGGRESSIVE_HYPHEN_SPLIT 

770 text = regexp.sub(substitution, text) 

771 if unescape: 

772 # Unescape the XML symbols. 

773 text = self.unescape_xml(text) 

774 # Keep track of no. of quotation marks. 

775 quote_counts = {"'": 0, '"': 0, "``": 0, "`": 0, "''": 0} 

776 

777 # The *prepend_space* variable is used to control the "effects" of 

778 # detokenization as the function loops through the list of tokens and 

779 # changes the *prepend_space* accordingly as it sequentially checks 

780 # through the language specific and language independent conditions. 

781 prepend_space = " " 

782 detokenized_text = "" 

783 tokens = text.split() 

784 # Iterate through every token and apply language specific detokenization rule(s). 

785 for i, token in enumerate(iter(tokens)): 

786 # Check if the first char is CJK. 

787 if is_cjk(token[0]) and self.lang != "ko": 

788 # Perform left shift if this is a second consecutive CJK word. 

789 if i > 0 and is_cjk(tokens[i - 1][-1]): 

790 detokenized_text += token 

791 # But do nothing special if this is a CJK word that doesn't follow a CJK word 

792 else: 

793 detokenized_text += prepend_space + token 

794 prepend_space = " " 

795 # If it's a currency symbol. 

796 elif self.IS_CURRENCY_SYMBOL.search(token): 

797 # Perform right shift on currency and other random punctuation items 

798 detokenized_text += prepend_space + token 

799 prepend_space = "" 

800 

801 elif self.IS_PUNCT.search(token): 

802 # In French, these punctuations are prefixed with a non-breakable space. 

803 if self.lang == "fr" and re.search(r"^[\?\!\:\;\\\%]$", token): 

804 detokenized_text += " " 

805 # Perform left shift on punctuation items. 

806 detokenized_text += token 

807 prepend_space = " " 

808 

809 elif ( 

810 self.lang == "en" 

811 and i > 0 

812 and self.IS_ENGLISH_CONTRACTION.search(token) 

813 ): 

814 # and re.search('[{}]$'.format(self.IsAlnum), tokens[i-1])): 

815 # For English, left-shift the contraction. 

816 detokenized_text += token 

817 prepend_space = " " 

818 

819 elif ( 

820 self.lang == "cs" 

821 and i > 1 

822 and re.search( 

823 r"^[0-9]+$", tokens[-2] 

824 ) # If the previous previous token is a number. 

825 and re.search(r"^[.,]$", tokens[-1]) # If previous token is a dot. 

826 and re.search(r"^[0-9]+$", token) 

827 ): # If the current token is a number. 

828 # In Czech, left-shift floats that are decimal numbers. 

829 detokenized_text += token 

830 prepend_space = " " 

831 

832 elif ( 

833 self.lang in ["fr", "it", "ga"] 

834 and i <= len(tokens) - 2 

835 and self.IS_FRENCH_CONRTACTION.search(token) 

836 and self.STARTS_WITH_ALPHA.search(tokens[i + 1]) 

837 ): # If the next token is alpha. 

838 # For French and Italian, right-shift the contraction. 

839 detokenized_text += prepend_space + token 

840 prepend_space = "" 

841 

842 elif ( 

843 self.lang == "cs" 

844 and i <= len(tokens) - 3 

845 and self.IS_FRENCH_CONRTACTION.search(token)