Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/sacremoses/util.py: 69%

1#!/usr/bin/env python3 

2# -*- coding: utf-8 -*- 

3 

4from itertools import tee, zip_longest 

5from xml.sax.saxutils import escape, unescape 

6 

7from joblib import Parallel, delayed 

8from tqdm import tqdm 

9 

10 

11class CJKChars(object): 

12 """ 

13 An object that enumerates the code points of the CJK characters as listed on 

14 http://en.wikipedia.org/wiki/Basic_Multilingual_Plane#Basic_Multilingual_Plane 

15 """ 

16 

17 # Hangul Jamo (1100–11FF) 

18 Hangul_Jamo = (4352, 4607) # (ord("\u1100"), ord("\u11ff")) 

19 

20 # CJK Radicals Supplement (2E80–2EFF) 

21 # Kangxi Radicals (2F00–2FDF) 

22 # Ideographic Description Characters (2FF0–2FFF) 

23 # CJK Symbols and Punctuation (3000–303F) 

24 # Hiragana (3040–309F) 

25 # Katakana (30A0–30FF) 

26 # Bopomofo (3100–312F) 

27 # Hangul Compatibility Jamo (3130–318F) 

28 # Kanbun (3190–319F) 

29 # Bopomofo Extended (31A0–31BF) 

30 # CJK Strokes (31C0–31EF) 

31 # Katakana Phonetic Extensions (31F0–31FF) 

32 # Enclosed CJK Letters and Months (3200–32FF) 

33 # CJK Compatibility (3300–33FF) 

34 # CJK Unified Ideographs Extension A (3400–4DBF) 

35 # Yijing Hexagram Symbols (4DC0–4DFF) 

36 # CJK Unified Ideographs (4E00–9FFF) 

37 # Yi Syllables (A000–A48F) 

38 # Yi Radicals (A490–A4CF) 

39 CJK_Radicals = (11904, 42191) # (ord("\u2e80"), ord("\ua4cf")) 

40 

41 # Phags-pa (A840–A87F) 

42 Phags_Pa = (43072, 43135) # (ord("\ua840"), ord("\ua87f")) 

43 

44 # Hangul Syllables (AC00–D7AF) 

45 Hangul_Syllables = (44032, 55215) # (ord("\uAC00"), ord("\uD7AF")) 

46 

47 # CJK Compatibility Ideographs (F900–FAFF) 

48 CJK_Compatibility_Ideographs = (63744, 64255) # (ord("\uF900"), ord("\uFAFF")) 

49 

50 # CJK Compatibility Forms (FE30–FE4F) 

51 CJK_Compatibility_Forms = (65072, 65103) # (ord("\uFE30"), ord("\uFE4F")) 

52 

53 # Range U+FF65–FFDC encodes halfwidth forms, of Katakana and Hangul characters 

54 Katakana_Hangul_Halfwidth = (65381, 65500) # (ord("\uFF65"), ord("\uFFDC")) 

55 

56 # Ideographic Symbols and Punctuation (16FE0–16FFF) 

57 Ideographic_Symbols_And_Punctuation = ( 

58 94176, 

59 94207, 

60 ) # (ord("\U00016FE0"), ord("\U00016FFF")) 

61 

62 # Tangut (17000-187FF) 

63 # Tangut Components (18800-18AFF) 

64 Tangut = (94208, 101119) # (ord("\U00017000"), ord("\U00018AFF")) 

65 

66 # Kana Supplement (1B000-1B0FF) 

67 # Kana Extended-A (1B100-1B12F) 

68 Kana_Supplement = (110592, 110895) # (ord("\U0001B000"), ord("\U0001B12F")) 

69 

70 # Nushu (1B170-1B2FF) 

71 Nushu = (110960, 111359) # (ord("\U0001B170"), ord("\U0001B2FF")) 

72 

73 # Supplementary Ideographic Plane (20000–2FFFF) 

74 Supplementary_Ideographic_Plane = ( 

75 131072, 

76 196607, 

77 ) # (ord("\U00020000"), ord("\U0002FFFF")) 

78 

79 ranges = [ 

80 Hangul_Jamo, 

81 CJK_Radicals, 

82 Phags_Pa, 

83 Hangul_Syllables, 

84 CJK_Compatibility_Ideographs, 

85 CJK_Compatibility_Forms, 

86 Katakana_Hangul_Halfwidth, 

87 Tangut, 

88 Kana_Supplement, 

89 Nushu, 

90 Supplementary_Ideographic_Plane, 

91 ] 

92 

93 

94_CJKChars_ranges = CJKChars().ranges 

95 

96 

97def is_cjk(character): 

98 """ 

99 This checks for CJK character. 

100 

101 >>> CJKChars().ranges 

102 [(4352, 4607), (11904, 42191), (43072, 43135), (44032, 55215), (63744, 64255), (65072, 65103), (65381, 65500), (94208, 101119), (110592, 110895), (110960, 111359), (131072, 196607)] 

103 >>> is_cjk('\u33fe') 

104 True 

105 >>> is_cjk('\uFE5F') 

106 False 

107 

108 :param character: The character that needs to be checked. 

109 :type character: char 

110 :return: bool 

111 """ 

112 char = ord(character) 

113 for start, end in _CJKChars_ranges: 

114 # Inclusive on both ends: the ranges are closed intervals. Comparing 

115 # exclusively excluded the first and last codepoint of all 11 ranges -- 

116 # 22 characters, including U+AC00 GA, the most common Korean syllable. 

117 # The ranges are sorted, so the first one ending at or after `char` 

118 # decides the answer and the loop can stop there. 

119 if char <= end: 

120 return char >= start 

121 return False 

122 

123 

124def xml_escape(text): 

125 """ 

126 This function transforms the input text into an "escaped" version suitable 

127 for well-formed XML formatting. 

128 Note that the default xml.sax.saxutils.escape() function don't escape 

129 some characters that Moses does so we have to manually add them to the 

130 entities dictionary. 

131 

132 >>> input_str = ''')| & < > ' " ] [''' 

133 >>> expected_output = ''')| &amp; &lt; &gt; ' " ] [''' 

134 >>> escape(input_str) == expected_output 

135 True 

136 >>> xml_escape(input_str) 

137 ')&#124; &amp; &lt; &gt; &apos; &quot; &#93; &#91;' 

138 

139 :param text: The text that needs to be escaped. 

140 :type text: str 

141 :rtype: str 

142 """ 

143 return escape( 

144 text, 

145 entities={ 

146 r"'": r"&apos;", 

147 r'"': r"&quot;", 

148 r"|": r"&#124;", 

149 r"[": r"&#91;", 

150 r"]": r"&#93;", 

151 }, 

152 ) 

153 

154 

155def xml_unescape(text): 

156 """ 

157 This function transforms the "escaped" version suitable 

158 for well-formed XML formatting into humanly-readable string. 

159 Note that the default xml.sax.saxutils.unescape() function don't unescape 

160 some characters that Moses does so we have to manually add them to the 

161 entities dictionary. 

162 

163 >>> from xml.sax.saxutils import unescape 

164 >>> s = ')&#124; &amp; &lt; &gt; &apos; &quot; &#93; &#91;' 

165 >>> expected = ''')| & < > \' " ] [''' 

166 >>> xml_unescape(s) == expected 

167 True 

168 

169 :param text: The text that needs to be unescaped. 

170 :type text: str 

171 :rtype: str 

172 """ 

173 return unescape( 

174 text, 

175 entities={ 

176 r"&apos;": r"'", 

177 r"&quot;": r'"', 

178 r"&#124;": r"|", 

179 r"&#91;": r"[", 

180 r"&#93;": r"]", 

181 }, 

182 ) 

183 

184 

185def pairwise(iterable): 

186 """ 

187 From https://docs.python.org/3/library/itertools.html#recipes 

188 s -> (s0,s1), (s1,s2), (s2, s3), ... 

189 """ 

190 a, b = tee(iterable) 

191 next(b, None) 

192 return zip(a, b) 

193 

194 

195def grouper(iterable, n, fillvalue=None): 

196 """Collect data into fixed-length chunks or blocks 

197 from https://stackoverflow.com/a/16789869/610569 

198 """ 

199 # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" 

200 args = [iter(iterable)] * n 

201 return zip_longest(*args, fillvalue=fillvalue) 

202 

203 

204def parallelize_preprocess(func, iterator, processes, progress_bar=False): 

205 iterator = tqdm(iterator) if progress_bar else iterator 

206 if processes <= 1: 

207 return map(func, iterator) 

208 return Parallel(n_jobs=processes)(delayed(func)(line) for line in iterator)