Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/sacremoses/normalize.py: 76%

1#!/usr/bin/env python3 

2# -*- coding: utf-8 -*- 

3 

4import re 

5import regex 

6 

7from itertools import chain 

8 

9 

10class MosesPunctNormalizer: 

11 """ 

12 This is a Python port of the Moses punctuation normalizer from 

13 https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl 

14 """ 

15 

16 EXTRA_WHITESPACE = [ # lines 21 - 30 

17 (r"\r", r""), 

18 (r"\(", r" ("), 

19 (r"\)", r") "), 

20 (r" +", r" "), 

21 (r"\) ([.!:?;,])", r")\g<1>"), 

22 (r"\( ", r"("), 

23 (r" \)", r")"), 

24 (r"(\d) %", r"\g<1>%"), 

25 (r" :", r":"), 

26 (r" ;", r";"), 

27 ] 

28 

29 NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')] # lines 33 - 34 

30 

31 NORMALIZE_UNICODE = [ # lines 37 - 50 

32 ("„", r'"'), 

33 ("“", r'"'), 

34 ("”", r'"'), 

35 ("–", r"-"), 

36 ("—", r" - "), 

37 (r" +", r" "), 

38 ("´", r"'"), 

39 ("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"), 

40 ("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"), 

41 ("‘", r"'"), 

42 ("‚", r"'"), 

43 ("’", r"'"), 

44 (r"''", r'"'), 

45 ("´´", r'"'), 

46 ("…", r"..."), 

47 ] 

48 

49 FRENCH_QUOTES = [ # lines 52 - 57 

50 ("\u00A0«\u00A0", r'"'), 

51 ("«\u00A0", r'"'), 

52 ("«", r'"'), 

53 ("\u00A0»\u00A0", r'"'), 

54 ("\u00A0»", r'"'), 

55 ("»", r'"'), 

56 ] 

57 

58 HANDLE_PSEUDO_SPACES = [ # lines 59 - 67 

59 ("\u00A0%", r"%"), 

60 ("nº\u00A0", "nº "), 

61 ("\u00A0:", r":"), 

62 ("\u00A0ºC", " ºC"), 

63 ("\u00A0cm", r" cm"), 

64 ("\u00A0\\?", "?"), 

65 ("\u00A0\\!", "!"), 

66 ("\u00A0;", r";"), 

67 (",\u00A0", r", "), 

68 (r" +", r" "), 

69 ] 

70 

71 EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')] 

72 

73 DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [ 

74 (r',"', r'",'), 

75 (r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'), # don't fix period at end of sentence 

76 ] 

77 

78 DE_ES_CZ_CS_FR = [ 

79 ("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"), 

80 ] 

81 

82 OTHER = [ 

83 ("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"), 

84 ] 

85 

86 # Regex substitutions from replace-unicode-punctuation.perl 

87 # https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl 

88 REPLACE_UNICODE_PUNCTUATION = [ 

89 ("，", ","), 

90 (r"。\s*", ". "), 

91 ("、", ","), 

92 ("”", '"'), 

93 ("“", '"'), 

94 ("∶", ":"), 

95 ("：", ":"), 

96 ("？", "?"), 

97 ("《", '"'), 

98 ("》", '"'), 

99 ("）", ")"), 

100 ("！", "!"), 

101 ("（", "("), 

102 ("；", ";"), 

103 ("」", '"'), 

104 ("「", '"'), 

105 ("０", "0"), 

106 ("１", "1"), 

107 ("２", "2"), 

108 ("３", "3"), 

109 ("４", "4"), 

110 ("５", "5"), 

111 ("６", "6"), 

112 ("７", "7"), 

113 ("８", "8"), 

114 ("９", "9"), 

115 (r"．\s*", ". "), 

116 ("～", "~"), 

117 ("’", "'"), 

118 ("…", "..."), 

119 ("━", "-"), 

120 ("〈", "<"), 

121 ("〉", ">"), 

122 ("【", "["), 

123 ("】", "]"), 

124 ("％", "%"), 

125 ] 

126 

127 def __init__( 

128 self, 

129 lang="en", 

130 penn=True, 

131 norm_quote_commas=True, 

132 norm_numbers=True, 

133 pre_replace_unicode_punct=False, 

134 post_remove_control_chars=False, 

135 perl_parity=False 

136 ): 

137 """ 

138 :param language: The two-letter language code. 

139 :type lang: str 

140 :param penn: Normalize Penn Treebank style quotations. 

141 :type penn: bool 

142 :param norm_quote_commas: Normalize quotations and commas 

143 :type norm_quote_commas: bool 

144 :param norm_numbers: Normalize numbers 

145 :type norm_numbers: bool 

146 :param perl_parity: exact parity with perl script 

147 :type perl_parity: bool 

148 """ 

149 

150 if perl_parity: 

151 # Copy first: these are class attributes, so assigning into them 

152 # in place would rewrite the defaults for every other normalizer in 

153 # the process, including ones built with perl_parity=False. 

154 self.NORMALIZE_UNICODE = list(self.NORMALIZE_UNICODE) 

155 self.FRENCH_QUOTES = list(self.FRENCH_QUOTES) 

156 self.NORMALIZE_UNICODE[11] = ("’", r'"') 

157 self.FRENCH_QUOTES[0] = ("\u00A0«\u00A0", r' "') 

158 self.FRENCH_QUOTES[3] = ("\u00A0»\u00A0", r'" ') 

159 

160 self.substitutions = [ 

161 self.EXTRA_WHITESPACE, 

162 self.NORMALIZE_UNICODE, 

163 self.FRENCH_QUOTES, 

164 self.HANDLE_PSEUDO_SPACES, 

165 ] 

166 

167 if penn: # Adds the penn substitutions after extra_whitespace regexes. 

168 self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN) 

169 

170 if norm_quote_commas: 

171 if lang == "en": 

172 self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA) 

173 elif lang in ["de", "es", "fr"]: 

174 self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA) 

175 

176 if norm_numbers: 

177 if lang in ["de", "es", "cz", "cs", "fr"]: 

178 self.substitutions.append(self.DE_ES_CZ_CS_FR) 

179 else: 

180 self.substitutions.append(self.OTHER) 

181 

182 self.substitutions = list(chain(*self.substitutions)) 

183 

184 self.pre_replace_unicode_punct = pre_replace_unicode_punct 

185 self.post_remove_control_chars = post_remove_control_chars 

186 

187 def normalize(self, text): 

188 """ 

189 Returns a string with normalized punctuation. 

190 """ 

191 # Optionally, replace unicode puncts BEFORE normalization. 

192 if self.pre_replace_unicode_punct: 

193 text = self.replace_unicode_punct(text) 

194 

195 # Actual normalization. 

196 for regexp, substitution in self.substitutions: 

197 # print(regexp, substitution) 

198 text = re.sub(regexp, substitution, str(text)) 

199 # print(text) 

200 

201 # Optionally, replace unicode puncts BEFORE normalization. 

202 if self.post_remove_control_chars: 

203 text = self.remove_control_chars(text) 

204 

205 return text.strip() 

206 

207 def replace_unicode_punct(self, text): 

208 for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION: 

209 text = re.sub(regexp, substitution, str(text)) 

210 return text 

211 

212 def remove_control_chars(self, text): 

213 return regex.sub(r"\p{C}", "", text)