Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/sacremoses/corpus.py: 54%

1#!/usr/bin/env python3 

2# -*- coding: utf-8 -*- 

3 

4from sacremoses._data_nonbreaking_prefixes import NONBREAKING_PREFIXES 

5from sacremoses._data_perluniprops import PERLUNIPROPS 

6 

7 

8class Perluniprops: 

9 """ 

10 This class is used to read lists of characters from the Perl Unicode 

11 Properties (see http://perldoc.perl.org/perluniprops.html). 

12 The files in the perluniprop.zip are extracted using the Unicode::Tussle 

13 module from http://search.cpan.org/~bdfoy/Unicode-Tussle-1.11/lib/Unicode/Tussle.pm 

14 """ 

15 

16 def __init__(self): 

17 # These are categories similar to the Perl Unicode Properties 

18 self.available_categories = [ 

19 "Close_Punctuation", 

20 "Currency_Symbol", 

21 "IsAlnum", 

22 "IsAlpha", 

23 "IsLower", 

24 "IsN", 

25 "IsSc", 

26 "IsSo", 

27 "IsUpper", 

28 "Line_Separator", 

29 "Number", 

30 "Open_Punctuation", 

31 "Punctuation", 

32 "Separator", 

33 "Symbol", 

34 "Lowercase_Letter", 

35 "Titlecase_Letter", 

36 "Uppercase_Letter", 

37 "IsPf", 

38 "IsPi", 

39 "CJKSymbols", 

40 "CJK", 

41 ] 

42 

43 def chars(self, category=None): 

44 """ 

45 This module returns a list of characters from the Perl Unicode Properties. 

46 They are very useful when porting Perl tokenizers to Python. 

47 

48 >>> from sacremoses.corpus import Perluniprops 

49 >>> pup = Perluniprops() 

50 >>> list(pup.chars('Open_Punctuation'))[:5] == ['(', '[', '{', '\u0f3a', '\u0f3c'] 

51 True 

52 >>> list(pup.chars('Currency_Symbol'))[:5] == ['$', '\xa2', '\xa3', '\xa4', '\xa5'] 

53 True 

54 >>> pup.available_categories[:5] 

55 ['Close_Punctuation', 'Currency_Symbol', 'IsAlnum', 'IsAlpha', 'IsLower'] 

56 

57 :return: a generator of characters given the specific unicode character category 

58 """ 

59 # Served from code, so no file is read and *category* is never 

60 # interpolated into a path. An unknown category raises KeyError rather 

61 # than reaching the filesystem. 

62 yield from PERLUNIPROPS[category] 

63 

64 

65class NonbreakingPrefixes: 

66 """ 

67 This is a class to read the nonbreaking prefixes textfiles from the 

68 Moses Machine Translation toolkit. These lists are used in the Python port 

69 of the Moses' word tokenizer. 

70 """ 

71 

72 def __init__(self): 

73 self.available_langs = { 

74 "assamese": "as", 

75 "bengali": "bn", 

76 "catalan": "ca", 

77 "czech": "cs", 

78 "german": "de", 

79 "greek": "el", 

80 "english": "en", 

81 "spanish": "es", 

82 "estonian": "et", 

83 "finnish": "fi", 

84 "french": "fr", 

85 "irish": "ga", 

86 "gujarati": "gu", 

87 "hindi": "hi", 

88 "hungarian": "hu", 

89 "icelandic": "is", 

90 "italian": "it", 

91 "kannada": "kn", 

92 "lithuanian": "lt", 

93 "latvian": "lv", 

94 "malayalam": "ml", 

95 "manipuri": "mni", 

96 "marathi": "mr", 

97 "dutch": "nl", 

98 "oriya": "or", 

99 "punjabi": "pa", 

100 "polish": "pl", 

101 "portuguese": "pt", 

102 "romanian": "ro", 

103 "russian": "ru", 

104 "slovak": "sk", 

105 "slovenian": "sl", 

106 "swedish": "sv", 

107 "tamil": "ta", 

108 "telugu": "te", 

109 "tetum": "tdt", 

110 "cantonese": "yue", 

111 "chinese": "zh", 

112 } 

113 # Also, add the lang IDs as the keys. 

114 self.available_langs.update({v: v for v in self.available_langs.values()}) 

115 

116 def words(self, lang=None, ignore_lines_startswith="#"): 

117 """ 

118 This module returns a list of nonbreaking prefixes for the specified 

119 language(s). 

120 

121 >>> from sacremoses.corpus import NonbreakingPrefixes 

122 >>> nbp = NonbreakingPrefixes() 

123 >>> list(nbp.words('en'))[:10] == ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'] 

124 True 

125 >>> list(nbp.words('ta'))[:5] == ['\u0bb0', '\u0bc2', '\u0ba4\u0bbf\u0bb0\u0bc1', '\u0b8f', '\u0baa\u0bc0'] 

126 True 

127 

128 :return: a generator words for the specified language(s). 

129 """ 

130 # If *lang* in list of languages available, allocate apt fileid. 

131 if lang in self.available_langs: 

132 filenames = ["nonbreaking_prefix." + self.available_langs[lang]] 

133 # Use non-breaking prefixes for all languages when lang==None. 

134 elif lang == None: 

135 filenames = [ 

136 "nonbreaking_prefix." + v for v in set(self.available_langs.values()) 

137 ] 

138 else: 

139 filenames = ["nonbreaking_prefix.en"] 

140 

141 for filename in filenames: 

142 # Served from code; *filename* is built from the vetted 

143 # available_langs mapping and never touches the filesystem. 

144 for line in NONBREAKING_PREFIXES[filename].splitlines(): 

145 line = line.strip() 

146 if line and not line.startswith(ignore_lines_startswith): 

147 yield line 

148 

149 

150__all__ = ["Perluniprops", "NonbreakingPrefixes"]