/src/skia/third_party/externals/icu/source/common/rbbirb.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // © 2016 and later: Unicode, Inc. and others. |
2 | | // License & terms of use: http://www.unicode.org/copyright.html |
3 | | // |
4 | | // file: rbbirb.cpp |
5 | | // |
6 | | // Copyright (C) 2002-2011, International Business Machines Corporation and others. |
7 | | // All Rights Reserved. |
8 | | // |
9 | | // This file contains the RBBIRuleBuilder class implementation. This is the main class for |
10 | | // building (compiling) break rules into the tables required by the runtime |
11 | | // RBBI engine. |
12 | | // |
13 | | |
14 | | #include "unicode/utypes.h" |
15 | | |
16 | | #if !UCONFIG_NO_BREAK_ITERATION |
17 | | |
18 | | #include "unicode/brkiter.h" |
19 | | #include "unicode/rbbi.h" |
20 | | #include "unicode/ubrk.h" |
21 | | #include "unicode/unistr.h" |
22 | | #include "unicode/uniset.h" |
23 | | #include "unicode/uchar.h" |
24 | | #include "unicode/uchriter.h" |
25 | | #include "unicode/ustring.h" |
26 | | #include "unicode/parsepos.h" |
27 | | #include "unicode/parseerr.h" |
28 | | |
29 | | #include "cmemory.h" |
30 | | #include "cstring.h" |
31 | | #include "rbbirb.h" |
32 | | #include "rbbinode.h" |
33 | | #include "rbbiscan.h" |
34 | | #include "rbbisetb.h" |
35 | | #include "rbbitblb.h" |
36 | | #include "rbbidata.h" |
37 | | #include "uassert.h" |
38 | | |
39 | | |
40 | | U_NAMESPACE_BEGIN |
41 | | |
42 | | |
43 | | //---------------------------------------------------------------------------------------- |
44 | | // |
45 | | // Constructor. |
46 | | // |
47 | | //---------------------------------------------------------------------------------------- |
48 | | RBBIRuleBuilder::RBBIRuleBuilder(const UnicodeString &rules, |
49 | | UParseError *parseErr, |
50 | | UErrorCode &status) |
51 | | : fRules(rules), fStrippedRules(rules) |
52 | 0 | { |
53 | 0 | fStatus = &status; // status is checked below |
54 | 0 | fParseError = parseErr; |
55 | 0 | fDebugEnv = NULL; |
56 | | #ifdef RBBI_DEBUG |
57 | | fDebugEnv = getenv("U_RBBIDEBUG"); |
58 | | #endif |
59 | | |
60 | |
|
61 | 0 | fForwardTree = NULL; |
62 | 0 | fReverseTree = NULL; |
63 | 0 | fSafeFwdTree = NULL; |
64 | 0 | fSafeRevTree = NULL; |
65 | 0 | fDefaultTree = &fForwardTree; |
66 | 0 | fForwardTable = NULL; |
67 | 0 | fRuleStatusVals = NULL; |
68 | 0 | fChainRules = FALSE; |
69 | 0 | fLBCMNoChain = FALSE; |
70 | 0 | fLookAheadHardBreak = FALSE; |
71 | 0 | fUSetNodes = NULL; |
72 | 0 | fRuleStatusVals = NULL; |
73 | 0 | fScanner = NULL; |
74 | 0 | fSetBuilder = NULL; |
75 | 0 | if (parseErr) { |
76 | 0 | uprv_memset(parseErr, 0, sizeof(UParseError)); |
77 | 0 | } |
78 | |
|
79 | 0 | if (U_FAILURE(status)) { |
80 | 0 | return; |
81 | 0 | } |
82 | | |
83 | 0 | fUSetNodes = new UVector(status); // bcos status gets overwritten here |
84 | 0 | fRuleStatusVals = new UVector(status); |
85 | 0 | fScanner = new RBBIRuleScanner(this); |
86 | 0 | fSetBuilder = new RBBISetBuilder(this); |
87 | 0 | if (U_FAILURE(status)) { |
88 | 0 | return; |
89 | 0 | } |
90 | 0 | if(fSetBuilder == 0 || fScanner == 0 || fUSetNodes == 0 || fRuleStatusVals == 0) { |
91 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
92 | 0 | } |
93 | 0 | } |
94 | | |
95 | | |
96 | | |
97 | | //---------------------------------------------------------------------------------------- |
98 | | // |
99 | | // Destructor |
100 | | // |
101 | | //---------------------------------------------------------------------------------------- |
102 | 0 | RBBIRuleBuilder::~RBBIRuleBuilder() { |
103 | |
|
104 | 0 | int i; |
105 | 0 | for (i=0; ; i++) { |
106 | 0 | RBBINode *n = (RBBINode *)fUSetNodes->elementAt(i); |
107 | 0 | if (n==NULL) { |
108 | 0 | break; |
109 | 0 | } |
110 | 0 | delete n; |
111 | 0 | } |
112 | |
|
113 | 0 | delete fUSetNodes; |
114 | 0 | delete fSetBuilder; |
115 | 0 | delete fForwardTable; |
116 | 0 | delete fForwardTree; |
117 | 0 | delete fReverseTree; |
118 | 0 | delete fSafeFwdTree; |
119 | 0 | delete fSafeRevTree; |
120 | 0 | delete fScanner; |
121 | 0 | delete fRuleStatusVals; |
122 | 0 | } |
123 | | |
124 | | |
125 | | |
126 | | |
127 | | |
128 | | //---------------------------------------------------------------------------------------- |
129 | | // |
130 | | // flattenData() - Collect up the compiled RBBI rule data and put it into |
131 | | // the format for saving in ICU data files, |
132 | | // which is also the format needed by the RBBI runtime engine. |
133 | | // |
134 | | //---------------------------------------------------------------------------------------- |
135 | 0 | static int32_t align8(int32_t i) {return (i+7) & 0xfffffff8;} |
136 | | |
137 | 0 | RBBIDataHeader *RBBIRuleBuilder::flattenData() { |
138 | 0 | int32_t i; |
139 | |
|
140 | 0 | if (U_FAILURE(*fStatus)) { |
141 | 0 | return NULL; |
142 | 0 | } |
143 | | |
144 | | // Remove whitespace from the rules to make it smaller. |
145 | | // The rule parser has already removed comments. |
146 | 0 | fStrippedRules = fScanner->stripRules(fStrippedRules); |
147 | | |
148 | | // Calculate the size of each section in the data. |
149 | | // Sizes here are padded up to a multiple of 8 for better memory alignment. |
150 | | // Sections sizes actually stored in the header are for the actual data |
151 | | // without the padding. |
152 | | // |
153 | 0 | int32_t headerSize = align8(sizeof(RBBIDataHeader)); |
154 | 0 | int32_t forwardTableSize = align8(fForwardTable->getTableSize()); |
155 | 0 | int32_t reverseTableSize = align8(fForwardTable->getSafeTableSize()); |
156 | 0 | int32_t trieSize = align8(fSetBuilder->getTrieSize()); |
157 | 0 | int32_t statusTableSize = align8(fRuleStatusVals->size() * sizeof(int32_t)); |
158 | |
|
159 | 0 | int32_t rulesLengthInUTF8 = 0; |
160 | 0 | u_strToUTF8WithSub(0, 0, &rulesLengthInUTF8, |
161 | 0 | fStrippedRules.getBuffer(), fStrippedRules.length(), |
162 | 0 | 0xfffd, nullptr, fStatus); |
163 | 0 | *fStatus = U_ZERO_ERROR; |
164 | |
|
165 | 0 | int32_t rulesSize = align8((rulesLengthInUTF8+1)); |
166 | |
|
167 | 0 | int32_t totalSize = headerSize |
168 | 0 | + forwardTableSize |
169 | 0 | + reverseTableSize |
170 | 0 | + statusTableSize + trieSize + rulesSize; |
171 | |
|
172 | | #ifdef RBBI_DEBUG |
173 | | if (fDebugEnv && uprv_strstr(fDebugEnv, "size")) { |
174 | | RBBIDebugPrintf("Header Size: %8d\n", headerSize); |
175 | | RBBIDebugPrintf("Forward Table Size: %8d\n", forwardTableSize); |
176 | | RBBIDebugPrintf("Reverse Table Size: %8d\n", reverseTableSize); |
177 | | RBBIDebugPrintf("Trie Size: %8d\n", trieSize); |
178 | | RBBIDebugPrintf("Status Table Size: %8d\n", statusTableSize); |
179 | | RBBIDebugPrintf("Rules Size: %8d\n", rulesSize); |
180 | | RBBIDebugPrintf("-----------------------------\n"); |
181 | | RBBIDebugPrintf("Total Size: %8d\n", totalSize); |
182 | | } |
183 | | #endif |
184 | |
|
185 | 0 | RBBIDataHeader *data = (RBBIDataHeader *)uprv_malloc(totalSize); |
186 | 0 | if (data == NULL) { |
187 | 0 | *fStatus = U_MEMORY_ALLOCATION_ERROR; |
188 | 0 | return NULL; |
189 | 0 | } |
190 | 0 | uprv_memset(data, 0, totalSize); |
191 | | |
192 | |
|
193 | 0 | data->fMagic = 0xb1a0; |
194 | 0 | data->fFormatVersion[0] = RBBI_DATA_FORMAT_VERSION[0]; |
195 | 0 | data->fFormatVersion[1] = RBBI_DATA_FORMAT_VERSION[1]; |
196 | 0 | data->fFormatVersion[2] = RBBI_DATA_FORMAT_VERSION[2]; |
197 | 0 | data->fFormatVersion[3] = RBBI_DATA_FORMAT_VERSION[3]; |
198 | 0 | data->fLength = totalSize; |
199 | 0 | data->fCatCount = fSetBuilder->getNumCharCategories(); |
200 | |
|
201 | 0 | data->fFTable = headerSize; |
202 | 0 | data->fFTableLen = forwardTableSize; |
203 | |
|
204 | 0 | data->fRTable = data->fFTable + data->fFTableLen; |
205 | 0 | data->fRTableLen = reverseTableSize; |
206 | |
|
207 | 0 | data->fTrie = data->fRTable + data->fRTableLen; |
208 | 0 | data->fTrieLen = trieSize; |
209 | 0 | data->fStatusTable = data->fTrie + data->fTrieLen; |
210 | 0 | data->fStatusTableLen= statusTableSize; |
211 | 0 | data->fRuleSource = data->fStatusTable + statusTableSize; |
212 | 0 | data->fRuleSourceLen = rulesLengthInUTF8; |
213 | |
|
214 | 0 | uprv_memset(data->fReserved, 0, sizeof(data->fReserved)); |
215 | |
|
216 | 0 | fForwardTable->exportTable((uint8_t *)data + data->fFTable); |
217 | 0 | fForwardTable->exportSafeTable((uint8_t *)data + data->fRTable); |
218 | 0 | fSetBuilder->serializeTrie ((uint8_t *)data + data->fTrie); |
219 | |
|
220 | 0 | int32_t *ruleStatusTable = (int32_t *)((uint8_t *)data + data->fStatusTable); |
221 | 0 | for (i=0; i<fRuleStatusVals->size(); i++) { |
222 | 0 | ruleStatusTable[i] = fRuleStatusVals->elementAti(i); |
223 | 0 | } |
224 | |
|
225 | 0 | u_strToUTF8WithSub((char *)data+data->fRuleSource, rulesSize, &rulesLengthInUTF8, |
226 | 0 | fStrippedRules.getBuffer(), fStrippedRules.length(), |
227 | 0 | 0xfffd, nullptr, fStatus); |
228 | 0 | if (U_FAILURE(*fStatus)) { |
229 | 0 | return NULL; |
230 | 0 | } |
231 | | |
232 | 0 | return data; |
233 | 0 | } |
234 | | |
235 | | |
236 | | //---------------------------------------------------------------------------------------- |
237 | | // |
238 | | // createRuleBasedBreakIterator construct from source rules that are passed in |
239 | | // in a UnicodeString |
240 | | // |
241 | | //---------------------------------------------------------------------------------------- |
242 | | BreakIterator * |
243 | | RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString &rules, |
244 | | UParseError *parseError, |
245 | | UErrorCode &status) |
246 | 0 | { |
247 | | // |
248 | | // Read the input rules, generate a parse tree, symbol table, |
249 | | // and list of all Unicode Sets referenced by the rules. |
250 | | // |
251 | 0 | RBBIRuleBuilder builder(rules, parseError, status); |
252 | 0 | if (U_FAILURE(status)) { // status checked here bcos build below doesn't |
253 | 0 | return NULL; |
254 | 0 | } |
255 | | |
256 | 0 | RBBIDataHeader *data = builder.build(status); |
257 | |
|
258 | 0 | if (U_FAILURE(status)) { |
259 | 0 | return nullptr; |
260 | 0 | } |
261 | | |
262 | | // |
263 | | // Create a break iterator from the compiled rules. |
264 | | // (Identical to creation from stored pre-compiled rules) |
265 | | // |
266 | | // status is checked after init in construction. |
267 | 0 | RuleBasedBreakIterator *This = new RuleBasedBreakIterator(data, status); |
268 | 0 | if (U_FAILURE(status)) { |
269 | 0 | delete This; |
270 | 0 | This = NULL; |
271 | 0 | } |
272 | 0 | else if(This == NULL) { // test for NULL |
273 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
274 | 0 | } |
275 | 0 | return This; |
276 | 0 | } |
277 | | |
278 | 0 | RBBIDataHeader *RBBIRuleBuilder::build(UErrorCode &status) { |
279 | 0 | if (U_FAILURE(status)) { |
280 | 0 | return nullptr; |
281 | 0 | } |
282 | | |
283 | 0 | fScanner->parse(); |
284 | 0 | if (U_FAILURE(status)) { |
285 | 0 | return nullptr; |
286 | 0 | } |
287 | | |
288 | | // |
289 | | // UnicodeSet processing. |
290 | | // Munge the Unicode Sets to create an initial set of character categories. |
291 | | // |
292 | 0 | fSetBuilder->buildRanges(); |
293 | | |
294 | | // |
295 | | // Generate the DFA state transition table. |
296 | | // |
297 | 0 | fForwardTable = new RBBITableBuilder(this, &fForwardTree, status); |
298 | 0 | if (fForwardTable == nullptr) { |
299 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
300 | 0 | return nullptr; |
301 | 0 | } |
302 | | |
303 | 0 | fForwardTable->buildForwardTable(); |
304 | | |
305 | | // State table and character category optimization. |
306 | | // Merge equivalent rows and columns. |
307 | | // Note that this process alters the initial set of character categories, |
308 | | // causing the representation of UnicodeSets in the parse tree to become invalid. |
309 | |
|
310 | 0 | optimizeTables(); |
311 | 0 | fForwardTable->buildSafeReverseTable(status); |
312 | | |
313 | |
|
314 | | #ifdef RBBI_DEBUG |
315 | | if (fDebugEnv && uprv_strstr(fDebugEnv, "states")) { |
316 | | fForwardTable->printStates(); |
317 | | fForwardTable->printRuleStatusTable(); |
318 | | fForwardTable->printReverseTable(); |
319 | | } |
320 | | #endif |
321 | | |
322 | | // Generate the mapping tables (TRIE) from input code points to |
323 | | // the character categories. |
324 | | // |
325 | 0 | fSetBuilder->buildTrie(); |
326 | | |
327 | | // |
328 | | // Package up the compiled data into a memory image |
329 | | // in the run-time format. |
330 | | // |
331 | 0 | RBBIDataHeader *data = flattenData(); // returns NULL if error |
332 | 0 | if (U_FAILURE(status)) { |
333 | 0 | return nullptr; |
334 | 0 | } |
335 | 0 | return data; |
336 | 0 | } |
337 | | |
338 | 0 | void RBBIRuleBuilder::optimizeTables() { |
339 | 0 | bool didSomething; |
340 | 0 | do { |
341 | 0 | didSomething = false; |
342 | | |
343 | | // Begin looking for duplicates with char class 3. |
344 | | // Classes 0, 1 and 2 are special; they are unused, {bof} and {eof} respectively, |
345 | | // and should not have other categories merged into them. |
346 | 0 | IntPair duplPair = {3, 0}; |
347 | 0 | while (fForwardTable->findDuplCharClassFrom(&duplPair)) { |
348 | 0 | fSetBuilder->mergeCategories(duplPair); |
349 | 0 | fForwardTable->removeColumn(duplPair.second); |
350 | 0 | didSomething = true; |
351 | 0 | } |
352 | |
|
353 | 0 | while (fForwardTable->removeDuplicateStates() > 0) { |
354 | 0 | didSomething = true; |
355 | 0 | } |
356 | 0 | } while (didSomething); |
357 | 0 | } |
358 | | |
359 | | U_NAMESPACE_END |
360 | | |
361 | | #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |