/src/icu/icu4c/source/common/rbbi.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // © 2016 and later: Unicode, Inc. and others. |
2 | | // License & terms of use: http://www.unicode.org/copyright.html |
3 | | /* |
4 | | *************************************************************************** |
5 | | * Copyright (C) 1999-2016 International Business Machines Corporation |
6 | | * and others. All rights reserved. |
7 | | *************************************************************************** |
8 | | */ |
9 | | // |
10 | | // file: rbbi.cpp Contains the implementation of the rule based break iterator |
11 | | // runtime engine and the API implementation for |
12 | | // class RuleBasedBreakIterator |
13 | | // |
14 | | |
15 | | #include "utypeinfo.h" // for 'typeid' to work |
16 | | |
17 | | #include "unicode/utypes.h" |
18 | | |
19 | | #if !UCONFIG_NO_BREAK_ITERATION |
20 | | |
21 | | #include <cinttypes> |
22 | | |
23 | | #include "unicode/rbbi.h" |
24 | | #include "unicode/schriter.h" |
25 | | #include "unicode/uchriter.h" |
26 | | #include "unicode/uclean.h" |
27 | | #include "unicode/udata.h" |
28 | | |
29 | | #include "brkeng.h" |
30 | | #include "ucln_cmn.h" |
31 | | #include "cmemory.h" |
32 | | #include "cstring.h" |
33 | | #include "localsvc.h" |
34 | | #include "rbbidata.h" |
35 | | #include "rbbi_cache.h" |
36 | | #include "rbbirb.h" |
37 | | #include "uassert.h" |
38 | | #include "umutex.h" |
39 | | #include "uvectr32.h" |
40 | | |
41 | | #ifdef RBBI_DEBUG |
42 | | static UBool gTrace = false; |
43 | | #endif |
44 | | |
45 | | U_NAMESPACE_BEGIN |
46 | | |
47 | | // The state number of the starting state |
48 | | constexpr int32_t START_STATE = 1; |
49 | | |
50 | | // The state-transition value indicating "stop" |
51 | | constexpr int32_t STOP_STATE = 0; |
52 | | |
53 | | |
54 | | UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator) |
55 | | |
56 | | |
57 | | //======================================================================= |
58 | | // constructors |
59 | | //======================================================================= |
60 | | |
61 | | /** |
62 | | * Constructs a RuleBasedBreakIterator that uses the already-created |
63 | | * tables object that is passed in as a parameter. |
64 | | */ |
65 | | RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) |
66 | 0 | : RuleBasedBreakIterator(&status) |
67 | 0 | { |
68 | 0 | fData = new RBBIDataWrapper(data, status); // status checked in constructor |
69 | 0 | if (U_FAILURE(status)) {return;} |
70 | 0 | if(fData == nullptr) { |
71 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
72 | 0 | return; |
73 | 0 | } |
74 | 0 | if (fData->fForwardTable->fLookAheadResultsSize > 0) { |
75 | 0 | fLookAheadMatches = static_cast<int32_t *>( |
76 | 0 | uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
77 | 0 | if (fLookAheadMatches == nullptr) { |
78 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
79 | 0 | return; |
80 | 0 | } |
81 | 0 | } |
82 | 0 | } |
83 | | |
84 | | //------------------------------------------------------------------------------- |
85 | | // |
86 | | // Constructor from a UDataMemory handle to precompiled break rules |
87 | | // stored in an ICU data file. This construcotr is private API, |
88 | | // only for internal use. |
89 | | // |
90 | | //------------------------------------------------------------------------------- |
91 | | RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UBool isPhraseBreaking, |
92 | 3.61k | UErrorCode &status) : RuleBasedBreakIterator(udm, status) |
93 | 3.61k | { |
94 | 3.61k | fIsPhraseBreaking = isPhraseBreaking; |
95 | 3.61k | } |
96 | | |
97 | | // |
98 | | // Construct from precompiled binary rules (tables). This constructor is public API, |
99 | | // taking the rules as a (const uint8_t *) to match the type produced by getBinaryRules(). |
100 | | // |
101 | | RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules, |
102 | | uint32_t ruleLength, |
103 | | UErrorCode &status) |
104 | 0 | : RuleBasedBreakIterator(&status) |
105 | 0 | { |
106 | 0 | if (U_FAILURE(status)) { |
107 | 0 | return; |
108 | 0 | } |
109 | 0 | if (compiledRules == nullptr || ruleLength < sizeof(RBBIDataHeader)) { |
110 | 0 | status = U_ILLEGAL_ARGUMENT_ERROR; |
111 | 0 | return; |
112 | 0 | } |
113 | 0 | const RBBIDataHeader* data = reinterpret_cast<const RBBIDataHeader*>(compiledRules); |
114 | 0 | if (data->fLength > ruleLength) { |
115 | 0 | status = U_ILLEGAL_ARGUMENT_ERROR; |
116 | 0 | return; |
117 | 0 | } |
118 | 0 | fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status); |
119 | 0 | if (U_FAILURE(status)) {return;} |
120 | 0 | if(fData == nullptr) { |
121 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
122 | 0 | return; |
123 | 0 | } |
124 | 0 | if (fData->fForwardTable->fLookAheadResultsSize > 0) { |
125 | 0 | fLookAheadMatches = static_cast<int32_t *>( |
126 | 0 | uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
127 | 0 | if (fLookAheadMatches == nullptr) { |
128 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
129 | 0 | return; |
130 | 0 | } |
131 | 0 | } |
132 | 0 | } |
133 | | |
134 | | |
135 | | //------------------------------------------------------------------------------- |
136 | | // |
137 | | // Constructor from a UDataMemory handle to precompiled break rules |
138 | | // stored in an ICU data file. |
139 | | // |
140 | | //------------------------------------------------------------------------------- |
141 | | RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status) |
142 | 3.61k | : RuleBasedBreakIterator(&status) |
143 | 3.61k | { |
144 | 3.61k | fData = new RBBIDataWrapper(udm, status); // status checked in constructor |
145 | 3.61k | if (U_FAILURE(status)) {return;} |
146 | 3.61k | if(fData == nullptr) { |
147 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
148 | 0 | return; |
149 | 0 | } |
150 | 3.61k | if (fData->fForwardTable->fLookAheadResultsSize > 0) { |
151 | 0 | fLookAheadMatches = static_cast<int32_t *>( |
152 | 0 | uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
153 | 0 | if (fLookAheadMatches == nullptr) { |
154 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
155 | 0 | return; |
156 | 0 | } |
157 | 0 | } |
158 | 3.61k | } |
159 | | |
160 | | |
161 | | |
162 | | //------------------------------------------------------------------------------- |
163 | | // |
164 | | // Constructor from a set of rules supplied as a string. |
165 | | // |
166 | | //------------------------------------------------------------------------------- |
167 | | RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules, |
168 | | UParseError &parseError, |
169 | | UErrorCode &status) |
170 | 0 | : RuleBasedBreakIterator(&status) |
171 | 0 | { |
172 | 0 | if (U_FAILURE(status)) {return;} |
173 | 0 | RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *) |
174 | 0 | RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status); |
175 | | // Note: This is a bit awkward. The RBBI ruleBuilder has a factory method that |
176 | | // creates and returns a complete RBBI. From here, in a constructor, we |
177 | | // can't just return the object created by the builder factory, hence |
178 | | // the assignment of the factory created object to "this". |
179 | 0 | if (U_SUCCESS(status)) { |
180 | 0 | *this = *bi; |
181 | 0 | delete bi; |
182 | 0 | } |
183 | 0 | } |
184 | | |
185 | | |
186 | | //------------------------------------------------------------------------------- |
187 | | // |
188 | | // Default Constructor. Create an empty shell that can be set up later. |
189 | | // Used when creating a RuleBasedBreakIterator from a set |
190 | | // of rules. |
191 | | //------------------------------------------------------------------------------- |
192 | | RuleBasedBreakIterator::RuleBasedBreakIterator() |
193 | 0 | : RuleBasedBreakIterator(nullptr) |
194 | 0 | { |
195 | 0 | } |
196 | | |
197 | | /** |
198 | | * Simple Constructor with an error code. |
199 | | * Handles common initialization for all other constructors. |
200 | | */ |
201 | 3.61k | RuleBasedBreakIterator::RuleBasedBreakIterator(UErrorCode *status) { |
202 | 3.61k | UErrorCode ec = U_ZERO_ERROR; |
203 | 3.61k | if (status == nullptr) { |
204 | 0 | status = &ec; |
205 | 0 | } |
206 | 3.61k | utext_openUChars(&fText, nullptr, 0, status); |
207 | 3.61k | LocalPointer<DictionaryCache> lpDictionaryCache(new DictionaryCache(this, *status), *status); |
208 | 3.61k | LocalPointer<BreakCache> lpBreakCache(new BreakCache(this, *status), *status); |
209 | 3.61k | if (U_FAILURE(*status)) { |
210 | 0 | fErrorCode = *status; |
211 | 0 | return; |
212 | 0 | } |
213 | 3.61k | fDictionaryCache = lpDictionaryCache.orphan(); |
214 | 3.61k | fBreakCache = lpBreakCache.orphan(); |
215 | | |
216 | | #ifdef RBBI_DEBUG |
217 | | static UBool debugInitDone = false; |
218 | | if (debugInitDone == false) { |
219 | | char *debugEnv = getenv("U_RBBIDEBUG"); |
220 | | if (debugEnv && uprv_strstr(debugEnv, "trace")) { |
221 | | gTrace = true; |
222 | | } |
223 | | debugInitDone = true; |
224 | | } |
225 | | #endif |
226 | 3.61k | } |
227 | | |
228 | | |
229 | | //------------------------------------------------------------------------------- |
230 | | // |
231 | | // Copy constructor. Will produce a break iterator with the same behavior, |
232 | | // and which iterates over the same text, as the one passed in. |
233 | | // |
234 | | //------------------------------------------------------------------------------- |
235 | | RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other) |
236 | 0 | : RuleBasedBreakIterator() |
237 | 0 | { |
238 | 0 | *this = other; |
239 | 0 | } |
240 | | |
241 | | |
242 | | /** |
243 | | * Destructor |
244 | | */ |
245 | 3.61k | RuleBasedBreakIterator::~RuleBasedBreakIterator() { |
246 | 3.61k | if (fCharIter != &fSCharIter) { |
247 | | // fCharIter was adopted from the outside. |
248 | 0 | delete fCharIter; |
249 | 0 | } |
250 | 3.61k | fCharIter = nullptr; |
251 | | |
252 | 3.61k | utext_close(&fText); |
253 | | |
254 | 3.61k | if (fData != nullptr) { |
255 | 3.61k | fData->removeReference(); |
256 | 3.61k | fData = nullptr; |
257 | 3.61k | } |
258 | 3.61k | delete fBreakCache; |
259 | 3.61k | fBreakCache = nullptr; |
260 | | |
261 | 3.61k | delete fDictionaryCache; |
262 | 3.61k | fDictionaryCache = nullptr; |
263 | | |
264 | 3.61k | delete fLanguageBreakEngines; |
265 | 3.61k | fLanguageBreakEngines = nullptr; |
266 | | |
267 | 3.61k | delete fUnhandledBreakEngine; |
268 | 3.61k | fUnhandledBreakEngine = nullptr; |
269 | | |
270 | 3.61k | uprv_free(fLookAheadMatches); |
271 | 3.61k | fLookAheadMatches = nullptr; |
272 | 3.61k | } |
273 | | |
274 | | /** |
275 | | * Assignment operator. Sets this iterator to have the same behavior, |
276 | | * and iterate over the same text, as the one passed in. |
277 | | * TODO: needs better handling of memory allocation errors. |
278 | | */ |
279 | | RuleBasedBreakIterator& |
280 | 0 | RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) { |
281 | 0 | if (this == &that) { |
282 | 0 | return *this; |
283 | 0 | } |
284 | 0 | BreakIterator::operator=(that); |
285 | |
|
286 | 0 | if (fLanguageBreakEngines != nullptr) { |
287 | 0 | delete fLanguageBreakEngines; |
288 | 0 | fLanguageBreakEngines = nullptr; // Just rebuild for now |
289 | 0 | } |
290 | | // TODO: clone fLanguageBreakEngines from "that" |
291 | 0 | UErrorCode status = U_ZERO_ERROR; |
292 | 0 | utext_clone(&fText, &that.fText, false, true, &status); |
293 | |
|
294 | 0 | if (fCharIter != &fSCharIter) { |
295 | 0 | delete fCharIter; |
296 | 0 | } |
297 | 0 | fCharIter = &fSCharIter; |
298 | |
|
299 | 0 | if (that.fCharIter != nullptr && that.fCharIter != &that.fSCharIter) { |
300 | | // This is a little bit tricky - it will initially appear that |
301 | | // this->fCharIter is adopted, even if that->fCharIter was |
302 | | // not adopted. That's ok. |
303 | 0 | fCharIter = that.fCharIter->clone(); |
304 | 0 | } |
305 | 0 | fSCharIter = that.fSCharIter; |
306 | 0 | if (fCharIter == nullptr) { |
307 | 0 | fCharIter = &fSCharIter; |
308 | 0 | } |
309 | |
|
310 | 0 | if (fData != nullptr) { |
311 | 0 | fData->removeReference(); |
312 | 0 | fData = nullptr; |
313 | 0 | } |
314 | 0 | if (that.fData != nullptr) { |
315 | 0 | fData = that.fData->addReference(); |
316 | 0 | } |
317 | |
|
318 | 0 | uprv_free(fLookAheadMatches); |
319 | 0 | fLookAheadMatches = nullptr; |
320 | 0 | if (fData && fData->fForwardTable->fLookAheadResultsSize > 0) { |
321 | 0 | fLookAheadMatches = static_cast<int32_t *>( |
322 | 0 | uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t))); |
323 | 0 | } |
324 | | |
325 | |
|
326 | 0 | fPosition = that.fPosition; |
327 | 0 | fRuleStatusIndex = that.fRuleStatusIndex; |
328 | 0 | fDone = that.fDone; |
329 | | |
330 | | // TODO: both the dictionary and the main cache need to be copied. |
331 | | // Current position could be within a dictionary range. Trying to continue |
332 | | // the iteration without the caches present would go to the rules, with |
333 | | // the assumption that the current position is on a rule boundary. |
334 | 0 | fBreakCache->reset(fPosition, fRuleStatusIndex); |
335 | 0 | fDictionaryCache->reset(); |
336 | |
|
337 | 0 | return *this; |
338 | 0 | } |
339 | | |
340 | | //----------------------------------------------------------------------------- |
341 | | // |
342 | | // clone - Returns a newly-constructed RuleBasedBreakIterator with the same |
343 | | // behavior, and iterating over the same text, as this one. |
344 | | // Virtual function: does the right thing with subclasses. |
345 | | // |
346 | | //----------------------------------------------------------------------------- |
347 | | RuleBasedBreakIterator* |
348 | 0 | RuleBasedBreakIterator::clone() const { |
349 | 0 | return new RuleBasedBreakIterator(*this); |
350 | 0 | } |
351 | | |
352 | | /** |
353 | | * Equality operator. Returns true if both BreakIterators are of the |
354 | | * same class, have the same behavior, and iterate over the same text. |
355 | | */ |
356 | | bool |
357 | 0 | RuleBasedBreakIterator::operator==(const BreakIterator& that) const { |
358 | 0 | if (typeid(*this) != typeid(that)) { |
359 | 0 | return false; |
360 | 0 | } |
361 | 0 | if (this == &that) { |
362 | 0 | return true; |
363 | 0 | } |
364 | | |
365 | | // The base class BreakIterator carries no state that participates in equality, |
366 | | // and does not implement an equality function that would otherwise be |
367 | | // checked at this point. |
368 | | |
369 | 0 | const RuleBasedBreakIterator& that2 = static_cast<const RuleBasedBreakIterator&>(that); |
370 | |
|
371 | 0 | if (!utext_equals(&fText, &that2.fText)) { |
372 | | // The two break iterators are operating on different text, |
373 | | // or have a different iteration position. |
374 | | // Note that fText's position is always the same as the break iterator's position. |
375 | 0 | return false; |
376 | 0 | } |
377 | | |
378 | 0 | if (!(fPosition == that2.fPosition && |
379 | 0 | fRuleStatusIndex == that2.fRuleStatusIndex && |
380 | 0 | fDone == that2.fDone)) { |
381 | 0 | return false; |
382 | 0 | } |
383 | | |
384 | 0 | if (that2.fData == fData || |
385 | 0 | (fData != nullptr && that2.fData != nullptr && *that2.fData == *fData)) { |
386 | | // The two break iterators are using the same rules. |
387 | 0 | return true; |
388 | 0 | } |
389 | 0 | return false; |
390 | 0 | } |
391 | | |
392 | | /** |
393 | | * Compute a hash code for this BreakIterator |
394 | | * @return A hash code |
395 | | */ |
396 | | int32_t |
397 | 0 | RuleBasedBreakIterator::hashCode() const { |
398 | 0 | int32_t hash = 0; |
399 | 0 | if (fData != nullptr) { |
400 | 0 | hash = fData->hashCode(); |
401 | 0 | } |
402 | 0 | return hash; |
403 | 0 | } |
404 | | |
405 | | |
406 | 3.61k | void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) { |
407 | 3.61k | if (U_FAILURE(status)) { |
408 | 0 | return; |
409 | 0 | } |
410 | 3.61k | fBreakCache->reset(); |
411 | 3.61k | fDictionaryCache->reset(); |
412 | 3.61k | utext_clone(&fText, ut, false, true, &status); |
413 | | |
414 | | // Set up a dummy CharacterIterator to be returned if anyone |
415 | | // calls getText(). With input from UText, there is no reasonable |
416 | | // way to return a characterIterator over the actual input text. |
417 | | // Return one over an empty string instead - this is the closest |
418 | | // we can come to signaling a failure. |
419 | | // (GetText() is obsolete, this failure is sort of OK) |
420 | 3.61k | fSCharIter.setText(u"", 0); |
421 | | |
422 | 3.61k | if (fCharIter != &fSCharIter) { |
423 | | // existing fCharIter was adopted from the outside. Delete it now. |
424 | 0 | delete fCharIter; |
425 | 0 | } |
426 | 3.61k | fCharIter = &fSCharIter; |
427 | | |
428 | 3.61k | this->first(); |
429 | 3.61k | } |
430 | | |
431 | | |
432 | 0 | UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const { |
433 | 0 | UText *result = utext_clone(fillIn, &fText, false, true, &status); |
434 | 0 | return result; |
435 | 0 | } |
436 | | |
437 | | |
438 | | //======================================================================= |
439 | | // BreakIterator overrides |
440 | | //======================================================================= |
441 | | |
442 | | /** |
443 | | * Return a CharacterIterator over the text being analyzed. |
444 | | */ |
445 | | CharacterIterator& |
446 | 0 | RuleBasedBreakIterator::getText() const { |
447 | 0 | return *fCharIter; |
448 | 0 | } |
449 | | |
450 | | /** |
451 | | * Set the iterator to analyze a new piece of text. This function resets |
452 | | * the current iteration position to the beginning of the text. |
453 | | * @param newText An iterator over the text to analyze. |
454 | | */ |
455 | | void |
456 | 0 | RuleBasedBreakIterator::adoptText(CharacterIterator* newText) { |
457 | | // If we are holding a CharacterIterator adopted from a |
458 | | // previous call to this function, delete it now. |
459 | 0 | if (fCharIter != &fSCharIter) { |
460 | 0 | delete fCharIter; |
461 | 0 | } |
462 | |
|
463 | 0 | fCharIter = newText; |
464 | 0 | UErrorCode status = U_ZERO_ERROR; |
465 | 0 | fBreakCache->reset(); |
466 | 0 | fDictionaryCache->reset(); |
467 | 0 | if (newText==nullptr || newText->startIndex() != 0) { |
468 | | // startIndex !=0 wants to be an error, but there's no way to report it. |
469 | | // Make the iterator text be an empty string. |
470 | 0 | utext_openUChars(&fText, nullptr, 0, &status); |
471 | 0 | } else { |
472 | 0 | utext_openCharacterIterator(&fText, newText, &status); |
473 | 0 | } |
474 | 0 | this->first(); |
475 | 0 | } |
476 | | |
477 | | /** |
478 | | * Set the iterator to analyze a new piece of text. This function resets |
479 | | * the current iteration position to the beginning of the text. |
480 | | * @param newText An iterator over the text to analyze. |
481 | | */ |
482 | | void |
483 | 0 | RuleBasedBreakIterator::setText(const UnicodeString& newText) { |
484 | 0 | UErrorCode status = U_ZERO_ERROR; |
485 | 0 | fBreakCache->reset(); |
486 | 0 | fDictionaryCache->reset(); |
487 | 0 | utext_openConstUnicodeString(&fText, &newText, &status); |
488 | | |
489 | | // Set up a character iterator on the string. |
490 | | // Needed in case someone calls getText(). |
491 | | // Can not, unfortunately, do this lazily on the (probably never) |
492 | | // call to getText(), because getText is const. |
493 | 0 | fSCharIter.setText(newText.getBuffer(), newText.length()); |
494 | |
|
495 | 0 | if (fCharIter != &fSCharIter) { |
496 | | // old fCharIter was adopted from the outside. Delete it. |
497 | 0 | delete fCharIter; |
498 | 0 | } |
499 | 0 | fCharIter = &fSCharIter; |
500 | |
|
501 | 0 | this->first(); |
502 | 0 | } |
503 | | |
504 | | |
505 | | /** |
506 | | * Provide a new UText for the input text. Must reference text with contents identical |
507 | | * to the original. |
508 | | * Intended for use with text data originating in Java (garbage collected) environments |
509 | | * where the data may be moved in memory at arbitrary times. |
510 | | */ |
511 | 0 | RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, UErrorCode &status) { |
512 | 0 | if (U_FAILURE(status)) { |
513 | 0 | return *this; |
514 | 0 | } |
515 | 0 | if (input == nullptr) { |
516 | 0 | status = U_ILLEGAL_ARGUMENT_ERROR; |
517 | 0 | return *this; |
518 | 0 | } |
519 | 0 | int64_t pos = utext_getNativeIndex(&fText); |
520 | | // Shallow read-only clone of the new UText into the existing input UText |
521 | 0 | utext_clone(&fText, input, false, true, &status); |
522 | 0 | if (U_FAILURE(status)) { |
523 | 0 | return *this; |
524 | 0 | } |
525 | 0 | utext_setNativeIndex(&fText, pos); |
526 | 0 | if (utext_getNativeIndex(&fText) != pos) { |
527 | | // Sanity check. The new input utext is supposed to have the exact same |
528 | | // contents as the old. If we can't set to the same position, it doesn't. |
529 | | // The contents underlying the old utext might be invalid at this point, |
530 | | // so it's not safe to check directly. |
531 | 0 | status = U_ILLEGAL_ARGUMENT_ERROR; |
532 | 0 | } |
533 | 0 | return *this; |
534 | 0 | } |
535 | | |
536 | | |
537 | | /** |
538 | | * Sets the current iteration position to the beginning of the text, position zero. |
539 | | * @return The new iterator position, which is zero. |
540 | | */ |
541 | 7.21k | int32_t RuleBasedBreakIterator::first() { |
542 | 7.21k | UErrorCode status = U_ZERO_ERROR; |
543 | 7.21k | if (!fBreakCache->seek(0)) { |
544 | 0 | fBreakCache->populateNear(0, status); |
545 | 0 | } |
546 | 7.21k | fBreakCache->current(); |
547 | 7.21k | U_ASSERT(fPosition == 0); |
548 | 7.21k | return 0; |
549 | 7.21k | } |
550 | | |
551 | | /** |
552 | | * Sets the current iteration position to the end of the text. |
553 | | * @return The text's past-the-end offset. |
554 | | */ |
555 | 0 | int32_t RuleBasedBreakIterator::last() { |
556 | 0 | int32_t endPos = static_cast<int32_t>(utext_nativeLength(&fText)); |
557 | 0 | UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position. |
558 | 0 | (void)endShouldBeBoundary; |
559 | 0 | U_ASSERT(endShouldBeBoundary); |
560 | 0 | U_ASSERT(fPosition == endPos); |
561 | 0 | return endPos; |
562 | 0 | } |
563 | | |
564 | | /** |
565 | | * Advances the iterator either forward or backward the specified number of steps. |
566 | | * Negative values move backward, and positive values move forward. This is |
567 | | * equivalent to repeatedly calling next() or previous(). |
568 | | * @param n The number of steps to move. The sign indicates the direction |
569 | | * (negative is backwards, and positive is forwards). |
570 | | * @return The character offset of the boundary position n boundaries away from |
571 | | * the current one. |
572 | | */ |
573 | 0 | int32_t RuleBasedBreakIterator::next(int32_t n) { |
574 | 0 | int32_t result = 0; |
575 | 0 | if (n > 0) { |
576 | 0 | for (; n > 0 && result != UBRK_DONE; --n) { |
577 | 0 | result = next(); |
578 | 0 | } |
579 | 0 | } else if (n < 0) { |
580 | 0 | for (; n < 0 && result != UBRK_DONE; ++n) { |
581 | 0 | result = previous(); |
582 | 0 | } |
583 | 0 | } else { |
584 | 0 | result = current(); |
585 | 0 | } |
586 | 0 | return result; |
587 | 0 | } |
588 | | |
589 | | /** |
590 | | * Advances the iterator to the next boundary position. |
591 | | * @return The position of the first boundary after this one. |
592 | | */ |
593 | 31.7M | int32_t RuleBasedBreakIterator::next() { |
594 | 31.7M | fBreakCache->next(); |
595 | 31.7M | return fDone ? UBRK_DONE : fPosition; |
596 | 31.7M | } |
597 | | |
598 | | /** |
599 | | * Move the iterator backwards, to the boundary preceding the current one. |
600 | | * |
601 | | * Starts from the current position within fText. |
602 | | * Starting position need not be on a boundary. |
603 | | * |
604 | | * @return The position of the boundary position immediately preceding the starting position. |
605 | | */ |
606 | 0 | int32_t RuleBasedBreakIterator::previous() { |
607 | 0 | UErrorCode status = U_ZERO_ERROR; |
608 | 0 | fBreakCache->previous(status); |
609 | 0 | return fDone ? UBRK_DONE : fPosition; |
610 | 0 | } |
611 | | |
612 | | /** |
613 | | * Sets the iterator to refer to the first boundary position following |
614 | | * the specified position. |
615 | | * @param startPos The position from which to begin searching for a break position. |
616 | | * @return The position of the first break after the current position. |
617 | | */ |
618 | 0 | int32_t RuleBasedBreakIterator::following(int32_t startPos) { |
619 | | // if the supplied position is before the beginning, return the |
620 | | // text's starting offset |
621 | 0 | if (startPos < 0) { |
622 | 0 | return first(); |
623 | 0 | } |
624 | | |
625 | | // Move requested offset to a code point start. It might be on a trail surrogate, |
626 | | // or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text. |
627 | 0 | utext_setNativeIndex(&fText, startPos); |
628 | 0 | startPos = static_cast<int32_t>(utext_getNativeIndex(&fText)); |
629 | |
|
630 | 0 | UErrorCode status = U_ZERO_ERROR; |
631 | 0 | fBreakCache->following(startPos, status); |
632 | 0 | return fDone ? UBRK_DONE : fPosition; |
633 | 0 | } |
634 | | |
635 | | /** |
636 | | * Sets the iterator to refer to the last boundary position before the |
637 | | * specified position. |
638 | | * @param offset The position to begin searching for a break from. |
639 | | * @return The position of the last boundary before the starting position. |
640 | | */ |
641 | 0 | int32_t RuleBasedBreakIterator::preceding(int32_t offset) { |
642 | 0 | if (offset > utext_nativeLength(&fText)) { |
643 | 0 | return last(); |
644 | 0 | } |
645 | | |
646 | | // Move requested offset to a code point start. It might be on a trail surrogate, |
647 | | // or on a trail byte if the input is UTF-8. |
648 | | |
649 | 0 | utext_setNativeIndex(&fText, offset); |
650 | 0 | int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText)); |
651 | |
|
652 | 0 | UErrorCode status = U_ZERO_ERROR; |
653 | 0 | fBreakCache->preceding(adjustedOffset, status); |
654 | 0 | return fDone ? UBRK_DONE : fPosition; |
655 | 0 | } |
656 | | |
657 | | /** |
658 | | * Returns true if the specified position is a boundary position. As a side |
659 | | * effect, leaves the iterator pointing to the first boundary position at |
660 | | * or after "offset". |
661 | | * |
662 | | * @param offset the offset to check. |
663 | | * @return True if "offset" is a boundary position. |
664 | | */ |
665 | 0 | UBool RuleBasedBreakIterator::isBoundary(int32_t offset) { |
666 | | // out-of-range indexes are never boundary positions |
667 | 0 | if (offset < 0) { |
668 | 0 | first(); // For side effects on current position, tag values. |
669 | 0 | return false; |
670 | 0 | } |
671 | | |
672 | | // Adjust offset to be on a code point boundary and not beyond the end of the text. |
673 | | // Note that isBoundary() is always false for offsets that are not on code point boundaries. |
674 | | // But we still need the side effect of leaving iteration at the following boundary. |
675 | | |
676 | 0 | utext_setNativeIndex(&fText, offset); |
677 | 0 | int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText)); |
678 | |
|
679 | 0 | bool result = false; |
680 | 0 | UErrorCode status = U_ZERO_ERROR; |
681 | 0 | if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) { |
682 | 0 | result = (fBreakCache->current() == offset); |
683 | 0 | } |
684 | |
|
685 | 0 | if (result && adjustedOffset < offset && utext_char32At(&fText, offset) == U_SENTINEL) { |
686 | | // Original offset is beyond the end of the text. Return false, it's not a boundary, |
687 | | // but the iteration position remains set to the end of the text, which is a boundary. |
688 | 0 | return false; |
689 | 0 | } |
690 | 0 | if (!result) { |
691 | | // Not on a boundary. isBoundary() must leave iterator on the following boundary. |
692 | | // Cache->seek(), above, left us on the preceding boundary, so advance one. |
693 | 0 | next(); |
694 | 0 | } |
695 | 0 | return result; |
696 | 0 | } |
697 | | |
698 | | |
699 | | /** |
700 | | * Returns the current iteration position. |
701 | | * @return The current iteration position. |
702 | | */ |
703 | 0 | int32_t RuleBasedBreakIterator::current() const { |
704 | 0 | return fPosition; |
705 | 0 | } |
706 | | |
707 | | |
708 | | //======================================================================= |
709 | | // implementation |
710 | | //======================================================================= |
711 | | |
712 | | // |
713 | | // RBBIRunMode - the state machine runs an extra iteration at the beginning and end |
714 | | // of user text. A variable with this enum type keeps track of where we |
715 | | // are. The state machine only fetches user input while in the RUN mode. |
716 | | // |
717 | | enum RBBIRunMode { |
718 | | RBBI_START, // state machine processing is before first char of input |
719 | | RBBI_RUN, // state machine processing is in the user text |
720 | | RBBI_END // state machine processing is after end of user text. |
721 | | }; |
722 | | |
723 | | |
724 | | // Wrapper functions to select the appropriate handleNext() or handleSafePrevious() |
725 | | // instantiation, based on whether an 8 or 16 bit table is required. |
726 | | // |
727 | | // These Trie access functions will be inlined within the handleNext()/Previous() instantions. |
728 | 83.1M | static inline uint16_t TrieFunc8(const UCPTrie *trie, UChar32 c) { |
729 | 83.1M | return UCPTRIE_FAST_GET(trie, UCPTRIE_8, c); |
730 | 83.1M | } |
731 | | |
732 | 0 | static inline uint16_t TrieFunc16(const UCPTrie *trie, UChar32 c) { |
733 | 0 | return UCPTRIE_FAST_GET(trie, UCPTRIE_16, c); |
734 | 0 | } |
735 | | |
736 | 18.1M | int32_t RuleBasedBreakIterator::handleNext() { |
737 | 18.1M | const RBBIStateTable *statetable = fData->fForwardTable; |
738 | 18.1M | bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8; |
739 | 18.1M | if (statetable->fFlags & RBBI_8BITS_ROWS) { |
740 | 18.1M | if (use8BitsTrie) { |
741 | 18.1M | return handleNext<RBBIStateTableRow8, TrieFunc8>(); |
742 | 18.1M | } else { |
743 | 0 | return handleNext<RBBIStateTableRow8, TrieFunc16>(); |
744 | 0 | } |
745 | 18.1M | } else { |
746 | 0 | if (use8BitsTrie) { |
747 | 0 | return handleNext<RBBIStateTableRow16, TrieFunc8>(); |
748 | 0 | } else { |
749 | 0 | return handleNext<RBBIStateTableRow16, TrieFunc16>(); |
750 | 0 | } |
751 | 0 | } |
752 | 18.1M | } |
753 | | |
754 | 0 | int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) { |
755 | 0 | const RBBIStateTable *statetable = fData->fReverseTable; |
756 | 0 | bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8; |
757 | 0 | if (statetable->fFlags & RBBI_8BITS_ROWS) { |
758 | 0 | if (use8BitsTrie) { |
759 | 0 | return handleSafePrevious<RBBIStateTableRow8, TrieFunc8>(fromPosition); |
760 | 0 | } else { |
761 | 0 | return handleSafePrevious<RBBIStateTableRow8, TrieFunc16>(fromPosition); |
762 | 0 | } |
763 | 0 | } else { |
764 | 0 | if (use8BitsTrie) { |
765 | 0 | return handleSafePrevious<RBBIStateTableRow16, TrieFunc8>(fromPosition); |
766 | 0 | } else { |
767 | 0 | return handleSafePrevious<RBBIStateTableRow16, TrieFunc16>(fromPosition); |
768 | 0 | } |
769 | 0 | } |
770 | 0 | } |
771 | | |
772 | | |
773 | | //----------------------------------------------------------------------------------- |
774 | | // |
775 | | // handleNext() |
776 | | // Run the state machine to find a boundary |
777 | | // |
778 | | //----------------------------------------------------------------------------------- |
779 | | template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc> |
780 | 18.1M | int32_t RuleBasedBreakIterator::handleNext() { |
781 | 18.1M | int32_t state; |
782 | 18.1M | uint16_t category = 0; |
783 | 18.1M | RBBIRunMode mode; |
784 | | |
785 | 18.1M | RowType *row; |
786 | 18.1M | UChar32 c; |
787 | 18.1M | int32_t result = 0; |
788 | 18.1M | int32_t initialPosition = 0; |
789 | 18.1M | const RBBIStateTable *statetable = fData->fForwardTable; |
790 | 18.1M | const char *tableData = statetable->fTableData; |
791 | 18.1M | uint32_t tableRowLen = statetable->fRowLen; |
792 | 18.1M | uint32_t dictStart = statetable->fDictCategoriesStart; |
793 | | #ifdef RBBI_DEBUG |
794 | | if (gTrace) { |
795 | | RBBIDebugPuts("Handle Next pos char state category"); |
796 | | } |
797 | | #endif |
798 | | |
799 | | // handleNext always sets the break tag value. |
800 | | // Set the default for it. |
801 | 18.1M | fRuleStatusIndex = 0; |
802 | | |
803 | 18.1M | fDictionaryCharCount = 0; |
804 | | |
805 | | // if we're already at the end of the text, return DONE. |
806 | 18.1M | initialPosition = fPosition; |
807 | 18.1M | UTEXT_SETNATIVEINDEX(&fText, initialPosition); |
808 | 18.1M | result = initialPosition; |
809 | 18.1M | c = UTEXT_NEXT32(&fText); |
810 | 18.1M | if (c==U_SENTINEL) { |
811 | 2.27k | fDone = true; |
812 | 2.27k | return UBRK_DONE; |
813 | 2.27k | } |
814 | | |
815 | | // Set the initial state for the state machine |
816 | 18.1M | state = START_STATE; |
817 | 18.1M | row = (RowType *) |
818 | | //(statetable->fTableData + (statetable->fRowLen * state)); |
819 | 18.1M | (tableData + tableRowLen * state); |
820 | | |
821 | | |
822 | 18.1M | mode = RBBI_RUN; |
823 | 18.1M | if (statetable->fFlags & RBBI_BOF_REQUIRED) { |
824 | 1.15M | category = 2; |
825 | 1.15M | mode = RBBI_START; |
826 | 1.15M | } |
827 | | |
828 | | |
829 | | // loop until we reach the end of the text or transition to state 0 |
830 | | // |
831 | 84.3M | for (;;) { |
832 | 84.3M | if (c == U_SENTINEL) { |
833 | | // Reached end of input string. |
834 | 5.02k | if (mode == RBBI_END) { |
835 | | // We have already run the loop one last time with the |
836 | | // character set to the psueudo {eof} value. Now it is time |
837 | | // to unconditionally bail out. |
838 | 333 | break; |
839 | 333 | } |
840 | | // Run the loop one last time with the fake end-of-input character category. |
841 | 4.68k | mode = RBBI_END; |
842 | 4.68k | category = 1; |
843 | 4.68k | } |
844 | | |
845 | | // |
846 | | // Get the char category. An incoming category of 1 or 2 means that |
847 | | // we are preset for doing the beginning or end of input, and |
848 | | // that we shouldn't get a category from an actual text input character. |
849 | | // |
850 | 84.3M | if (mode == RBBI_RUN) { |
851 | | // look up the current character's character category, which tells us |
852 | | // which column in the state table to look at. |
853 | 83.1M | category = trieFunc(fData->fTrie, c); |
854 | 83.1M | fDictionaryCharCount += (category >= dictStart); |
855 | 83.1M | } |
856 | | |
857 | | #ifdef RBBI_DEBUG |
858 | | if (gTrace) { |
859 | | RBBIDebugPrintf(" %4" PRId64 " ", utext_getNativeIndex(&fText)); |
860 | | if (0x20<=c && c<0x7f) { |
861 | | RBBIDebugPrintf("\"%c\" ", c); |
862 | | } else { |
863 | | RBBIDebugPrintf("%5x ", c); |
864 | | } |
865 | | RBBIDebugPrintf("%3d %3d\n", state, category); |
866 | | } |
867 | | #endif |
868 | | |
869 | | // State Transition - move machine to its next state |
870 | | // |
871 | | |
872 | | // fNextState is a variable-length array. |
873 | 84.3M | U_ASSERT(category<fData->fHeader->fCatCount); |
874 | 84.3M | state = row->fNextState[category]; /*Not accessing beyond memory*/ |
875 | 84.3M | row = (RowType *) |
876 | | // (statetable->fTableData + (statetable->fRowLen * state)); |
877 | 84.3M | (tableData + tableRowLen * state); |
878 | | |
879 | | |
880 | 84.3M | uint16_t accepting = row->fAccepting; |
881 | 84.3M | if (accepting == ACCEPTING_UNCONDITIONAL) { |
882 | | // Match found, common case. |
883 | 66.0M | if (mode != RBBI_START) { |
884 | 64.8M | result = static_cast<int32_t>(UTEXT_GETNATIVEINDEX(&fText)); |
885 | 64.8M | } |
886 | 66.0M | fRuleStatusIndex = row->fTagsIdx; // Remember the break status (tag) values. |
887 | 66.0M | } else if (accepting > ACCEPTING_UNCONDITIONAL) { |
888 | | // Lookahead match is completed. |
889 | 0 | U_ASSERT(accepting < fData->fForwardTable->fLookAheadResultsSize); |
890 | 0 | int32_t lookaheadResult = fLookAheadMatches[accepting]; |
891 | 0 | if (lookaheadResult >= 0) { |
892 | 0 | fRuleStatusIndex = row->fTagsIdx; |
893 | 0 | fPosition = lookaheadResult; |
894 | 0 | return lookaheadResult; |
895 | 0 | } |
896 | 0 | } |
897 | | |
898 | | // If we are at the position of the '/' in a look-ahead (hard break) rule; |
899 | | // record the current position, to be returned later, if the full rule matches. |
900 | | // TODO: Move this check before the previous check of fAccepting. |
901 | | // This would enable hard-break rules with no following context. |
902 | | // But there are line break test failures when trying this. Investigate. |
903 | | // Issue ICU-20837 |
904 | 84.3M | uint16_t rule = row->fLookAhead; |
905 | 84.3M | U_ASSERT(rule == 0 || rule > ACCEPTING_UNCONDITIONAL); |
906 | 84.3M | U_ASSERT(rule == 0 || rule < fData->fForwardTable->fLookAheadResultsSize); |
907 | 84.3M | if (rule > ACCEPTING_UNCONDITIONAL) { |
908 | 0 | int32_t pos = static_cast<int32_t>(UTEXT_GETNATIVEINDEX(&fText)); |
909 | 0 | fLookAheadMatches[rule] = pos; |
910 | 0 | } |
911 | | |
912 | 84.3M | if (state == STOP_STATE) { |
913 | | // This is the normal exit from the lookup state machine. |
914 | | // We have advanced through the string until it is certain that no |
915 | | // longer match is possible, no matter what characters follow. |
916 | 18.1M | break; |
917 | 18.1M | } |
918 | | |
919 | | // Advance to the next character. |
920 | | // If this is a beginning-of-input loop iteration, don't advance |
921 | | // the input position. The next iteration will be processing the |
922 | | // first real input character. |
923 | 66.1M | if (mode == RBBI_RUN) { |
924 | 65.0M | c = UTEXT_NEXT32(&fText); |
925 | 65.0M | } else { |
926 | 1.15M | if (mode == RBBI_START) { |
927 | 1.15M | mode = RBBI_RUN; |
928 | 1.15M | } |
929 | 1.15M | } |
930 | 66.1M | } |
931 | | |
932 | | // The state machine is done. Check whether it found a match... |
933 | | |
934 | | // If the iterator failed to advance in the match engine, force it ahead by one. |
935 | | // (This really indicates a defect in the break rules. They should always match |
936 | | // at least one character.) |
937 | 18.1M | if (result == initialPosition) { |
938 | 0 | utext_setNativeIndex(&fText, initialPosition); |
939 | 0 | utext_next32(&fText); |
940 | 0 | result = static_cast<int32_t>(utext_getNativeIndex(&fText)); |
941 | 0 | fRuleStatusIndex = 0; |
942 | 0 | } |
943 | | |
944 | | // Leave the iterator at our result position. |
945 | 18.1M | fPosition = result; |
946 | | #ifdef RBBI_DEBUG |
947 | | if (gTrace) { |
948 | | RBBIDebugPrintf("result = %d\n\n", result); |
949 | | } |
950 | | #endif |
951 | 18.1M | return result; |
952 | 18.1M | } rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleNext<icu_78::RBBIStateTableRowT<unsigned char>, &icu_78::TrieFunc8>() Line | Count | Source | 780 | 18.1M | int32_t RuleBasedBreakIterator::handleNext() { | 781 | 18.1M | int32_t state; | 782 | 18.1M | uint16_t category = 0; | 783 | 18.1M | RBBIRunMode mode; | 784 | | | 785 | 18.1M | RowType *row; | 786 | 18.1M | UChar32 c; | 787 | 18.1M | int32_t result = 0; | 788 | 18.1M | int32_t initialPosition = 0; | 789 | 18.1M | const RBBIStateTable *statetable = fData->fForwardTable; | 790 | 18.1M | const char *tableData = statetable->fTableData; | 791 | 18.1M | uint32_t tableRowLen = statetable->fRowLen; | 792 | 18.1M | uint32_t dictStart = statetable->fDictCategoriesStart; | 793 | | #ifdef RBBI_DEBUG | 794 | | if (gTrace) { | 795 | | RBBIDebugPuts("Handle Next pos char state category"); | 796 | | } | 797 | | #endif | 798 | | | 799 | | // handleNext always sets the break tag value. | 800 | | // Set the default for it. | 801 | 18.1M | fRuleStatusIndex = 0; | 802 | | | 803 | 18.1M | fDictionaryCharCount = 0; | 804 | | | 805 | | // if we're already at the end of the text, return DONE. | 806 | 18.1M | initialPosition = fPosition; | 807 | 18.1M | UTEXT_SETNATIVEINDEX(&fText, initialPosition); | 808 | 18.1M | result = initialPosition; | 809 | 18.1M | c = UTEXT_NEXT32(&fText); | 810 | 18.1M | if (c==U_SENTINEL) { | 811 | 2.27k | fDone = true; | 812 | 2.27k | return UBRK_DONE; | 813 | 2.27k | } | 814 | | | 815 | | // Set the initial state for the state machine | 816 | 18.1M | state = START_STATE; | 817 | 18.1M | row = (RowType *) | 818 | | //(statetable->fTableData + (statetable->fRowLen * state)); | 819 | 18.1M | (tableData + tableRowLen * state); | 820 | | | 821 | | | 822 | 18.1M | mode = RBBI_RUN; | 823 | 18.1M | if (statetable->fFlags & RBBI_BOF_REQUIRED) { | 824 | 1.15M | category = 2; | 825 | 1.15M | mode = RBBI_START; | 826 | 1.15M | } | 827 | | | 828 | | | 829 | | // loop until we reach the end of the text or transition to state 0 | 830 | | // | 831 | 84.3M | for (;;) { | 832 | 84.3M | if (c == U_SENTINEL) { | 833 | | // Reached end of input string. | 834 | 5.02k | if (mode == RBBI_END) { | 835 | | // We have already run the loop one last time with the | 836 | | // character set to the psueudo {eof} value. Now it is time | 837 | | // to unconditionally bail out. | 838 | 333 | break; | 839 | 333 | } | 840 | | // Run the loop one last time with the fake end-of-input character category. | 841 | 4.68k | mode = RBBI_END; | 842 | 4.68k | category = 1; | 843 | 4.68k | } | 844 | | | 845 | | // | 846 | | // Get the char category. An incoming category of 1 or 2 means that | 847 | | // we are preset for doing the beginning or end of input, and | 848 | | // that we shouldn't get a category from an actual text input character. | 849 | | // | 850 | 84.3M | if (mode == RBBI_RUN) { | 851 | | // look up the current character's character category, which tells us | 852 | | // which column in the state table to look at. | 853 | 83.1M | category = trieFunc(fData->fTrie, c); | 854 | 83.1M | fDictionaryCharCount += (category >= dictStart); | 855 | 83.1M | } | 856 | | | 857 | | #ifdef RBBI_DEBUG | 858 | | if (gTrace) { | 859 | | RBBIDebugPrintf(" %4" PRId64 " ", utext_getNativeIndex(&fText)); | 860 | | if (0x20<=c && c<0x7f) { | 861 | | RBBIDebugPrintf("\"%c\" ", c); | 862 | | } else { | 863 | | RBBIDebugPrintf("%5x ", c); | 864 | | } | 865 | | RBBIDebugPrintf("%3d %3d\n", state, category); | 866 | | } | 867 | | #endif | 868 | | | 869 | | // State Transition - move machine to its next state | 870 | | // | 871 | | | 872 | | // fNextState is a variable-length array. | 873 | 84.3M | U_ASSERT(category<fData->fHeader->fCatCount); | 874 | 84.3M | state = row->fNextState[category]; /*Not accessing beyond memory*/ | 875 | 84.3M | row = (RowType *) | 876 | | // (statetable->fTableData + (statetable->fRowLen * state)); | 877 | 84.3M | (tableData + tableRowLen * state); | 878 | | | 879 | | | 880 | 84.3M | uint16_t accepting = row->fAccepting; | 881 | 84.3M | if (accepting == ACCEPTING_UNCONDITIONAL) { | 882 | | // Match found, common case. | 883 | 66.0M | if (mode != RBBI_START) { | 884 | 64.8M | result = static_cast<int32_t>(UTEXT_GETNATIVEINDEX(&fText)); | 885 | 64.8M | } | 886 | 66.0M | fRuleStatusIndex = row->fTagsIdx; // Remember the break status (tag) values. | 887 | 66.0M | } else if (accepting > ACCEPTING_UNCONDITIONAL) { | 888 | | // Lookahead match is completed. | 889 | 0 | U_ASSERT(accepting < fData->fForwardTable->fLookAheadResultsSize); | 890 | 0 | int32_t lookaheadResult = fLookAheadMatches[accepting]; | 891 | 0 | if (lookaheadResult >= 0) { | 892 | 0 | fRuleStatusIndex = row->fTagsIdx; | 893 | 0 | fPosition = lookaheadResult; | 894 | 0 | return lookaheadResult; | 895 | 0 | } | 896 | 0 | } | 897 | | | 898 | | // If we are at the position of the '/' in a look-ahead (hard break) rule; | 899 | | // record the current position, to be returned later, if the full rule matches. | 900 | | // TODO: Move this check before the previous check of fAccepting. | 901 | | // This would enable hard-break rules with no following context. | 902 | | // But there are line break test failures when trying this. Investigate. | 903 | | // Issue ICU-20837 | 904 | 84.3M | uint16_t rule = row->fLookAhead; | 905 | 84.3M | U_ASSERT(rule == 0 || rule > ACCEPTING_UNCONDITIONAL); | 906 | 84.3M | U_ASSERT(rule == 0 || rule < fData->fForwardTable->fLookAheadResultsSize); | 907 | 84.3M | if (rule > ACCEPTING_UNCONDITIONAL) { | 908 | 0 | int32_t pos = static_cast<int32_t>(UTEXT_GETNATIVEINDEX(&fText)); | 909 | 0 | fLookAheadMatches[rule] = pos; | 910 | 0 | } | 911 | | | 912 | 84.3M | if (state == STOP_STATE) { | 913 | | // This is the normal exit from the lookup state machine. | 914 | | // We have advanced through the string until it is certain that no | 915 | | // longer match is possible, no matter what characters follow. | 916 | 18.1M | break; | 917 | 18.1M | } | 918 | | | 919 | | // Advance to the next character. | 920 | | // If this is a beginning-of-input loop iteration, don't advance | 921 | | // the input position. The next iteration will be processing the | 922 | | // first real input character. | 923 | 66.1M | if (mode == RBBI_RUN) { | 924 | 65.0M | c = UTEXT_NEXT32(&fText); | 925 | 65.0M | } else { | 926 | 1.15M | if (mode == RBBI_START) { | 927 | 1.15M | mode = RBBI_RUN; | 928 | 1.15M | } | 929 | 1.15M | } | 930 | 66.1M | } | 931 | | | 932 | | // The state machine is done. Check whether it found a match... | 933 | | | 934 | | // If the iterator failed to advance in the match engine, force it ahead by one. | 935 | | // (This really indicates a defect in the break rules. They should always match | 936 | | // at least one character.) | 937 | 18.1M | if (result == initialPosition) { | 938 | 0 | utext_setNativeIndex(&fText, initialPosition); | 939 | 0 | utext_next32(&fText); | 940 | 0 | result = static_cast<int32_t>(utext_getNativeIndex(&fText)); | 941 | 0 | fRuleStatusIndex = 0; | 942 | 0 | } | 943 | | | 944 | | // Leave the iterator at our result position. | 945 | 18.1M | fPosition = result; | 946 | | #ifdef RBBI_DEBUG | 947 | | if (gTrace) { | 948 | | RBBIDebugPrintf("result = %d\n\n", result); | 949 | | } | 950 | | #endif | 951 | 18.1M | return result; | 952 | 18.1M | } |
Unexecuted instantiation: rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleNext<icu_78::RBBIStateTableRowT<unsigned char>, &icu_78::TrieFunc16>() Unexecuted instantiation: rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleNext<icu_78::RBBIStateTableRowT<unsigned short>, &icu_78::TrieFunc8>() Unexecuted instantiation: rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleNext<icu_78::RBBIStateTableRowT<unsigned short>, &icu_78::TrieFunc16>() |
953 | | |
954 | | |
955 | | //----------------------------------------------------------------------------------- |
956 | | // |
957 | | // handleSafePrevious() |
958 | | // |
959 | | // Iterate backwards using the safe reverse rules. |
960 | | // The logic of this function is similar to handleNext(), but simpler |
961 | | // because the safe table does not require as many options. |
962 | | // |
963 | | //----------------------------------------------------------------------------------- |
964 | | template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc> |
965 | 0 | int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) { |
966 | |
|
967 | 0 | int32_t state; |
968 | 0 | uint16_t category = 0; |
969 | 0 | RowType *row; |
970 | 0 | UChar32 c; |
971 | 0 | int32_t result = 0; |
972 | |
|
973 | 0 | const RBBIStateTable *stateTable = fData->fReverseTable; |
974 | 0 | UTEXT_SETNATIVEINDEX(&fText, fromPosition); |
975 | | #ifdef RBBI_DEBUG |
976 | | if (gTrace) { |
977 | | RBBIDebugPuts("Handle Previous pos char state category"); |
978 | | } |
979 | | #endif |
980 | | |
981 | | // if we're already at the start of the text, return DONE. |
982 | 0 | if (fData == nullptr || UTEXT_GETNATIVEINDEX(&fText)==0) { |
983 | 0 | return BreakIterator::DONE; |
984 | 0 | } |
985 | | |
986 | | // Set the initial state for the state machine |
987 | 0 | c = UTEXT_PREVIOUS32(&fText); |
988 | 0 | state = START_STATE; |
989 | 0 | row = (RowType *) |
990 | 0 | (stateTable->fTableData + (stateTable->fRowLen * state)); |
991 | | |
992 | | // loop until we reach the start of the text or transition to state 0 |
993 | | // |
994 | 0 | for (; c != U_SENTINEL; c = UTEXT_PREVIOUS32(&fText)) { |
995 | | |
996 | | // look up the current character's character category, which tells us |
997 | | // which column in the state table to look at. |
998 | | // |
999 | | // Off the dictionary flag bit. For reverse iteration it is not used. |
1000 | 0 | category = trieFunc(fData->fTrie, c); |
1001 | |
|
1002 | | #ifdef RBBI_DEBUG |
1003 | | if (gTrace) { |
1004 | | RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(&fText)); |
1005 | | if (0x20<=c && c<0x7f) { |
1006 | | RBBIDebugPrintf("\"%c\" ", c); |
1007 | | } else { |
1008 | | RBBIDebugPrintf("%5x ", c); |
1009 | | } |
1010 | | RBBIDebugPrintf("%3d %3d\n", state, category); |
1011 | | } |
1012 | | #endif |
1013 | | |
1014 | | // State Transition - move machine to its next state |
1015 | | // |
1016 | | // fNextState is a variable-length array. |
1017 | 0 | U_ASSERT(category<fData->fHeader->fCatCount); |
1018 | 0 | state = row->fNextState[category]; /*Not accessing beyond memory*/ |
1019 | 0 | row = (RowType *) |
1020 | 0 | (stateTable->fTableData + (stateTable->fRowLen * state)); |
1021 | |
|
1022 | 0 | if (state == STOP_STATE) { |
1023 | | // This is the normal exit from the lookup state machine. |
1024 | | // Transition to state zero means we have found a safe point. |
1025 | 0 | break; |
1026 | 0 | } |
1027 | 0 | } |
1028 | | |
1029 | | // The state machine is done. Check whether it found a match... |
1030 | 0 | result = static_cast<int32_t>(UTEXT_GETNATIVEINDEX(&fText)); |
1031 | | #ifdef RBBI_DEBUG |
1032 | | if (gTrace) { |
1033 | | RBBIDebugPrintf("result = %d\n\n", result); |
1034 | | } |
1035 | | #endif |
1036 | 0 | return result; |
1037 | 0 | } Unexecuted instantiation: rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleSafePrevious<icu_78::RBBIStateTableRowT<unsigned char>, &icu_78::TrieFunc8>(int) Unexecuted instantiation: rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleSafePrevious<icu_78::RBBIStateTableRowT<unsigned char>, &icu_78::TrieFunc16>(int) Unexecuted instantiation: rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleSafePrevious<icu_78::RBBIStateTableRowT<unsigned short>, &icu_78::TrieFunc8>(int) Unexecuted instantiation: rbbi.cpp:int icu_78::RuleBasedBreakIterator::handleSafePrevious<icu_78::RBBIStateTableRowT<unsigned short>, &icu_78::TrieFunc16>(int) |
1038 | | |
1039 | | |
1040 | | //------------------------------------------------------------------------------- |
1041 | | // |
1042 | | // getRuleStatus() Return the break rule tag associated with the current |
1043 | | // iterator position. If the iterator arrived at its current |
1044 | | // position by iterating forwards, the value will have been |
1045 | | // cached by the handleNext() function. |
1046 | | // |
1047 | | //------------------------------------------------------------------------------- |
1048 | | |
1049 | 0 | int32_t RuleBasedBreakIterator::getRuleStatus() const { |
1050 | | |
1051 | | // fLastRuleStatusIndex indexes to the start of the appropriate status record |
1052 | | // (the number of status values.) |
1053 | | // This function returns the last (largest) of the array of status values. |
1054 | 0 | int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex]; |
1055 | 0 | int32_t tagVal = fData->fRuleStatusTable[idx]; |
1056 | |
|
1057 | 0 | return tagVal; |
1058 | 0 | } |
1059 | | |
1060 | | |
1061 | | int32_t RuleBasedBreakIterator::getRuleStatusVec( |
1062 | 0 | int32_t *fillInVec, int32_t capacity, UErrorCode &status) { |
1063 | 0 | if (U_FAILURE(status)) { |
1064 | 0 | return 0; |
1065 | 0 | } |
1066 | | |
1067 | 0 | int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex]; |
1068 | 0 | int32_t numValsToCopy = numVals; |
1069 | 0 | if (numVals > capacity) { |
1070 | 0 | status = U_BUFFER_OVERFLOW_ERROR; |
1071 | 0 | numValsToCopy = capacity; |
1072 | 0 | } |
1073 | 0 | int i; |
1074 | 0 | for (i=0; i<numValsToCopy; i++) { |
1075 | 0 | fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1]; |
1076 | 0 | } |
1077 | 0 | return numVals; |
1078 | 0 | } |
1079 | | |
1080 | | |
1081 | | |
1082 | | //------------------------------------------------------------------------------- |
1083 | | // |
1084 | | // getBinaryRules Access to the compiled form of the rules, |
1085 | | // for use by build system tools that save the data |
1086 | | // for standard iterator types. |
1087 | | // |
1088 | | //------------------------------------------------------------------------------- |
1089 | 0 | const uint8_t *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) { |
1090 | 0 | const uint8_t *retPtr = nullptr; |
1091 | 0 | length = 0; |
1092 | |
|
1093 | 0 | if (fData != nullptr) { |
1094 | 0 | retPtr = reinterpret_cast<const uint8_t*>(fData->fHeader); |
1095 | 0 | length = fData->fHeader->fLength; |
1096 | 0 | } |
1097 | 0 | return retPtr; |
1098 | 0 | } |
1099 | | |
1100 | | |
1101 | | RuleBasedBreakIterator *RuleBasedBreakIterator::createBufferClone( |
1102 | 0 | void * /*stackBuffer*/, int32_t &bufferSize, UErrorCode &status) { |
1103 | 0 | if (U_FAILURE(status)){ |
1104 | 0 | return nullptr; |
1105 | 0 | } |
1106 | | |
1107 | 0 | if (bufferSize == 0) { |
1108 | 0 | bufferSize = 1; // preflighting for deprecated functionality |
1109 | 0 | return nullptr; |
1110 | 0 | } |
1111 | | |
1112 | 0 | BreakIterator *clonedBI = clone(); |
1113 | 0 | if (clonedBI == nullptr) { |
1114 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
1115 | 0 | } else { |
1116 | 0 | status = U_SAFECLONE_ALLOCATED_WARNING; |
1117 | 0 | } |
1118 | 0 | return (RuleBasedBreakIterator *)clonedBI; |
1119 | 0 | } |
1120 | | |
1121 | | U_NAMESPACE_END |
1122 | | |
1123 | | |
1124 | | static icu::UStack *gLanguageBreakFactories = nullptr; |
1125 | | static const icu::UnicodeString *gEmptyString = nullptr; |
1126 | | static icu::UInitOnce gLanguageBreakFactoriesInitOnce {}; |
1127 | | static icu::UInitOnce gRBBIInitOnce {}; |
1128 | | static icu::ICULanguageBreakFactory *gICULanguageBreakFactory = nullptr; |
1129 | | |
1130 | | /** |
1131 | | * Release all static memory held by breakiterator. |
1132 | | */ |
1133 | | U_CDECL_BEGIN |
1134 | 0 | UBool U_CALLCONV rbbi_cleanup() { |
1135 | 0 | delete gLanguageBreakFactories; |
1136 | 0 | gLanguageBreakFactories = nullptr; |
1137 | 0 | delete gEmptyString; |
1138 | 0 | gEmptyString = nullptr; |
1139 | 0 | gLanguageBreakFactoriesInitOnce.reset(); |
1140 | 0 | gRBBIInitOnce.reset(); |
1141 | 0 | return true; |
1142 | 0 | } |
1143 | | U_CDECL_END |
1144 | | |
1145 | | U_CDECL_BEGIN |
1146 | 0 | static void U_CALLCONV _deleteFactory(void *obj) { |
1147 | 0 | delete (icu::LanguageBreakFactory *) obj; |
1148 | 0 | } |
1149 | | U_CDECL_END |
1150 | | U_NAMESPACE_BEGIN |
1151 | | |
1152 | 0 | static void U_CALLCONV rbbiInit() { |
1153 | 0 | gEmptyString = new UnicodeString(); |
1154 | 0 | ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup); |
1155 | 0 | } |
1156 | | |
1157 | 1 | static void U_CALLCONV initLanguageFactories(UErrorCode& status) { |
1158 | 1 | U_ASSERT(gLanguageBreakFactories == nullptr); |
1159 | 1 | gLanguageBreakFactories = new UStack(_deleteFactory, nullptr, status); |
1160 | 1 | if (gLanguageBreakFactories != nullptr && U_SUCCESS(status)) { |
1161 | 1 | LocalPointer<ICULanguageBreakFactory> factory(new ICULanguageBreakFactory(status), status); |
1162 | 1 | if (U_SUCCESS(status)) { |
1163 | 1 | gICULanguageBreakFactory = factory.orphan(); |
1164 | 1 | gLanguageBreakFactories->push(gICULanguageBreakFactory, status); |
1165 | | #ifdef U_LOCAL_SERVICE_HOOK |
1166 | | LanguageBreakFactory *extra = (LanguageBreakFactory *)uprv_svc_hook("languageBreakFactory", &status); |
1167 | | if (extra != nullptr) { |
1168 | | gLanguageBreakFactories->push(extra, status); |
1169 | | } |
1170 | | #endif |
1171 | 1 | } |
1172 | 1 | } |
1173 | 1 | ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup); |
1174 | 1 | } |
1175 | | |
1176 | 69.5k | void ensureLanguageFactories(UErrorCode& status) { |
1177 | 69.5k | umtx_initOnce(gLanguageBreakFactoriesInitOnce, &initLanguageFactories, status); |
1178 | 69.5k | } |
1179 | | |
1180 | | static const LanguageBreakEngine* |
1181 | | getLanguageBreakEngineFromFactory(UChar32 c, const char* locale) |
1182 | 69.5k | { |
1183 | 69.5k | UErrorCode status = U_ZERO_ERROR; |
1184 | 69.5k | ensureLanguageFactories(status); |
1185 | 69.5k | if (U_FAILURE(status)) return nullptr; |
1186 | | |
1187 | 69.5k | int32_t i = gLanguageBreakFactories->size(); |
1188 | 69.5k | const LanguageBreakEngine *lbe = nullptr; |
1189 | 135k | while (--i >= 0) { |
1190 | 69.5k | LanguageBreakFactory* factory = static_cast<LanguageBreakFactory*>(gLanguageBreakFactories->elementAt(i)); |
1191 | 69.5k | lbe = factory->getEngineFor(c, locale); |
1192 | 69.5k | if (lbe != nullptr) { |
1193 | 3.47k | break; |
1194 | 3.47k | } |
1195 | 69.5k | } |
1196 | 69.5k | return lbe; |
1197 | 69.5k | } |
1198 | | |
1199 | | |
1200 | | //------------------------------------------------------------------------------- |
1201 | | // |
1202 | | // getLanguageBreakEngine Find an appropriate LanguageBreakEngine for the |
1203 | | // the character c. |
1204 | | // |
1205 | | //------------------------------------------------------------------------------- |
1206 | | const LanguageBreakEngine * |
1207 | 359k | RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c, const char* locale) { |
1208 | 359k | const LanguageBreakEngine *lbe = nullptr; |
1209 | 359k | UErrorCode status = U_ZERO_ERROR; |
1210 | | |
1211 | 359k | if (fLanguageBreakEngines == nullptr) { |
1212 | 2.92k | fLanguageBreakEngines = new UStack(status); |
1213 | 2.92k | if (fLanguageBreakEngines == nullptr || U_FAILURE(status)) { |
1214 | 0 | delete fLanguageBreakEngines; |
1215 | 0 | fLanguageBreakEngines = nullptr; |
1216 | 0 | return nullptr; |
1217 | 0 | } |
1218 | 2.92k | } |
1219 | | |
1220 | 359k | int32_t i = fLanguageBreakEngines->size(); |
1221 | 1.37M | while (--i >= 0) { |
1222 | 1.30M | lbe = static_cast<const LanguageBreakEngine*>(fLanguageBreakEngines->elementAt(i)); |
1223 | 1.30M | if (lbe->handles(c, locale)) { |
1224 | 289k | return lbe; |
1225 | 289k | } |
1226 | 1.30M | } |
1227 | | |
1228 | | // No existing dictionary took the character. See if a factory wants to |
1229 | | // give us a new LanguageBreakEngine for this character. |
1230 | 69.5k | lbe = getLanguageBreakEngineFromFactory(c, locale); |
1231 | | |
1232 | | // If we got one, use it and push it on our stack. |
1233 | 69.5k | if (lbe != nullptr) { |
1234 | 3.47k | fLanguageBreakEngines->push((void *)lbe, status); |
1235 | | // Even if we can't remember it, we can keep looking it up, so |
1236 | | // return it even if the push fails. |
1237 | 3.47k | return lbe; |
1238 | 3.47k | } |
1239 | | |
1240 | | // No engine is forthcoming for this character. Add it to the |
1241 | | // reject set. Create the reject break engine if needed. |
1242 | 66.0k | if (fUnhandledBreakEngine == nullptr) { |
1243 | 394 | fUnhandledBreakEngine = new UnhandledEngine(status); |
1244 | 394 | if (U_SUCCESS(status) && fUnhandledBreakEngine == nullptr) { |
1245 | 0 | status = U_MEMORY_ALLOCATION_ERROR; |
1246 | 0 | return nullptr; |
1247 | 0 | } |
1248 | | // Put it last so that scripts for which we have an engine get tried |
1249 | | // first. |
1250 | 394 | fLanguageBreakEngines->insertElementAt(fUnhandledBreakEngine, 0, status); |
1251 | | // If we can't insert it, or creation failed, get rid of it |
1252 | 394 | U_ASSERT(!fLanguageBreakEngines->hasDeleter()); |
1253 | 394 | if (U_FAILURE(status)) { |
1254 | 0 | delete fUnhandledBreakEngine; |
1255 | 0 | fUnhandledBreakEngine = nullptr; |
1256 | 0 | return nullptr; |
1257 | 0 | } |
1258 | 394 | } |
1259 | | |
1260 | | // Tell the reject engine about the character; at its discretion, it may |
1261 | | // add more than just the one character. |
1262 | 66.0k | fUnhandledBreakEngine->handleCharacter(c); |
1263 | | |
1264 | 66.0k | return fUnhandledBreakEngine; |
1265 | 66.0k | } |
1266 | | |
1267 | | #ifndef U_HIDE_DRAFT_API |
1268 | | void U_EXPORT2 RuleBasedBreakIterator::registerExternalBreakEngine( |
1269 | 0 | ExternalBreakEngine* toAdopt, UErrorCode& status) { |
1270 | 0 | LocalPointer<ExternalBreakEngine> engine(toAdopt, status); |
1271 | 0 | if (U_FAILURE(status)) return; |
1272 | 0 | ensureLanguageFactories(status); |
1273 | 0 | if (U_FAILURE(status)) return; |
1274 | 0 | gICULanguageBreakFactory->addExternalEngine(engine.orphan(), status); |
1275 | 0 | } |
1276 | | #endif /* U_HIDE_DRAFT_API */ |
1277 | | |
1278 | | |
1279 | 0 | void RuleBasedBreakIterator::dumpCache() { |
1280 | 0 | fBreakCache->dumpCache(); |
1281 | 0 | } |
1282 | | |
1283 | 0 | void RuleBasedBreakIterator::dumpTables() { |
1284 | 0 | fData->printData(); |
1285 | 0 | } |
1286 | | |
1287 | | /** |
1288 | | * Returns the description used to create this iterator |
1289 | | */ |
1290 | | |
1291 | | const UnicodeString& |
1292 | 0 | RuleBasedBreakIterator::getRules() const { |
1293 | 0 | if (fData != nullptr) { |
1294 | 0 | return fData->getRuleSourceString(); |
1295 | 0 | } else { |
1296 | 0 | umtx_initOnce(gRBBIInitOnce, &rbbiInit); |
1297 | 0 | return *gEmptyString; |
1298 | 0 | } |
1299 | 0 | } |
1300 | | |
1301 | | U_NAMESPACE_END |
1302 | | |
1303 | | #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |