/src/icu/icu4c/source/i18n/numparse_impl.cpp
Line | Count | Source |
1 | | // © 2018 and later: Unicode, Inc. and others. |
2 | | // License & terms of use: http://www.unicode.org/copyright.html |
3 | | |
4 | | #include "unicode/utypes.h" |
5 | | |
6 | | #if !UCONFIG_NO_FORMATTING |
7 | | |
8 | | // Allow implicit conversion from char16_t* to UnicodeString for this file: |
9 | | // Helpful in toString methods and elsewhere. |
10 | | #define UNISTR_FROM_STRING_EXPLICIT |
11 | | |
12 | | #include <typeinfo> |
13 | | #include <array> |
14 | | #include "number_types.h" |
15 | | #include "number_patternstring.h" |
16 | | #include "numparse_types.h" |
17 | | #include "numparse_impl.h" |
18 | | #include "numparse_symbols.h" |
19 | | #include "numparse_decimal.h" |
20 | | #include "unicode/numberformatter.h" |
21 | | #include "cstr.h" |
22 | | #include "number_mapper.h" |
23 | | #include "static_unicode_sets.h" |
24 | | |
25 | | using namespace icu; |
26 | | using namespace icu::number; |
27 | | using namespace icu::number::impl; |
28 | | using namespace icu::numparse; |
29 | | using namespace icu::numparse::impl; |
30 | | |
31 | | |
32 | 65.3M | NumberParseMatcher::~NumberParseMatcher() = default; |
33 | | |
34 | | |
35 | | NumberParserImpl* |
36 | | NumberParserImpl::createSimpleParser(const Locale& locale, const UnicodeString& patternString, |
37 | 0 | parse_flags_t parseFlags, UErrorCode& status) { |
38 | |
|
39 | 0 | LocalPointer<NumberParserImpl> parser(new NumberParserImpl(parseFlags)); |
40 | 0 | DecimalFormatSymbols symbols(locale, status); |
41 | |
|
42 | 0 | parser->fLocalMatchers.ignorables = {parseFlags}; |
43 | 0 | IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables; |
44 | |
|
45 | 0 | DecimalFormatSymbols dfs(locale, status); |
46 | 0 | dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$"); |
47 | 0 | dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU"); |
48 | 0 | CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status); |
49 | |
|
50 | 0 | ParsedPatternInfo patternInfo; |
51 | 0 | PatternParser::parseToPatternInfo(patternString, patternInfo, status); |
52 | | |
53 | | // The following statements set up the affix matchers. |
54 | 0 | AffixTokenMatcherSetupData affixSetupData = { |
55 | 0 | currencySymbols, symbols, ignorables, locale, parseFlags}; |
56 | 0 | parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData}; |
57 | 0 | parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse}; |
58 | 0 | parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers( |
59 | 0 | patternInfo, *parser, ignorables, parseFlags, status); |
60 | |
|
61 | 0 | Grouper grouper = Grouper::forStrategy(UNUM_GROUPING_AUTO); |
62 | 0 | grouper.setLocaleData(patternInfo, locale); |
63 | |
|
64 | 0 | parser->addMatcher(parser->fLocalMatchers.ignorables); |
65 | 0 | parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags}); |
66 | 0 | parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false}); |
67 | 0 | parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false}); |
68 | 0 | parser->addMatcher(parser->fLocalMatchers.approximatelySign = {symbols, false}); |
69 | 0 | parser->addMatcher(parser->fLocalMatchers.percent = {symbols}); |
70 | 0 | parser->addMatcher(parser->fLocalMatchers.permille = {symbols}); |
71 | 0 | parser->addMatcher(parser->fLocalMatchers.nan = {symbols}); |
72 | 0 | parser->addMatcher(parser->fLocalMatchers.infinity = {symbols}); |
73 | 0 | parser->addMatcher(parser->fLocalMatchers.padding = {u"@"}); |
74 | 0 | parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper}); |
75 | 0 | parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status}); |
76 | 0 | parser->addMatcher(parser->fLocalValidators.number = {}); |
77 | |
|
78 | 0 | parser->freeze(); |
79 | 0 | return parser.orphan(); |
80 | 0 | } |
81 | | |
82 | | NumberParserImpl* |
83 | | NumberParserImpl::createParserFromProperties(const number::impl::DecimalFormatProperties& properties, |
84 | | const DecimalFormatSymbols& symbols, bool parseCurrency, |
85 | 83.7k | UErrorCode& status) { |
86 | 83.7k | Locale locale = symbols.getLocale(); |
87 | 83.7k | AutoAffixPatternProvider affixProvider(properties, status); |
88 | 83.7k | if (U_FAILURE(status)) { return nullptr; } |
89 | 83.7k | CurrencyUnit currency = resolveCurrency(properties, locale, status); |
90 | 83.7k | CurrencySymbols currencySymbols(currency, locale, symbols, status); |
91 | 83.7k | bool isStrict = properties.parseMode.getOrDefault(PARSE_MODE_STRICT) == PARSE_MODE_STRICT; |
92 | 83.7k | Grouper grouper = Grouper::forProperties(properties); |
93 | 83.7k | int parseFlags = 0; |
94 | 83.7k | if (U_FAILURE(status)) { return nullptr; } |
95 | 83.7k | if (!properties.parseCaseSensitive) { |
96 | 83.7k | parseFlags |= PARSE_FLAG_IGNORE_CASE; |
97 | 83.7k | } |
98 | 83.7k | if (properties.parseIntegerOnly) { |
99 | 0 | parseFlags |= PARSE_FLAG_INTEGER_ONLY; |
100 | 0 | } |
101 | 83.7k | if (properties.signAlwaysShown) { |
102 | 0 | parseFlags |= PARSE_FLAG_PLUS_SIGN_ALLOWED; |
103 | 0 | } |
104 | 83.7k | if (isStrict) { |
105 | 83.7k | parseFlags |= PARSE_FLAG_STRICT_GROUPING_SIZE; |
106 | 83.7k | parseFlags |= PARSE_FLAG_STRICT_SEPARATORS; |
107 | 83.7k | parseFlags |= PARSE_FLAG_USE_FULL_AFFIXES; |
108 | 83.7k | parseFlags |= PARSE_FLAG_EXACT_AFFIX; |
109 | 83.7k | parseFlags |= PARSE_FLAG_STRICT_IGNORABLES; |
110 | 83.7k | } else { |
111 | 0 | parseFlags |= PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES; |
112 | 0 | } |
113 | 83.7k | if (grouper.getPrimary() <= 0) { |
114 | 54.1k | parseFlags |= PARSE_FLAG_GROUPING_DISABLED; |
115 | 54.1k | } |
116 | 83.7k | if (parseCurrency || affixProvider.get().hasCurrencySign()) { |
117 | 12.7k | parseFlags |= PARSE_FLAG_MONETARY_SEPARATORS; |
118 | 12.7k | } |
119 | 83.7k | if (!parseCurrency) { |
120 | 83.7k | parseFlags |= PARSE_FLAG_NO_FOREIGN_CURRENCY; |
121 | 83.7k | } |
122 | | |
123 | 83.7k | LocalPointer<NumberParserImpl> parser(new NumberParserImpl(parseFlags)); |
124 | | |
125 | 83.7k | parser->fLocalMatchers.ignorables = {parseFlags}; |
126 | 83.7k | IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables; |
127 | | |
128 | | ////////////////////// |
129 | | /// AFFIX MATCHERS /// |
130 | | ////////////////////// |
131 | | |
132 | | // The following statements set up the affix matchers. |
133 | 83.7k | AffixTokenMatcherSetupData affixSetupData = { |
134 | 83.7k | currencySymbols, symbols, ignorables, locale, parseFlags}; |
135 | 83.7k | parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData}; |
136 | 83.7k | parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse}; |
137 | 83.7k | parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers( |
138 | 83.7k | affixProvider.get(), *parser, ignorables, parseFlags, status); |
139 | | |
140 | | //////////////////////// |
141 | | /// CURRENCY MATCHER /// |
142 | | //////////////////////// |
143 | | |
144 | 83.7k | if (parseCurrency || affixProvider.get().hasCurrencySign()) { |
145 | 12.7k | parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status}); |
146 | 12.7k | } |
147 | | |
148 | | /////////////// |
149 | | /// PERCENT /// |
150 | | /////////////// |
151 | | |
152 | | // ICU-TC meeting, April 11, 2018: accept percent/permille only if it is in the pattern, |
153 | | // and to maintain regressive behavior, divide by 100 even if no percent sign is present. |
154 | 83.7k | if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERCENT, status)) { |
155 | 0 | parser->addMatcher(parser->fLocalMatchers.percent = {symbols}); |
156 | 0 | } |
157 | 83.7k | if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERMILLE, status)) { |
158 | 0 | parser->addMatcher(parser->fLocalMatchers.permille = {symbols}); |
159 | 0 | } |
160 | | |
161 | | /////////////////////////////// |
162 | | /// OTHER STANDARD MATCHERS /// |
163 | | /////////////////////////////// |
164 | | |
165 | 83.7k | if (!isStrict) { |
166 | 0 | parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false}); |
167 | 0 | parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false}); |
168 | 0 | parser->addMatcher(parser->fLocalMatchers.approximatelySign = {symbols, false}); |
169 | 0 | } |
170 | 83.7k | parser->addMatcher(parser->fLocalMatchers.nan = {symbols}); |
171 | 83.7k | parser->addMatcher(parser->fLocalMatchers.infinity = {symbols}); |
172 | 83.7k | UnicodeString padString = properties.padString; |
173 | 83.7k | if (!padString.isBogus() && !ignorables.getSet()->contains(padString)) { |
174 | 745 | parser->addMatcher(parser->fLocalMatchers.padding = {padString}); |
175 | 745 | } |
176 | 83.7k | parser->addMatcher(parser->fLocalMatchers.ignorables); |
177 | 83.7k | parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags}); |
178 | | // NOTE: parseNoExponent doesn't disable scientific parsing if we have a scientific formatter |
179 | 83.7k | if (!properties.parseNoExponent || properties.minimumExponentDigits > 0) { |
180 | 83.7k | parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper}); |
181 | 83.7k | } |
182 | | |
183 | | ////////////////// |
184 | | /// VALIDATORS /// |
185 | | ////////////////// |
186 | | |
187 | 83.7k | parser->addMatcher(parser->fLocalValidators.number = {}); |
188 | 83.7k | if (isStrict) { |
189 | 83.7k | parser->addMatcher(parser->fLocalValidators.affix = {}); |
190 | 83.7k | } |
191 | 83.7k | if (parseCurrency) { |
192 | 0 | parser->addMatcher(parser->fLocalValidators.currency = {}); |
193 | 0 | } |
194 | 83.7k | if (properties.decimalPatternMatchRequired) { |
195 | 0 | bool patternHasDecimalSeparator = |
196 | 0 | properties.decimalSeparatorAlwaysShown || properties.maximumFractionDigits != 0; |
197 | 0 | parser->addMatcher(parser->fLocalValidators.decimalSeparator = {patternHasDecimalSeparator}); |
198 | 0 | } |
199 | | // The multiplier takes care of scaling percentages. |
200 | 83.7k | Scale multiplier = scaleFromProperties(properties); |
201 | 83.7k | if (multiplier.isValid()) { |
202 | 10.4k | parser->addMatcher(parser->fLocalValidators.multiplier = {multiplier}); |
203 | 10.4k | } |
204 | | |
205 | 83.7k | parser->freeze(); |
206 | 83.7k | return parser.orphan(); |
207 | 83.7k | } |
208 | | |
209 | | NumberParserImpl::NumberParserImpl(parse_flags_t parseFlags) |
210 | 83.7k | : fParseFlags(parseFlags) { |
211 | 83.7k | } |
212 | | |
213 | 83.7k | NumberParserImpl::~NumberParserImpl() { |
214 | 83.7k | fNumMatchers = 0; |
215 | 83.7k | } |
216 | | |
217 | 777k | void NumberParserImpl::addMatcher(NumberParseMatcher& matcher) { |
218 | 777k | if (fNumMatchers + 1 > fMatchers.getCapacity()) { |
219 | 54 | fMatchers.resize(fNumMatchers * 2, fNumMatchers); |
220 | 54 | } |
221 | 777k | fMatchers[fNumMatchers] = &matcher; |
222 | 777k | fNumMatchers++; |
223 | 777k | } |
224 | | |
225 | 83.7k | void NumberParserImpl::freeze() { |
226 | 83.7k | fFrozen = true; |
227 | 83.7k | } |
228 | | |
229 | 360k | parse_flags_t NumberParserImpl::getParseFlags() const { |
230 | 360k | return fParseFlags; |
231 | 360k | } |
232 | | |
233 | | void NumberParserImpl::parse(const UnicodeString& input, bool greedy, ParsedNumber& result, |
234 | 0 | UErrorCode& status) const { |
235 | 0 | return parse(input, 0, greedy, result, status); |
236 | 0 | } |
237 | | |
238 | | void NumberParserImpl::parse(const UnicodeString& input, int32_t start, bool greedy, ParsedNumber& result, |
239 | 2.32M | UErrorCode& status) const { |
240 | 2.32M | if (U_FAILURE(status)) { |
241 | 0 | return; |
242 | 0 | } |
243 | 2.32M | U_ASSERT(fFrozen); |
244 | | // TODO: Check start >= 0 and start < input.length() |
245 | 2.32M | StringSegment segment(input, 0 != (fParseFlags & PARSE_FLAG_IGNORE_CASE)); |
246 | 2.32M | segment.adjustOffset(start); |
247 | 2.32M | if (greedy) { |
248 | 2.32M | parseGreedy(segment, result, status); |
249 | 2.32M | } else if (0 != (fParseFlags & PARSE_FLAG_ALLOW_INFINITE_RECURSION)) { |
250 | | // Start at 1 so that recursionLevels never gets to 0 |
251 | 0 | parseLongestRecursive(segment, result, 1, status); |
252 | 0 | } else { |
253 | | // Arbitrary recursion safety limit: 100 levels. |
254 | 0 | parseLongestRecursive(segment, result, -100, status); |
255 | 0 | } |
256 | 23.4M | for (int32_t i = 0; i < fNumMatchers; i++) { |
257 | 21.1M | fMatchers[i]->postProcess(result); |
258 | 21.1M | } |
259 | 2.32M | result.postProcess(); |
260 | 2.32M | } |
261 | | |
262 | | void NumberParserImpl::parseGreedy(StringSegment& segment, ParsedNumber& result, |
263 | 2.32M | UErrorCode& status) const { |
264 | | // Note: this method is not recursive in order to avoid stack overflow. |
265 | 27.3M | for (int i = 0; i <fNumMatchers;) { |
266 | | // Base Case |
267 | 25.0M | if (segment.length() == 0) { |
268 | 12.6k | return; |
269 | 12.6k | } |
270 | 25.0M | const NumberParseMatcher* matcher = fMatchers[i]; |
271 | 25.0M | if (!matcher->smokeTest(segment)) { |
272 | | // Matcher failed smoke test: try the next one |
273 | 22.5M | i++; |
274 | 22.5M | continue; |
275 | 22.5M | } |
276 | 2.49M | int32_t initialOffset = segment.getOffset(); |
277 | 2.49M | matcher->match(segment, result, status); |
278 | 2.49M | if (U_FAILURE(status)) { |
279 | 0 | return; |
280 | 0 | } |
281 | 2.49M | if (segment.getOffset() != initialOffset) { |
282 | | // Greedy heuristic: accept the match and loop back |
283 | 712k | i = 0; |
284 | 712k | continue; |
285 | 1.78M | } else { |
286 | | // Matcher did not match: try the next one |
287 | 1.78M | i++; |
288 | 1.78M | continue; |
289 | 1.78M | } |
290 | 2.49M | UPRV_UNREACHABLE_EXIT; |
291 | 2.49M | } |
292 | | |
293 | | // NOTE: If we get here, the greedy parse completed without consuming the entire string. |
294 | 2.32M | } |
295 | | |
296 | | void NumberParserImpl::parseLongestRecursive(StringSegment& segment, ParsedNumber& result, |
297 | | int32_t recursionLevels, |
298 | 0 | UErrorCode& status) const { |
299 | | // Base Case |
300 | 0 | if (segment.length() == 0) { |
301 | 0 | return; |
302 | 0 | } |
303 | | |
304 | | // Safety against stack overflow |
305 | 0 | if (recursionLevels == 0) { |
306 | 0 | return; |
307 | 0 | } |
308 | | |
309 | | // TODO: Give a nice way for the matcher to reset the ParsedNumber? |
310 | 0 | ParsedNumber initial(result); |
311 | 0 | ParsedNumber candidate; |
312 | |
|
313 | 0 | int initialOffset = segment.getOffset(); |
314 | 0 | for (int32_t i = 0; i < fNumMatchers; i++) { |
315 | 0 | const NumberParseMatcher* matcher = fMatchers[i]; |
316 | 0 | if (!matcher->smokeTest(segment)) { |
317 | 0 | continue; |
318 | 0 | } |
319 | | |
320 | | // In a non-greedy parse, we attempt all possible matches and pick the best. |
321 | 0 | for (int32_t charsToConsume = 0; charsToConsume < segment.length();) { |
322 | 0 | charsToConsume += U16_LENGTH(segment.codePointAt(charsToConsume)); |
323 | | |
324 | | // Run the matcher on a segment of the current length. |
325 | 0 | candidate = initial; |
326 | 0 | segment.setLength(charsToConsume); |
327 | 0 | bool maybeMore = matcher->match(segment, candidate, status); |
328 | 0 | segment.resetLength(); |
329 | 0 | if (U_FAILURE(status)) { |
330 | 0 | return; |
331 | 0 | } |
332 | | |
333 | | // If the entire segment was consumed, recurse. |
334 | 0 | if (segment.getOffset() - initialOffset == charsToConsume) { |
335 | 0 | parseLongestRecursive(segment, candidate, recursionLevels + 1, status); |
336 | 0 | if (U_FAILURE(status)) { |
337 | 0 | return; |
338 | 0 | } |
339 | 0 | if (candidate.isBetterThan(result)) { |
340 | 0 | result = candidate; |
341 | 0 | } |
342 | 0 | } |
343 | | |
344 | | // Since the segment can be re-used, reset the offset. |
345 | | // This does not have an effect if the matcher did not consume any chars. |
346 | 0 | segment.setOffset(initialOffset); |
347 | | |
348 | | // Unless the matcher wants to see the next char, continue to the next matcher. |
349 | 0 | if (!maybeMore) { |
350 | 0 | break; |
351 | 0 | } |
352 | 0 | } |
353 | 0 | } |
354 | 0 | } |
355 | | |
356 | 0 | UnicodeString NumberParserImpl::toString() const { |
357 | 0 | UnicodeString result(u"<NumberParserImpl matchers:["); |
358 | 0 | for (int32_t i = 0; i < fNumMatchers; i++) { |
359 | 0 | result.append(u' '); |
360 | 0 | result.append(fMatchers[i]->toString()); |
361 | 0 | } |
362 | 0 | result.append(u" ]>", -1); |
363 | 0 | return result; |
364 | 0 | } |
365 | | |
366 | | |
367 | | #endif /* #if !UCONFIG_NO_FORMATTING */ |