/src/icu/source/i18n/numparse_impl.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | // © 2018 and later: Unicode, Inc. and others. |
2 | | // License & terms of use: http://www.unicode.org/copyright.html |
3 | | |
4 | | #include "unicode/utypes.h" |
5 | | |
6 | | #if !UCONFIG_NO_FORMATTING |
7 | | |
8 | | // Allow implicit conversion from char16_t* to UnicodeString for this file: |
9 | | // Helpful in toString methods and elsewhere. |
10 | | #define UNISTR_FROM_STRING_EXPLICIT |
11 | | |
12 | | #include <typeinfo> |
13 | | #include <array> |
14 | | #include "number_types.h" |
15 | | #include "number_patternstring.h" |
16 | | #include "numparse_types.h" |
17 | | #include "numparse_impl.h" |
18 | | #include "numparse_symbols.h" |
19 | | #include "numparse_decimal.h" |
20 | | #include "unicode/numberformatter.h" |
21 | | #include "cstr.h" |
22 | | #include "number_mapper.h" |
23 | | #include "static_unicode_sets.h" |
24 | | |
25 | | using namespace icu; |
26 | | using namespace icu::number; |
27 | | using namespace icu::number::impl; |
28 | | using namespace icu::numparse; |
29 | | using namespace icu::numparse::impl; |
30 | | |
31 | | |
32 | 0 | NumberParseMatcher::~NumberParseMatcher() = default; |
33 | | |
34 | | |
35 | | NumberParserImpl* |
36 | | NumberParserImpl::createSimpleParser(const Locale& locale, const UnicodeString& patternString, |
37 | 0 | parse_flags_t parseFlags, UErrorCode& status) { |
38 | |
|
39 | 0 | LocalPointer<NumberParserImpl> parser(new NumberParserImpl(parseFlags)); |
40 | 0 | DecimalFormatSymbols symbols(locale, status); |
41 | |
|
42 | 0 | parser->fLocalMatchers.ignorables = {parseFlags}; |
43 | 0 | IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables; |
44 | |
|
45 | 0 | DecimalFormatSymbols dfs(locale, status); |
46 | 0 | dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$"); |
47 | 0 | dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU"); |
48 | 0 | CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status); |
49 | |
|
50 | 0 | ParsedPatternInfo patternInfo; |
51 | 0 | PatternParser::parseToPatternInfo(patternString, patternInfo, status); |
52 | | |
53 | | // The following statements set up the affix matchers. |
54 | 0 | AffixTokenMatcherSetupData affixSetupData = { |
55 | 0 | currencySymbols, symbols, ignorables, locale, parseFlags}; |
56 | 0 | parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData}; |
57 | 0 | parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse}; |
58 | 0 | parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers( |
59 | 0 | patternInfo, *parser, ignorables, parseFlags, status); |
60 | |
|
61 | 0 | Grouper grouper = Grouper::forStrategy(UNUM_GROUPING_AUTO); |
62 | 0 | grouper.setLocaleData(patternInfo, locale); |
63 | |
|
64 | 0 | parser->addMatcher(parser->fLocalMatchers.ignorables); |
65 | 0 | parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags}); |
66 | 0 | parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false}); |
67 | 0 | parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false}); |
68 | 0 | parser->addMatcher(parser->fLocalMatchers.percent = {symbols}); |
69 | 0 | parser->addMatcher(parser->fLocalMatchers.permille = {symbols}); |
70 | 0 | parser->addMatcher(parser->fLocalMatchers.nan = {symbols}); |
71 | 0 | parser->addMatcher(parser->fLocalMatchers.infinity = {symbols}); |
72 | 0 | parser->addMatcher(parser->fLocalMatchers.padding = {u"@"}); |
73 | 0 | parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper}); |
74 | 0 | parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status}); |
75 | 0 | parser->addMatcher(parser->fLocalValidators.number = {}); |
76 | |
|
77 | 0 | parser->freeze(); |
78 | 0 | return parser.orphan(); |
79 | 0 | } |
80 | | |
81 | | NumberParserImpl* |
82 | | NumberParserImpl::createParserFromProperties(const number::impl::DecimalFormatProperties& properties, |
83 | | const DecimalFormatSymbols& symbols, bool parseCurrency, |
84 | 0 | UErrorCode& status) { |
85 | 0 | Locale locale = symbols.getLocale(); |
86 | 0 | AutoAffixPatternProvider affixProvider(properties, status); |
87 | 0 | if (U_FAILURE(status)) { return nullptr; } |
88 | 0 | CurrencyUnit currency = resolveCurrency(properties, locale, status); |
89 | 0 | CurrencySymbols currencySymbols(currency, locale, symbols, status); |
90 | 0 | bool isStrict = properties.parseMode.getOrDefault(PARSE_MODE_STRICT) == PARSE_MODE_STRICT; |
91 | 0 | Grouper grouper = Grouper::forProperties(properties); |
92 | 0 | int parseFlags = 0; |
93 | 0 | if (U_FAILURE(status)) { return nullptr; } |
94 | 0 | if (!properties.parseCaseSensitive) { |
95 | 0 | parseFlags |= PARSE_FLAG_IGNORE_CASE; |
96 | 0 | } |
97 | 0 | if (properties.parseIntegerOnly) { |
98 | 0 | parseFlags |= PARSE_FLAG_INTEGER_ONLY; |
99 | 0 | } |
100 | 0 | if (properties.signAlwaysShown) { |
101 | 0 | parseFlags |= PARSE_FLAG_PLUS_SIGN_ALLOWED; |
102 | 0 | } |
103 | 0 | if (isStrict) { |
104 | 0 | parseFlags |= PARSE_FLAG_STRICT_GROUPING_SIZE; |
105 | 0 | parseFlags |= PARSE_FLAG_STRICT_SEPARATORS; |
106 | 0 | parseFlags |= PARSE_FLAG_USE_FULL_AFFIXES; |
107 | 0 | parseFlags |= PARSE_FLAG_EXACT_AFFIX; |
108 | 0 | parseFlags |= PARSE_FLAG_STRICT_IGNORABLES; |
109 | 0 | } else { |
110 | 0 | parseFlags |= PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES; |
111 | 0 | } |
112 | 0 | if (grouper.getPrimary() <= 0) { |
113 | 0 | parseFlags |= PARSE_FLAG_GROUPING_DISABLED; |
114 | 0 | } |
115 | 0 | if (parseCurrency || affixProvider.get().hasCurrencySign()) { |
116 | 0 | parseFlags |= PARSE_FLAG_MONETARY_SEPARATORS; |
117 | 0 | } |
118 | 0 | if (!parseCurrency) { |
119 | 0 | parseFlags |= PARSE_FLAG_NO_FOREIGN_CURRENCY; |
120 | 0 | } |
121 | |
|
122 | 0 | LocalPointer<NumberParserImpl> parser(new NumberParserImpl(parseFlags)); |
123 | |
|
124 | 0 | parser->fLocalMatchers.ignorables = {parseFlags}; |
125 | 0 | IgnorablesMatcher& ignorables = parser->fLocalMatchers.ignorables; |
126 | | |
127 | | ////////////////////// |
128 | | /// AFFIX MATCHERS /// |
129 | | ////////////////////// |
130 | | |
131 | | // The following statements set up the affix matchers. |
132 | 0 | AffixTokenMatcherSetupData affixSetupData = { |
133 | 0 | currencySymbols, symbols, ignorables, locale, parseFlags}; |
134 | 0 | parser->fLocalMatchers.affixTokenMatcherWarehouse = {&affixSetupData}; |
135 | 0 | parser->fLocalMatchers.affixMatcherWarehouse = {&parser->fLocalMatchers.affixTokenMatcherWarehouse}; |
136 | 0 | parser->fLocalMatchers.affixMatcherWarehouse.createAffixMatchers( |
137 | 0 | affixProvider.get(), *parser, ignorables, parseFlags, status); |
138 | | |
139 | | //////////////////////// |
140 | | /// CURRENCY MATCHER /// |
141 | | //////////////////////// |
142 | |
|
143 | 0 | if (parseCurrency || affixProvider.get().hasCurrencySign()) { |
144 | 0 | parser->addMatcher(parser->fLocalMatchers.currency = {currencySymbols, symbols, parseFlags, status}); |
145 | 0 | } |
146 | | |
147 | | /////////////// |
148 | | /// PERCENT /// |
149 | | /////////////// |
150 | | |
151 | | // ICU-TC meeting, April 11, 2018: accept percent/permille only if it is in the pattern, |
152 | | // and to maintain regressive behavior, divide by 100 even if no percent sign is present. |
153 | 0 | if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERCENT, status)) { |
154 | 0 | parser->addMatcher(parser->fLocalMatchers.percent = {symbols}); |
155 | 0 | } |
156 | 0 | if (!isStrict && affixProvider.get().containsSymbolType(AffixPatternType::TYPE_PERMILLE, status)) { |
157 | 0 | parser->addMatcher(parser->fLocalMatchers.permille = {symbols}); |
158 | 0 | } |
159 | | |
160 | | /////////////////////////////// |
161 | | /// OTHER STANDARD MATCHERS /// |
162 | | /////////////////////////////// |
163 | |
|
164 | 0 | if (!isStrict) { |
165 | 0 | parser->addMatcher(parser->fLocalMatchers.plusSign = {symbols, false}); |
166 | 0 | parser->addMatcher(parser->fLocalMatchers.minusSign = {symbols, false}); |
167 | 0 | } |
168 | 0 | parser->addMatcher(parser->fLocalMatchers.nan = {symbols}); |
169 | 0 | parser->addMatcher(parser->fLocalMatchers.infinity = {symbols}); |
170 | 0 | UnicodeString padString = properties.padString; |
171 | 0 | if (!padString.isBogus() && !ignorables.getSet()->contains(padString)) { |
172 | 0 | parser->addMatcher(parser->fLocalMatchers.padding = {padString}); |
173 | 0 | } |
174 | 0 | parser->addMatcher(parser->fLocalMatchers.ignorables); |
175 | 0 | parser->addMatcher(parser->fLocalMatchers.decimal = {symbols, grouper, parseFlags}); |
176 | | // NOTE: parseNoExponent doesn't disable scientific parsing if we have a scientific formatter |
177 | 0 | if (!properties.parseNoExponent || properties.minimumExponentDigits > 0) { |
178 | 0 | parser->addMatcher(parser->fLocalMatchers.scientific = {symbols, grouper}); |
179 | 0 | } |
180 | | |
181 | | ////////////////// |
182 | | /// VALIDATORS /// |
183 | | ////////////////// |
184 | |
|
185 | 0 | parser->addMatcher(parser->fLocalValidators.number = {}); |
186 | 0 | if (isStrict) { |
187 | 0 | parser->addMatcher(parser->fLocalValidators.affix = {}); |
188 | 0 | } |
189 | 0 | if (parseCurrency) { |
190 | 0 | parser->addMatcher(parser->fLocalValidators.currency = {}); |
191 | 0 | } |
192 | 0 | if (properties.decimalPatternMatchRequired) { |
193 | 0 | bool patternHasDecimalSeparator = |
194 | 0 | properties.decimalSeparatorAlwaysShown || properties.maximumFractionDigits != 0; |
195 | 0 | parser->addMatcher(parser->fLocalValidators.decimalSeparator = {patternHasDecimalSeparator}); |
196 | 0 | } |
197 | | // The multiplier takes care of scaling percentages. |
198 | 0 | Scale multiplier = scaleFromProperties(properties); |
199 | 0 | if (multiplier.isValid()) { |
200 | 0 | parser->addMatcher(parser->fLocalValidators.multiplier = {multiplier}); |
201 | 0 | } |
202 | |
|
203 | 0 | parser->freeze(); |
204 | 0 | return parser.orphan(); |
205 | 0 | } |
206 | | |
207 | | NumberParserImpl::NumberParserImpl(parse_flags_t parseFlags) |
208 | 0 | : fParseFlags(parseFlags) { |
209 | 0 | } |
210 | | |
211 | 0 | NumberParserImpl::~NumberParserImpl() { |
212 | 0 | fNumMatchers = 0; |
213 | 0 | } |
214 | | |
215 | 0 | void NumberParserImpl::addMatcher(NumberParseMatcher& matcher) { |
216 | 0 | if (fNumMatchers + 1 > fMatchers.getCapacity()) { |
217 | 0 | fMatchers.resize(fNumMatchers * 2, fNumMatchers); |
218 | 0 | } |
219 | 0 | fMatchers[fNumMatchers] = &matcher; |
220 | 0 | fNumMatchers++; |
221 | 0 | } |
222 | | |
223 | 0 | void NumberParserImpl::freeze() { |
224 | 0 | fFrozen = true; |
225 | 0 | } |
226 | | |
227 | 0 | parse_flags_t NumberParserImpl::getParseFlags() const { |
228 | 0 | return fParseFlags; |
229 | 0 | } |
230 | | |
231 | | void NumberParserImpl::parse(const UnicodeString& input, bool greedy, ParsedNumber& result, |
232 | 0 | UErrorCode& status) const { |
233 | 0 | return parse(input, 0, greedy, result, status); |
234 | 0 | } |
235 | | |
236 | | void NumberParserImpl::parse(const UnicodeString& input, int32_t start, bool greedy, ParsedNumber& result, |
237 | 0 | UErrorCode& status) const { |
238 | 0 | if (U_FAILURE(status)) { |
239 | 0 | return; |
240 | 0 | } |
241 | 0 | U_ASSERT(fFrozen); |
242 | | // TODO: Check start >= 0 and start < input.length() |
243 | 0 | StringSegment segment(input, 0 != (fParseFlags & PARSE_FLAG_IGNORE_CASE)); |
244 | 0 | segment.adjustOffset(start); |
245 | 0 | if (greedy) { |
246 | 0 | parseGreedy(segment, result, status); |
247 | 0 | } else if (0 != (fParseFlags & PARSE_FLAG_ALLOW_INFINITE_RECURSION)) { |
248 | | // Start at 1 so that recursionLevels never gets to 0 |
249 | 0 | parseLongestRecursive(segment, result, 1, status); |
250 | 0 | } else { |
251 | | // Arbitrary recursion safety limit: 100 levels. |
252 | 0 | parseLongestRecursive(segment, result, -100, status); |
253 | 0 | } |
254 | 0 | for (int32_t i = 0; i < fNumMatchers; i++) { |
255 | 0 | fMatchers[i]->postProcess(result); |
256 | 0 | } |
257 | 0 | result.postProcess(); |
258 | 0 | } |
259 | | |
260 | | void NumberParserImpl::parseGreedy(StringSegment& segment, ParsedNumber& result, |
261 | 0 | UErrorCode& status) const { |
262 | | // Note: this method is not recursive in order to avoid stack overflow. |
263 | 0 | for (int i = 0; i <fNumMatchers;) { |
264 | | // Base Case |
265 | 0 | if (segment.length() == 0) { |
266 | 0 | return; |
267 | 0 | } |
268 | 0 | const NumberParseMatcher* matcher = fMatchers[i]; |
269 | 0 | if (!matcher->smokeTest(segment)) { |
270 | | // Matcher failed smoke test: try the next one |
271 | 0 | i++; |
272 | 0 | continue; |
273 | 0 | } |
274 | 0 | int32_t initialOffset = segment.getOffset(); |
275 | 0 | matcher->match(segment, result, status); |
276 | 0 | if (U_FAILURE(status)) { |
277 | 0 | return; |
278 | 0 | } |
279 | 0 | if (segment.getOffset() != initialOffset) { |
280 | | // Greedy heuristic: accept the match and loop back |
281 | 0 | i = 0; |
282 | 0 | continue; |
283 | 0 | } else { |
284 | | // Matcher did not match: try the next one |
285 | 0 | i++; |
286 | 0 | continue; |
287 | 0 | } |
288 | 0 | UPRV_UNREACHABLE; |
289 | 0 | } |
290 | | |
291 | | // NOTE: If we get here, the greedy parse completed without consuming the entire string. |
292 | 0 | } |
293 | | |
294 | | void NumberParserImpl::parseLongestRecursive(StringSegment& segment, ParsedNumber& result, |
295 | | int32_t recursionLevels, |
296 | 0 | UErrorCode& status) const { |
297 | | // Base Case |
298 | 0 | if (segment.length() == 0) { |
299 | 0 | return; |
300 | 0 | } |
301 | | |
302 | | // Safety against stack overflow |
303 | 0 | if (recursionLevels == 0) { |
304 | 0 | return; |
305 | 0 | } |
306 | | |
307 | | // TODO: Give a nice way for the matcher to reset the ParsedNumber? |
308 | 0 | ParsedNumber initial(result); |
309 | 0 | ParsedNumber candidate; |
310 | |
|
311 | 0 | int initialOffset = segment.getOffset(); |
312 | 0 | for (int32_t i = 0; i < fNumMatchers; i++) { |
313 | 0 | const NumberParseMatcher* matcher = fMatchers[i]; |
314 | 0 | if (!matcher->smokeTest(segment)) { |
315 | 0 | continue; |
316 | 0 | } |
317 | | |
318 | | // In a non-greedy parse, we attempt all possible matches and pick the best. |
319 | 0 | for (int32_t charsToConsume = 0; charsToConsume < segment.length();) { |
320 | 0 | charsToConsume += U16_LENGTH(segment.codePointAt(charsToConsume)); |
321 | | |
322 | | // Run the matcher on a segment of the current length. |
323 | 0 | candidate = initial; |
324 | 0 | segment.setLength(charsToConsume); |
325 | 0 | bool maybeMore = matcher->match(segment, candidate, status); |
326 | 0 | segment.resetLength(); |
327 | 0 | if (U_FAILURE(status)) { |
328 | 0 | return; |
329 | 0 | } |
330 | | |
331 | | // If the entire segment was consumed, recurse. |
332 | 0 | if (segment.getOffset() - initialOffset == charsToConsume) { |
333 | 0 | parseLongestRecursive(segment, candidate, recursionLevels + 1, status); |
334 | 0 | if (U_FAILURE(status)) { |
335 | 0 | return; |
336 | 0 | } |
337 | 0 | if (candidate.isBetterThan(result)) { |
338 | 0 | result = candidate; |
339 | 0 | } |
340 | 0 | } |
341 | | |
342 | | // Since the segment can be re-used, reset the offset. |
343 | | // This does not have an effect if the matcher did not consume any chars. |
344 | 0 | segment.setOffset(initialOffset); |
345 | | |
346 | | // Unless the matcher wants to see the next char, continue to the next matcher. |
347 | 0 | if (!maybeMore) { |
348 | 0 | break; |
349 | 0 | } |
350 | 0 | } |
351 | 0 | } |
352 | 0 | } |
353 | | |
354 | 0 | UnicodeString NumberParserImpl::toString() const { |
355 | 0 | UnicodeString result(u"<NumberParserImpl matchers:["); |
356 | 0 | for (int32_t i = 0; i < fNumMatchers; i++) { |
357 | 0 | result.append(u' '); |
358 | 0 | result.append(fMatchers[i]->toString()); |
359 | 0 | } |
360 | 0 | result.append(u" ]>", -1); |
361 | 0 | return result; |
362 | 0 | } |
363 | | |
364 | | |
365 | | #endif /* #if !UCONFIG_NO_FORMATTING */ |