1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
|
// ยฉ 2018 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
#include "unicode/utypes.h"
#if !UCONFIG_NO_FORMATTING
#include "numbertest.h"
#include "numparse_impl.h"
#include "static_unicode_sets.h"
#include "unicode/dcfmtsym.h"
#include "unicode/testlog.h"
#include <cmath>
#include <numparse_affixes.h>
void NumberParserTest::runIndexedTest(int32_t index, UBool exec, const char*& name, char*) {
if (exec) {
logln("TestSuite NumberParserTest: ");
}
TESTCASE_AUTO_BEGIN;
TESTCASE_AUTO(testBasic);
TESTCASE_AUTO(testSeriesMatcher);
TESTCASE_AUTO(testCombinedCurrencyMatcher);
TESTCASE_AUTO(testAffixPatternMatcher);
TESTCASE_AUTO(test20360_BidiOverflow);
TESTCASE_AUTO(testInfiniteRecursion);
TESTCASE_AUTO_END;
}
void NumberParserTest::testBasic() {
IcuTestErrorCode status(*this, "testBasic");
static const struct TestCase {
int32_t flags;
const char16_t* inputString;
const char16_t* patternString;
int32_t expectedCharsConsumed;
double expectedResultDouble;
} cases[] = {{3, u"51423", u"0", 5, 51423.},
{3, u"51423x", u"0", 5, 51423.},
{3, u" 51423", u"0", 6, 51423.},
{3, u"51423 ", u"0", 5, 51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏ", u"0", 10, 51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏx", u"0", 10, 51423.},
{3, u" ๐ฑ๐ญ๐ฐ๐ฎ๐ฏ", u"0", 11, 51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏ ", u"0", 10, 51423.},
{7, u"51,423", u"#,##,##0", 6, 51423.},
{7, u" 51,423", u"#,##,##0", 7, 51423.},
{7, u"51,423 ", u"#,##,##0", 6, 51423.},
{7, u"51,423,", u"#,##,##0", 6, 51423.},
{7, u"51,423,,", u"#,##,##0", 6, 51423.},
{7, u"51,423.5", u"#,##,##0", 8, 51423.5},
{7, u"51,423.5,", u"#,##,##0", 8, 51423.5},
{7, u"51,423.5,,", u"#,##,##0", 8, 51423.5},
{7, u"51,423.5.", u"#,##,##0", 8, 51423.5},
{7, u"51,423.5..", u"#,##,##0", 8, 51423.5},
{7, u"๐ฑ๐ญ,๐ฐ๐ฎ๐ฏ", u"#,##,##0", 11, 51423.},
{7, u"๐ณ,๐ด๐ต,๐ฑ๐ญ,๐ฐ๐ฎ๐ฏ", u"#,##,##0", 19, 78951423.},
{7, u"๐ณ๐ด,๐ต๐ฑ๐ญ.๐ฐ๐ฎ๐ฏ", u"#,##,##0", 18, 78951.423},
{7, u"๐ณ๐ด,๐ฌ๐ฌ๐ฌ", u"#,##,##0", 11, 78000.},
{7, u"๐ณ๐ด,๐ฌ๐ฌ๐ฌ.๐ฌ๐ฌ๐ฌ", u"#,##,##0", 18, 78000.},
{7, u"๐ณ๐ด,๐ฌ๐ฌ๐ฌ.๐ฌ๐ฎ๐ฏ", u"#,##,##0", 18, 78000.023},
{7, u"๐ณ๐ด.๐ฌ๐ฌ๐ฌ.๐ฌ๐ฎ๐ฏ", u"#,##,##0", 11, 78.},
{7, u"1,", u"#,##,##0", 1, 1.},
{7, u"1,,", u"#,##,##0", 1, 1.},
{7, u"1.,", u"#,##,##0", 2, 1.},
{3, u"1,.", u"#,##,##0", 3, 1.},
{7, u"1..", u"#,##,##0", 2, 1.},
{3, u",1", u"#,##,##0", 2, 1.},
{3, u"1,1", u"#,##,##0", 1, 1.},
{3, u"1,1,", u"#,##,##0", 1, 1.},
{3, u"1,1,,", u"#,##,##0", 1, 1.},
{3, u"1,1,1", u"#,##,##0", 1, 1.},
{3, u"1,1,1,", u"#,##,##0", 1, 1.},
{3, u"1,1,1,,", u"#,##,##0", 1, 1.},
{3, u"-51423", u"0", 6, -51423.},
{3, u"51423-", u"0", 5, 51423.}, // plus and minus sign by default do NOT match after
{3, u"+51423", u"0", 6, 51423.},
{3, u"51423+", u"0", 5, 51423.}, // plus and minus sign by default do NOT match after
{3, u"%51423", u"0", 6, 51423.},
{3, u"51423%", u"0", 6, 51423.},
{3, u"51423%%", u"0", 6, 51423.},
{3, u"โฐ51423", u"0", 6, 51423.},
{3, u"51423โฐ", u"0", 6, 51423.},
{3, u"51423โฐโฐ", u"0", 6, 51423.},
{3, u"โ", u"0", 1, INFINITY},
{3, u"-โ", u"0", 2, -INFINITY},
{3, u"@@@123 @@", u"0", 6, 123.}, // TODO: Should padding be strong instead of weak?
{3, u"@@@123@@ ", u"0", 6, 123.}, // TODO: Should padding be strong instead of weak?
{3, u"a51423US dollars", u"a0ยคยคยค", 16, 51423.},
{3, u"a 51423 US dollars", u"a0ยคยคยค", 18, 51423.},
{3, u"514.23 USD", u"ยค0", 10, 514.23},
{3, u"514.23 GBP", u"ยค0", 10, 514.23},
{3, u"a ๐ฑ๐ญ๐ฐ๐ฎ๐ฏ b", u"a0b", 14, 51423.},
{3, u"-a ๐ฑ๐ญ๐ฐ๐ฎ๐ฏ b", u"a0b", 15, -51423.},
{3, u"a -๐ฑ๐ญ๐ฐ๐ฎ๐ฏ b", u"a0b", 15, -51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏ", u"[0];(0)", 10, 51423.},
{3, u"[๐ฑ๐ญ๐ฐ๐ฎ๐ฏ", u"[0];(0)", 11, 51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏ]", u"[0];(0)", 11, 51423.},
{3, u"[๐ฑ๐ญ๐ฐ๐ฎ๐ฏ]", u"[0];(0)", 12, 51423.},
{3, u"(๐ฑ๐ญ๐ฐ๐ฎ๐ฏ", u"[0];(0)", 11, -51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏ)", u"[0];(0)", 11, -51423.},
{3, u"(๐ฑ๐ญ๐ฐ๐ฎ๐ฏ)", u"[0];(0)", 12, -51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏ", u"{0};{0}", 10, 51423.},
{3, u"{๐ฑ๐ญ๐ฐ๐ฎ๐ฏ", u"{0};{0}", 11, 51423.},
{3, u"๐ฑ๐ญ๐ฐ๐ฎ๐ฏ}", u"{0};{0}", 11, 51423.},
{3, u"{๐ฑ๐ญ๐ฐ๐ฎ๐ฏ}", u"{0};{0}", 12, 51423.},
{1, u"a40b", u"a0'0b'", 3, 40.}, // greedy code path thinks "40" is the number
{2, u"a40b", u"a0'0b'", 4, 4.}, // slow code path finds the suffix "0b"
{3, u"๐ฑ.๐ญ๐ฐ๐ฎE๐ฏ", u"0", 12, 5142.},
{3, u"๐ฑ.๐ญ๐ฐ๐ฎE-๐ฏ", u"0", 13, 0.005142},
{3, u"๐ฑ.๐ญ๐ฐ๐ฎe-๐ฏ", u"0", 13, 0.005142},
{3, u"5.142e+3", u"0", 8, 5142.0 },
{3, u"5.142\u200Ee+3", u"0", 9, 5142.0},
{3, u"5.142e\u200E+3", u"0", 9, 5142.0},
{3, u"5.142e+\u200E3", u"0", 9, 5142.0},
{7, u"5,142.50 Canadian dollars", u"#,##,##0 ยคยคยค", 25, 5142.5},
{3, u"a$ b5", u"a ยค b0", 5, 5.0},
{3, u"๐บ1.23", u"๐บ0;๐ป0", 6, 1.23},
{3, u"๐ป1.23", u"๐บ0;๐ป0", 6, -1.23},
{3, u".00", u"0", 3, 0.0},
{3, u" 1,234", u"a0", 35, 1234.}, // should not hang
{3, u"NaN", u"0", 3, NAN},
{3, u"NaN E5", u"0", 6, NAN},
{3, u"~100", u"~0", 4, 100.0},
{3, u" ~ 100", u"~0", 6, 100.0},
{3, u"โ100", u"~0", 4, 100.0},
{3, u"100โ", u"~0", 3, 100.0},
{3, u"0", u"0", 1, 0.0}};
parse_flags_t parseFlags = PARSE_FLAG_IGNORE_CASE | PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES;
for (const auto& cas : cases) {
UnicodeString inputString(cas.inputString);
UnicodeString patternString(cas.patternString);
LocalPointer<const NumberParserImpl> parser(
NumberParserImpl::createSimpleParser(
Locale("en"), patternString, parseFlags, status));
if (status.errDataIfFailureAndReset("createSimpleParser() failed")) {
continue;
}
UnicodeString message =
UnicodeString("Input <") + inputString + UnicodeString("> Parser ") + parser->toString();
if (0 != (cas.flags & 0x01)) {
// Test greedy code path
ParsedNumber resultObject;
parser->parse(inputString, true, resultObject, status);
assertTrue("Greedy Parse failed: " + message, resultObject.success());
assertEquals("Greedy Parse failed: " + message,
cas.expectedCharsConsumed, resultObject.charEnd);
assertEquals("Greedy Parse failed: " + message,
cas.expectedResultDouble, resultObject.getDouble(status));
}
if (0 != (cas.flags & 0x02)) {
// Test slow code path
ParsedNumber resultObject;
parser->parse(inputString, false, resultObject, status);
assertTrue("Non-Greedy Parse failed: " + message, resultObject.success());
assertEquals(
"Non-Greedy Parse failed: " + message,
cas.expectedCharsConsumed,
resultObject.charEnd);
assertEquals(
"Non-Greedy Parse failed: " + message,
cas.expectedResultDouble,
resultObject.getDouble(status));
}
if (0 != (cas.flags & 0x04)) {
// Test with strict separators
parser.adoptInstead(
NumberParserImpl::createSimpleParser(
Locale("en"),
patternString,
parseFlags | PARSE_FLAG_STRICT_GROUPING_SIZE,
status));
ParsedNumber resultObject;
parser->parse(inputString, true, resultObject, status);
assertTrue("Strict Parse failed: " + message, resultObject.success());
assertEquals("Strict Parse failed: " + message,
cas.expectedCharsConsumed, resultObject.charEnd);
assertEquals("Strict Parse failed: " + message,
cas.expectedResultDouble, resultObject.getDouble(status));
}
if (status.errDataIfFailureAndReset("parsing test failed")) {
continue;
}
}
}
void NumberParserTest::testSeriesMatcher() {
IcuTestErrorCode status(*this, "testSeriesMatcher");
DecimalFormatSymbols symbols("en", status);
if (status.errDataIfFailureAndReset("Failure in DecimalFormtSymbols constructor")) {
return;
}
PlusSignMatcher m0(symbols, false);
MinusSignMatcher m1(symbols, false);
IgnorablesMatcher m2(0);
PercentMatcher m3(symbols);
IgnorablesMatcher m4(0);
ArraySeriesMatcher::MatcherArray matchers(5, status);
status.assertSuccess();
matchers[0] = &m0;
matchers[1] = &m1;
matchers[2] = &m2;
matchers[3] = &m3;
matchers[4] = &m4;
ArraySeriesMatcher series(matchers, 5);
assertFalse("", series.smokeTest(StringSegment(u"x", false)));
assertFalse("", series.smokeTest(StringSegment(u"-", false)));
assertTrue("", series.smokeTest(StringSegment(u"+", false)));
static const struct TestCase {
const char16_t* input;
int32_t expectedOffset;
bool expectedMaybeMore;
} cases[] = {{u"", 0, true},
{u" ", 0, false},
{u"$", 0, false},
{u"+", 0, true},
{u" +", 0, false},
{u"+-", 0, true},
{u"+ -", 0, false},
{u"+- ", 0, true},
{u"+- $", 0, false},
{u"+-%", 3, true},
{u" +- % ", 0, false},
{u"+- % ", 7, true},
{u"+-%$", 3, false}};
for (const auto& cas : cases) {
UnicodeString input(cas.input);
StringSegment segment(input, false);
ParsedNumber result;
bool actualMaybeMore = series.match(segment, result, status);
int actualOffset = segment.getOffset();
assertEquals("'" + input + "'", cas.expectedOffset, actualOffset);
assertEquals("'" + input + "'", cas.expectedMaybeMore, actualMaybeMore);
}
}
void NumberParserTest::testCombinedCurrencyMatcher() {
IcuTestErrorCode status(*this, "testCombinedCurrencyMatcher");
IgnorablesMatcher ignorables(0);
Locale locale = Locale::getEnglish();
DecimalFormatSymbols dfs(locale, status);
if (status.errDataIfFailureAndReset("Failure in DecimalFormtSymbols constructor")) {
return;
}
dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$", status);
dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU", status);
CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status);
AffixTokenMatcherSetupData affixSetupData = {
currencySymbols, {"en", status}, ignorables, "en", 0};
AffixTokenMatcherWarehouse warehouse(&affixSetupData);
NumberParseMatcher& matcher = warehouse.currency(status);
affixSetupData.parseFlags = PARSE_FLAG_NO_FOREIGN_CURRENCY;
AffixTokenMatcherWarehouse warehouseNoForeign(&affixSetupData);
NumberParseMatcher& matcherNoForeign = warehouseNoForeign.currency(status);
static const struct TestCase {
const char16_t* input;
const char16_t* expectedCurrencyCode;
const char16_t* expectedNoForeignCurrencyCode;
} cases[]{{u"", u"", u""},
{u"FOO", u"", u""},
{u"USD", u"USD", u""},
{u"$", u"USD", u""},
{u"US dollars", u"USD", u""},
{u"eu", u"", u""},
{u"euros", u"EUR", u""},
{u"ICU", u"ICU", u"ICU"},
{u"IU$", u"ICU", u"ICU"}};
for (const auto& cas : cases) {
UnicodeString input(cas.input);
{
StringSegment segment(input, false);
ParsedNumber result;
matcher.match(segment, result, status);
assertEquals(
"Parsing " + input,
cas.expectedCurrencyCode,
result.currencyCode);
assertEquals(
"Whole string on " + input,
cas.expectedCurrencyCode[0] == 0 ? 0 : input.length(),
result.charEnd);
}
{
StringSegment segment(input, false);
ParsedNumber result;
matcherNoForeign.match(segment, result, status);
assertEquals(
"[no foreign] Parsing " + input,
cas.expectedNoForeignCurrencyCode,
result.currencyCode);
assertEquals(
"[no foreign] Whole string on " + input,
cas.expectedNoForeignCurrencyCode[0] == 0 ? 0 : input.length(),
result.charEnd);
}
}
}
void NumberParserTest::testAffixPatternMatcher() {
IcuTestErrorCode status(*this, "testAffixPatternMatcher");
Locale locale = Locale::getEnglish();
IgnorablesMatcher ignorables(0);
DecimalFormatSymbols dfs(locale, status);
dfs.setSymbol(DecimalFormatSymbols::kCurrencySymbol, u"IU$", status);
dfs.setSymbol(DecimalFormatSymbols::kIntlCurrencySymbol, u"ICU", status);
CurrencySymbols currencySymbols({u"ICU", status}, locale, dfs, status);
AffixTokenMatcherSetupData affixSetupData = {
currencySymbols, {"en", status}, ignorables, "en", 0};
AffixTokenMatcherWarehouse warehouse(&affixSetupData);
static const struct TestCase {
bool exactMatch;
const char16_t* affixPattern;
int32_t expectedMatcherLength;
const char16_t* sampleParseableString;
} cases[] = {{false, u"-", 1, u"-"},
{false, u"+-%", 5, u"+-%"},
{true, u"+-%", 3, u"+-%"},
{false, u"ab c", 5, u"a bc"},
{true, u"abc", 3, u"abc"},
{false, u"hello-to+this%veryยคlongโฐstring", 59, u"hello-to+this%very USD longโฐstring"}};
for (const auto& cas : cases) {
UnicodeString affixPattern(cas.affixPattern);
UnicodeString sampleParseableString(cas.sampleParseableString);
int parseFlags = cas.exactMatch ? PARSE_FLAG_EXACT_AFFIX : 0;
bool success;
AffixPatternMatcher matcher = AffixPatternMatcher::fromAffixPattern(
affixPattern, warehouse, parseFlags, &success, status);
if (!status.errDataIfFailureAndReset("Creation should be successful")) {
// Check that the matcher has the expected number of children
assertEquals(affixPattern + " " + cas.exactMatch, cas.expectedMatcherLength, matcher.length());
// Check that the matcher works on a sample string
StringSegment segment(sampleParseableString, false);
ParsedNumber result;
matcher.match(segment, result, status);
assertEquals(affixPattern + " " + cas.exactMatch, sampleParseableString.length(), result.charEnd);
}
}
}
void NumberParserTest::test20360_BidiOverflow() {
IcuTestErrorCode status(*this, "test20360_BidiOverflow");
UnicodeString inputString;
inputString.append(u'-');
for (int32_t i=0; i<100000; i++) {
inputString.append(u'\u061C');
}
inputString.append(u'5');
LocalPointer<const NumberParserImpl> parser(NumberParserImpl::createSimpleParser("en", u"0", 0, status));
if (status.errDataIfFailureAndReset("createSimpleParser() failed")) {
return;
}
ParsedNumber resultObject;
parser->parse(inputString, true, resultObject, status);
assertTrue("Greedy Parse, success", resultObject.success());
assertEquals("Greedy Parse, chars consumed", 100002, resultObject.charEnd);
assertEquals("Greedy Parse, expected double", -5.0, resultObject.getDouble(status));
resultObject.clear();
parser->parse(inputString, false, resultObject, status);
assertFalse("Non-Greedy Parse, success", resultObject.success());
assertEquals("Non-Greedy Parse, chars consumed", 1, resultObject.charEnd);
}
void NumberParserTest::testInfiniteRecursion() {
IcuTestErrorCode status(*this, "testInfiniteRecursion");
UnicodeString inputString;
inputString.append(u'-');
for (int32_t i=0; i<200; i++) {
inputString.append(u'\u061C');
}
inputString.append(u'5');
LocalPointer<const NumberParserImpl> parser(NumberParserImpl::createSimpleParser("en", u"0", 0, status));
if (status.errDataIfFailureAndReset("createSimpleParser() failed")) {
return;
}
ParsedNumber resultObject;
parser->parse(inputString, false, resultObject, status);
assertFalse("Default recursion limit, success", resultObject.success());
assertEquals("Default recursion limit, chars consumed", 1, resultObject.charEnd);
parser.adoptInstead(NumberParserImpl::createSimpleParser(
"en", u"0", PARSE_FLAG_ALLOW_INFINITE_RECURSION, status));
resultObject.clear();
parser->parse(inputString, false, resultObject, status);
assertTrue("Unlimited recursion, success", resultObject.success());
assertEquals("Unlimited recursion, chars consumed", 202, resultObject.charEnd);
assertEquals("Unlimited recursion, expected double", -5.0, resultObject.getDouble(status));
}
#endif
|
