Skip to content

Commit c6010ed

Browse files
mihnitamihnita
committed
ICU-22908 MF2 ICU4J: Update spec tests and update implementation for recent spec changes
1 parent 2f348f4 commit c6010ed

25 files changed

+461
-359
lines changed

‎icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelValidator.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ private boolean validateDeclarations(List<Declaration> declarations) throws MFPa
140140
private void validateExpression(Expression expression, boolean fromInput)
141141
throws MFParseException {
142142
String argName = null;
143+
boolean wasLiteral = false;
143144
Annotation annotation = null;
144145
if (expression instanceof Literal) {
145146
// ...{foo}... or ...{|foo|}... or ...{123}...
@@ -148,6 +149,7 @@ private void validateExpression(Expression expression, boolean fromInput)
148149
LiteralExpression le = (LiteralExpression) expression;
149150
argName = le.arg.value;
150151
annotation = le.annotation;
152+
wasLiteral = true;
151153
} else if (expression instanceof VariableExpression) {
152154
VariableExpression ve = (VariableExpression) expression;
153155
// ...{$foo :bar opt1=|str| opt2=$x opt3=$y}...
@@ -184,7 +186,10 @@ private void validateExpression(Expression expression, boolean fromInput)
184186
addVariableDeclaration(argName);
185187
} else {
186188
// Remember that we've seen it, to complain if there is a declaration later
187-
declaredVars.add(argName);
189+
if (!wasLiteral) {
190+
// We don't consider {|bar| :func} to be a declaration of a "bar" variable
191+
declaredVars.add(argName);
192+
}
188193
}
189194
}
190195
}

‎icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -559,9 +559,9 @@ private MFDataModel.Message getComplexMessage() throws MFParseException {
559559
}
560560
}
561561

562-
// abnf: matcher = match-statement 1*([s] variant)
563-
// abnf: match-statement = match 1*([s] selector)
564-
// abnf: selector = expression
562+
// abnf: matcher = match-statement s variant *([s] variant)
563+
// abnf: match-statement = match 1*(s selector)
564+
// abnf: selector = variable
565565
// abnf: variant = key *(s key) [s] quoted-pattern
566566
// abnf: key = literal / "*"
567567
// abnf: match = %s".match"
@@ -571,17 +571,18 @@ private MFDataModel.SelectMessage getMatch(List<MFDataModel.Declaration> declara
571571
// Look for selectors
572572
List<MFDataModel.Expression> expressions = new ArrayList<>();
573573
while (true) {
574-
// Whitespace not required between selectors:
575-
// match 1*([s] selector)
576-
// Whitespace not required before first variant:
577-
// matcher = match-statement 1*([s] variant)
578-
skipOptionalWhitespaces();
579-
MFDataModel.Expression expression = getPlaceholder();
580-
if (expression == null) {
574+
// Whitespace required between selectors but not required before first variant.
575+
skipMandatoryWhitespaces();
576+
int cp = input.peekChar();
577+
if (cp != '$') {
581578
break;
582579
}
583-
checkCondition(
584-
!(expression instanceof MFDataModel.Markup), "Cannot do selection on markup");
580+
MFDataModel.VariableRef variableRef = getVariableRef();
581+
if (variableRef == null) {
582+
break;
583+
}
584+
MFDataModel.Expression expression =
585+
new MFDataModel.VariableExpression(variableRef, null, new ArrayList<>());
585586
expressions.add(expression);
586587
}
587588

‎icu4j/main/core/src/main/java/com/ibm/icu/message2/NumberFormatterFactory.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,14 +163,16 @@ public FormattedPlaceholder format(Object toFormat, Map<String, Object> variable
163163
private static class PluralSelectorImpl implements Selector {
164164
private static final String NO_MATCH = "\uFFFDNO_MATCH\uFFFE"; // Unlikely to show in a key
165165
private final PluralRules rules;
166-
private Map<String, Object> fixedOptions;
167-
private LocalizedNumberFormatter icuFormatter;
166+
private final Map<String, Object> fixedOptions;
167+
private final LocalizedNumberFormatter icuFormatter;
168+
private final String kind;
168169

169170
private PluralSelectorImpl(
170171
Locale locale, PluralRules rules, Map<String, Object> fixedOptions, String kind) {
171172
this.rules = rules;
172173
this.fixedOptions = fixedOptions;
173174
this.icuFormatter = formatterForOptions(locale, fixedOptions, kind);
175+
this.kind = kind;
174176
}
175177

176178
/**
@@ -252,6 +254,9 @@ private boolean matches(Object value, String key, Map<String, Object> variableOp
252254
} else {
253255
return false;
254256
}
257+
if ("integer".equals(kind)) {
258+
valToCheck = valToCheck.intValue();
259+
}
255260

256261
Number keyNrVal = OptUtils.asNumber(key);
257262
if (keyNrVal != null && valToCheck.doubleValue() == keyNrVal.doubleValue()) {

‎icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java

Lines changed: 33 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -14,37 +14,39 @@
1414
@SuppressWarnings({"static-method", "javadoc"})
1515
@RunWith(JUnit4.class)
1616
public class CoreTest extends CoreTestFmwk {
17-
private static final String[] JSON_FILES = {"alias-selector-annotations.json",
18-
"duplicate-declarations.json",
19-
"icu-parser-tests.json",
20-
"icu-test-functions.json",
21-
"icu-test-previous-release.json",
22-
"icu-test-selectors.json",
23-
"invalid-number-literals-diagnostics.json",
24-
"invalid-options.json",
25-
"markup.json",
26-
"matches-whitespace.json",
27-
"more-data-model-errors.json",
28-
"more-functions.json",
29-
"resolution-errors.json",
30-
"runtime-errors.json",
31-
"spec/data-model-errors.json",
32-
"spec/syntax-errors.json",
33-
"spec/syntax.json",
34-
"spec/functions/date.json",
35-
"spec/functions/datetime.json",
36-
"spec/functions/integer.json",
37-
"spec/functions/number.json",
38-
"spec/functions/string.json",
39-
"spec/functions/time.json",
40-
"syntax-errors-diagnostics.json",
41-
"syntax-errors-diagnostics-multiline.json",
42-
"syntax-errors-end-of-input.json",
43-
"syntax-errors-reserved.json",
44-
"tricky-declarations.json",
45-
"unsupported-expressions.json",
46-
"unsupported-statements.json",
47-
"valid-tests.json"};
17+
private static final String[] JSON_FILES = {
18+
"alias-selector-annotations.json",
19+
"duplicate-declarations.json",
20+
"icu-parser-tests.json",
21+
"icu-test-functions.json",
22+
"icu-test-previous-release.json",
23+
"icu-test-selectors.json",
24+
"invalid-number-literals-diagnostics.json",
25+
"invalid-options.json",
26+
"markup.json",
27+
"matches-whitespace.json",
28+
"more-data-model-errors.json",
29+
"more-functions.json",
30+
"resolution-errors.json",
31+
"runtime-errors.json",
32+
"spec/data-model-errors.json",
33+
"spec/syntax-errors.json",
34+
"spec/syntax.json",
35+
"spec/functions/date.json",
36+
"spec/functions/datetime.json",
37+
"spec/functions/integer.json",
38+
"spec/functions/number.json",
39+
"spec/functions/string.json",
40+
"spec/functions/time.json",
41+
"syntax-errors-diagnostics.json",
42+
"syntax-errors-diagnostics-multiline.json",
43+
"syntax-errors-end-of-input.json",
44+
"syntax-errors-reserved.json",
45+
"tricky-declarations.json",
46+
"unsupported-expressions.json",
47+
"unsupported-statements.json",
48+
"valid-tests.json"
49+
};
4850

4951
@Test
5052
public void test() throws Exception {

‎icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DefaultTestProperties.java

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,37 @@
33

44
package com.ibm.icu.dev.test.message2;
55

6+
import com.google.gson.JsonArray;
7+
import com.google.gson.JsonElement;
8+
69
// See https://github.com/unicode-org/conformance/blob/main/schema/message_fmt2/testgen_schema.json
710

811
// Class corresponding to the json test files.
912
// Since this is serialized by Gson, the field names should match the keys in the .json files.
1013
class DefaultTestProperties {
14+
private static final Object[] NO_ERRORS = {};
1115
// Unused fields ignored
12-
final String locale;
13-
final Object[] expErrors;
16+
private final String locale;
17+
private final JsonElement expErrors;
1418

15-
DefaultTestProperties(String locale, Object[] expErrors) {
19+
DefaultTestProperties(String locale, JsonElement expErrors) {
1620
this.locale = locale;
1721
this.expErrors = expErrors;
1822
}
23+
24+
String getLocale() {
25+
return this.locale;
26+
}
27+
28+
Object[] getExpErrors() {
29+
if (expErrors == null || !expErrors.isJsonArray()) {
30+
return NO_ERRORS;
31+
}
32+
JsonArray arr = expErrors.getAsJsonArray();
33+
Object [] result = new Object[arr.size()];
34+
for (int i = 0; i < result.length; i++) {
35+
result[i] = arr.get(i);
36+
}
37+
return result;
38+
}
1939
}

‎icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ static Map<String, Object> paramsToMap(Param[] params) {
129129

130130
static boolean expectsErrors(DefaultTestProperties defaults, Unit unit) {
131131
return (unit.expErrors != null && !unit.expErrors.isEmpty())
132-
|| (defaults.expErrors != null && defaults.expErrors.length > 0);
132+
|| (defaults.getExpErrors().length > 0);
133133
}
134134

135135
static void runTestCase(DefaultTestProperties defaults, Unit unit) {
@@ -154,8 +154,8 @@ static void runTestCase(DefaultTestProperties defaults, Unit unit, Param[] param
154154
MessageFormatter.builder().setPattern(pattern.toString());
155155
if (unit.locale != null && !unit.locale.isEmpty()) {
156156
mfBuilder.setLocale(Locale.forLanguageTag(unit.locale));
157-
} else if (defaults.locale != null) {
158-
mfBuilder.setLocale(Locale.forLanguageTag(defaults.locale));
157+
} else if (defaults.getLocale() != null) {
158+
mfBuilder.setLocale(Locale.forLanguageTag(defaults.getLocale()));
159159
} else {
160160
mfBuilder.setLocale(Locale.US);
161161
}

‎icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class Unit {
3232
this.expErrors = expErrors;
3333
}
3434

35-
class Error {
35+
static class Error {
3636
final String name;
3737
final String type;
3838

@@ -62,6 +62,9 @@ public String toString() {
6262
if (exp != null) {
6363
result.add("exp=" + escapeString(exp));
6464
}
65+
if (ignoreJava != null) {
66+
result.add("ignoreJava=" + ignoreJava);
67+
}
6568
return result.toString();
6669
}
6770

‎testdata/message2/alias-selector-annotations.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66
},
77
"tests": [
88
{
9-
"src": ".local $one = {|The one| :string}\n .match {$one}\n 1 {{Value is one}}\n * {{Value is not one}}",
9+
"src": ".local $one = {|The one| :string}\n .match $one\n 1 {{Value is one}}\n * {{Value is not one}}",
1010
"exp": "Value is not one"
1111
},
1212
{
13-
"src": ".local $one = {|The one| :string}\n .local $two = {$one}\n .match {$two}\n 1 {{Value is one}}\n * {{Value is not one}}",
13+
"src": ".local $one = {|The one| :string}\n .local $two = {$one}\n .match $two\n 1 {{Value is one}}\n * {{Value is not one}}",
1414
"exp": "Value is not one"
1515
}
1616
]

‎testdata/message2/icu-test-previous-release.json

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,50 +31,50 @@
3131
"exp": "bar foo"
3232
},
3333
{
34-
"src": ".match {$foo :number} 1 {{one}} * {{other}}",
34+
"src": ".input {$foo :number} .match $foo 1 {{one}} * {{other}}",
3535
"params": [{ "name": "foo", "value": "1" }],
3636
"exp": "one",
3737
"ignoreJava": "See ICU-22809"
3838
},
3939
{
40-
"src": ".match {$foo :string} 1 {{one}} * {{other}}",
40+
"src": ".input {$foo :string} .match $foo 1 {{one}} * {{other}}",
4141
"params": [{ "name": "foo", "value": "1" }],
4242
"exp": "one"
4343
},
4444
{
45-
"src": ".match {$foo :number} 1 {{one}} * {{other}}",
45+
"src": ".input {$foo :number} .match $foo 1 {{one}} * {{other}}",
4646
"params": [{ "name": "foo", "value": 1 }],
4747
"exp": "one"
4848
},
4949
{
5050
"ignoreJava": "Can't pass null in a map",
5151
"ignoreCpp": "Same as Java",
52-
"src": ".match {$foo} 1 {{one}} * {{other}}",
52+
"src": ".match $foo 1 {{one}} * {{other}}",
5353
"params": [{ "name": "foo", "value": null }],
5454
"exp": "other"
5555
},
5656
{
57-
"src": ".match {$foo :number} 1 {{one}} * {{other}}",
57+
"src": ".input {$foo :number} .match $foo 1 {{one}} * {{other}}",
5858
"exp": "other",
5959
"expErrors": [{ "type": "unresolved-variable" }]
6060
},
6161
{
62-
"src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}",
62+
"src": ".local $foo = {$bar :number} .match $foo one {{one}} * {{other}}",
6363
"params": [{ "name": "bar", "value": 1 }],
6464
"exp": "one"
6565
},
6666
{
67-
"src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}",
67+
"src": ".local $foo = {$bar :number} .match $foo one {{one}} * {{other}}",
6868
"params": [{ "name": "bar", "value": 2 }],
6969
"exp": "other"
7070
},
7171
{
72-
"src": ".local $bar = {$none} .match {$foo :number} one {{one}} * {{{$bar}}}",
72+
"src": ".local $bar = {$none} .input {$foo :number} .match $foo one {{one}} * {{{$bar}}}",
7373
"params": [{ "name": "foo", "value": 1 }, {"name": "none", "value": "" }],
7474
"exp": "one"
7575
},
7676
{
77-
"src": ".local $bar = {$none :number} .match {$foo :string} one {{one}} * {{{$bar}}}",
77+
"src": ".local $bar = {$none :number} .input {$foo :string} .match $foo one {{one}} * {{{$bar}}}",
7878
"params": [{ "name": "foo", "value": 2 }],
7979
"exp": "{$none}",
8080
"expErrors": [{ "type": "unresolved-variable" }],
@@ -120,17 +120,17 @@
120120
"ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/703"
121121
},
122122
{
123-
"src": ".match {|foo| :string} *{{foo}}",
123+
"src": ".local $f = {|foo| :string} .match $f *{{foo}}",
124124
"exp": "foo"
125125
},
126126
{
127-
"src": ".match {$foo :string} * * {{foo}}",
127+
"src": ".input {$foo :string} .match $foo * * {{foo}}",
128128
"exp": "foo",
129129
"expErrors": [{ "type": "variant-key-mismatch" }, { "type": "unresolved-variable" }],
130130
"ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/735"
131131
},
132132
{
133-
"src": ".match {$foo :string} {$bar :string} * {{foo}}",
133+
"src": ".input {$foo :string} .input {$bar :string} .match $foo $bar * {{foo}}",
134134
"exp": "foo",
135135
"expErrors": [{ "type": "variant-key-mismatch" }, { "type": "unresolved-variable" }],
136136
"ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/735"

0 commit comments

Comments
 (0)