ICU-22953 MF2: Allow unpaired surrogates in text and quoted literals

See unicode-org/message-format-wg#906
unicode-org · Nov 6, 2024 · 376da67 · 376da67
1 parent 842899d
commit 376da67
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 4 deletions.
diff --git a/icu4c/source/i18n/messageformat2_parser.cpp b/icu4c/source/i18n/messageformat2_parser.cpp
@@ -121,8 +121,8 @@ static bool isContentChar(UChar32 c) {
            || inRange(c, 0x002F, 0x003F) // Omit '@'
            || inRange(c, 0x0041, 0x005B) // Omit '\'
            || inRange(c, 0x005D, 0x007A) // Omit { | }
-           || inRange(c, 0x007E, 0xD7FF) // Omit surrogates
-           || inRange(c, 0xE000, 0x10FFFF);
+           || inRange(c, 0x007E, 0x2FFF) // Omit IDEOGRAPHIC_SPACE
+           || inRange(c, 0x3001, 0x10FFFF); // Allowing surrogates is intentional
 }
 
 // See `s` in the MessageFormat 2 grammar

diff --git a/icu4c/source/test/intltest/messageformat2test.cpp b/icu4c/source/test/intltest/messageformat2test.cpp
@@ -33,6 +33,7 @@ TestMessageFormat2::runIndexedTest(int32_t index, UBool exec,
     TESTCASE_AUTO(testFormatterAPI);
     TESTCASE_AUTO(testHighLoneSurrogate);
     TESTCASE_AUTO(testLowLoneSurrogate);
+    TESTCASE_AUTO(testLoneSurrogateInQuotedLiteral);
     TESTCASE_AUTO(dataDrivenTests);
     TESTCASE_AUTO_END;
 }
@@ -350,7 +351,8 @@ void TestMessageFormat2::testHighLoneSurrogate() {
       .setPattern(loneSurrogate, pe, errorCode)
       .build(errorCode);
     UnicodeString result = msgfmt1.formatToString({}, errorCode);
-    errorCode.expectErrorAndReset(U_MF_SYNTAX_ERROR, "testHighLoneSurrogate");
+    assertEquals("testHighLoneSurrogate", loneSurrogate, result);
+    errorCode.errIfFailureAndReset("testHighLoneSurrogate");
 }
 
 // ICU-22890 lone surrogate cause infinity loop
@@ -364,7 +366,25 @@ void TestMessageFormat2::testLowLoneSurrogate() {
       .setPattern(loneSurrogate, pe, errorCode)
       .build(errorCode);
     UnicodeString result = msgfmt2.formatToString({}, errorCode);
-    errorCode.expectErrorAndReset(U_MF_SYNTAX_ERROR, "testLowLoneSurrogate");
+    assertEquals("testLowLoneSurrogate", loneSurrogate, result);
+    errorCode.errIfFailureAndReset("testLowLoneSurrogate");
+}
+
+void TestMessageFormat2::testLoneSurrogateInQuotedLiteral() {
+    IcuTestErrorCode errorCode(*this, "testLoneSurrogateInQuotedLiteral");
+    UParseError pe = { 0, 0, {0}, {0} };
+    // |\udc02|
+    UnicodeString literal("{|");
+    literal += 0xdc02;
+    literal += "|}";
+    UnicodeString expectedResult({0xdc02, 0});
+    icu::message2::MessageFormatter msgfmt2 =
+      icu::message2::MessageFormatter::Builder(errorCode)
+      .setPattern(literal, pe, errorCode)
+      .build(errorCode);
+    UnicodeString result = msgfmt2.formatToString({}, errorCode);
+    assertEquals("testLoneSurrogateInQuotedLiteral", expectedResult, result);
+    errorCode.errIfFailureAndReset("testLoneSurrogateInQuotedLiteral");
 }
 
 void TestMessageFormat2::dataDrivenTests() {

diff --git a/icu4c/source/test/intltest/messageformat2test.h b/icu4c/source/test/intltest/messageformat2test.h
@@ -91,6 +91,7 @@ class TestMessageFormat2: public IntlTest {
 
     void testHighLoneSurrogate(void);
     void testLowLoneSurrogate(void);
+    void testLoneSurrogateInQuotedLiteral(void);
 }; // class TestMessageFormat2
 
 U_NAMESPACE_BEGIN