diff --git a/icu4c/source/common/uniset_props.cpp b/icu4c/source/common/uniset_props.cpp index 8d926370a7bc..374e6000e7c3 100644 --- a/icu4c/source/common/uniset_props.cpp +++ b/icu4c/source/common/uniset_props.cpp @@ -356,10 +356,37 @@ class UnicodeSet::Lexer { u"(end of text)", }}; LexicalElement(Category category, UnicodeString string, RuleCharacterIterator::Pos after, - UErrorCode errorCode, const UnicodeSet *precomputedSet, UnicodeSet set, - std::u16string_view sourceText) + const UErrorCode errorCode, const UnicodeSet *precomputedSet, UnicodeSet set, + std::u16string_view sourceText, bool separated, + std::optional const &preceding) : category_(category), string_(std::move(string)), after_(after), errorCode_(errorCode), - precomputedSet_(precomputedSet), set_(set), sourceText_(sourceText) {} + precomputedSet_(precomputedSet), set_(set), sourceText_(sourceText) { + if (U_FAILURE(errorCode_) || separated || !preceding.has_value()) { + return; + } + if (preceding->isSetOperator(u'[') && category == LITERAL_ELEMENT && + *codePoint() == u':') { + errorCode_ = U_MALFORMED_SET; + } else if (preceding->category_ == ESCAPED_ELEMENT) { + if (U_IS_LEAD(*preceding->codePoint()) && category_ == ESCAPED_ELEMENT && + U_IS_TRAIL(*codePoint())) { + errorCode_ = U_MALFORMED_SET; + } else if (preceding->sourceText_.length() <= 3 && category == LITERAL_ELEMENT) { + UChar32 cp = *codePoint(); + if (preceding->sourceText_[1] == u'x') { + if ((cp >= u'0' && cp <= u'9') || (cp >= u'A' && cp <= u'F') || + (cp >= u'a' && cp <= u'f')) { + errorCode_ = U_MALFORMED_SET; + } + } else if (preceding->sourceText_[1] >= u'0' && + preceding->sourceText_[1] <= u'7') { + if (cp >= u'0' && cp <= u'7') { + errorCode_ = U_MALFORMED_SET; + } + } + } + } + } Category category_; UnicodeString string_; RuleCharacterIterator::Pos after_; @@ -387,7 +414,7 @@ class UnicodeSet::Lexer { const LexicalElement &lookahead() { if (!ahead_.has_value()) { const RuleCharacterIterator::Pos before = getPos(); - ahead_.emplace(nextToken()); + ahead_.emplace(nextToken(behind_)); chars_.setPos(before); } return *ahead_; @@ -400,7 +427,7 @@ class UnicodeSet::Lexer { // since we start from ahead_.after_. const RuleCharacterIterator::Pos before = getPos(); chars_.setPos(lookahead().after_); - ahead2_.emplace(nextToken()); + ahead2_.emplace(nextToken(ahead_)); chars_.setPos(before); } return *ahead2_; @@ -433,6 +460,7 @@ class UnicodeSet::Lexer { // working on incorrect values of `getPos`. This is why the result of `getCharacterIterator` // must no longer be used. chars_.setPos(lookahead().after_); + behind_ = ahead_; ahead_ = ahead2_; ahead2_.reset(); } @@ -446,14 +474,33 @@ class UnicodeSet::Lexer { return result; } - LexicalElement nextToken() { + LexicalElement nextToken(std::optional const& preceding) { UErrorCode errorCode = U_ZERO_ERROR; + bool separated = false; + if (charsOptions_ & RuleCharacterIterator::SKIP_WHITESPACE) { + UChar32 s; + RuleCharacterIterator::Pos pos; + for (;;) { + UBool unusedEscaped; + chars_.getPos(pos); + s = chars_.next(charsOptions_ & ~(RuleCharacterIterator::PARSE_ESCAPES | + RuleCharacterIterator::SKIP_WHITESPACE), + unusedEscaped, errorCode); + if (!PatternProps::isWhiteSpace(s)) { + chars_.setPos(pos); + break; + } + if (s != u'\u200E' && s != u'\u200F') { + separated = true; + } + } + } chars_.skipIgnored(charsOptions_); if (chars_.atEnd()) { return LexicalElement(LexicalElement::END_OF_TEXT, {}, getPos(), errorCode, /*precomputedSet=*/nullptr, /*set=*/{}, - u""); + u"", separated, preceding); } const int32_t start = parsePosition_.getIndex(); const RuleCharacterIterator::Pos before = getPos(); @@ -475,13 +522,15 @@ class UnicodeSet::Lexer { LexicalElement::NAMED_ELEMENT, UnicodeString(queryResult), getPos(), errorCode, /*precomputedSet=*/nullptr, /*set=*/{}, - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); } else { UnicodeSet queryResult = scanPropertyQueryAfterStart(first, second, start, errorCode); return LexicalElement( LexicalElement::PROPERTY_QUERY, {}, getPos(), errorCode, /*precomputedSet=*/nullptr, /*set=*/std::move(queryResult), - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); } } // Not a property-query. @@ -491,6 +540,10 @@ class UnicodeSet::Lexer { auto nameEnd = parsePosition_; // The SymbolTable defines the lexing of variable names past the $. if (UnicodeString name = symbols_->parseReference(pattern_, nameEnd, pattern_.length()); + // LexicalElements returned here and from evaluateVariable are always constructed + // with separated=true, preceding=nullopt: the separation constraints are lexical, + // and variables are their own lexical elements; their values are not affected by + // lexical constraints. !name.isEmpty()) { chars_.jumpahead(nameEnd.getIndex() - (start + 1)); const std::u16string_view source = @@ -498,7 +551,8 @@ class UnicodeSet::Lexer { const UnicodeSet *precomputedSet = symbols_->lookupSet(name); if (precomputedSet != nullptr) { return LexicalElement(LexicalElement::VARIABLE, {}, getPos(), U_ZERO_ERROR, - precomputedSet, /*set=*/{}, source); + precomputedSet, /*set=*/{}, source, /*separated=*/true, + /*preceding=*/std::nullopt); } // The variable was not a precomputed set. Use the old-fashioned `lookup`, which // should give us its source text; if that parses as a single set or element, use @@ -512,7 +566,9 @@ class UnicodeSet::Lexer { LexicalElement::VARIABLE, {}, getPos(), U_UNDEFINED_VARIABLE, /*precomputedSet=*/nullptr, /*set=*/{}, - source); + source, + /*separated=*/true, + /*preceding=*/std::nullopt); } return evaluateVariable(*expression, source); } @@ -523,8 +579,16 @@ class UnicodeSet::Lexer { LexicalElement::SET_OPERATOR, UnicodeString(u'['), getPos(), errorCode, /*precomputedSet=*/nullptr, /*set=*/{}, - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); case u'\\': { + const UChar32 second = chars_.next(charsOptions_ & ~(RuleCharacterIterator::PARSE_ESCAPES | + RuleCharacterIterator::SKIP_WHITESPACE), + unusedEscaped, errorCode); + if (second == u'\u200E' || second == u'\u200F') { + // Prohibit \ and \. + errorCode = U_MALFORMED_UNICODE_ESCAPE; + } // Now try to parse the escape. chars_.setPos(before); UChar32 codePoint = chars_.next(charsOptions_, unusedEscaped, errorCode); @@ -533,7 +597,8 @@ class UnicodeSet::Lexer { UnicodeString(codePoint), getPos(), errorCode, nullptr, /*set=*/{}, - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); } case u'&': case u'-': @@ -545,7 +610,8 @@ class UnicodeSet::Lexer { LexicalElement::SET_OPERATOR, UnicodeString(first), getPos(), errorCode, /*precomputedSet=*/nullptr, /*set=*/{}, - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); case u'{': { UnicodeString string; UBool escaped; @@ -564,13 +630,15 @@ class UnicodeSet::Lexer { if (afterBackslash == u'N') { next = scanNamedElementBrackets(errorCode); escaped = true; - } else if (afterBackslash == u'p' || afterBackslash == u'P') { + } else if (afterBackslash == u'p' || afterBackslash == u'P' || + afterBackslash == u'\u200E' || afterBackslash == u'\u200F') { return LexicalElement(LexicalElement::STRING_LITERAL, {}, getPos(), U_MALFORMED_SET, /*precomputedSet=*/nullptr, /*set=*/{}, std::u16string_view(pattern_).substr( - start, parsePosition_.getIndex() - start)); + start, parsePosition_.getIndex() - start), + separated, preceding); } else { chars_.setPos(beforeNext); // Parse the escape. @@ -596,7 +664,8 @@ class UnicodeSet::Lexer { std::move(string), getPos(), errorCode, /*precomputedSet=*/nullptr, /*set=*/{}, - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); } string.append(next); codePointCount += 1; @@ -605,13 +674,15 @@ class UnicodeSet::Lexer { LexicalElement::STRING_LITERAL, {}, getPos(), U_MALFORMED_SET, /*precomputedSet=*/nullptr, /*set=*/{}, - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); } default: return LexicalElement( LexicalElement::LITERAL_ELEMENT, UnicodeString(first), getPos(), errorCode, nullptr, /*set=*/{}, - std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start)); + std::u16string_view(pattern_).substr(start, parsePosition_.getIndex() - start), + separated, preceding); } } @@ -695,7 +766,8 @@ class UnicodeSet::Lexer { return {}; } - LexicalElement evaluateVariable(const UnicodeString &expression, const std::u16string_view source) { + LexicalElement evaluateVariable(const UnicodeString &expression, + const std::u16string_view source) { UErrorCode errorCode = U_ZERO_ERROR; ParsePosition expressionPosition; RuleCharacterIterator expressionIterator(expression, symbols_, expressionPosition); @@ -714,13 +786,17 @@ class UnicodeSet::Lexer { LexicalElement::VARIABLE, {}, getPos(), U_MALFORMED_VARIABLE_DEFINITION, /*precomputedSet=*/nullptr, /*set=*/{}, - source); + source, + /*separated=*/true, + /*preceding=*/std::nullopt); } return LexicalElement( LexicalElement::VARIABLE, {}, getPos(), errorCode, /*precomputedSet=*/nullptr, /*set=*/std::move(expressionValue), - source); + source, + /*separated=*/true, + /*preceding=*/std::nullopt); } else { expressionLexer.advance(); if (!expressionLexer.atEnd()) { @@ -728,7 +804,9 @@ class UnicodeSet::Lexer { LexicalElement::VARIABLE, {}, getPos(), U_MALFORMED_VARIABLE_DEFINITION, /*precomputedSet=*/nullptr, /*set=*/{}, - source); + source, + /*separated=*/true, + /*preceding=*/std::nullopt); } switch (variableToken.category_) { case LexicalElement::LITERAL_ELEMENT: @@ -743,12 +821,16 @@ class UnicodeSet::Lexer { // expansion. return LexicalElement( variableToken.category_, std::move(variableToken.string_), getPos(), - variableToken.errorCode_, variableToken.precomputedSet_, std::move(variableToken.set_), source); + variableToken.errorCode_, variableToken.precomputedSet_, std::move(variableToken.set_), source, + /*separated=*/true, + /*preceding=*/std::nullopt); default: return LexicalElement(LexicalElement::VARIABLE, {}, getPos(), U_MALFORMED_VARIABLE_DEFINITION, /*precomputedSet=*/nullptr, - /*set=*/{}, source); + /*set=*/{}, source, + /*separated=*/true, + /*preceding=*/std::nullopt); } } } @@ -864,6 +946,7 @@ class UnicodeSet::Lexer { const int32_t charsOptions_; const SymbolTable *const symbols_; UnicodeSet &(UnicodeSet::* const caseClosure_)(int32_t attribute); + std::optional behind_; std::optional ahead_; std::optional ahead2_; }; diff --git a/icu4c/source/common/util.cpp b/icu4c/source/common/util.cpp index d80ccb50ed0e..f95843c9f78a 100644 --- a/icu4c/source/common/util.cpp +++ b/icu4c/source/common/util.cpp @@ -76,6 +76,10 @@ UBool ICU_Utility::shouldAlwaysBeEscaped(UChar32 c) { return false; // printable ASCII } else if (c <= 0x9f) { return true; // C1 control codes + } else if (c == 0x200E || c == 0x200F) { + // LRM and RLM are ignorable-format-controls and thus not escapable-characters in the + // UnicodeSet grammar: https://www.unicode.org/reports/tr61/#escapable-character. + return true; } else if (c < 0xd800) { return false; // most of the BMP } else if (c <= 0xdfff || (0xfdd0 <= c && c <= 0xfdef) || (c & 0xfffe) == 0xfffe) { diff --git a/icu4c/source/test/intltest/usettest.cpp b/icu4c/source/test/intltest/usettest.cpp index 453851570ed9..22a7a19bf762 100644 --- a/icu4c/source/test/intltest/usettest.cpp +++ b/icu4c/source/test/intltest/usettest.cpp @@ -1794,6 +1794,15 @@ void UnicodeSetTest::TestSymbolTable() { "us", "[a-z]", "[0-1$us]", "[0-1[a-z]]", nullptr, // Variables do not expand inside string literals. "us", "[a-z]", "[$us{$us}]", R"([a-z{\$us}])", nullptr, + // Variables are their own lexical elements added to the syntax; their + // values are not subject to the lexical separation constraints (this is + // not macro replacement anymore). + "colon", ":", "[$colon]", R"([\:])", nullptr, + "DBFF", R"(\uDBFF)", + "DFFF", R"(\uDFFF)", + "[$DBFF$DFFF]", R"([\uDFFF\uDBFF])", nullptr, + "nul", R"(\00)", "seven", "7", "[$nul$seven]", R"([\u00007])", nullptr, + "cr", R"(\xD)", "F", "F", "[$cr$F]", R"([\u000DF])", nullptr, "privateUse", "[[:Co:]]", "$privateUse", "[[:Co:]]", nullptr, nullptr }; @@ -4735,6 +4744,27 @@ void UnicodeSetTest::TestToPatternOutput() { {uR"([\N{CJK unified ideograph 5-5-b-5}])", u"[喵]"}, {uR"([{\N{Hangul syllable YA}\N{Hangul syllable ONG}}])", u"[{야옹}]"}, {uR"([{\N{Hangul-syllable-y-a}\N{Hangul-syllable-o-ng}}])", u"[{야옹}]"}, + {uR"([\xDF])", u"[ß]"}, + {uR"([\xDFF])", u"[Fß]"}, + {uR"([\xD F])", uR"([\u000DF])"}, + {uR"([\0007])", uR"([\u00007])"}, + {uR"([\007])", uR"([\u0007])"}, + {uR"([\00 7])", uR"([\u00007])"}, + {uR"([\0 07])", uR"([\u000007])"}, + {uR"([\ 007])", uR"([\ 07])"}, + {uR"([\800])", uR"([08])"}, + {uR"([\0008])", uR"([\u00008])"}, + {uR"([\008])", uR"([\u00008])"}, + {uR"([\08])", uR"([\u00008])"}, + {uR"([\8])", uR"([8])"}, + {uR"([\007F])", uR"([\u0007F])"}, + {uR"([\07F])", uR"([\u0007F])"}, + {uR"([\7F])", uR"([\u0007F])"}, + {uR"([\u200E007])", uR"([07\u200E])"}, + {uR"([ :Greek:])", uR"([\:Gekr])"}, + {uR"([\uDBFF \uDFFF])", uR"([\uDFFF\uDBFF])"}, + // ICU-2906, ICU extension to UnicodeSet. + {uR"([\uDBFF\uDFFF])", uR"([\U0010FFFF])"}, }) { UErrorCode errorCode = U_ZERO_ERROR; const UnicodeSet set(expression, errorCode); @@ -4752,6 +4782,8 @@ void UnicodeSetTest::TestParseErrors() { for (const auto expression : std::vector{ uR"([\u])", uR"([\x{}])", + // ICU-23497: the ignorable-format-control characters are not escapable-characters. + u"[\\\u200E007]", }) { UErrorCode errorCode = U_ZERO_ERROR; const UnicodeSet set(expression, errorCode); @@ -4797,10 +4829,11 @@ void UnicodeSetTest::TestParseErrors() { // This was a well-formed string in ICU 78 and earlier, with the value // "N{LATINCAPITALLETTERZ". uR"([{\N{LATIN CAPITAL LETTER Z}])", - // These three were well-formed in ICU 78 and earlier. + // These four were well-formed in ICU 78 and earlier. uR"([{\Normandie}])", uR"([{\Picardie}])", uR"([{Provence-Al\pes-Côte d'Azur}])", + u"[{\\\u200E}]", // This was a well-formed set in ICU 78 and earlier; now it must be enclosed in square // brackets. uR"(\N{ latin small letter a })", @@ -4811,6 +4844,20 @@ void UnicodeSetTest::TestParseErrors() { uR"([ { \x5A e i c h e n k e t t e } \x5Aeichenmenge ])", u"[ { Z e i c h e n k e t t e } [] Zeichenmenge ]", uR"([ { \x5A e i c h e n k e t t e } [] \x5Aeichenmenge ])", + // ICU-23497 Implicit Directional Marks should not separate UnicodeSet lexical elements. + u"[\\xD\u200EF]", + u"[\\00\u200E7]", + u"[\\0\u200E07]", + u"[\\uDBFF\u200E\\uDFFF]", + u"[\\x{DBFF}\u200E\\U0000DFFF]", + u"[\u200E:Greek:]", + // ICU4J interprets those like [\uDBFF\uDFFF], as [\U0010FFFF]. ICU4C < 79 interpreted + // them as [\uDBFF \uDFFF], because the sequence of two escapes is longer than + // MAX_U_NOTATION_LEN defined in ruleiter.cpp. They were thus never portable; they + // become ill-formed in ICU4C 79 because of the requirement to separate escaped-elements + // for lead and trail surrogates. + uR"([\x{DBFF}\x{DFFF}])", + uR"([\U0000DBFF\U0000DFFF])", }) { UErrorCode errorCode = U_ZERO_ERROR; const UnicodeSet set(expression, errorCode); diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/impl/Utility.java b/icu4j/main/core/src/main/java/com/ibm/icu/impl/Utility.java index b40a54e9e62f..7236279e8331 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/impl/Utility.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/impl/Utility.java @@ -1444,6 +1444,10 @@ public static boolean shouldAlwaysBeEscaped(int c) { return false; // printable ASCII } else if (c <= 0x9f) { return true; // C1 control codes + } else if (c == 0x200E || c == 0x200F) { + // LRM and RLM are ignorable-format-controls and thus not escapable-characters in the + // UnicodeSet grammar: https://www.unicode.org/reports/tr61/#escapable-character. + return true; } else if (c < 0xd800) { return false; // most of the BMP } else if (c <= 0xdfff || (0xfdd0 <= c && c <= 0xfdef) || (c & 0xfffe) == 0xfffe) { diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/text/UnicodeSetLexer.java b/icu4j/main/core/src/main/java/com/ibm/icu/text/UnicodeSetLexer.java index 755979bd9a0a..eaacf1d21650 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/text/UnicodeSetLexer.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/text/UnicodeSetLexer.java @@ -5,6 +5,7 @@ import com.ibm.icu.impl.PatternProps; import com.ibm.icu.impl.RuleCharacterIterator; import com.ibm.icu.impl.Utility; +import com.ibm.icu.lang.UCharacter; import com.ibm.icu.text.UnicodeSet.XSymbolTable; import com.ibm.icu.util.VersionInfo; import java.text.ParsePosition; @@ -128,12 +129,65 @@ public String toString() { String string, RuleCharacterIterator.Position after, UnicodeSet set, - CharSequence sourceText) { + CharSequence sourceText, + boolean separated, + LexicalElement preceding, + UnicodeSetLexer lexer) { category_ = category; string_ = string; after_ = after; set_ = set; sourceText_ = sourceText; + if (separated || preceding == null) { + return; + } + if (preceding.isSetOperator('[') + && category == Category.LITERAL_ELEMENT + && codePoint() == ':') { + // Not in the C++ version since we do not have lexer error messages there: + // set the position between the two unseparated lexical elements so the pointing + // hand shows up where a space is needed. + lexer.chars_.setPos(preceding.after_); + throw lexer.lexicalError( + "white-space other than ignorable-format-control is required between" + + " set-operator [ and literal-element :"); + } else if (preceding.category_ == Category.ESCAPED_ELEMENT) { + if (UCharacter.isHighSurrogate(preceding.codePoint()) + && category_ == Category.ESCAPED_ELEMENT + && UCharacter.isLowSurrogate(codePoint())) { + lexer.chars_.setPos(preceding.after_); + throw lexer.lexicalError( + "white-space other than ignorable-format-control is required between" + + " escaped-elements representing high and low surrogates"); + } else if (preceding.sourceText_.length() <= 3 + && category == Category.LITERAL_ELEMENT) { + int cp = codePoint(); + if (preceding.sourceText_.charAt(1) == 'x') { + if ((cp >= '0' && cp <= '9') + || (cp >= 'A' && cp <= 'F') + || (cp >= 'a' && cp <= 'f')) { + lexer.chars_.setPos(preceding.after_); + throw lexer.lexicalError( + "white-space other than ignorable-format-control is required" + + " between escaped-element " + + preceding.sourceText_ + + " and literal-element " + + sourceText_); + } + } else if (preceding.sourceText_.charAt(1) >= '0' + && preceding.sourceText_.charAt(1) <= '7') { + if (cp >= '0' && cp <= '7') { + lexer.chars_.setPos(preceding.after_); + throw lexer.lexicalError( + "white-space other than ignorable-format-control is required" + + " between escaped-element " + + preceding.sourceText_ + + " and literal-element " + + sourceText_); + } + } + } + } } Category category_; @@ -173,7 +227,7 @@ boolean acceptSetOperator(char op) { LexicalElement lookahead() { if (ahead_ == null) { RuleCharacterIterator.Position before = getPos(); - ahead_ = nextToken(); + ahead_ = nextToken(behind_); chars_.setPos(before); } return ahead_; @@ -186,7 +240,7 @@ LexicalElement lookahead2() { // here, since we start from ahead_.after_. RuleCharacterIterator.Position before = getPos(); chars_.setPos(lookahead().after_); - ahead2_ = nextToken(); + ahead2_ = nextToken(ahead_); chars_.setPos(before); } return ahead2_; @@ -215,6 +269,7 @@ void advance() { // we would be working on incorrect values of `getPos`. This is why the result of // `getCharacterIterator` must no longer be used. chars_.setPos(lookahead().after_); + behind_ = ahead_; ahead_ = ahead2_; ahead2_ = null; } @@ -226,11 +281,38 @@ private RuleCharacterIterator.Position getPos() { return result; } - private LexicalElement nextToken() { + private LexicalElement nextToken(LexicalElement preceding) { + boolean separated = false; + if ((charsOptions_ & RuleCharacterIterator.SKIP_WHITESPACE) != 0) { + int s; + RuleCharacterIterator.Position pos = new RuleCharacterIterator.Position(); + for (; ; ) { + chars_.getPos(pos); + s = + chars_.next( + charsOptions_ + & ~(RuleCharacterIterator.PARSE_ESCAPES + | RuleCharacterIterator.SKIP_WHITESPACE)); + if (!PatternProps.isWhiteSpace(s)) { + chars_.setPos(pos); + break; + } + if (s != '\u200E' && s != '\u200F') { + separated = true; + } + } + } chars_.skipIgnored(charsOptions_); if (chars_.atEnd()) { return new LexicalElement( - LexicalElement.Category.END_OF_TEXT, null, getPos(), /* set= */ null, ""); + LexicalElement.Category.END_OF_TEXT, + null, + getPos(), + /* set= */ null, + "", + separated, + preceding, + this); } final int start = parsePosition_.getIndex(); final RuleCharacterIterator.Position before = getPos(); @@ -266,7 +348,10 @@ private LexicalElement nextToken() { Character.toString(queryResult), getPos(), /* set= */ null, - pattern_.subSequence(start, parsePosition_.getIndex())); + pattern_.subSequence(start, parsePosition_.getIndex()), + separated, + preceding, + this); } else { UnicodeSet queryResult = scanPropertyQueryAfterStart(first, second, start); return new LexicalElement( @@ -274,7 +359,10 @@ private LexicalElement nextToken() { null, getPos(), /* set= */ queryResult, - pattern_.subSequence(start, parsePosition_.getIndex())); + pattern_.subSequence(start, parsePosition_.getIndex()), + separated, + preceding, + this); } } // Not a property-query. @@ -295,9 +383,24 @@ private LexicalElement nextToken() { "[", getPos(), /* set= */ null, - pattern_.subSequence(start, parsePosition_.getIndex())); + pattern_.subSequence(start, parsePosition_.getIndex()), + separated, + preceding, + this); case '\\': { + int second = + chars_.next( + charsOptions_ + & ~(RuleCharacterIterator.PARSE_ESCAPES + | RuleCharacterIterator.SKIP_WHITESPACE)); + if (second == '\u200E' || second == '\u200F') { + // Prohibit \ and \. + throw lexicalError( + "ignorable-format-control U+" + + Utility.hex(second) + + " is not an escapable-character"); + } // Now try to parse the escape. chars_.setPos(before); int codePoint = chars_.next(charsOptions_); @@ -306,7 +409,10 @@ private LexicalElement nextToken() { Character.toString(codePoint), getPos(), /* set= */ null, - pattern_.subSequence(start, parsePosition_.getIndex())); + pattern_.subSequence(start, parsePosition_.getIndex()), + separated, + preceding, + this); } case '&': case '-': @@ -319,7 +425,10 @@ private LexicalElement nextToken() { Character.toString(first), getPos(), /* set= */ null, - pattern_.subSequence(start, parsePosition_.getIndex())); + pattern_.subSequence(start, parsePosition_.getIndex()), + separated, + preceding, + this); case '{': { final var string = new StringBuilder(); @@ -348,6 +457,11 @@ private LexicalElement nextToken() { "Invalid escape sequence \\" + Character.toString(afterBackslash) + " in UnicodeSet string"); + } else if (afterBackslash == '\u200E' || afterBackslash == '\u200F') { + throw lexicalError( + "Invalid escape sequence \\ in UnicodeSet string"); } else { chars_.setPos(beforeNext); // Parse the escape. @@ -382,7 +496,10 @@ private LexicalElement nextToken() { string.toString(), getPos(), /* set= */ null, - pattern_.subSequence(start, parsePosition_.getIndex())); + pattern_.subSequence(start, parsePosition_.getIndex()), + separated, + preceding, + this); } string.append(Character.toString(next)); codePointCount += 1; @@ -397,18 +514,32 @@ private LexicalElement nextToken() { Character.toString(first), getPos(), /* set= */ null, - pattern_.subSequence(start, parsePosition_.getIndex())); + pattern_.subSequence(start, parsePosition_.getIndex()), + separated, + preceding, + this); } } private LexicalElement lookupVariable( String name, int lexicalElementStart, int nameStart, ParsePosition nameEnd) { + // LexicalElements returned from this function and from evaluateVariable are always + // constructed with separated=true, preceding=null: the separation constraints are lexical, + // and variables are their own lexical elements; their values are not affected by lexical + // constraints. chars_.jumpahead(nameEnd.getIndex() - nameStart); final var source = pattern_.subSequence(lexicalElementStart, parsePosition_.getIndex()); UnicodeSet precomputedSet = symbols_.lookupSet(name); if (precomputedSet != null) { return new LexicalElement( - LexicalElement.Category.VARIABLE, null, getPos(), precomputedSet, source); + LexicalElement.Category.VARIABLE, + null, + getPos(), + precomputedSet, + source, + /* separated= */ true, + /* preceding= */ null, + this); } // The variable was not a precomputed set. Use the old-fashioned `lookup`, which // should give us its source text; if that parses as a single set or element, use @@ -568,7 +699,10 @@ private LexicalElement evaluateVariable(String expression, CharSequence source) null, getPos(), /* set= */ expressionValue, - source); + source, + /* separated= */ true, + /* preceding= */ null, + this); } else { expressionLexer.advance(); if (!expressionLexer.atEnd()) { @@ -595,7 +729,10 @@ private LexicalElement evaluateVariable(String expression, CharSequence source) variableToken.string_, getPos(), variableToken.set_, - source); + source, + /* separated= */ true, + /* preceding= */ null, + this); default: throw lexicalError( "Value of variable " @@ -716,6 +853,7 @@ private UnicodeSet scanPropertyQueryAfterStart(int first, int second, int queryS final int charsOptions_; final SymbolTable symbols_; final XSymbolTable xsymbols_; + LexicalElement behind_; LexicalElement ahead_; LexicalElement ahead2_; } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/lang/UnicodeSetTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/lang/UnicodeSetTest.java index 4ca63acda98b..b4c2cbaee288 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/lang/UnicodeSetTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/lang/UnicodeSetTest.java @@ -1787,12 +1787,22 @@ public void TestSymbolTable() { // Multiple test cases can be set up here. Each test case // is terminated by null: // var, value, var, value,..., input pat., exp. output pat., null + // spotless:off String DATA[] = { "us", "[a-z]", "[0-1$us]", "[0-1[a-z]]", null, // Variables do not expand inside string literals. "us", "[a-z]", "[$us{$us}]", "[a-z{\\$us}]", null, + // Variables are their own lexical elements added to the syntax; their values are not + // subject to the lexical separation constraints (this is not macro replacement + // anymore). + "colon", ":", "[$colon]", "[\\:]", null, + "dbff", "\uDBFF", "dfff", "\uDFFF", "[$dbff$dfff]", "[\\uDFFF\\uDBFF]", null, + "DBFF", "\\uDBFF", "DFFF", "\\uDFFF", "[$DBFF$DFFF]", "[\\uDFFF\\uDBFF]", null, + "nul", "\\00", "seven", "7", "[$nul$seven]", "[\\u00007]", null, + "cr", "\\xD", "F", "F", "[$cr$F]", "[\\u000DF]", null, "privateUse", "[[:Co:]]", "$privateUse", "[[:Co:]]", null, }; + // spotless:on for (int i = 0; i < DATA.length; ++i) { TokenSymbolTable sym = new TokenSymbolTable(); @@ -4318,6 +4328,36 @@ public TestCase(String expression, String expected) { new TestCase("[{\\N{Hangul syllable YA}\\N{Hangul syllable ONG}}]", "[{야옹}]"), new TestCase("[{\\N{Hangul-syllable-y-a}\\N{Hangul-syllable-o-ng}}]", "[{야옹}]"), */ + new TestCase("[\\xDF]", "[ß]"), + new TestCase("[\\xDFF]", "[Fß]"), + new TestCase("[\\xD F]", "[\\u000DF]"), + new TestCase("[\\0007]", "[\\u00007]"), + new TestCase("[\\007]", "[\\u0007]"), + new TestCase("[\\00 7]", "[\\u00007]"), + new TestCase("[\\0 07]", "[\\u000007]"), + new TestCase("[\\ 007]", "[\\ 07]"), + new TestCase("[\\800]", "[08]"), + new TestCase("[\\0008]", "[\\u00008]"), + new TestCase("[\\008]", "[\\u00008]"), + new TestCase("[\\08]", "[\\u00008]"), + new TestCase("[\\8]", "[8]"), + new TestCase("[\\007F]", "[\\u0007F]"), + new TestCase("[\\07F]", "[\\u0007F]"), + new TestCase("[\\7F]", "[\\u0007F]"), + new TestCase("[\\u200E007]", "[07\\u200E]"), + new TestCase("[ :Greek:]", "[\\:Gekr]"), + new TestCase("[\\uDBFF \\uDFFF]", "[\\uDFFF\\uDBFF]"), + // ICU-2906, ICU extension to UnicodeSet. + new TestCase("[\\uDBFF\\uDFFF]", "[\\U0010FFFF]"), + // Nonportable ICU4J-only extension. + // ICU4J interprets those like [\uDBFF\uDFFF], as [\U0010FFFF]. ICU4C < 79 + // interpreted them as [\uDBFF \uDFFF], because the sequence of two escapes is + // longer than MAX_U_NOTATION_LEN defined in ruleiter.cpp. They became + // ill-formed in ICU4C 79 because of the requirement to separate + // escaped-elements for lead and trail surrogates. + // TODO(egg): Propose making them ill-formed in ICU4J too. + new TestCase("[\\x{DBFF}\\x{DFFF}]", "[\\U0010FFFF]"), + new TestCase("[\\U0000DBFF\\U0000DFFF]", "[\\U0010FFFF]"), }) { final var set = new UnicodeSet(testCase.expression); final String actual = set.toPattern(false); @@ -4396,7 +4436,7 @@ public void testParseErrors() { "[{\\N{LATIN CAPITAL LETTER Z}]", "String literal was not terminated: {\\N{LATIN CAPITAL LETTER Z}] [{\\N{LATIN CAPITAL LETTER Z}]☜" }, - // These three were well-formed in ICU 78 and earlier. + // These four were well-formed in ICU 78 and earlier. {"[{\\Normandie}]", "Ill-formed named-element [{\\No☜rmandie}]"}, { "[{\\Picardie}]", @@ -4406,6 +4446,10 @@ public void testParseErrors() { "[{Provence-Al\\pes-Côte d'Azur}]", "Invalid escape sequence \\p in UnicodeSet string [{Provence-Al\\p☜es-Côte d'Azur}]" }, + { + "[{\\\u200E}]", + "Invalid escape sequence \\ in UnicodeSet string [{\\‎☜}]" + }, // This was a well-formed set in ICU 78 and earlier; now it must be enclosed in // square // brackets. @@ -4432,6 +4476,32 @@ public void testParseErrors() { "[ { \\x5A e i c h e n k e t t e } [] \\x5Aeichenmenge ]", "Unescaped Pattern_White_Space in UnicodeSet string literals is prohibited until ICU 81. Escape U+0020. [ { ☜\\x5A e i c h e n k e t t e } [] \\x5Aeichenmenge ]" }, + // ICU-23497 Implicit Directional Marks should not separate UnicodeSet lexical + // elements. + { + "[\\xD\u200EF]", + "white-space other than ignorable-format-control is required between escaped-element \\xD and literal-element F [\\xD☜‎F]" + }, + { + "[\\00\u200E7]", + "white-space other than ignorable-format-control is required between escaped-element \\00 and literal-element 7 [\\00☜‎7]" + }, + { + "[\\0\u200E07]", + "white-space other than ignorable-format-control is required between escaped-element \\0 and literal-element 0 [\\0☜‎07]" + }, + { + "[\\uDBFF\u200E\\uDFFF]", + "white-space other than ignorable-format-control is required between escaped-elements representing high and low surrogates [\\uDBFF☜‎\\uDFFF]" + }, + { + "[\\x{DBFF}\u200E\\U0000DFFF]", + "white-space other than ignorable-format-control is required between escaped-elements representing high and low surrogates [\\x{DBFF}☜‎\\U0000DFFF]" + }, + { + "[\u200E:Greek:]", + "white-space other than ignorable-format-control is required between set-operator [ and literal-element : [☜‎:Greek:]" + }, }) { final String expression = testCase[0]; final String errorMessage = testCase[1];