From 4af6707b3bc601bf61d7460837f5aab90267d2ca Mon Sep 17 00:00:00 2001 From: Khaled Hosny Date: Wed, 8 Jul 2026 02:29:19 +0300 Subject: [PATCH] tdf#172646: Fix text search match offsets around surrogate pairs Use OUString::iterateCodePoints() to get the next code point. Just incrementing the index does not work with surrogate pairs. This is an existing bug that was exposed by properly handling surrogate pairs while case folding in: commit 89229524298398ca1b5239c2a7ca798900733f5d Author: Khaled Hosny Date: Mon Jul 24 18:45:38 2023 +0300 i18nutil: Fix ICU case folding of surrogate pairs Change-Id: I51328ae6d17ef935d5be00f7327949a26724b9ab Reviewed-on: https://gerrit.libreoffice.org/c/core/+/207750 Tested-by: Jenkins Reviewed-by: Khaled Hosny --- i18npool/qa/cppunit/test_textsearch.cxx | 51 +++++++++++++++++++++++++ i18npool/source/search/textsearch.cxx | 27 +++++++++---- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/i18npool/qa/cppunit/test_textsearch.cxx b/i18npool/qa/cppunit/test_textsearch.cxx index 1ea1bf64d66f..7f56548f297d 100644 --- a/i18npool/qa/cppunit/test_textsearch.cxx +++ b/i18npool/qa/cppunit/test_textsearch.cxx @@ -37,6 +37,7 @@ public: void testICU(); void testSearches(); void testWildcardSearch(); + void testTdf172646_wildcardBeforeSurrogatePair(); void testApostropheSearch(); void testQuotationMarkSearch(); void testTdf91764_searchArabicDiacritics(); @@ -45,6 +46,7 @@ public: CPPUNIT_TEST(testICU); CPPUNIT_TEST(testSearches); CPPUNIT_TEST(testWildcardSearch); + CPPUNIT_TEST(testTdf172646_wildcardBeforeSurrogatePair); CPPUNIT_TEST(testApostropheSearch); CPPUNIT_TEST(testQuotationMarkSearch); CPPUNIT_TEST(testTdf91764_searchArabicDiacritics); @@ -293,6 +295,55 @@ void TestTextSearch::testWildcardSearch() CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.endOffset[0]); } +void TestTextSearch::testTdf172646_wildcardBeforeSurrogatePair() +{ + util::SearchOptions2 aOptions; + aOptions.AlgorithmType2 = util::SearchAlgorithms2::WILDCARD; + aOptions.transliterateFlags = sal_Int32(::css::i18n::TransliterationModules::TransliterationModules_IGNORE_CASE); + util::SearchResult aRes; + + // A match ending at a supplementary character (a surrogate pair in + // UTF-16) must report the end offset behind the whole pair. + OUString aText = u"a\U0001F334"_ustr; + + aOptions.searchString = u"*\U0001F334"_ustr; + m_xSearch2->setOptions2( aOptions ); + // match the whole string, [0,3) + aRes = m_xSearch2->searchForward( aText, 0, aText.getLength()); + CPPUNIT_ASSERT_EQUAL(static_cast(1), aRes.subRegExpressions); + CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.startOffset[0]); + CPPUNIT_ASSERT_EQUAL(sal_Int32(3), aRes.endOffset[0]); + // match the whole string, (3,0] + aRes = m_xSearch2->searchBackward( aText, aText.getLength(), 0); + CPPUNIT_ASSERT_EQUAL(static_cast(1), aRes.subRegExpressions); + CPPUNIT_ASSERT_EQUAL(sal_Int32(3), aRes.startOffset[0]); + CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.endOffset[0]); + + // The same in whole-cell matching mode (as used e.g. by SUMIF), matching + // "abc" then the surrogate pair. + aText = u"abc\U0001F334"_ustr; + aOptions.searchFlag |= util::SearchFlags::WILD_MATCH_SELECTION; + m_xSearch2->setOptions2( aOptions ); + // match the whole string, [0,5) + aRes = m_xSearch2->searchForward( aText, 0, aText.getLength()); + CPPUNIT_ASSERT_EQUAL(static_cast(1), aRes.subRegExpressions); + CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.startOffset[0]); + CPPUNIT_ASSERT_EQUAL(sal_Int32(5), aRes.endOffset[0]); + // match the whole string, (5,0] + aRes = m_xSearch2->searchBackward( aText, aText.getLength(), 0); + CPPUNIT_ASSERT_EQUAL(static_cast(1), aRes.subRegExpressions); + CPPUNIT_ASSERT_EQUAL(sal_Int32(5), aRes.startOffset[0]); + CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.endOffset[0]); + + aOptions.searchString = u"*\U0001F334*"_ustr; + m_xSearch2->setOptions2( aOptions ); + // match the whole string, [0,5) + aRes = m_xSearch2->searchForward( aText, 0, aText.getLength()); + CPPUNIT_ASSERT_EQUAL(static_cast(1), aRes.subRegExpressions); + CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.startOffset[0]); + CPPUNIT_ASSERT_EQUAL(sal_Int32(5), aRes.endOffset[0]); +} + void TestTextSearch::testApostropheSearch() { // A) find typographic apostrophes also by using ASCII apostrophe in searchString diff --git a/i18npool/source/search/textsearch.cxx b/i18npool/source/search/textsearch.cxx index ae0d1fab4f0d..cd0721657944 100644 --- a/i18npool/source/search/textsearch.cxx +++ b/i18npool/source/search/textsearch.cxx @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -322,6 +323,12 @@ static sal_Int32 FindPosInSeq_Impl( const Sequence & rOff, sal_Int32 return static_cast(std::distance(rOff.begin(), pOff)); } +static auto nextCodePoint(std::u16string_view rStr, sal_Int32 nPos) +{ + o3tl::iterateCodePoints(rStr, &nPos); + return nPos; +} + SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 startPos, sal_Int32 endPos ) { std::unique_lock g(m_aMutex); @@ -391,7 +398,8 @@ SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 sta // Result offsets are negative (-1) if a group expression was // not matched. if (nStart >= 0) - sres_startOffsetRange[k] = (nStart < nOffsets ? offset[nStart] : (offset[nOffsets - 1] + 1)); + sres_startOffsetRange[k] = (nStart < nOffsets ? offset[nStart] + : nextCodePoint(searchStr, offset[nOffsets - 1])); // JP 20.6.2001: end is ever exclusive and then don't return // the position of the next character - return the // next position behind the last found character! @@ -400,7 +408,8 @@ SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 sta if (nStop >= 0) { if (nStop > 0) - sres_endOffsetRange[k] = offset[(nStop <= nOffsets ? nStop : nOffsets) - 1] + 1; + sres_endOffsetRange[k] = nextCodePoint(searchStr, + offset[(nStop <= nOffsets ? nStop : nOffsets) - 1]); else sres_endOffsetRange[k] = offset[0]; } @@ -440,9 +449,9 @@ SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 sta for ( int k = 0; k < sres2.startOffset.getLength(); k++ ) { if (sres2.startOffset[k]) - sres2_startOffsetRange[k] = offset[sres2.startOffset[k]-1] + 1; + sres2_startOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.startOffset[k]-1]); if (sres2.endOffset[k]) - sres2_endOffsetRange[k] = offset[sres2.endOffset[k]-1] + 1; + sres2_endOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.endOffset[k]-1]); } // pick first and long one @@ -515,7 +524,8 @@ SearchResult TextSearch::searchBackward( const OUString& searchStr, sal_Int32 st if (nStart >= 0) { if (nStart > 0) - sres_startOffsetRange[k] = offset[(nStart <= nOffsets ? nStart : nOffsets) - 1] + 1; + sres_startOffsetRange[k] = nextCodePoint(searchStr, + offset[(nStart <= nOffsets ? nStart : nOffsets) - 1]); else sres_startOffsetRange[k] = offset[0]; } @@ -525,7 +535,8 @@ SearchResult TextSearch::searchBackward( const OUString& searchStr, sal_Int32 st // "a b c" find "b" must return 2,3 and not 2,4!!! const sal_Int32 nStop = sres.endOffset[k]; if (nStop >= 0) - sres_endOffsetRange[k] = (nStop < nOffsets ? offset[nStop] : (offset[nOffsets - 1] + 1)); + sres_endOffsetRange[k] = (nStop < nOffsets ? offset[nStop] + : nextCodePoint(searchStr, offset[nOffsets - 1])); } } } @@ -562,9 +573,9 @@ SearchResult TextSearch::searchBackward( const OUString& searchStr, sal_Int32 st for( int k = 0; k < sres2.startOffset.getLength(); k++ ) { if (sres2.startOffset[k]) - sres2_startOffsetRange[k] = offset[sres2.startOffset[k]-1]+1; + sres2_startOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.startOffset[k]-1]); if (sres2.endOffset[k]) - sres2_endOffsetRange[k] = offset[sres2.endOffset[k]-1]+1; + sres2_endOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.endOffset[k]-1]); } // pick last and long one