ODFstudio/odfstudio
2
0
Fork 0
Code Issues Packages Activity

tdf#172646: Fix text search match offsets around surrogate pairs

Browse Source 
Use OUString::iterateCodePoints() to get the next code point. Just
incrementing the index does not work with surrogate pairs.

This is an existing bug that was exposed by properly handling surrogate
pairs while case folding in:

commit 8922952429
Author: Khaled Hosny <khaled@libreoffice.org>
Date:   Mon Jul 24 18:45:38 2023 +0300

    i18nutil: Fix ICU case folding of surrogate pairs

Change-Id: I51328ae6d17ef935d5be00f7327949a26724b9ab
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/207750
Tested-by: Jenkins
Reviewed-by: Khaled Hosny <khaled@libreoffice.org>
This commit is contained in:
Khaled Hosny
2026-07-08 13:14:20 +02:00
committed by Khaled Hosny
parent 208f2d6a7b
commit 4af6707b3b
2 changed files with 70 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
i18npool/qa/cppunit/test_textsearch.cxx
+51
View File
@@ -37,6 +37,7 @@ public:
void testICU();
void testSearches();
void testWildcardSearch();
void testTdf172646_wildcardBeforeSurrogatePair();
void testApostropheSearch();
void testQuotationMarkSearch();
void testTdf91764_searchArabicDiacritics();
@@ -45,6 +46,7 @@ public:
CPPUNIT_TEST(testICU);
CPPUNIT_TEST(testSearches);
CPPUNIT_TEST(testWildcardSearch);
CPPUNIT_TEST(testTdf172646_wildcardBeforeSurrogatePair);
CPPUNIT_TEST(testApostropheSearch);
CPPUNIT_TEST(testQuotationMarkSearch);
CPPUNIT_TEST(testTdf91764_searchArabicDiacritics);
@@ -293,6 +295,55 @@ void TestTextSearch::testWildcardSearch()
CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.endOffset[0]);
}
void TestTextSearch::testTdf172646_wildcardBeforeSurrogatePair()
{
util::SearchOptions2 aOptions;
aOptions.AlgorithmType2 = util::SearchAlgorithms2::WILDCARD;
aOptions.transliterateFlags = sal_Int32(::css::i18n::TransliterationModules::TransliterationModules_IGNORE_CASE);
util::SearchResult aRes;
// A match ending at a supplementary character (a surrogate pair in
// UTF-16) must report the end offset behind the whole pair.
OUString aText = u"a\U0001F334"_ustr;
aOptions.searchString = u"*\U0001F334"_ustr;
m_xSearch2->setOptions2( aOptions );
// match the whole string, [0,3)
aRes = m_xSearch2->searchForward( aText, 0, aText.getLength());
CPPUNIT_ASSERT_EQUAL(static_cast<sal_Int32>(1), aRes.subRegExpressions);
CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.startOffset[0]);
CPPUNIT_ASSERT_EQUAL(sal_Int32(3), aRes.endOffset[0]);
// match the whole string, (3,0]
aRes = m_xSearch2->searchBackward( aText, aText.getLength(), 0);
CPPUNIT_ASSERT_EQUAL(static_cast<sal_Int32>(1), aRes.subRegExpressions);
CPPUNIT_ASSERT_EQUAL(sal_Int32(3), aRes.startOffset[0]);
CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.endOffset[0]);
// The same in whole-cell matching mode (as used e.g. by SUMIF), matching
// "abc" then the surrogate pair.
aText = u"abc\U0001F334"_ustr;
aOptions.searchFlag |= util::SearchFlags::WILD_MATCH_SELECTION;
m_xSearch2->setOptions2( aOptions );
// match the whole string, [0,5)
aRes = m_xSearch2->searchForward( aText, 0, aText.getLength());
CPPUNIT_ASSERT_EQUAL(static_cast<sal_Int32>(1), aRes.subRegExpressions);
CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.startOffset[0]);
CPPUNIT_ASSERT_EQUAL(sal_Int32(5), aRes.endOffset[0]);
// match the whole string, (5,0]
aRes = m_xSearch2->searchBackward( aText, aText.getLength(), 0);
CPPUNIT_ASSERT_EQUAL(static_cast<sal_Int32>(1), aRes.subRegExpressions);
CPPUNIT_ASSERT_EQUAL(sal_Int32(5), aRes.startOffset[0]);
CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.endOffset[0]);
aOptions.searchString = u"*\U0001F334*"_ustr;
m_xSearch2->setOptions2( aOptions );
// match the whole string, [0,5)
aRes = m_xSearch2->searchForward( aText, 0, aText.getLength());
CPPUNIT_ASSERT_EQUAL(static_cast<sal_Int32>(1), aRes.subRegExpressions);
CPPUNIT_ASSERT_EQUAL(sal_Int32(0), aRes.startOffset[0]);
CPPUNIT_ASSERT_EQUAL(sal_Int32(5), aRes.endOffset[0]);
}
void TestTextSearch::testApostropheSearch()
{
// A) find typographic apostrophes also by using ASCII apostrophe in searchString
i18npool/source/search/textsearch.cxx
+19 -8
View File
@@ -31,6 +31,7 @@
#include <cppuhelper/supportsservice.hxx>
#include <cppuhelper/weak.hxx>
#include <i18nutil/transliteration.hxx>
#include <o3tl/string_view.hxx>
#include <rtl/ustrbuf.hxx>
#include <sal/log.hxx>
@@ -322,6 +323,12 @@ static sal_Int32 FindPosInSeq_Impl( const Sequence <sal_Int32>& rOff, sal_Int32
return static_cast<sal_Int32>(std::distance(rOff.begin(), pOff));
}
static auto nextCodePoint(std::u16string_view rStr, sal_Int32 nPos)
{
o3tl::iterateCodePoints(rStr, &nPos);
return nPos;
}
SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 startPos, sal_Int32 endPos )
{
std::unique_lock g(m_aMutex);
@@ -391,7 +398,8 @@ SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 sta
// Result offsets are negative (-1) if a group expression was
// not matched.
if (nStart >= 0)
sres_startOffsetRange[k] = (nStart < nOffsets ? offset[nStart] : (offset[nOffsets - 1] + 1));
sres_startOffsetRange[k] = (nStart < nOffsets ? offset[nStart]
: nextCodePoint(searchStr, offset[nOffsets - 1]));
// JP 20.6.2001: end is ever exclusive and then don't return
// the position of the next character - return the
// next position behind the last found character!
@@ -400,7 +408,8 @@ SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 sta
if (nStop >= 0)
{
if (nStop > 0)
sres_endOffsetRange[k] = offset[(nStop <= nOffsets ? nStop : nOffsets) - 1] + 1;
sres_endOffsetRange[k] = nextCodePoint(searchStr,
offset[(nStop <= nOffsets ? nStop : nOffsets) - 1]);
else
sres_endOffsetRange[k] = offset[0];
}
@@ -440,9 +449,9 @@ SearchResult TextSearch::searchForward( const OUString& searchStr, sal_Int32 sta
for ( int k = 0; k < sres2.startOffset.getLength(); k++ )
{
if (sres2.startOffset[k])
sres2_startOffsetRange[k] = offset[sres2.startOffset[k]-1] + 1;
sres2_startOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.startOffset[k]-1]);
if (sres2.endOffset[k])
sres2_endOffsetRange[k] = offset[sres2.endOffset[k]-1] + 1;
sres2_endOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.endOffset[k]-1]);
}
// pick first and long one
@@ -515,7 +524,8 @@ SearchResult TextSearch::searchBackward( const OUString& searchStr, sal_Int32 st
if (nStart >= 0)
{
if (nStart > 0)
sres_startOffsetRange[k] = offset[(nStart <= nOffsets ? nStart : nOffsets) - 1] + 1;
sres_startOffsetRange[k] = nextCodePoint(searchStr,
offset[(nStart <= nOffsets ? nStart : nOffsets) - 1]);
else
sres_startOffsetRange[k] = offset[0];
}
@@ -525,7 +535,8 @@ SearchResult TextSearch::searchBackward( const OUString& searchStr, sal_Int32 st
// "a b c" find "b" must return 2,3 and not 2,4!!!
const sal_Int32 nStop = sres.endOffset[k];
if (nStop >= 0)
sres_endOffsetRange[k] = (nStop < nOffsets ? offset[nStop] : (offset[nOffsets - 1] + 1));
sres_endOffsetRange[k] = (nStop < nOffsets ? offset[nStop]
: nextCodePoint(searchStr, offset[nOffsets - 1]));
}
}
}
@@ -562,9 +573,9 @@ SearchResult TextSearch::searchBackward( const OUString& searchStr, sal_Int32 st
for( int k = 0; k < sres2.startOffset.getLength(); k++ )
{
if (sres2.startOffset[k])
sres2_startOffsetRange[k] = offset[sres2.startOffset[k]-1]+1;
sres2_startOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.startOffset[k]-1]);
if (sres2.endOffset[k])
sres2_endOffsetRange[k] = offset[sres2.endOffset[k]-1]+1;
sres2_endOffsetRange[k] = nextCodePoint(searchStr, offset[sres2.endOffset[k]-1]);
}
// pick last and long one