tdf#172513 ko char conv: Preserve char represented by >1 UTF-16 code unit

In order to represent the (codepoint of a) Unicode character
in UTF-16, multiple UTF-16 code units may be needed.

So far, the Korean Hangul/Hanja character conversion
was assuming that each Unicode character was only
consisting of a single UTF-16 code unit, and splitting
into single sal_Unicode characters accordingly in
TextConversion_ko::getCharConversions.

This breaks for characters represented by multiple
code units, like "𡽜" (CJK Unified Ideograph [1],
unicode codepoint: 139100/U+21F5C) which is represented
by 2 UTF-16 code units.

As a consequence, an invalid OUString/UTF-16 string would
be passed to HangulHanjaConversionDialog::SetCurrentString
for some of the suggestions.

Fix this by no longer splitting into single sal_Unicode,
but use o3tl::iterateCodePoints to split into actual
code points.

Example: When using "Tools" -> "Language" ->
"Hangul/Hanja Conversion..." for a Korean text of
"한글맞춤법", the string of all characters for potential
conversions in TextConversion_ko::getCharConversions is:
"韓漢寒限閑恨旱汗翰邯罕悍澣閒瀚桿捍閈暵鼾犴僩嫺熯嫻駻鷳扞狠僴覵瞷猂邗骭撊鷴垾憪橌㸁䛞䦘闲鹇娴汉焊闬韩仠佷傼兯哻娨嫨屽晘橺浫澖皔蓒蔊蛝螒豻貋釬銲雗鶾鷼麲㒏㛠㡾㢨㪋㯗㲦㵄䁂䂅䈨䉯䍐䍑䎯䏷䓍䓳䕿䗙䥜䦥䮧𡽜忓"

Previously, the last 3 of them ("䮧𡽜忓") would incorrectly be
split into (stdout output, showing something is wrong):
"䮧", " " (something wrong, but a valid UTF-16 string), "?" (wrong, and invalid UTF-16 string), "忓"

With the fix in place, they remain intact:
"䮧", "𡽜", "忓"

Also adjust HangulHanjaConversionDialog::OnSuggestionModified
accordingly to compare the number of code points the
string consists of, not the number of UTF-16 code units.
Otherwise, the "Replace"/"Always Replace" buttons in the
"Hangul/Hanja Conversion" dialog would remain greyed
out for the above example when selecting the "𡽜" character
(that now correctly gets displayed with the fix in
TextConversion_ko::getCharConversions in place).
Thanks to Jonathan Clark for pointing out that additional
aspect in review!

Add a corresponding test case.
While the fix is in i18npool, the new test is implemented
in editeng, where the HangulHanjaConversion class uses
the i18npool logic via a UNO service.
The test is implemented there because the component context
availabe in i18npool doesn't supply the required service.
The same test implemented there would fail like this:

    An uncaught UNO exception
    - com.sun.star.uno.DeploymentException: {Message: "component context fails to supply service com.sun.star.linguistic2.ConversionDictionaryList of type com.sun.star.linguistic2.XConversionDictionaryList at /home/michi/development/git/libreoffice/workdir/UnoApiHeadersTarget/offapi/normal/com/sun/star/linguistic2/ConversionDictionaryList.hpp:45", Context: @0x7fff17acecf0}

[1] https://codepoints.net/U+21F5C

Change-Id: I1ab202cc0d3b5a6ff5d220f4b8e90c578f591158
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/207017
Reviewed-by: Jonathan Clark <jonathan@libreoffice.org>
Tested-by: Jenkins

cui/source/dialogs/hangulhanjadlg.cxx

@@ -28,6 +28,7 @@
 #include <osl/diagnose.h>
 #include <tools/debug.hxx>
 #include <i18nlangtag/languagetag.hxx>
+#include <o3tl/string_view.hxx>
 #include <vcl/svapp.hxx>
 #include <vcl/virdev.hxx>
 #include <vcl/weld/ScrolledWindow.hxx>
@@ -538,12 +539,25 @@ IMPL_LINK_NOARG(HangulHanjaConversionDialog, OnSuggestionSelected, SuggestionDis
     OnSuggestionModified(*m_xWordInput);
 }
 
+static int GetCodePointCount(std::u16string_view sText)
+{
+    sal_Int32 nCount = 0;
+    sal_Int32 nIndex = 0;
+    while (o3tl::make_unsigned(nIndex) < sText.size())
+    {
+        o3tl::iterateCodePoints(sText, &nIndex);
+        nCount++;
+    }
+
+    return nCount;
+}
+
 IMPL_LINK_NOARG(HangulHanjaConversionDialog, OnSuggestionModified, weld::Entry&, void)
 {
     m_xFind->set_sensitive(m_xWordInput->get_value_changed_from_saved());
 
-    bool bSameLen
-        = m_xWordInput->get_text().getLength() == m_xOriginalWord->get_label().getLength();
+    bool bSameLen = GetCodePointCount(m_xWordInput->get_text())
+                    == GetCodePointCount(m_xOriginalWord->get_label());
     m_xReplace->set_sensitive(m_bDocumentMode && bSameLen);
     m_xReplaceAll->set_sensitive(m_bDocumentMode && bSameLen);
 }

editeng/CppunitTest_editeng_editeng.mk

@@ -12,6 +12,7 @@
 $(eval $(call gb_CppunitTest_CppunitTest,editeng_editeng))
 
 $(eval $(call gb_CppunitTest_add_exception_objects,editeng_editeng, \
+    editeng/qa/editeng/CharacterConversionTest \
     editeng/qa/editeng/editeng \
 ))
 

editeng/qa/editeng/CharacterConversionTest.cxx

@@ -0,0 +1,61 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <test/bootstrapfixture.hxx>
+
+#include <cppuhelper/bootstrap.hxx>
+
+#include <com/sun/star/i18n/TextConversion.hpp>
+#include <com/sun/star/i18n/TextConversionOption.hpp>
+#include <com/sun/star/i18n/TextConversionType.hpp>
+
+namespace
+{
+class CharacterConversionTest : public test::BootstrapFixture
+{
+};
+
+bool isValidOUString(const OUString& rString)
+{
+    OString sDummy;
+    return rString.convertToString(&sDummy, RTL_TEXTENCODING_UTF8,
+                                   RTL_UNICODETOTEXT_FLAGS_UNDEFINED_ERROR
+                                       | RTL_UNICODETOTEXT_FLAGS_INVALID_ERROR);
+}
+
+// Test that Korean Hangul/Hanja character conversion only gives conversion
+// candidate strings consisting of valid UTF-16 strings.
+// Underlying logic is implemented in i18npool and gets used in class HangulHanjaConversion in editeng.
+// Test is done in editeng (and not i18npool) tests to have access to a component context
+// that supplies the required service com.sun.star.linguistic2.ConversionDictionaryList .
+CPPUNIT_TEST_FIXTURE(CharacterConversionTest, testKoreanHangulHanjaCharacterConversion)
+{
+    css::lang::Locale aKoreanLocale;
+    aKoreanLocale.Language = u"ko"_ustr;
+    aKoreanLocale.Country = u"KR"_ustr;
+
+    css::uno::Reference<css::i18n::XExtendedTextConversion> xConverter
+        = css::i18n::TextConversion::create(m_xContext);
+    CPPUNIT_ASSERT_MESSAGE("Creating text converter for Korean must succeed", xConverter);
+    css::i18n::TextConversionResult aResult = xConverter->getConversions(
+        u"한글맞춤법"_ustr, 0, 5, aKoreanLocale, css::i18n::TextConversionType::TO_HANJA,
+        css::i18n::TextConversionOption::CHARACTER_BY_CHARACTER
+            | css::i18n::TextConversionOption::IGNORE_POST_POSITIONAL_WORD);
+
+    CPPUNIT_ASSERT_MESSAGE("Converter must returns character conversion candidates",
+                           aResult.Candidates.size() > 0);
+
+    for (const OUString& rCandidate : aResult.Candidates)
+        CPPUNIT_ASSERT_MESSAGE("Conversion candidate must be a valid UTF-16 string",
+                               isValidOUString(rCandidate));
+}
+
+} // end anonymous namespace
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */

i18npool/source/textconversion/textconversion_ko.cxx

@@ -25,6 +25,7 @@
 #include <com/sun/star/linguistic2/ConversionDictionaryType.hpp>
 #include <com/sun/star/linguistic2/ConversionDictionaryList.hpp>
 #include <comphelper/sequence.hxx>
+#include <o3tl/string_view.hxx>
 #include <rtl/ustrbuf.hxx>
 #include <unicode/uchar.h>
 #include <memory>
@@ -132,8 +133,20 @@ TextConversion_ko::getCharConversions(std::u16string_view aText, sal_Int32 nStar
                 sal_Int16 count = Hangul_ko[current].count;
                 output.realloc(count);
                 auto poutput = output.getArray();
-                for (sal_Int16 i = 0; i < count; i++)
-                    poutput[i] = OUString(ptr + i, 1);
+
+                // split into single characters/code points
+                // (one code point can be represented by multiple UTF-16 code units)
+                const std::u16string_view sAllChars(ptr, count);
+                sal_Int32 nIndex = 0;
+                sal_Int32 nCodePointCount = 0;
+                while (o3tl::make_unsigned(nIndex) < sAllChars.size())
+                {
+                    auto nChar = o3tl::iterateCodePoints(sAllChars, &nIndex);
+                    poutput[nCodePointCount] = OUString{&nChar, 1};
+                    nCodePointCount++;
+                }
+                output.realloc(nCodePointCount);
+
                 break;
             }
         }