From 5936fab3e6293101a536f55904ff2072f2bb133e Mon Sep 17 00:00:00 2001 From: AnamayNarkar Date: Fri, 6 Feb 2026 02:24:09 +0530 Subject: [PATCH] tdf#106137 i18nutil: Add helper to process Unicode escapes Adds a new utility function processUnicodeEscapes() to i18nutil. This supports parsing: * Standard escapes: \uhhhh (e.g. \u0041) * Extended escapes: \Uhhhhhhhh (e.g. \U0001F600) * Escaped backslashes: \\ Change-Id: I589eb2403376c8cef694f52f1c785d42fba94ff9 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/198777 Tested-by: Jenkins Reviewed-by: Mike Kaganski --- .../CppunitTest_i18nutil_unicodeescape.mk | 25 ++++ i18nutil/Library_i18nutil.mk | 1 + i18nutil/Module_i18nutil.mk | 1 + i18nutil/qa/cppunit/test_unicodeescape.cxx | 121 ++++++++++++++++++ i18nutil/source/utility/unicodeescape.cxx | 102 +++++++++++++++ include/i18nutil/unicodeescape.hxx | 31 +++++ 6 files changed, 281 insertions(+) create mode 100644 i18nutil/CppunitTest_i18nutil_unicodeescape.mk create mode 100644 i18nutil/qa/cppunit/test_unicodeescape.cxx create mode 100644 i18nutil/source/utility/unicodeescape.cxx create mode 100644 include/i18nutil/unicodeescape.hxx diff --git a/i18nutil/CppunitTest_i18nutil_unicodeescape.mk b/i18nutil/CppunitTest_i18nutil_unicodeescape.mk new file mode 100644 index 000000000000..7efe55097331 --- /dev/null +++ b/i18nutil/CppunitTest_i18nutil_unicodeescape.mk @@ -0,0 +1,25 @@ +# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +$(eval $(call gb_CppunitTest_CppunitTest,i18nutil_unicodeescape)) + +$(eval $(call gb_CppunitTest_use_external,i18nutil_unicodeescape,boost_headers)) + +$(eval $(call gb_CppunitTest_add_exception_objects,i18nutil_unicodeescape, \ + i18nutil/qa/cppunit/test_unicodeescape \ +)) + +$(eval $(call gb_CppunitTest_use_libraries,i18nutil_unicodeescape, \ + cppu \ + cppuhelper \ + i18nutil \ + sal \ +)) + +# vim: set noet sw=4 ts=4: diff --git a/i18nutil/Library_i18nutil.mk b/i18nutil/Library_i18nutil.mk index cf166a476401..4c6923e8a621 100644 --- a/i18nutil/Library_i18nutil.mk +++ b/i18nutil/Library_i18nutil.mk @@ -52,6 +52,7 @@ $(eval $(call gb_Library_add_exception_objects,i18nutil,\ i18nutil/source/utility/scriptclass \ i18nutil/source/utility/scripttypedetector \ i18nutil/source/utility/unicode \ + i18nutil/source/utility/unicodeescape \ i18nutil/source/utility/widthfolding \ )) diff --git a/i18nutil/Module_i18nutil.mk b/i18nutil/Module_i18nutil.mk index 3fac872e83ed..dc9bc57c62cf 100644 --- a/i18nutil/Module_i18nutil.mk +++ b/i18nutil/Module_i18nutil.mk @@ -14,6 +14,7 @@ $(eval $(call gb_Module_add_targets,i18nutil,\ $(eval $(call gb_Module_add_check_targets,i18nutil,\ CppunitTest_i18nutil \ + CppunitTest_i18nutil_unicodeescape \ )) # vim: set noet sw=4: diff --git a/i18nutil/qa/cppunit/test_unicodeescape.cxx b/i18nutil/qa/cppunit/test_unicodeescape.cxx new file mode 100644 index 000000000000..2cb098e692b8 --- /dev/null +++ b/i18nutil/qa/cppunit/test_unicodeescape.cxx @@ -0,0 +1,121 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include +#include +#include +#include +#include + +using namespace o3tl; + +class TestUnicodeEscape : public CppUnit::TestFixture +{ +public: + void testBasicUnicodeEscape(); + void testExtendedUnicodeEscape(); + void testEscapedBackslash(); + void testInvalidEscape(); + void testControlCharFiltering(); + void testSurrogatePair(); + + CPPUNIT_TEST_SUITE(TestUnicodeEscape); + CPPUNIT_TEST(testBasicUnicodeEscape); + CPPUNIT_TEST(testExtendedUnicodeEscape); + CPPUNIT_TEST(testEscapedBackslash); + CPPUNIT_TEST(testInvalidEscape); + CPPUNIT_TEST(testControlCharFiltering); + CPPUNIT_TEST(testSurrogatePair); + CPPUNIT_TEST_SUITE_END(); +}; + +void TestUnicodeEscape::testBasicUnicodeEscape() +{ + // \u0041 should become 'A' + OUString input(u"\\u0041"_ustr); + OUString result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(u"A"_ustr, result); + + // Multiple escapes + input = u"\\u0041\\u0042\\u0043"_ustr; + result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(u"ABC"_ustr, result); + + // Mixed with text + input = u"Hello \\u0041 World"_ustr; + result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(u"Hello A World"_ustr, result); +} + +void TestUnicodeEscape::testExtendedUnicodeEscape() +{ + // \U00000041 should become 'A' + OUString result = i18nutil::processUnicodeEscapes(u"\\U00000041"); + CPPUNIT_ASSERT_EQUAL(u"A"_ustr, result); +} + +void TestUnicodeEscape::testEscapedBackslash() +{ + // \\u0041 should become literal \u0041 + OUString result = i18nutil::processUnicodeEscapes(u"\\\\u0041"); + CPPUNIT_ASSERT_EQUAL(u"\\u0041"_ustr, result); +} + +void TestUnicodeEscape::testInvalidEscape() +{ + // Invalid hex should keep literal + OUString input(u"\\uXYZW"_ustr); + OUString result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(input, result); + + // Incomplete escape should keep literal + input = u"\\u004"_ustr; + result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(input, result); +} + +void TestUnicodeEscape::testControlCharFiltering() +{ + // \u000A (LF) should be allowed + OUString input(u"\\u000A"_ustr); + OUString result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(u"\n"_ustr, result); + + // \u0009 (TAB) should be allowed + input = u"\\u0009"_ustr; + result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(u"\t"_ustr, result); + + // \u0000 (NULL) should be filtered + input = u"\\u0000"_ustr; + result = i18nutil::processUnicodeEscapes(input); + CPPUNIT_ASSERT_EQUAL(input, result); // Should remain literal + + input = u"\\\\uXYZW"_ustr; + result = i18nutil::processUnicodeEscapes(input); + + CPPUNIT_ASSERT_EQUAL(u"\\uXYZW"_ustr, result); +} + +void TestUnicodeEscape::testSurrogatePair() +{ + // \U0001F600 (emoji) should work with surrogate pairs + OUString result = i18nutil::processUnicodeEscapes(u"\\U0001F600"); + + // U+1F600 = UTF-16: 0xD83D 0xDE00 + sal_Unicode surrogates[2] = { 0xD83D, 0xDE00 }; + OUString expected(surrogates, 2); + + CPPUNIT_ASSERT_EQUAL(expected, result); +} + +CPPUNIT_TEST_SUITE_REGISTRATION(TestUnicodeEscape); +CPPUNIT_PLUGIN_IMPLEMENT(); + +/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */ diff --git a/i18nutil/source/utility/unicodeescape.cxx b/i18nutil/source/utility/unicodeescape.cxx new file mode 100644 index 000000000000..cf9fee576565 --- /dev/null +++ b/i18nutil/source/utility/unicodeescape.cxx @@ -0,0 +1,102 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include +#include +#include +#include +#include + +namespace i18nutil +{ +namespace +{ +// Convert hex string to codepoint +sal_uInt32 hexToCodepoint(std::u16string_view hexStr) +{ + assert(hexStr.size() <= 8); + + sal_uInt32 result = 0; + for (sal_Unicode c : hexStr) + { + sal_Int32 hexValue = o3tl::convertToHex(c); + if (hexValue < 0) + return 0xFFFFFFFF; // Invalid + result = (result << 4) | hexValue; + } + return result; +} + +// Helper: Check if codepoint is allowed +bool isAllowedCodepoint(sal_uInt32 cp) +{ + if (cp < 0x20) + { + return cp == 0x09 || cp == 0x0A || cp == 0x0D; // TAB, LF, CR + } + return rtl::isUnicodeCodePoint(cp); +} + +} // local namespace + +OUString processUnicodeEscapes(std::u16string_view input) +{ + OUStringBuffer result; + sal_Int32 len = input.size(); + + for (sal_Int32 i = 0; i < len; ++i) + { + // Check for backslash and ensure we have at least one char after it + if (input[i] == '\\' && i + 1 < len) + { + sal_Unicode next = input[i + 1]; + + // Handle escaped backslash: (backslash)(backslash) -> (backslash) + if (next == '\\') + { + result.append('\\'); + ++i; // Skip the second backslash + continue; + } + + // Handle \uhhhh (4 hex digits) + else if (next == 'u' && i + 5 < len) + { + sal_uInt32 codepoint = hexToCodepoint(input.substr(i + 2, 4)); + if (isAllowedCodepoint(codepoint)) + { + result.append(sal_Unicode(codepoint)); + i += 5; // Skip \uhhhh + continue; + } + } + + // Handle \Uhhhhhhhh (8 hex digits) + else if (next == 'U' && i + 9 < len) + { + sal_uInt32 codepoint = hexToCodepoint(input.substr(i + 2, 8)); + if (isAllowedCodepoint(codepoint)) + { + result.appendUtf32(codepoint); + i += 9; // Skip \Uhhhhhhhh + continue; + } + } + } + + // If not a valid escape, or just a regular character, append it as is + result.append(input[i]); + } + + return result.makeStringAndClear(); +} + +} // namespace i18nutil + +/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */ diff --git a/include/i18nutil/unicodeescape.hxx b/include/i18nutil/unicodeescape.hxx new file mode 100644 index 000000000000..064d6d82e380 --- /dev/null +++ b/include/i18nutil/unicodeescape.hxx @@ -0,0 +1,31 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include +#include + +namespace i18nutil +{ +/** + * Process Unicode escape sequences in a string. + * Converts \uhhhh (4 hex digits) and \Uhhhhhhhh (8 hex digits) to their + * corresponding Unicode characters. + * Handles escaped backslashes: \\ becomes \ + * Filters out control characters < 0x20 except TAB, LF, and CR. + * + * @param input String potentially containing Unicode escape sequences + * @return String with escape sequences converted to actual Unicode characters + */ +I18NUTIL_DLLPUBLIC OUString processUnicodeEscapes(std::u16string_view input); + +} // namespace i18nutil + +/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */