tdf#106137 i18nutil: Add helper to process Unicode escapes
Adds a new utility function processUnicodeEscapes() to i18nutil. This supports parsing: * Standard escapes: \uhhhh (e.g. \u0041) * Extended escapes: \Uhhhhhhhh (e.g. \U0001F600) * Escaped backslashes: \\ Change-Id: I589eb2403376c8cef694f52f1c785d42fba94ff9 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/198777 Tested-by: Jenkins Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
This commit is contained in:
AnamayNarkar
committed by
Mike Kaganski
Mike Kaganski
parent
12ef3e0880
commit
5936fab3e6
@@ -0,0 +1,25 @@
|
||||
# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
|
||||
#
|
||||
# This file is part of the LibreOffice project.
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
|
||||
$(eval $(call gb_CppunitTest_CppunitTest,i18nutil_unicodeescape))
|
||||
|
||||
$(eval $(call gb_CppunitTest_use_external,i18nutil_unicodeescape,boost_headers))
|
||||
|
||||
$(eval $(call gb_CppunitTest_add_exception_objects,i18nutil_unicodeescape, \
|
||||
i18nutil/qa/cppunit/test_unicodeescape \
|
||||
))
|
||||
|
||||
$(eval $(call gb_CppunitTest_use_libraries,i18nutil_unicodeescape, \
|
||||
cppu \
|
||||
cppuhelper \
|
||||
i18nutil \
|
||||
sal \
|
||||
))
|
||||
|
||||
# vim: set noet sw=4 ts=4:
|
||||
@@ -52,6 +52,7 @@ $(eval $(call gb_Library_add_exception_objects,i18nutil,\
|
||||
i18nutil/source/utility/scriptclass \
|
||||
i18nutil/source/utility/scripttypedetector \
|
||||
i18nutil/source/utility/unicode \
|
||||
i18nutil/source/utility/unicodeescape \
|
||||
i18nutil/source/utility/widthfolding \
|
||||
))
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ $(eval $(call gb_Module_add_targets,i18nutil,\
|
||||
|
||||
$(eval $(call gb_Module_add_check_targets,i18nutil,\
|
||||
CppunitTest_i18nutil \
|
||||
CppunitTest_i18nutil_unicodeescape \
|
||||
))
|
||||
|
||||
# vim: set noet sw=4:
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
|
||||
/*
|
||||
* This file is part of the LibreOffice project.
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
*/
|
||||
|
||||
#include <cppunit/TestFixture.h>
|
||||
#include <cppunit/extensions/HelperMacros.h>
|
||||
#include <cppunit/plugin/TestPlugIn.h>
|
||||
#include <i18nutil/unicodeescape.hxx>
|
||||
#include <o3tl/string_view.hxx>
|
||||
|
||||
using namespace o3tl;
|
||||
|
||||
class TestUnicodeEscape : public CppUnit::TestFixture
|
||||
{
|
||||
public:
|
||||
void testBasicUnicodeEscape();
|
||||
void testExtendedUnicodeEscape();
|
||||
void testEscapedBackslash();
|
||||
void testInvalidEscape();
|
||||
void testControlCharFiltering();
|
||||
void testSurrogatePair();
|
||||
|
||||
CPPUNIT_TEST_SUITE(TestUnicodeEscape);
|
||||
CPPUNIT_TEST(testBasicUnicodeEscape);
|
||||
CPPUNIT_TEST(testExtendedUnicodeEscape);
|
||||
CPPUNIT_TEST(testEscapedBackslash);
|
||||
CPPUNIT_TEST(testInvalidEscape);
|
||||
CPPUNIT_TEST(testControlCharFiltering);
|
||||
CPPUNIT_TEST(testSurrogatePair);
|
||||
CPPUNIT_TEST_SUITE_END();
|
||||
};
|
||||
|
||||
void TestUnicodeEscape::testBasicUnicodeEscape()
|
||||
{
|
||||
// \u0041 should become 'A'
|
||||
OUString input(u"\\u0041"_ustr);
|
||||
OUString result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(u"A"_ustr, result);
|
||||
|
||||
// Multiple escapes
|
||||
input = u"\\u0041\\u0042\\u0043"_ustr;
|
||||
result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(u"ABC"_ustr, result);
|
||||
|
||||
// Mixed with text
|
||||
input = u"Hello \\u0041 World"_ustr;
|
||||
result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(u"Hello A World"_ustr, result);
|
||||
}
|
||||
|
||||
void TestUnicodeEscape::testExtendedUnicodeEscape()
|
||||
{
|
||||
// \U00000041 should become 'A'
|
||||
OUString result = i18nutil::processUnicodeEscapes(u"\\U00000041");
|
||||
CPPUNIT_ASSERT_EQUAL(u"A"_ustr, result);
|
||||
}
|
||||
|
||||
void TestUnicodeEscape::testEscapedBackslash()
|
||||
{
|
||||
// \\u0041 should become literal \u0041
|
||||
OUString result = i18nutil::processUnicodeEscapes(u"\\\\u0041");
|
||||
CPPUNIT_ASSERT_EQUAL(u"\\u0041"_ustr, result);
|
||||
}
|
||||
|
||||
void TestUnicodeEscape::testInvalidEscape()
|
||||
{
|
||||
// Invalid hex should keep literal
|
||||
OUString input(u"\\uXYZW"_ustr);
|
||||
OUString result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(input, result);
|
||||
|
||||
// Incomplete escape should keep literal
|
||||
input = u"\\u004"_ustr;
|
||||
result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(input, result);
|
||||
}
|
||||
|
||||
void TestUnicodeEscape::testControlCharFiltering()
|
||||
{
|
||||
// \u000A (LF) should be allowed
|
||||
OUString input(u"\\u000A"_ustr);
|
||||
OUString result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(u"\n"_ustr, result);
|
||||
|
||||
// \u0009 (TAB) should be allowed
|
||||
input = u"\\u0009"_ustr;
|
||||
result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(u"\t"_ustr, result);
|
||||
|
||||
// \u0000 (NULL) should be filtered
|
||||
input = u"\\u0000"_ustr;
|
||||
result = i18nutil::processUnicodeEscapes(input);
|
||||
CPPUNIT_ASSERT_EQUAL(input, result); // Should remain literal
|
||||
|
||||
input = u"\\\\uXYZW"_ustr;
|
||||
result = i18nutil::processUnicodeEscapes(input);
|
||||
|
||||
CPPUNIT_ASSERT_EQUAL(u"\\uXYZW"_ustr, result);
|
||||
}
|
||||
|
||||
void TestUnicodeEscape::testSurrogatePair()
|
||||
{
|
||||
// \U0001F600 (emoji) should work with surrogate pairs
|
||||
OUString result = i18nutil::processUnicodeEscapes(u"\\U0001F600");
|
||||
|
||||
// U+1F600 = UTF-16: 0xD83D 0xDE00
|
||||
sal_Unicode surrogates[2] = { 0xD83D, 0xDE00 };
|
||||
OUString expected(surrogates, 2);
|
||||
|
||||
CPPUNIT_ASSERT_EQUAL(expected, result);
|
||||
}
|
||||
|
||||
CPPUNIT_TEST_SUITE_REGISTRATION(TestUnicodeEscape);
|
||||
CPPUNIT_PLUGIN_IMPLEMENT();
|
||||
|
||||
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */
|
||||
@@ -0,0 +1,102 @@
|
||||
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
|
||||
/*
|
||||
* This file is part of the LibreOffice project.
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
*/
|
||||
|
||||
#include <i18nutil/unicodeescape.hxx>
|
||||
#include <rtl/character.hxx>
|
||||
#include <rtl/ustrbuf.hxx>
|
||||
#include <o3tl/numeric.hxx>
|
||||
#include <cassert>
|
||||
|
||||
namespace i18nutil
|
||||
{
|
||||
namespace
|
||||
{
|
||||
// Convert hex string to codepoint
|
||||
sal_uInt32 hexToCodepoint(std::u16string_view hexStr)
|
||||
{
|
||||
assert(hexStr.size() <= 8);
|
||||
|
||||
sal_uInt32 result = 0;
|
||||
for (sal_Unicode c : hexStr)
|
||||
{
|
||||
sal_Int32 hexValue = o3tl::convertToHex<sal_Int32>(c);
|
||||
if (hexValue < 0)
|
||||
return 0xFFFFFFFF; // Invalid
|
||||
result = (result << 4) | hexValue;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Helper: Check if codepoint is allowed
|
||||
bool isAllowedCodepoint(sal_uInt32 cp)
|
||||
{
|
||||
if (cp < 0x20)
|
||||
{
|
||||
return cp == 0x09 || cp == 0x0A || cp == 0x0D; // TAB, LF, CR
|
||||
}
|
||||
return rtl::isUnicodeCodePoint(cp);
|
||||
}
|
||||
|
||||
} // local namespace
|
||||
|
||||
OUString processUnicodeEscapes(std::u16string_view input)
|
||||
{
|
||||
OUStringBuffer result;
|
||||
sal_Int32 len = input.size();
|
||||
|
||||
for (sal_Int32 i = 0; i < len; ++i)
|
||||
{
|
||||
// Check for backslash and ensure we have at least one char after it
|
||||
if (input[i] == '\\' && i + 1 < len)
|
||||
{
|
||||
sal_Unicode next = input[i + 1];
|
||||
|
||||
// Handle escaped backslash: (backslash)(backslash) -> (backslash)
|
||||
if (next == '\\')
|
||||
{
|
||||
result.append('\\');
|
||||
++i; // Skip the second backslash
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle \uhhhh (4 hex digits)
|
||||
else if (next == 'u' && i + 5 < len)
|
||||
{
|
||||
sal_uInt32 codepoint = hexToCodepoint(input.substr(i + 2, 4));
|
||||
if (isAllowedCodepoint(codepoint))
|
||||
{
|
||||
result.append(sal_Unicode(codepoint));
|
||||
i += 5; // Skip \uhhhh
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle \Uhhhhhhhh (8 hex digits)
|
||||
else if (next == 'U' && i + 9 < len)
|
||||
{
|
||||
sal_uInt32 codepoint = hexToCodepoint(input.substr(i + 2, 8));
|
||||
if (isAllowedCodepoint(codepoint))
|
||||
{
|
||||
result.appendUtf32(codepoint);
|
||||
i += 9; // Skip \Uhhhhhhhh
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not a valid escape, or just a regular character, append it as is
|
||||
result.append(input[i]);
|
||||
}
|
||||
|
||||
return result.makeStringAndClear();
|
||||
}
|
||||
|
||||
} // namespace i18nutil
|
||||
|
||||
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */
|
||||
@@ -0,0 +1,31 @@
|
||||
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
|
||||
/*
|
||||
* This file is part of the LibreOffice project.
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <rtl/ustring.hxx>
|
||||
#include <i18nutil/i18nutildllapi.h>
|
||||
|
||||
namespace i18nutil
|
||||
{
|
||||
/**
|
||||
* Process Unicode escape sequences in a string.
|
||||
* Converts \uhhhh (4 hex digits) and \Uhhhhhhhh (8 hex digits) to their
|
||||
* corresponding Unicode characters.
|
||||
* Handles escaped backslashes: \\ becomes \
|
||||
* Filters out control characters < 0x20 except TAB, LF, and CR.
|
||||
*
|
||||
* @param input String potentially containing Unicode escape sequences
|
||||
* @return String with escape sequences converted to actual Unicode characters
|
||||
*/
|
||||
I18NUTIL_DLLPUBLIC OUString processUnicodeEscapes(std::u16string_view input);
|
||||
|
||||
} // namespace i18nutil
|
||||
|
||||
/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */
|
||||
Reference in New Issue
Block a user