ODFstudio/odfstudio
2
0
Fork 0
Code Issues Packages Activity

Move encoding detection to SvStream

Browse Source 
Allows to avoid similar code in SwIoSystem and SfxObjectShell, and
unifies the code.

Change-Id: I16817f2aea5bdabb9bf3505f2f3c2f697e3106e8
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/200284
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
This commit is contained in:
Mike Kaganski
2026-02-25 10:33:49 +01:00
parent 0fc80694c5
commit 610168656a
12 changed files with 133 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/sfx2/objsh.hxx
-1
View File
@@ -455,7 +455,6 @@ public:
const css::uno::Sequence< css::beans::PropertyValue >& GetModifyPasswordInfo() const;
bool SetModifyPasswordInfo( const css::uno::Sequence< css::beans::PropertyValue >& aInfo );
static void DetectCharSet(SvStream& stream, rtl_TextEncoding& eCharSet, SvStreamEndian& endian);
static void DetectCsvSeparators(SvStream& stream, rtl_TextEncoding eCharSet, OUString& separators, sal_Unicode cStringDelimiter);
static void DetectCsvFilterOptions(SvStream& stream, OUString& aFilterOptions);
static void DetectFilterOptions(SfxMedium* pMedium);
include/tools/stream.hxx
+6
View File
@@ -326,6 +326,12 @@ public:
encoding and switch endian swapping if UTF-16 and 0xfffe. */
void StartReadingUnicodeText( rtl_TextEncoding eReadBomCharSet );
/** Try StartWritingUnicodeText; if failed, use ICU to detect encoding from data.
After the function, the position is after BOM (if any); GetStreamCharSet
returns the detected encoding; GetEndian returns the detected endianness
(for UTF-16). */
void DetectEncoding();
/** Read a line of Unicode.
@param nMaxCodepointsToRead
sc/source/ui/dbgui/asciiopt.cxx
+2 -4
View File
@@ -121,13 +121,11 @@ void ScAsciiOptions::ReadFromString( std::u16string_view rString, SvStream* pStr
if ( nPos >= 0 )
{
const std::u16string_view aToken = o3tl::getToken(rString, 0, ',', nPos);
SvStreamEndian endian;
bool bDetectCharSet = aToken == pStrDet;
if ( bDetectCharSet && pStream4Detect )
{
SfxObjectShell::DetectCharSet(*pStream4Detect, eCharSet, endian);
if (eCharSet == RTL_TEXTENCODING_UNICODE)
pStream4Detect->SetEndian(endian);
pStream4Detect->DetectEncoding();
eCharSet = pStream4Detect->GetStreamCharSet();
}
else if (!bDetectCharSet)
eCharSet = ScGlobal::GetCharsetValue( aToken );
sc/source/ui/dbgui/scuiasciiopt.cxx
+3 -5
View File
@@ -372,11 +372,9 @@ ScImportAsciiDlg::ScImportAsciiDlg(weld::Window* pParent, std::u16string_view aD
mxRbDetectSep->set_active(true);
// Detect character set only once and then use it for "Detect" option.
SvStreamEndian eEndian;
SfxObjectShell::DetectCharSet(*mpDatStream, meDetectedCharSet, eEndian);
if (meDetectedCharSet == RTL_TEXTENCODING_UNICODE)
mpDatStream->SetEndian(eEndian);
else if ( meDetectedCharSet == RTL_TEXTENCODING_DONTKNOW )
mpDatStream->DetectEncoding();
meDetectedCharSet = mpDatStream->GetStreamCharSet();
if (meDetectedCharSet == RTL_TEXTENCODING_DONTKNOW)
{
meDetectedCharSet = osl_getThreadTextEncoding();
// Prefer UTF-8, as UTF-16 would have already been detected from the stream.
sfx2/source/doc/objstor.cxx
+2 -77
View File
@@ -125,7 +125,6 @@
#include <guisaveas.hxx>
#include "objstor.hxx"
#include "exoticfileloadexception.hxx"
#include <unicode/ucsdet.h>
#include <o3tl/string_view.hxx>
using namespace ::com::sun::star;
@@ -881,78 +880,6 @@ bool SfxObjectShell::DoLoadExternal( SfxMedium *pMed )
return LoadExternal(*pMedium);
}
const ::std::unordered_map<std::string, rtl_TextEncoding> mapCharSets =
{{"UTF-8", RTL_TEXTENCODING_UTF8},
{"UTF-16BE", RTL_TEXTENCODING_UCS2},
{"UTF-16LE", RTL_TEXTENCODING_UCS2},
{"UTF-32BE", RTL_TEXTENCODING_UCS4},
{"UTF-32LE", RTL_TEXTENCODING_UCS4},
{"Shift_JIS", RTL_TEXTENCODING_SHIFT_JIS},
{"ISO-2022-JP", RTL_TEXTENCODING_ISO_2022_JP},
{"ISO-2022-CN", RTL_TEXTENCODING_ISO_2022_CN},
{"ISO-2022-KR", RTL_TEXTENCODING_ISO_2022_KR},
{"GB18030", RTL_TEXTENCODING_GB_18030},
{"Big5", RTL_TEXTENCODING_BIG5},
{"EUC-JP", RTL_TEXTENCODING_EUC_JP},
{"EUC-KR", RTL_TEXTENCODING_EUC_KR},
{"ISO-8859-1", RTL_TEXTENCODING_ISO_8859_1},
{"ISO-8859-2", RTL_TEXTENCODING_ISO_8859_2},
{"ISO-8859-5", RTL_TEXTENCODING_ISO_8859_5},
{"ISO-8859-6", RTL_TEXTENCODING_ISO_8859_6},
{"ISO-8859-7", RTL_TEXTENCODING_ISO_8859_7},
{"ISO-8859-8", RTL_TEXTENCODING_ISO_8859_8},
{"ISO-8859-9", RTL_TEXTENCODING_ISO_8859_9},
{"windows-1250", RTL_TEXTENCODING_MS_1250},
{"windows-1251", RTL_TEXTENCODING_MS_1251},
{"windows-1252", RTL_TEXTENCODING_MS_1252},
{"windows-1253", RTL_TEXTENCODING_MS_1253},
{"windows-1254", RTL_TEXTENCODING_MS_1254},
{"windows-1255", RTL_TEXTENCODING_MS_1255},
{"windows-1256", RTL_TEXTENCODING_MS_1256},
{"KOI8-R", RTL_TEXTENCODING_KOI8_R}};
void SfxObjectShell::DetectCharSet(SvStream& stream, rtl_TextEncoding& eCharSet, SvStreamEndian &endian)
{
constexpr size_t buffsize = 4096;
sal_Int8 bytes[buffsize] = { 0 };
sal_uInt64 nInitPos = stream.Tell();
sal_Int32 nRead = stream.ReadBytes(bytes, buffsize);
stream.Seek(nInitPos);
eCharSet = RTL_TEXTENCODING_DONTKNOW;
if (!nRead)
return;
UErrorCode uerr = U_ZERO_ERROR;
UCharsetDetector* ucd = ucsdet_open(&uerr);
if (!U_SUCCESS(uerr))
return;
const UCharsetMatch* match = nullptr;
const char* pEncodingName = nullptr;
ucsdet_setText(ucd, reinterpret_cast<const char*>(bytes), nRead, &uerr);
if (U_SUCCESS(uerr))
match = ucsdet_detect(ucd, &uerr);
if (U_SUCCESS(uerr))
pEncodingName = ucsdet_getName(match, &uerr);
if (U_SUCCESS(uerr) && pEncodingName)
{
const auto it = mapCharSets.find(pEncodingName);
if (it != mapCharSets.end())
eCharSet = it->second;
if (eCharSet == RTL_TEXTENCODING_UNICODE && !strcmp("UTF-16LE", pEncodingName))
endian = SvStreamEndian::LITTLE;
else if (eCharSet == RTL_TEXTENCODING_UNICODE && !strcmp("UTF-16BE", pEncodingName))
endian = SvStreamEndian::BIG;
}
ucsdet_close(ucd);
}
void SfxObjectShell::DetectCsvSeparators(SvStream& stream, rtl_TextEncoding eCharSet, OUString& separators, sal_Unicode cStringDelimiter)
{
OUString sLine;
@@ -1123,10 +1050,8 @@ void SfxObjectShell::DetectCsvFilterOptions(SvStream& stream, OUString& aFilterO
// Detect charset
if (aCharSet == aDetect)
{
SvStreamEndian endian;
DetectCharSet(stream, eCharSet, endian);
if (eCharSet == RTL_TEXTENCODING_UNICODE)
stream.SetEndian(endian);
stream.DetectEncoding();
eCharSet = stream.GetStreamCharSet();
}
else if (!aCharSet.empty())
eCharSet = o3tl::toInt32(aCharSet);
sw/inc/iodetect.hxx
-2
View File
@@ -108,8 +108,6 @@ public:
static bool IsValidStgFilter( SotStorage& , const SfxFilter& );
static bool IsValidStgFilter( const css::uno::Reference < css::embed::XStorage >& rStg, const SfxFilter& rFilter);
// tdf#106899 - wrapper around IsDetectableText to retrieve the text encoding for a given stream
SW_DLLPUBLIC static rtl_TextEncoding GetTextEncoding(SvStream&);
static bool IsDetectableText( const char* pBuf, sal_uLong &rLen,
rtl_TextEncoding *pCharSet, bool *pSwap, LineEnd *pLineEnd, bool *pBom);
sw/source/core/edit/edtox.cxx
+2 -1
View File
@@ -296,7 +296,8 @@ void SwEditShell::ApplyAutoMark()
SvStream& rStrm = *aMedium.GetInStream();
Push();
// tdf#106899 - import tox concordance file using the appropriate character set
rtl_TextEncoding eChrSet = SwIoSystem::GetTextEncoding(rStrm);
rStrm.DetectEncoding();
rtl_TextEncoding eChrSet = rStrm.GetStreamCharSet();
if (eChrSet == RTL_TEXTENCODING_DONTKNOW)
eChrSet = ::osl_getThreadTextEncoding();
sw/source/filter/basflt/iodetect.cxx
+9 -87
View File
@@ -240,87 +240,15 @@ std::shared_ptr<const SfxFilter> SwIoSystem::GetFileFilter(const OUString& rFile
return SwIoSystem::GetFilterOfFormat(FILTER_TEXT);
}
rtl_TextEncoding SwIoSystem::GetTextEncoding(SvStream& rStrm)
{
sal_uLong nLen, nOrig;
char aBuf[4096];
nOrig = nLen = rStrm.ReadBytes(aBuf, sizeof(aBuf));
rtl_TextEncoding eCharSet = RTL_TEXTENCODING_DONTKNOW;
const bool bRet = SwIoSystem::IsDetectableText(aBuf, nLen, &eCharSet, nullptr, nullptr, nullptr);
if (bRet && eCharSet != RTL_TEXTENCODING_DONTKNOW)
rStrm.SeekRel(-(tools::Long(nLen)));
else
rStrm.SeekRel(-(tools::Long(nOrig)));
return eCharSet;
}
bool SwIoSystem::IsDetectableText(const char* pBuf, sal_uLong &rLen,
rtl_TextEncoding *pCharSet, bool *pSwap, LineEnd *pLineEnd, bool *pBom)
{
bool bSwap = false;
rtl_TextEncoding eCharSet = RTL_TEXTENCODING_DONTKNOW;
bool bLE = true;
bool bBom = false;
/*See if it's a known unicode type*/
if (rLen >= 2)
{
sal_uLong nHead=0;
if (rLen > 2 && sal_uInt8(pBuf[0]) == 0xEF && sal_uInt8(pBuf[1]) == 0xBB &&
sal_uInt8(pBuf[2]) == 0xBF)
{
eCharSet = RTL_TEXTENCODING_UTF8;
nHead = 3;
bBom = true;
}
else if (sal_uInt8(pBuf[0]) == 0xFE && sal_uInt8(pBuf[1]) == 0xFF)
{
eCharSet = RTL_TEXTENCODING_UCS2;
bLE = false;
nHead = 2;
bBom = true;
}
else if (sal_uInt8(pBuf[1]) == 0xFE && sal_uInt8(pBuf[0]) == 0xFF)
{
eCharSet = RTL_TEXTENCODING_UCS2;
nHead = 2;
bBom = true;
}
pBuf+=nHead;
rLen-=nHead;
}
/*See unicode type again without BOM*/
if (rLen >= 1 && eCharSet == RTL_TEXTENCODING_DONTKNOW)
{
UErrorCode uerr = U_ZERO_ERROR;
UCharsetDetector* ucd = ucsdet_open(&uerr);
ucsdet_setText(ucd, pBuf, rLen, &uerr);
if (const UCharsetMatch* match = ucsdet_detect(ucd, &uerr))
{
const char* pEncodingName = ucsdet_getName(match, &uerr);
if (U_SUCCESS(uerr) && !strcmp("UTF-8", pEncodingName))
{
eCharSet = RTL_TEXTENCODING_UTF8; // UTF-8
}
else if (U_SUCCESS(uerr) && !strcmp("UTF-16BE", pEncodingName))
{
eCharSet = RTL_TEXTENCODING_UCS2; // UTF-16BE
bLE = false;
}
else if (U_SUCCESS(uerr) && !strcmp("UTF-16LE", pEncodingName))
{
eCharSet = RTL_TEXTENCODING_UCS2; // UTF-16LE
}
else if (U_SUCCESS(uerr) && !strcmp("GB18030", pEncodingName))
{
eCharSet = RTL_TEXTENCODING_GB_18030;
}
}
ucsdet_close(ucd);
}
SvMemoryStream aStream(const_cast<char*>(pBuf), rLen, StreamMode::READ);
aStream.DetectEncoding();
rtl_TextEncoding eCharSet = aStream.GetStreamCharSet();
auto nBomSize = aStream.Tell();
pBuf += nBomSize;
rLen -= nBomSize;
bool bCR = false, bLF = false, bIsBareUnicode = false;
@@ -352,14 +280,8 @@ bool SwIoSystem::IsDetectableText(const char* pBuf, sal_uLong &rLen,
{
nNewLen = rLen/2;
memcpy(pNewBuf, pBuf, rLen);
#ifdef OSL_LITENDIAN
bool const bNativeLE = true;
#else
bool const bNativeLE = false;
#endif
if (bLE != bNativeLE)
if (aStream.IsEndianSwap())
{
bSwap = true;
for (sal_uLong n = 0; n < nNewLen; ++n)
pNewBuf[n] = OSL_SWAPWORD(pNewBuf[n]);
}
@@ -417,11 +339,11 @@ bool SwIoSystem::IsDetectableText(const char* pBuf, sal_uLong &rLen,
if (pCharSet)
*pCharSet = eCharSet;
if (pSwap)
*pSwap = bSwap;
*pSwap = aStream.IsEndianSwap();
if (pLineEnd)
*pLineEnd = eLineEnd;
if (pBom)
*pBom = bBom;
*pBom = nBomSize != 0;
return !bIsBareUnicode;
}
sw/source/filter/md/swmd.cxx
+3 -18
View File
@@ -842,28 +842,13 @@ ErrCodeMsg MarkdownReader::Read(SwDoc& rDoc, const OUString& rBaseURL, SwPaM& rP
ErrCode SwMarkdownParser::CallParser()
{
// use utf8
m_rInput.StartReadingUnicodeText(RTL_TEXTENCODING_DONTKNOW);
if (!m_rInput.good())
m_rInput.DetectEncoding();
const rtl_TextEncoding eSrcEnc = m_rInput.GetStreamCharSet();
if (eSrcEnc == RTL_TEXTENCODING_DONTKNOW)
{
return ERRCODE_IO_INVALIDCHAR;
}
rtl_TextEncoding eSrcEnc;
const sal_uInt64 nPos = m_rInput.Tell(); //bom size
if (nPos == 2)
eSrcEnc = RTL_TEXTENCODING_UCS2;
else if (nPos == 3)
eSrcEnc = RTL_TEXTENCODING_UTF8;
else
{
SvStreamEndian eEndian;
SfxObjectShell::DetectCharSet(m_rInput, eSrcEnc, eEndian);
if (eSrcEnc == RTL_TEXTENCODING_DONTKNOW)
return ERRCODE_IO_INVALIDCHAR;
m_rInput.SetEndian(eEndian);
}
m_rInput.ResetError();
const sal_uInt64 nFilesize = m_rInput.remainingSize();
OString sUtf8Data;
sw/source/ui/index/cnttab.cxx
+2 -1
View File
@@ -4006,7 +4006,8 @@ void SwEntryBrowseBox::ReadEntries(SvStream& rInStr)
{
AutoMarkEntry* pToInsert = nullptr;
// tdf#108910, tdf#125496 - read index entries using the appropriate character set
rtl_TextEncoding eTEnc = SwIoSystem::GetTextEncoding(rInStr);
rInStr.DetectEncoding();
rtl_TextEncoding eTEnc = rInStr.GetStreamCharSet();
if (eTEnc == RTL_TEXTENCODING_DONTKNOW)
eTEnc = osl_getThreadTextEncoding();
while (rInStr.good())
tools/Library_tl.mk
+4
View File
@@ -108,6 +108,10 @@ $(eval $(call gb_Library_add_generated_exception_objects,tl,\
$(eval $(call gb_Library_use_externals,tl,\
boost_headers \
frozen \
icu_headers \
icui18n \
icuuc \
zlib \
libxml2 \
))
tools/source/stream/stream.cxx
+100
View File
@@ -39,6 +39,12 @@
#include <comphelper/fileformat.h>
#include <comphelper/fileurl.hxx>
#include <comphelper/scopeguard.hxx>
#include <unicode/ucsdet.h>
#include <frozen/bits/elsa_std.h>
#include <frozen/unordered_map.h>
static void swapNibbles(unsigned char &c)
{
@@ -742,6 +748,100 @@ void SvStream::StartReadingUnicodeText( rtl_TextEncoding eReadBomCharSet )
Seek(nOldPos); // no BOM, pure data
}
void SvStream::DetectEncoding()
{
static constexpr auto mapEncodings
= frozen::make_unordered_map<std::string_view, rtl_TextEncoding>({
{ "UTF-8", RTL_TEXTENCODING_UTF8 },
{ "UTF-16BE", RTL_TEXTENCODING_UCS2 },
{ "UTF-16LE", RTL_TEXTENCODING_UCS2 },
//{ "UTF-32BE", RTL_TEXTENCODING_UCS4 },
//{ "UTF-32LE", RTL_TEXTENCODING_UCS4 },
{ "Shift_JIS", RTL_TEXTENCODING_SHIFT_JIS },
{ "ISO-2022-JP", RTL_TEXTENCODING_ISO_2022_JP },
{ "ISO-2022-CN", RTL_TEXTENCODING_ISO_2022_CN },
{ "ISO-2022-KR", RTL_TEXTENCODING_ISO_2022_KR },
{ "GB18030", RTL_TEXTENCODING_GB_18030 },
{ "Big5", RTL_TEXTENCODING_BIG5 },
{ "EUC-JP", RTL_TEXTENCODING_EUC_JP },
{ "EUC-KR", RTL_TEXTENCODING_EUC_KR },
{ "ISO-8859-1", RTL_TEXTENCODING_ISO_8859_1 },
{ "ISO-8859-2", RTL_TEXTENCODING_ISO_8859_2 },
{ "ISO-8859-5", RTL_TEXTENCODING_ISO_8859_5 },
{ "ISO-8859-6", RTL_TEXTENCODING_ISO_8859_6 },
{ "ISO-8859-7", RTL_TEXTENCODING_ISO_8859_7 },
{ "ISO-8859-8", RTL_TEXTENCODING_ISO_8859_8 },
{ "ISO-8859-9", RTL_TEXTENCODING_ISO_8859_9 },
{ "windows-1250", RTL_TEXTENCODING_MS_1250 },
{ "windows-1251", RTL_TEXTENCODING_MS_1251 },
{ "windows-1252", RTL_TEXTENCODING_MS_1252 },
{ "windows-1253", RTL_TEXTENCODING_MS_1253 },
{ "windows-1254", RTL_TEXTENCODING_MS_1254 },
{ "windows-1255", RTL_TEXTENCODING_MS_1255 },
{ "windows-1256", RTL_TEXTENCODING_MS_1256 },
{ "KOI8-R", RTL_TEXTENCODING_KOI8_R },
});
const sal_uInt64 nOrigPos = Tell();
SetStreamCharSet(RTL_TEXTENCODING_DONTKNOW);
StartReadingUnicodeText(RTL_TEXTENCODING_DONTKNOW);
if (!good())
return;
const sal_uInt64 nBomSize = Tell() - nOrigPos;
if (nBomSize == 2)
{
SetStreamCharSet(RTL_TEXTENCODING_UCS2);
return;
}
if (nBomSize == 3)
{
SetStreamCharSet(RTL_TEXTENCODING_UTF8);
return;
}
assert(nBomSize == 0); // we are at nOrigPos
char bytes[4096] = { 0 };
size_t nRead = ReadBytes(bytes, sizeof(bytes));
Seek(nOrigPos);
ResetError();
if (nRead == 0)
return;
UErrorCode uerr = U_ZERO_ERROR;
UCharsetDetector* ucd = ucsdet_open(&uerr);
if (!U_SUCCESS(uerr))
return;
comphelper::ScopeGuard ucsdet_close_guard([ucd] { ucsdet_close(ucd); });
ucsdet_setText(ucd, bytes, nRead, &uerr);
if (!U_SUCCESS(uerr))
return;
const UCharsetMatch* match = ucsdet_detect(ucd, &uerr);
if (!U_SUCCESS(uerr))
return;
const char* pEncodingName = ucsdet_getName(match, &uerr);
if (!U_SUCCESS(uerr) || !pEncodingName)
return;
const auto it = mapEncodings.find(pEncodingName);
if (it == mapEncodings.end())
return;
rtl_TextEncoding eEncoding = it->second;
SetStreamCharSet(eEncoding);
if (eEncoding == RTL_TEXTENCODING_UCS2)
{
if (it->first == "UTF-16LE")
SetEndian(SvStreamEndian::LITTLE);
else if (it->first == "UTF-16BE")
SetEndian(SvStreamEndian::BIG);
}
}
sal_uInt64 SvStream::SeekRel(sal_Int64 const nPos)
{
sal_uInt64 nActualPos = Tell();