From d71b9934b32e9389109d0e9f7052425df0271bd6 Mon Sep 17 00:00:00 2001 From: Julien Nabet Date: Sun, 16 Aug 2026 12:21:08 +0200 Subject: [PATCH] tdf#166299 tdf#172587: try to improve encoding detection Instead of selecting the first encoding returned by ucsdet_detectAll (except the 5 CJK when they have confidence of 10), recalculate confidence by removing 10 only when one of the 5 CJK encodings then check each encoding returned by ucsdet_detectAll and select the one with the max confidence with this new rule. I also tested this again on tdf#172905, no regression. Change-Id: I6a8e2b30e8467fefd53c02296484e48ca467c662 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/209143 Tested-by: Jenkins Reviewed-by: Julien Nabet --- tools/source/stream/stream.cxx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/source/stream/stream.cxx b/tools/source/stream/stream.cxx index bce41a48f18e..89b1edafc12a 100644 --- a/tools/source/stream/stream.cxx +++ b/tools/source/stream/stream.cxx @@ -839,6 +839,8 @@ void SvStream::DetectEncoding(size_t maxBytes) // by default, let's use the first character set detected const UCharsetMatch* match = matches[0]; constexpr int CONFIDENCE_MIN_SKIP_CHARSETS = 10; + // initialize to -1 so we're sure to retrieve an encoding when matchCount != 0 + int max_confidence = -1; for (int i=0; i < matchCount; i++) { const char* pName = ucsdet_getName(matches[i], &uerr); @@ -846,13 +848,15 @@ void SvStream::DetectEncoding(size_t maxBytes) SAL_INFO("tools.stream", "\tcharacter set name=" << pName << " has a confidence of " << confidence); - if (confidence == CONFIDENCE_MIN_SKIP_CHARSETS && - std::find(aSkipCharsets.begin(), aSkipCharsets.end(), pName) != aSkipCharsets.end()) + if (std::find(aSkipCharsets.begin(), aSkipCharsets.end(), pName) != aSkipCharsets.end()) { - continue; + confidence -= CONFIDENCE_MIN_SKIP_CHARSETS; + } + if (confidence > max_confidence) + { + match = matches[i]; + max_confidence = confidence; } - match = matches[i]; - break; } const char* pEncodingName = ucsdet_getName(match, &uerr);