From abda72eeac19b18c22f57d5443c3955a463605d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20Sz=C5=B1cs?= Date: Mon, 20 Feb 2023 00:32:22 +0100 Subject: [PATCH] tdf#82984 tdf#94915 zip64 support (import + export) Implemented import + export for "Zip64 Extended Information Extra Field", (in "Local file header" and "Central directory file header") and for Data descriptor. Focused only to be able to handle files with over 4GB uncompressed size, in the zip archive. The 64k filecount, and the 4GB compressed size limit is probably still present Tried to follow pkware .ZIP File Format Specification, Some cases were not clear to me and/or some zip compressing tool may not perfectly follow the standard, like 'extra field' should be 28 bytes long, but its reader now can read shorter (or longer) 'extra field'. Replaced some 32bit codes with 64bit codes, in stream handling, in deflater. Tested with an ods file that contained a content.xml that bigger then 4BG+ (import + export + reimport) on windows. I think 4GB+ files import/export would be too slow fot unittest. So, for unit test, used the small but zip64 format files, that was attached to the bugzilla tickets Note: It helps with Bug 128244 too (1 of the unittest tests it), but that ods file missing manifest.xml, so LO won't be able to import it. Change-Id: Idfeb90594388fd34ae719677f5d268ca9a484fb1 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/147306 Tested-by: Jenkins Reviewed-by: Michael Meeks --- include/package/Deflater.hxx | 2 + package/inc/ByteChucker.hxx | 30 +++- package/inc/ZipFile.hxx | 4 + package/inc/ZipOutputStream.hxx | 3 +- package/qa/cppunit/data/export64.zip | Bin 0 -> 3556 bytes package/qa/cppunit/test_package.cxx | 49 ++++++ package/source/zipapi/ByteChucker.cxx | 2 + package/source/zipapi/Deflater.cxx | 18 ++- package/source/zipapi/MemoryByteGrabber.hxx | 32 ++++ package/source/zipapi/ZipFile.cxx | 143 +++++++++++++----- package/source/zipapi/ZipOutputStream.cxx | 69 +++++---- .../data/xlsx/tdf82984_zip64XLSXImport.xlsx | Bin 0 -> 4708 bytes sc/qa/unit/subsequent_filters_test3.cxx | 6 + tools/source/stream/strmwnt.cxx | 25 +-- unotools/source/streaming/streamwrap.cxx | 2 +- 15 files changed, 304 insertions(+), 81 deletions(-) create mode 100644 package/qa/cppunit/data/export64.zip create mode 100644 sc/qa/unit/data/xlsx/tdf82984_zip64XLSXImport.xlsx diff --git a/include/package/Deflater.hxx b/include/package/Deflater.hxx index 3cd528bf5afd..2a5c9d103260 100644 --- a/include/package/Deflater.hxx +++ b/include/package/Deflater.hxx @@ -36,6 +36,8 @@ class DLLPUBLIC_PACKAGE Deflater final bool bFinish; bool bFinished; sal_Int64 nOffset, nLength; + // zlib total_in / total_out may be stored in 32bit, so they can overflow in case of 4gb files + sal_uInt64 nTotalOut64, nTotalIn64; // save the overflowed value here. std::unique_ptr pStream; void init (sal_Int32 nLevel, bool bNowrap); diff --git a/package/inc/ByteChucker.hxx b/package/inc/ByteChucker.hxx index 707b678ff2a5..c502ad6b5c42 100644 --- a/package/inc/ByteChucker.hxx +++ b/package/inc/ByteChucker.hxx @@ -29,8 +29,8 @@ class ByteChucker final { css::uno::Reference < css::io::XOutputStream > xStream; css::uno::Reference < css::io::XSeekable > xSeek; - css::uno::Sequence < sal_Int8 > a2Sequence, a4Sequence; - sal_Int8 * const p2Sequence, * const p4Sequence; + css::uno::Sequence < sal_Int8 > a2Sequence, a4Sequence, a8Sequence; + sal_Int8 * const p2Sequence, * const p4Sequence, * const p8Sequence; public: ByteChucker (css::uno::Reference const & xOstream); @@ -70,6 +70,32 @@ public: p4Sequence[3] = static_cast < sal_Int8 > ((nuInt32 >> 24 ) & 0xFF); WriteBytes( a4Sequence ); } + + void WriteInt64(sal_Int64 nInt64) + { + p8Sequence[0] = static_cast((nInt64 >> 0) & 0xFF); + p8Sequence[1] = static_cast((nInt64 >> 8) & 0xFF); + p8Sequence[2] = static_cast((nInt64 >> 16) & 0xFF); + p8Sequence[3] = static_cast((nInt64 >> 24) & 0xFF); + p8Sequence[4] = static_cast((nInt64 >> 32) & 0xFF); + p8Sequence[5] = static_cast((nInt64 >> 40) & 0xFF); + p8Sequence[6] = static_cast((nInt64 >> 48) & 0xFF); + p8Sequence[7] = static_cast((nInt64 >> 56) & 0xFF); + WriteBytes( a8Sequence ); + } + + void WriteUInt64(sal_uInt64 nuInt64) + { + p8Sequence[0] = static_cast((nuInt64 >> 0) & 0xFF); + p8Sequence[1] = static_cast((nuInt64 >> 8) & 0xFF); + p8Sequence[2] = static_cast((nuInt64 >> 16) & 0xFF); + p8Sequence[3] = static_cast((nuInt64 >> 24) & 0xFF); + p8Sequence[4] = static_cast((nuInt64 >> 32) & 0xFF); + p8Sequence[5] = static_cast((nuInt64 >> 40) & 0xFF); + p8Sequence[6] = static_cast((nuInt64 >> 48) & 0xFF); + p8Sequence[7] = static_cast((nuInt64 >> 56) & 0xFF); + WriteBytes( a8Sequence ); + } }; #endif diff --git a/package/inc/ZipFile.hxx b/package/inc/ZipFile.hxx index 7fe15f70ff99..f6b184994729 100644 --- a/package/inc/ZipFile.hxx +++ b/package/inc/ZipFile.hxx @@ -29,6 +29,7 @@ #include "HashMaps.hxx" #include "EncryptionData.hxx" +class MemoryByteGrabber; namespace com::sun::star { namespace uno { class XComponentContext; } namespace ucb { class XProgressHandler; } @@ -81,6 +82,9 @@ class ZipFile sal_Int32 readCEN(); sal_Int32 findEND(); void recover(); + static void readExtraFields(MemoryByteGrabber& aMemGrabber, sal_Int16 nExtraLen, + sal_uInt64& nSize, sal_uInt64& nCompressedSize, + sal_uInt64* nOffset); public: diff --git a/package/inc/ZipOutputStream.hxx b/package/inc/ZipOutputStream.hxx index f55ef59a8880..b31af9d3d5c4 100644 --- a/package/inc/ZipOutputStream.hxx +++ b/package/inc/ZipOutputStream.hxx @@ -78,7 +78,8 @@ private: void writeCEN( const ZipEntry &rEntry ); /// @throws css::io::IOException /// @throws css::uno::RuntimeException - void writeEXT( const ZipEntry &rEntry ); + void writeDataDescriptor( const ZipEntry &rEntry ); + void writeExtraFields( const ZipEntry& rEntry ); // ScheduledThread handling helpers void consumeScheduledThreadTaskEntry(std::unique_ptr pCandidate); diff --git a/package/qa/cppunit/data/export64.zip b/package/qa/cppunit/data/export64.zip new file mode 100644 index 0000000000000000000000000000000000000000..b30326696794740e932f2955530bf69cfc55ce45 GIT binary patch literal 3556 zcmaJ^XH-*Nvrgy{kRm}Th9U%M0V&d}^xk_BLV!p&pn!!A;!7`~7y%&^X;MQ+TIfZp zg$~jQsGuMUd~n~(dds@seP*3C`>eBP_MSQO<9Q6UiAfj$6lb=jZKMzQ8z|1!9!L*_ zZ-6%f2m_3sI(U1#BjFCdNH0&J2cAwaF9#G71@rPocshB({X7t!zA%*ceT0J($`yg| zHP9xfh;WyBE_#**cJ>(j{|6y_9bo<*?y(8_Zp0k4__zH_v$nNj*f6)82nKKlUB|mh26AQ(!TPkom z;KUJoA#o@eOt0So&|V+U+ZWLgrsMm0p{H zDg#Y7w}ciT?`CZaciv`ah@MwpaO1L8?^)&Ez(1=O{70Kl{}+@11_0d4)Azi6XAJ-X zXaN_WE;7~-_+&n#l~_$?{Nu}L0V5o%c5U)kEuUrNw!gc}G_r6!MAM)nlFI^+ zQYx=xpCj~vX9K8cm{UqgsO9i7i(aN(3*B2>SXk&7`15!{q^wUal@%zxT- z4Lu8apG>9Q7`!B3u<<^*Tcf}9{XJc1a!cO{WVsVPPk3ORX-#xlt3hC`5`8I*F*;fV zYITw|pVE)Y9L_Q487-=8h|#9_&Qn zJVRGpRkK-ZVZpD1Y#(R#5b1AzGO7ZblVY4}Um&p_Pr5BYro8FE;zgX?ZVsX!z3)_Z z7;dat&kZRB(|EI{KwSDn3PUjSMA1;D5@A7Ndnx5o1J^pLCI-gE{^HZXuN%;7i6kQt z8_tVR-H5K&-*ghk_@WucpqVSW7;Q2-rLclH)GPn>4_(bCv4gdnb;)XMWF#i{H0bnP zW#y3TpU0^*SVtt8R{6kN%Hyt=HNy8ShPfu%ioXc?4V6)^a_idlHkUWV1$etBwXl*I zw~*bRu^LyeK52q4PUOHSIG0JH995o8`1-m-$&Xnw+~B)4^QZGcCyzd|+2qN`9G09& zNyB{xp#f9E#o$?}OLC}9eu;CMPvkO2Y(*^WDB`nlSqg{ib6r87vHH*$HR4Q%^x_fn z_1)IB06A>YT3C?S-1V&u;YW35+x4az?RcWBG+-d;sqwYnL}7ey08J(npoMG`yOnj> z@WVVhZkzUX09B_XjbPUcgspO`Ln5ZF9=IiK%_|(meT`jSrPUIzNnGBSK!j*q=&K8M z>MlNP$!fM`C~%CYhZ}z*=VCU9rYi;)WZaS0sEj8;}g~S8G7n=UeGo0vh!C zL4%ack-`VVv^79r0;54W-F>2OtE=?3IK5QD6Y*d#n2^4pzxK!BVBV4}TqB^*QPPT< zf!H@khb?n$}X(T`}sUa8Msp>ReMDHlfhHV~mw!>{-&7{SzC0sKJ-yD}NF?4X)9 z!L_Dn>HBZoi)%@2J{g)LB!PPpnXlxHlf#)pqN3zT#8iu^#=hUzBi8__wy&oMqp1>1 zljeWqwr|&D+KcRS=b!e2jzlwx@y2!21n_i)5YxNufYS2k%Vd~KO5p};l6ZsfLomX2 zHY(0hz(dOeAu&2&so25#;|lX&nKVN>`SzUxxW0YzGw&oalA+2OjSsfWDw5RVe9J2K z1a405BGr^|M;IZ(<@JYX(cy7g`Y)fJTJcuZ&K)Hfz7mY@Xi1hr}Y==rD+_U04P4_h}EM9eo~_&Y(sr*?j{9J0^$F(AHVJerb`-DA5c@S7)N zbLO+BxuI5%++S=uLzpU4a^&OSZbvtfp0{6eVS!XdZuG(sD>n>jw${nlAG2Ldu9t?h{LnUlg;fS5{Ux_UE}U zomH=Qa2b@3w0ioO(VQYzoyTMh2Gs9ZWQLpfNr%j^(Q%A|3wRqdpc&!sDyQ0*jny4C z?Pq6LA}1qxZp!SC%Mn);UBfYtzEZ{x@IM5;sGZrT9>#OV)p1`Qs+Gx$tQnU(h)M}l z*L=ndSJdr!7O^qyZv?7g3S188qwJ1z%nmt{xr8lf5WLU!<>`6W*pv^py#1g{O)+^c*` z$SZ%laLCWV@hUvVC&c2pqX_gC^L=#T;xDNJ-6Xw*6VP4)R?$DKTe5dg#a<+iXz-^= zJ4-3Rt1Q+Ll#C)ua#tKLmwSSqpx!1TY8F@%mm-ZY@61 zq9cPslmT%cNVuhRj&BG+MoAdP!?n_7YA$I8IUBcNBPXxN03Cn&lnBqTR;u;OvcCu3 zm_TJVw%dd_C$<+X-c6-$D&FiBHs#^G7;$* z`*!Sh=ZDA0bY63M17m^13QqHu@fuTyM%O3%gd=!KbryC9>ma%%t&!de%%Vf%yHu!N zv?)T8#J5s-%ob{VEQZ^iB#Xu|v3zmN>}vQ-Vl#T>Hb2s>J3IW4NYn1$pilPucRd(H z1z5ntnt3HBtb~AS1ys==tP<+HTwR#_RzDjf1DnjCyR*?g^V54jN-ql|BqU~!jtc^Z zi!5bFXSY8nqJ6R{r^a&j)9`t_k_>C;?17o<@LTOjHtZ`sq56XzNis$8sy#C z4MSC)NG}K5{QCTj(2A}3HC+M1aoF(UbY8i5x-*vqK%i5P3U?rb)-?@;Rj=w6guc3j zU};jf8{br2R!2RW&~A7w@92|cZz z`!>yXKr?k%vQ&Ib?CV#_Vq1Ry)9}oHGZ7TOCxclpOY2(o5$I#uN%{zX(CvdJskmLt zH5-4=#zV%0-2mG?9FX>E>xqFmoaGBai0N?k4>eM3$tDjCq;Sn&XE4j-fdr!+etcpW ze%ciZ_?RF1f9{l zGi+M^>CZXt|IP9)ov}QWZ-6@j_1{QdP0E00m@uvCLFy3=`k;$YUKPCExLtewObKT7ppmBz7Z7%_SVyRcqGgwb aNVs2{ { "URL", uno::Any(aURL2) } }; + uno::Sequence aArgs2{ uno::Any(aNVs2) }; + + uno::Reference xCxt = comphelper::getProcessComponentContext(); + uno::Reference xSvcMgr = xCxt->getServiceManager(); + + // Without Zip64 support, it would crash here + uno::Reference xZip2( + xSvcMgr->createInstanceWithArgumentsAndContext( + "com.sun.star.packages.zip.ZipFileAccess", aArgs2, xCxt), + uno::UNO_QUERY); + + CPPUNIT_ASSERT(xZip2.is()); + + uno::Reference xNA; + xNA = xZip2; + CPPUNIT_ASSERT(xNA.is()); + + // Check if the styles.xml seems to be right + uno::Reference xStrm; + xNA->getByName("styles.xml") >>= xStrm; + CPPUNIT_ASSERT(xStrm.is()); + // Filesize check + sal_Int32 nSize = xStrm->available(); + CPPUNIT_ASSERT_EQUAL(sal_Int32(1112), nSize); + + uno::Sequence aBytes; + sal_Int32 nBytesRead = xStrm->readBytes(aBytes, nSize); + const sal_Int8* p = aBytes.getArray(); + CPPUNIT_ASSERT_EQUAL(sal_Int32(1112), nBytesRead); + + // Check the uncompressed styles.xml file content. + OString aFile(static_cast(static_cast(p)), nSize); + CPPUNIT_ASSERT(aFile.startsWith( + "\r\n\r\n \r\n\r\n")); + } + CPPUNIT_TEST_SUITE_REGISTRATION(PackageTest); } diff --git a/package/source/zipapi/ByteChucker.cxx b/package/source/zipapi/ByteChucker.cxx index 6c5994aa6a2f..fe1f6a868503 100644 --- a/package/source/zipapi/ByteChucker.cxx +++ b/package/source/zipapi/ByteChucker.cxx @@ -29,8 +29,10 @@ ByteChucker::ByteChucker(Reference const & xOstream) , xSeek (xOstream, UNO_QUERY ) , a2Sequence ( 2 ) , a4Sequence ( 4 ) +, a8Sequence ( 8 ) , p2Sequence ( a2Sequence.getArray() ) , p4Sequence ( a4Sequence.getArray() ) +, p8Sequence ( a8Sequence.getArray() ) { } diff --git a/package/source/zipapi/Deflater.cxx b/package/source/zipapi/Deflater.cxx index 5c0fd70121fe..2c66253fd0ed 100644 --- a/package/source/zipapi/Deflater.cxx +++ b/package/source/zipapi/Deflater.cxx @@ -62,6 +62,8 @@ Deflater::Deflater(sal_Int32 nSetLevel, bool bNowrap) , bFinished(false) , nOffset(0) , nLength(0) +, nTotalOut64(0) +, nTotalIn64(0) { init(nSetLevel, bNowrap); } @@ -73,12 +75,24 @@ sal_Int32 Deflater::doDeflateBytes (uno::Sequence < sal_Int8 > &rBuffer, sal_Int pStream->next_out = reinterpret_cast(rBuffer.getArray())+nNewOffset; pStream->avail_in = nLength; pStream->avail_out = nNewLength; + auto nLastTotalIn = pStream->total_in; + auto nLastTotalOut = pStream->total_out; #if !defined Z_PREFIX nResult = deflate(pStream.get(), bFinish ? Z_FINISH : Z_NO_FLUSH); #else nResult = z_deflate(pStream.get(), bFinish ? Z_FINISH : Z_NO_FLUSH); #endif + // total_in / total_out may stored only in 32bit, and can owerflow during deflate + // 1 deflate call, uncompress only a few data, so only 1 overflow can happen at once. + if (pStream->total_in < nLastTotalIn) + { + nTotalIn64 += 0x100000000; + } + if (pStream->total_out < nLastTotalOut) + { + nTotalOut64 += 0x100000000; + } switch (nResult) { case Z_STREAM_END: @@ -115,11 +129,11 @@ sal_Int32 Deflater::doDeflateSegment( uno::Sequence< sal_Int8 >& rBuffer, sal_In } sal_Int64 Deflater::getTotalIn() const { - return pStream->total_in; // FIXME64: zlib doesn't look 64bit clean here + return pStream->total_in + nTotalIn64; } sal_Int64 Deflater::getTotalOut() const { - return pStream->total_out; // FIXME64: zlib doesn't look 64bit clean here + return pStream->total_out + nTotalOut64; } void Deflater::reset( ) { diff --git a/package/source/zipapi/MemoryByteGrabber.hxx b/package/source/zipapi/MemoryByteGrabber.hxx index 8dcf7f067064..a4d9f0b1ba49 100644 --- a/package/source/zipapi/MemoryByteGrabber.hxx +++ b/package/source/zipapi/MemoryByteGrabber.hxx @@ -81,6 +81,38 @@ public: nInt32 |= ( mpBuffer [mnCurrent++] & 0xFF ) << 24; return nInt32; } + + sal_Int64 ReadInt64() + { + if (mnCurrent + 8 > mnEnd) + return 0; + + sal_Int64 nInt64 = mpBuffer[mnCurrent++] & 0xFF; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 8; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 16; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 24; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 32; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 40; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 48; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 56; + return nInt64; + } + + sal_uInt64 ReadUInt64() + { + if (mnCurrent + 8 > mnEnd) + return 0; + + sal_uInt64 nInt64 = mpBuffer[mnCurrent++] & 0xFF; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 8; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 16; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 24; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 32; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 40; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 48; + nInt64 |= static_cast(mpBuffer[mnCurrent++] & 0xFF) << 56; + return nInt64; + } }; #endif diff --git a/package/source/zipapi/ZipFile.cxx b/package/source/zipapi/ZipFile.cxx index f7cb9df4d309..41325f47e38f 100644 --- a/package/source/zipapi/ZipFile.cxx +++ b/package/source/zipapi/ZipFile.cxx @@ -799,8 +799,6 @@ void ZipFile::readLOC( ZipEntry &rEntry ) rEntry.nOffset = aGrabber.getPosition() + nPathLen + nExtraLen; - // FIXME64: need to read 64bit LOC - bool bBroken = false; try @@ -942,26 +940,13 @@ sal_Int32 ZipFile::readCEN() aEntry.nTime = aMemGrabber.ReadInt32(); aEntry.nCrc = aMemGrabber.ReadInt32(); - sal_uInt32 nCompressedSize = aMemGrabber.ReadUInt32(); - sal_uInt32 nSize = aMemGrabber.ReadUInt32(); + sal_uInt64 nCompressedSize = aMemGrabber.ReadUInt32(); + sal_uInt64 nSize = aMemGrabber.ReadUInt32(); aEntry.nPathLen = aMemGrabber.ReadInt16(); aEntry.nExtraLen = aMemGrabber.ReadInt16(); nCommentLen = aMemGrabber.ReadInt16(); aMemGrabber.skipBytes ( 8 ); - sal_uInt32 nOffset = aMemGrabber.ReadUInt32(); - - // FIXME64: need to read the 64bit header instead - if ( nSize == 0xffffffff || - nOffset == 0xffffffff || - nCompressedSize == 0xffffffff ) { - throw ZipException("PK64 zip file entry" ); - } - aEntry.nCompressedSize = nCompressedSize; - aEntry.nSize = nSize; - aEntry.nOffset = nOffset; - - aEntry.nOffset += nLocPos; - aEntry.nOffset *= -1; + sal_uInt64 nOffset = aMemGrabber.ReadUInt32(); if ( aEntry.nPathLen < 0 ) throw ZipException("unexpected name length" ); @@ -983,7 +968,20 @@ sal_Int32 ZipFile::readCEN() if ( !::comphelper::OStorageHelper::IsValidZipEntryFileName( aEntry.sPath, true ) ) throw ZipException("Zip entry has an invalid name." ); - aMemGrabber.skipBytes( aEntry.nPathLen + aEntry.nExtraLen + nCommentLen ); + aMemGrabber.skipBytes(aEntry.nPathLen); + + if (aEntry.nExtraLen>0) + { + readExtraFields(aMemGrabber, aEntry.nExtraLen, nSize, nCompressedSize, &nOffset); + } + aEntry.nCompressedSize = nCompressedSize; + aEntry.nSize = nSize; + aEntry.nOffset = nOffset; + + aEntry.nOffset += nLocPos; + aEntry.nOffset *= -1; + + aMemGrabber.skipBytes(nCommentLen); aEntries[aEntry.sPath] = aEntry; } @@ -998,6 +996,40 @@ sal_Int32 ZipFile::readCEN() return nCenPos; } +void ZipFile::readExtraFields(MemoryByteGrabber& aMemGrabber, sal_Int16 nExtraLen, + sal_uInt64& nSize, sal_uInt64& nCompressedSize, sal_uInt64* nOffset) +{ + while (nExtraLen > 0) // Extensible data fields + { + sal_Int16 nheaderID = aMemGrabber.ReadInt16(); + sal_Int16 dataSize = aMemGrabber.ReadInt16(); + if (nheaderID == 1) // Load Zip64 Extended Information Extra Field + { + // Datasize should be 28byte but some files have less (maybe non standard?) + nSize = aMemGrabber.ReadUInt64(); + sal_Int16 nReadSize = 8; + if (dataSize >= 16) + { + nCompressedSize = aMemGrabber.ReadUInt64(); + nReadSize = 16; + if (dataSize >= 24 && nOffset) + { + *nOffset = aMemGrabber.ReadUInt64(); + nReadSize = 24; + // 4 byte should be "Disk Start Number" but we not need it + } + } + if (dataSize > nReadSize) + aMemGrabber.skipBytes(dataSize - nReadSize); + } + else + { + aMemGrabber.skipBytes(dataSize); + } + nExtraLen -= dataSize + 4; + } +} + void ZipFile::recover() { ::osl::MutexGuard aGuard( m_aMutexHolder->GetMutex() ); @@ -1028,6 +1060,7 @@ void ZipFile::recover() { if ( nPos < nBufSize - 30 && pBuffer[nPos] == 'P' && pBuffer[nPos+1] == 'K' && pBuffer[nPos+2] == 3 && pBuffer[nPos+3] == 4 ) { + //PK34: Local file header ZipEntry aEntry; Sequence aTmpBuffer(&(pBuffer[nPos+4]), 26); MemoryByteGrabber aMemGrabber(aTmpBuffer); @@ -1043,19 +1076,11 @@ void ZipFile::recover() { aEntry.nTime = aMemGrabber.ReadInt32(); aEntry.nCrc = aMemGrabber.ReadInt32(); - sal_uInt32 nCompressedSize = aMemGrabber.ReadUInt32(); - sal_uInt32 nSize = aMemGrabber.ReadUInt32(); + sal_uInt64 nCompressedSize = aMemGrabber.ReadUInt32(); + sal_uInt64 nSize = aMemGrabber.ReadUInt32(); aEntry.nPathLen = aMemGrabber.ReadInt16(); aEntry.nExtraLen = aMemGrabber.ReadInt16(); - // FIXME64: need to read the 64bit header instead - if ( nSize == 0xffffffff || - nCompressedSize == 0xffffffff ) { - throw ZipException("PK64 zip file entry" ); - } - aEntry.nCompressedSize = nCompressedSize; - aEntry.nSize = nSize; - sal_Int32 nDescrLength = ( aEntry.nMethod == DEFLATED && ( aEntry.nFlag & 8 ) ) ? 16 : 0; @@ -1080,6 +1105,35 @@ void ZipFile::recover() aEntry.nPathLen = static_cast< sal_Int16 >(aFileName.getLength()); } + // read 64bit header + if (aEntry.nExtraLen > 0) + { + Sequence* aExtraBuffer; + if (nPos + 30 + aEntry.nPathLen + aEntry.nExtraLen <= nBufSize) + { + Sequence aTmpBuffer2( + &(pBuffer[nPos + 30 + aEntry.nPathLen]), + aEntry.nExtraLen); + aExtraBuffer = &aTmpBuffer2; + } + else + { + Sequence aExtraFields; + aGrabber.seek(nGenPos + nPos + 30 + aEntry.nExtraLen); + aGrabber.readBytes(aExtraFields, aEntry.nExtraLen); + aExtraBuffer = &aExtraFields; + } + MemoryByteGrabber aMemGrabberExtra(*aExtraBuffer); + if (aEntry.nExtraLen > 0) + { + readExtraFields(aMemGrabberExtra, aEntry.nExtraLen, nSize, + nCompressedSize, nullptr); + } + } + + aEntry.nCompressedSize = nCompressedSize; + aEntry.nSize = nSize; + aEntry.nOffset = nGenPos + nPos + 30 + aEntry.nPathLen + aEntry.nExtraLen; if ( ( aEntry.nSize || aEntry.nCompressedSize ) && !checkSizeAndCRC( aEntry ) ) @@ -1098,16 +1152,35 @@ void ZipFile::recover() } else if (pBuffer[nPos] == 'P' && pBuffer[nPos+1] == 'K' && pBuffer[nPos+2] == 7 && pBuffer[nPos+3] == 8 ) { + //PK78: Data descriptor sal_Int64 nCompressedSize, nSize; - Sequence aTmpBuffer(&(pBuffer[nPos+4]), 12); + Sequence aTmpBuffer(&(pBuffer[nPos + 4]), 12 + 8 + 4); MemoryByteGrabber aMemGrabber(aTmpBuffer); sal_Int32 nCRC32 = aMemGrabber.ReadInt32(); - sal_uInt32 nCompressedSize32 = aMemGrabber.ReadUInt32(); - sal_uInt32 nSize32 = aMemGrabber.ReadUInt32(); - // FIXME64: work to be done here ... - nCompressedSize = nCompressedSize32; - nSize = nSize32; + // FIXME64: find a better way to recognize if Zip64 mode is used + // Now we check if the memory at +16 byte seems to be a signature + // if not, then probably Zip64 mode is used here, except + // if memory at +24 byte seems not to be a signature. + // Normally Data Descriptor should followed by the next Local File header + // that should start with PK34, except for the last file, then it may + // followed by Central directory that start with PK12, or + // followed by "archive decryption header" that don't have a signature. + if ((pBuffer[nPos + 16] == 'P' && pBuffer[nPos + 17] == 'K' + && pBuffer[nPos + 19] == pBuffer[nPos + 18] + 1 + && (pBuffer[nPos + 18] == 3 || pBuffer[nPos + 18] == 1)) + || !(pBuffer[nPos + 24] == 'P' && pBuffer[nPos + 25] == 'K' + && pBuffer[nPos + 27] == pBuffer[nPos + 26] + 1 + && (pBuffer[nPos + 26] == 3 || pBuffer[nPos + 26] == 1))) + { + nCompressedSize = aMemGrabber.ReadUInt32(); + nSize = aMemGrabber.ReadUInt32(); + } + else + { + nCompressedSize = aMemGrabber.ReadUInt64(); + nSize = aMemGrabber.ReadUInt64(); + } for( auto& rEntry : aEntries ) { diff --git a/package/source/zipapi/ZipOutputStream.cxx b/package/source/zipapi/ZipOutputStream.cxx index df21f1ffeb13..402a2930c0e2 100644 --- a/package/source/zipapi/ZipOutputStream.cxx +++ b/package/source/zipapi/ZipOutputStream.cxx @@ -84,7 +84,7 @@ void ZipOutputStream::rawCloseEntry( bool bEncrypt ) { assert(m_pCurrentEntry && "Forgot to call writeLOC()?"); if ( m_pCurrentEntry->nMethod == DEFLATED && ( m_pCurrentEntry->nFlag & 8 ) ) - writeEXT(*m_pCurrentEntry); + writeDataDescriptor(*m_pCurrentEntry); if (bEncrypt) m_pCurrentEntry->nMethod = STORED; @@ -235,42 +235,54 @@ void ZipOutputStream::writeCEN( const ZipEntry &rEntry ) m_aChucker.WriteUInt32( rEntry.nCrc ); m_aChucker.WriteUInt32( getTruncated( rEntry.nCompressedSize, &bWrite64Header ) ); m_aChucker.WriteUInt32( getTruncated( rEntry.nSize, &bWrite64Header ) ); - m_aChucker.WriteInt16( nNameLength ); - m_aChucker.WriteInt16( 0 ); + sal_uInt32 nOffset32bit = getTruncated( rEntry.nOffset, &bWrite64Header ); + m_aChucker.WriteInt16(nNameLength); + m_aChucker.WriteInt16( bWrite64Header? 32 : 0 ); //in ZIP64 case extra field is 32byte m_aChucker.WriteInt16( 0 ); m_aChucker.WriteInt16( 0 ); m_aChucker.WriteInt16( 0 ); m_aChucker.WriteInt32( 0 ); - m_aChucker.WriteUInt32( getTruncated( rEntry.nOffset, &bWrite64Header ) ); - - if( bWrite64Header ) - { - // FIXME64: need to append a ZIP64 header instead of throwing - // We're about to silently lose people's data - which they are - // unlikely to appreciate so fail instead: - throw IOException( "File contains streams that are too large." ); - } + m_aChucker.WriteUInt32( nOffset32bit ); Sequence < sal_Int8 > aSequence( reinterpret_cast(sUTF8Name.getStr()), sUTF8Name.getLength() ); m_aChucker.WriteBytes( aSequence ); + + if (bWrite64Header) + { + writeExtraFields( rEntry ); + } } -void ZipOutputStream::writeEXT( const ZipEntry &rEntry ) +void ZipOutputStream::writeDataDescriptor(const ZipEntry& rEntry) { bool bWrite64Header = false; m_aChucker.WriteInt32( EXTSIG ); m_aChucker.WriteUInt32( rEntry.nCrc ); - m_aChucker.WriteUInt32( getTruncated( rEntry.nCompressedSize, &bWrite64Header ) ); - m_aChucker.WriteUInt32( getTruncated( rEntry.nSize, &bWrite64Header ) ); - - if( bWrite64Header ) + // For ZIP64(tm) format archives, the compressed and uncompressed sizes are 8 bytes each. + // TODO: Not sure if this is the "when ZIP64(tm) format is used" + bWrite64Header = rEntry.nCompressedSize >= 0x100000000 || rEntry.nSize >= 0x100000000; + if (!bWrite64Header) { - // FIXME64: need to append a ZIP64 header instead of throwing - // We're about to silently lose people's data - which they are - // unlikely to appreciate so fail instead: - throw IOException( "File contains streams that are too large." ); + m_aChucker.WriteUInt32( static_cast(rEntry.nCompressedSize) ); + m_aChucker.WriteUInt32( static_cast(rEntry.nSize) ); } + else + { + m_aChucker.WriteUInt64( rEntry.nCompressedSize ); + m_aChucker.WriteUInt64( rEntry.nSize ); + } +} + +void ZipOutputStream::writeExtraFields(const ZipEntry& rEntry) +{ + //Could contain more fields, now we only save Zip64 extended information + m_aChucker.WriteInt16( 1 ); //id of Zip64 extended information extra field + m_aChucker.WriteInt16( 28 ); //data size of this field = 3*8+4 byte + m_aChucker.WriteUInt64( rEntry.nSize ); + m_aChucker.WriteUInt64( rEntry.nCompressedSize ); + m_aChucker.WriteUInt64( rEntry.nOffset ); + m_aChucker.WriteInt32( 0 ); //Number of the disk on which this file starts } void ZipOutputStream::writeLOC( ZipEntry *pEntry, bool bEncrypt ) @@ -312,20 +324,17 @@ void ZipOutputStream::writeLOC( ZipEntry *pEntry, bool bEncrypt ) m_aChucker.WriteUInt32( getTruncated( rEntry.nSize, &bWrite64Header ) ); } m_aChucker.WriteInt16( nNameLength ); - m_aChucker.WriteInt16( 0 ); - - if( bWrite64Header ) - { - // FIXME64: need to append a ZIP64 header instead of throwing - // We're about to silently lose people's data - which they are - // unlikely to appreciate so fail instead: - throw IOException( "File contains streams that are too large." ); - } + m_aChucker.WriteInt16( bWrite64Header ? 32 : 0 ); Sequence < sal_Int8 > aSequence( reinterpret_cast(sUTF8Name.getStr()), sUTF8Name.getLength() ); m_aChucker.WriteBytes( aSequence ); m_pCurrentEntry->nOffset = m_aChucker.GetPosition() - (LOCHDR + nNameLength); + + if (bWrite64Header) + { + writeExtraFields(rEntry); + } } sal_uInt32 ZipOutputStream::getCurrentDosTime() diff --git a/sc/qa/unit/data/xlsx/tdf82984_zip64XLSXImport.xlsx b/sc/qa/unit/data/xlsx/tdf82984_zip64XLSXImport.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1e03f68b7f5762040bdb82bb3cb72c0a934f26a6 GIT binary patch literal 4708 zcmb7|1z3~&+sA@*AV*S4$&n%{AT2dYNl{7|jg*vx zv>c=d9A5kb$0OJE{@?RHyRO~Ww&(e7zt8Xc+%YHwKq!odhj#%Ff}&RkYRRk}ogOrJ zba)5{3#hY$ql>78gR`Z`V>=W+9yQ)P{_*+82jOWfjOx76Bt{o#nuVC0$Yw`eu(M9T z@Kg~%$zMjcnDP>y6fhgXp4J%{IV+ejcu)61HRStYS1ZuN$4k&CrzkwNcdun3dY#9T z=Aq@1#u~)!HLMm0N_qZDM}`jNHS7VEX>Dl8YiwM_K!^XDE^%5& zzHlj%{Il2hec>decUW%G*=EhKL6^jL&^-~upf{u2EI%BA6cG6-;@8*4^a@&s)aFax zE!SqrR_-h5MuhFg(%{Pv;#l$yDxa z)!=2j>MbsLW*xax`4O^{KApRn6If3>sp05IoOJtlI(W^EZfA5r$8_$iI@r5f+Pj(> zcsN?Rz|L&ONO*!r`pjlAowo?UKW(-+cr#Qa3`{OGP#C>HHn2_E-_x+k5Mb@{m3xnL zf|)Q=`F&eHa!qDl*3N3ebTVir%N>Q!Y)5`gkrB{!alrtZ}S9vv|> zOin3NbC*oksMGL9*F!@a5yr-YR|aW>%3UV@g*uNl2)br(sVU;(O27&|5QMF-j)BNd zd(6$mTt1o0u?XsY3pcuO9gAzwma!OhrJ*+8JX&=X5r+DEYk|uNU)MG}#UFjQh=TBy z8Zu)oK5f!ZHLQgR_+K3+pj&IITJcwJZ*Tm>x|tr_BMR@TvcKhnqV(o{=v;B|dlR&y*-d2Ldp8^UjS!%AA$v zAxd`Wg?uxctbFz9ZVp}rk%uDBY5k|Z46#kWT54I?G4Xao$;i9bk6MU`1U-Mu@fn`a z!8v=WHQ*&%dcn0$8y!`DJ=!FMaDKfdyL!yQf&a3K)S%)T}JzJ_J_r zrV$iXK^uO@7 zLEB8(es)m!Rfb1TXrDPTC8yN;xT6XI(dF9<6Y+eFDK5*IpAa?U5moDVk{;3qrYDS< zc-M=3H{kZSww#UA9KOf*MTH2uQ1pneQHcDN?YeaiL(Ov`&+=VVCnaR{PO%{*I!-d& z)vd&SNaIuAguq;4^^vh8lBW`X6)-hC{b#nQJB}iyXluP`RV_wyge=8&OQ{IDAIDJL zq~e!9r^mV|9TWIw&O453ICqfahef6!w=sXZ+MRwSnH)cGQD_8KR%V5~+ zELMvHoU8D{6wHNr2kWGeC^N4lOntaIPEk^9XH=;Pqto0_CR{r_eOQnohoy~Q)w};Z z&^0BhL}O$+M5Bgf85`X)*c+e;&qkT+G)q|Q6#;z{FURH=?PE-tktvI8`1)%ZNWuav zF=vmzQ#;tI3=v-zCS|naX(n4K-VhR1R@9#3(Vp{C!rqbYO%?`|-$P>6LT{JaocCCD zj9PM-p}NKTjY1Z+0T6Q+KG|;@In~THUvW3qYTB%r%kxErWRszd?0e8_l6W(JZS6Sr zXNCgpxgn(JM0I(#gE-@BE#M*HCZR5>EZiU7%5tW0%A|ra%S8P2NF>5|-wD(DQfmWE zwM0Mukn$}7Y+Q~2%UI=g+wxWhjsNAQl_}SPTn>d#yn9_y9YF*7f9kr`1TE{nb!w6t zIxiwmbr(CrRBq6JdM;Yb3m%WSIn^#^rZ} z77xeg)RE)-=Kc86(`4m!>?r--u6KXMmwxED{G>l!a2TVp)$8l zf5xOHqzJA<$|t+&NHm+$5u)HIP^r-~z+wB2);VnNvygtKnreW(8D7K7q`Q6IP4O8C z!5__@>j1;8+T(>{wr>9d9y#s&DEd~kH}lC zqNb4Y;iTZ~{-7qVJ)VI)c+Szxz*hZ!Qh#)YOz(uAo}RO2*R8r^f>VI?ei zH2D_umD_c8^(IEer-R~~LOp+wU|#dA{AGh1iDt%+i4l1*_a0=}g28ddr>FFK0rUSH5m)}*I!~2 zi)2gtGB=E@L#dc&jc4t=Rd7;<Spkh4mn zkfLTX$GsZ-zPRfBu0bNrY0_&#{87;6r*YRT_7APQ7w6wT@QL)tsZEOz0}OSaewc!f z2M)eBNgNS;W8A?`Y4iiQ4THxO6G~30_azOk$VJ45CWy_ZT<){seKD|vfH~YB*uZ%9 zx_0;q=-b%>$_oXdoWTK+B^Ldzlv2H`n&i%WDrHpxTQyK`GV78ohUWLLVTddIUXxaT z8L-NbKQ3^}JL)u*6F(Bv>*mn?i>|0}!*njIdUWB_d*gG}@4JL5_6!fxFMGJySX#Qe z92dgl06y~xlJSHJDc~fgB#cKSiBSix32w1dQ&Bta0^evE>5Sypqwu(Q5lyJ!wgt8>RU}@K_n+86x}`U=Q6jjRrv%B z*?Bdv-ndb{>L;Zx1=5^Ys?cl8hW^37lgi)mu*0QP{#vc!#Mq9-?H|?9YB%@Xh?FM< zlG#S?v9-gns-A%=JssVOqKV`gM9mG^&HHXVqA$MleY-wkHbP_5Ha~*MJo@a->dUF5s@_byH$9z@8PUTwpGM6yxpf^WfS8&B1 zT9Ro{fdZ*oBe@|_=YDaXxb0m9BH&_q0h}-uR-sffVhAL_7TsvgBqcZSJ8Uv*6~0(X zfJJY-B;mQ^JK`x<*z<~h)b}W8Ja8S2#9W;aZ)EPYBK>LBOa-JAWGnB z<%zU;i#HNr5=QZ$^bi=f1;2Uxws{>>zazdqDACD$>fW#|yz}kFuLRf zBz~_lcttOaS$G0~=}CK!y}66!2v7|IXIgr;QF=wa&hl&%NqCRN?qO4dK2lp~KyYYS z0GJGUtOC57_R>6Ugxl#^mX}88KAJs#hYJ>>T(+Dc(23OWwxVjG8BH$lp&Ci=WbUVA zF_X%Y=^X1`S(jExiitnqk}a1(j&Nd<`jV}1*4t>d^iJ)LBQ*Q*B~p0DU(M?CZn$o; z%c>RL-N5K)9|lG?emi72hCrSn#bw;{}?;Zi=Cg#eu-HgC%}I> zrJWZ&KX3dJojy(&e=qvKOdsbJ&$s5k6oEu||GH5>uX?`c`K3yH;Y9Vnc0lLl&KKBU zatpx!$o;d>{QOVge7QX@c)nu&Ly+`8g8z^Dbzb{?y8oq3ay;G~k3K(t?7vF>PzVw6 R&#QsQkBy(Eq5Apie*hWU?lJ%X literal 0 HcmV?d00001 diff --git a/sc/qa/unit/subsequent_filters_test3.cxx b/sc/qa/unit/subsequent_filters_test3.cxx index dd9f92eea7ba..c2e76d2e071f 100644 --- a/sc/qa/unit/subsequent_filters_test3.cxx +++ b/sc/qa/unit/subsequent_filters_test3.cxx @@ -1706,6 +1706,12 @@ CPPUNIT_TEST_FIXTURE(ScFiltersTest3, testTdf104502_hiddenColsCountedInPageCount) CPPUNIT_ASSERT_EQUAL(SCROW(55), nEndRow); } +CPPUNIT_TEST_FIXTURE(ScFiltersTest3, testTdf82984_zip64XLSXImport) +{ + // Without the fix in place, it would have crashed at import time + createScDoc("xlsx/tdf82984_zip64XLSXImport.xlsx"); +} + CPPUNIT_TEST_FIXTURE(ScFiltersTest3, testTdf108188_pagestyle) { createScDoc("ods/tdf108188_pagestyle.ods"); diff --git a/tools/source/stream/strmwnt.cxx b/tools/source/stream/strmwnt.cxx index 57f7c8b50c07..d7d3a73ed2ce 100644 --- a/tools/source/stream/strmwnt.cxx +++ b/tools/source/stream/strmwnt.cxx @@ -152,24 +152,29 @@ sal_uInt64 SvFileStream::SeekPos(sal_uInt64 const nPos) { // check if a truncated STREAM_SEEK_TO_END was passed assert(nPos != SAL_MAX_UINT32); - DWORD nNewPos = 0; + LARGE_INTEGER nNewPos, nActPos; + nNewPos.QuadPart = 0; + nActPos.QuadPart = nPos; + bool result = false; if( IsOpen() ) { if( nPos != STREAM_SEEK_TO_END ) - // 64-Bit files are not supported - nNewPos=SetFilePointer(mxFileHandle,nPos,nullptr,FILE_BEGIN); - else - nNewPos=SetFilePointer(mxFileHandle,0L,nullptr,FILE_END); - - if( nNewPos == 0xFFFFFFFF ) { - SetError(::GetSvError( GetLastError() ) ); - nNewPos = 0; + result = SetFilePointerEx(mxFileHandle, nActPos, &nNewPos, FILE_BEGIN); + } + else + { + result = SetFilePointerEx(mxFileHandle, nNewPos, &nNewPos, FILE_END); + } + if (!result) + { + SetError(::GetSvError(GetLastError())); + return 0; } } else SetError( SVSTREAM_GENERALERROR ); - return static_cast(nNewPos); + return static_cast(nNewPos.QuadPart); } void SvFileStream::FlushData() diff --git a/unotools/source/streaming/streamwrap.cxx b/unotools/source/streaming/streamwrap.cxx index 7cb90aa3a63f..593f7a1f0b28 100644 --- a/unotools/source/streaming/streamwrap.cxx +++ b/unotools/source/streaming/streamwrap.cxx @@ -176,7 +176,7 @@ void SAL_CALL OSeekableInputStreamWrapper::seek( sal_Int64 _nLocation ) std::scoped_lock aGuard( m_aMutex ); checkConnected(); - m_pSvStream->Seek(static_cast(_nLocation)); + m_pSvStream->Seek(static_cast(_nLocation)); checkError(); }