From ca81051fe70ec2f20a1d4272a9478fe981723060 Mon Sep 17 00:00:00 2001 From: cielavenir Date: Fri, 1 Apr 2022 17:40:54 +0900 Subject: [PATCH] add FastLzma2 --- .gitmodules | 3 + CPP/7zip/7zip_gcc_additional.mak | 36 ++- CPP/7zip/Bundles/Format7zF/Arc_gcc.mak | 22 +- CPP/7zip/Compress/FastLzma2Encoder.cpp | 294 ++++++++++++++++++++++++ CPP/7zip/Compress/FastLzma2Encoder.h | 66 ++++++ CPP/7zip/Compress/FastLzma2Register.cpp | 18 ++ Codecs/fast-lzma2 | 1 + 7 files changed, 437 insertions(+), 3 deletions(-) create mode 100644 CPP/7zip/Compress/FastLzma2Encoder.cpp create mode 100644 CPP/7zip/Compress/FastLzma2Encoder.h create mode 100644 CPP/7zip/Compress/FastLzma2Register.cpp create mode 160000 Codecs/fast-lzma2 diff --git a/.gitmodules b/.gitmodules index b542d044c..621cfa69c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "Codecs/StormLib"] path = Codecs/StormLib url = https://github.com/ladislav-zezula/StormLib.git +[submodule "Codecs/fast-lzma2"] + path = Codecs/fast-lzma2 + url = https://github.com/conor42/fast-lzma2.git diff --git a/CPP/7zip/7zip_gcc_additional.mak b/CPP/7zip/7zip_gcc_additional.mak index bb19e5ffb..211b2b12e 100644 --- a/CPP/7zip/7zip_gcc_additional.mak +++ b/CPP/7zip/7zip_gcc_additional.mak @@ -160,6 +160,36 @@ $O/PKImplodeEncoder.o: ../../Compress/PKImplodeEncoder.cpp $O/PKImplodeRegister.o: ../../Compress/PKImplodeRegister.cpp $(CXX) $(CXXFLAGS) $< +# Build fast-lzma2 lib +$O/dict_buffer.o: ../../../../Codecs/fast-lzma2/dict_buffer.c + $(CC) $(CFLAGS) $< +$O/fl2_common.o: ../../../../Codecs/fast-lzma2/fl2_common.c + $(CC) $(CFLAGS) $< +$O/fl2_compress.o: ../../../../Codecs/fast-lzma2/fl2_compress.c + $(CC) $(CFLAGS) $< -Wno-sign-compare +$O/fl2_pool.o: ../../../../Codecs/fast-lzma2/fl2_pool.c + $(CC) $(CFLAGS) $< +$O/fl2_threading.o: ../../../../Codecs/fast-lzma2/fl2_threading.c + $(CC) $(CFLAGS) $< +$O/lzma2_enc.o: ../../../../Codecs/fast-lzma2/lzma2_enc.c + $(CC) $(CFLAGS) $< +$O/radix_bitpack.o: ../../../../Codecs/fast-lzma2/radix_bitpack.c + $(CC) $(CFLAGS) $< +$O/radix_mf.o: ../../../../Codecs/fast-lzma2/radix_mf.c + $(CC) $(CFLAGS) $< +$O/radix_struct.o: ../../../../Codecs/fast-lzma2/radix_struct.c + $(CC) $(CFLAGS) $< +$O/range_enc.o: ../../../../Codecs/fast-lzma2/range_enc.c + $(CC) $(CFLAGS) $< +$O/fl2util.o: ../../../../Codecs/fast-lzma2/util.c + $(CC) $(CFLAGS) $< + +# Compile fast-lzma2 method +$O/FastLzma2Encoder.o: ../../Compress/FastLzma2Encoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/FastLzma2Register.o: ../../Compress/FastLzma2Register.cpp + $(CXX) $(CXXFLAGS) $< + # Build hashes lib $O/md2.o: ../../../../Codecs/hashes/md2.c $(CC) $(CFLAGS) $< @@ -181,7 +211,11 @@ $O/Sha384Reg.o: ../../../Common/Sha384Reg.cpp $(CXX) $(CXXFLAGS) $< $O/Sha512Reg.o: ../../../Common/Sha512Reg.cpp $(CXX) $(CXXFLAGS) $< - +$O/XXH32Reg.o: ../../../Common/XXH32Reg.cpp + $(CXX) $(CXXFLAGS) $< +$O/XXH64Reg.o: ../../../Common/XXH64Reg.cpp + $(CXX) $(CXXFLAGS) $< + clean2: $(RM) zstd_build # $(RM) lz4_build diff --git a/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak index 14cc39725..e79c9fa2c 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak @@ -344,6 +344,8 @@ HASHES_OBJS = \ $O/Md5Reg.o \ $O/Sha384Reg.o \ $O/Sha512Reg.o \ + $O/XXH32Reg.o \ + $O/XXH64Reg.o \ $O/md2.o \ $O/md4.o \ $O/md5.o \ @@ -394,13 +396,28 @@ C_OBJS = \ $O/Sha1.o \ $O/Sha1Opt.o \ +FASTLZMA2_OBJS = \ + $O/FastLzma2Register.o \ + $O/FastLzma2Encoder.o \ + $O/dict_buffer.o \ + $O/fl2_common.o \ + $O/fl2_compress.o \ + $O/fl2_pool.o \ + $O/fl2_threading.o \ + $O/lzma2_enc.o \ + $O/radix_bitpack.o \ + $O/radix_mf.o \ + $O/radix_struct.o \ + $O/range_enc.o \ + $O/fl2util.o \ + ZSTD_STATIC_LIB = $O/libzstd.a -LZ4_STATIC_LIB = $O/liblz4.a +# LZ4_STATIC_LIB = $O/liblz4.a BROTLI_STATIC_LIB = $O/libbrotlienc-static.a \ $O/libbrotlidec-static.a \ $O/libbrotlicommon-static.a LIZARD_STATIC_LIB = $O/liblizard.a -LZ5_STATIC_LIB = $O/liblz5.a +# LZ5_STATIC_LIB = $O/liblz5.a LZHAM_STATIC_LIB = $O/lzham_lib.o $O/liblzhamcomp.a $O/liblzhamdecomp.a ARC_OBJS = \ @@ -427,6 +444,7 @@ ARC_OBJS = \ $(7ZIP_COMMON_OBJS) \ $(ADDITIONAL_CODECS_OBJS) \ $(HASHES_OBJS) \ + $(FASTLZMA2_OBJS) \ $(ZSTD_STATIC_LIB) \ $(BROTLI_STATIC_LIB) \ $(LIZARD_STATIC_LIB) \ diff --git a/CPP/7zip/Compress/FastLzma2Encoder.cpp b/CPP/7zip/Compress/FastLzma2Encoder.cpp new file mode 100644 index 000000000..1d9e68211 --- /dev/null +++ b/CPP/7zip/Compress/FastLzma2Encoder.cpp @@ -0,0 +1,294 @@ +// Lzma2Encoder.cpp + +#include "StdAfx.h" + +#include "../../../C/Alloc.h" + +#include "../Common/CWrappers.h" +#include "../Common/StreamUtils.h" + +#include "Lzma2Encoder.h" +#include "FastLzma2Encoder.h" +#include "../../../Codecs/fast-lzma2/fl2_errors.h" + +#ifndef max +#define max(a, b) (((a) > (b)) ? (a) : (b)) +#endif + +#ifndef min +#define min(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +namespace NCompress { + +namespace NLzma { + +HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep); + +} + +namespace NLzma2 { + +HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props); + +static HRESULT TranslateError(size_t res) +{ + if (FL2_getErrorCode(res) == FL2_error_memory_allocation) + return E_OUTOFMEMORY; + return S_FALSE; +} + +#define CHECK_S(f_) do { \ + size_t r_ = f_; \ + if (FL2_isError(r_)) \ + return TranslateError(r_); \ +} while (false) + +#define CHECK_H(f_) do { \ + HRESULT r_ = f_; \ + if (r_ != S_OK) \ + return r_; \ +} while (false) + +#define CHECK_P(f) if (FL2_isError(f)) return E_INVALIDARG; /* check and convert error code */ + +#define MIN_BLOCK_SIZE (1U << 20) +#define MAX_BLOCK_SIZE (1U << 28) + +CFastEncoder::FastLzma2::FastLzma2() + : fcs(NULL), + dict_pos(0) +{ +} + +CFastEncoder::FastLzma2::~FastLzma2() +{ + FL2_freeCCtx(fcs); +} + +HRESULT CFastEncoder::FastLzma2::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps) +{ + CLzma2EncProps lzma2Props; + Lzma2EncProps_Init(&lzma2Props); + + for (UInt32 i = 0; i < numProps; i++) + { + RINOK(SetLzma2Prop(propIDs[i], coderProps[i], lzma2Props)); + } + if (fcs == NULL) { + fcs = FL2_createCStreamMt(lzma2Props.numTotalThreads, 1); + if (fcs == NULL) + return E_OUTOFMEMORY; + } + if (lzma2Props.lzmaProps.algo > 2) { + if (lzma2Props.lzmaProps.algo > 3) + return E_INVALIDARG; + lzma2Props.lzmaProps.algo = 2; + FL2_CCtx_setParameter(fcs, FL2_p_highCompression, 1); + FL2_CCtx_setParameter(fcs, FL2_p_compressionLevel, lzma2Props.lzmaProps.level); + } + else { + FL2_CCtx_setParameter(fcs, FL2_p_compressionLevel, lzma2Props.lzmaProps.level); + } + size_t dictSize = lzma2Props.lzmaProps.dictSize; + if (!dictSize) { + dictSize = (UInt32)FL2_CCtx_getParameter(fcs, FL2_p_dictionarySize); + } + UInt64 reduceSize = lzma2Props.lzmaProps.reduceSize; + reduceSize += (reduceSize < (UInt64)-1); /* prevent extra buffer shift after read */ + dictSize = (UInt32)min(dictSize, reduceSize); + dictSize = max(dictSize, FL2_DICTSIZE_MIN); + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_dictionarySize, dictSize)); + if (lzma2Props.lzmaProps.algo >= 0) { + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_strategy, (unsigned)lzma2Props.lzmaProps.algo)); + } + if (lzma2Props.lzmaProps.fb > 0) + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_fastLength, lzma2Props.lzmaProps.fb)); + if (lzma2Props.lzmaProps.mc > 0) + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_hybridCycles, lzma2Props.lzmaProps.mc)); + if (lzma2Props.lzmaProps.lc >= 0) + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_literalCtxBits, lzma2Props.lzmaProps.lc)); + if (lzma2Props.lzmaProps.lp >= 0) + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_literalPosBits, lzma2Props.lzmaProps.lp)); + if (lzma2Props.lzmaProps.pb >= 0) + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_posBits, lzma2Props.lzmaProps.pb)); + if (lzma2Props.blockSize == 0) + lzma2Props.blockSize = min(max(MIN_BLOCK_SIZE, dictSize * 4U), MAX_BLOCK_SIZE); + else if (lzma2Props.blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2Props.blockSize = 0; + unsigned r = 0; + if (lzma2Props.blockSize != 0) { + r = 1; + // Do not exceed the block size. TODO: the lib should support setting a value instead of a multiplier. + while (r < FL2_RESET_INTERVAL_MAX && (r + 1) * (UInt64)dictSize <= lzma2Props.blockSize) + ++r; + } + CHECK_P(FL2_CCtx_setParameter(fcs, FL2_p_resetInterval, r)); + FL2_CCtx_setParameter(fcs, FL2_p_omitProperties, 1); + FL2_setCStreamTimeout(fcs, 500); + return S_OK; +} + +size_t CFastEncoder::FastLzma2::GetDictSize() const +{ + return FL2_CCtx_getParameter(fcs, FL2_p_dictionarySize); +} + +HRESULT CFastEncoder::FastLzma2::Begin() +{ + CHECK_S(FL2_initCStream(fcs, 0)); + CHECK_S(FL2_getDictionaryBuffer(fcs, &dict)); + dict_pos = 0; + return S_OK; +} + +BYTE* CFastEncoder::FastLzma2::GetAvailableBuffer(unsigned long& size) +{ + size = static_cast(dict.size - dict_pos); + return reinterpret_cast(dict.dst) + dict_pos; +} + +HRESULT CFastEncoder::FastLzma2::WaitAndReport(size_t& res, ICompressProgressInfo *progress) +{ + while (FL2_isTimedOut(res)) { + if (!UpdateProgress(progress)) + return S_FALSE; + res = FL2_waitCStream(fcs); + } + CHECK_S(res); + return S_OK; +} + +HRESULT CFastEncoder::FastLzma2::AddByteCount(size_t count, ISequentialOutStream *outStream, ICompressProgressInfo *progress) +{ + dict_pos += count; + if (dict_pos == dict.size) { + size_t res = FL2_updateDictionary(fcs, dict_pos); + CHECK_H(WaitAndReport(res, progress)); + if (res != 0) + CHECK_H(WriteBuffers(outStream)); + res = FL2_getDictionaryBuffer(fcs, &dict); + while (FL2_isTimedOut(res)) { + if (!UpdateProgress(progress)) + return S_FALSE; + res = FL2_getDictionaryBuffer(fcs, &dict); + } + CHECK_S(res); + dict_pos = 0; + } + if (!UpdateProgress(progress)) + return S_FALSE; + return S_OK; +} + +bool CFastEncoder::FastLzma2::UpdateProgress(ICompressProgressInfo *progress) +{ + if (progress) { + UInt64 outProcessed; + UInt64 inProcessed = FL2_getCStreamProgress(fcs, &outProcessed); + HRESULT err = progress->SetRatioInfo(&inProcessed, &outProcessed); + if (err != S_OK) { + FL2_cancelCStream(fcs); + return false; + } + } + return true; +} + +HRESULT CFastEncoder::FastLzma2::WriteBuffers(ISequentialOutStream *outStream) +{ + size_t csize; + for (;;) { + FL2_cBuffer cbuf; + do { + csize = FL2_getNextCompressedBuffer(fcs, &cbuf); + } while (FL2_isTimedOut(csize)); + CHECK_S(csize); + if (csize == 0) + break; + HRESULT err = WriteStream(outStream, cbuf.src, cbuf.size); + if (err != S_OK) + return err; + } + return S_OK; +} + +HRESULT CFastEncoder::FastLzma2::End(ISequentialOutStream *outStream, ICompressProgressInfo *progress) +{ + if (dict_pos) { + size_t res = FL2_updateDictionary(fcs, dict_pos); + CHECK_H(WaitAndReport(res, progress)); + } + + size_t res = FL2_endStream(fcs, nullptr); + CHECK_H(WaitAndReport(res, progress)); + while (res) { + CHECK_H(WriteBuffers(outStream)); + res = FL2_endStream(fcs, nullptr); + CHECK_H(WaitAndReport(res, progress)); + } + return S_OK; +} + +void CFastEncoder::FastLzma2::Cancel() +{ + FL2_cancelCStream(fcs); +} + +CFastEncoder::CFastEncoder() +{ +} + +CFastEncoder::~CFastEncoder() +{ +} + + +STDMETHODIMP CFastEncoder::SetCoderProperties(const PROPID *propIDs, + const PROPVARIANT *coderProps, UInt32 numProps) +{ + return _encoder.SetCoderProperties(propIDs, coderProps, numProps); +} + + +#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) + +STDMETHODIMP CFastEncoder::WriteCoderProperties(ISequentialOutStream *outStream) +{ + Byte prop; + unsigned i; + size_t dictSize = _encoder.GetDictSize(); + for (i = 0; i < 40; i++) + if (dictSize <= LZMA2_DIC_SIZE_FROM_PROP(i)) + break; + prop = (Byte)i; + return WriteStream(outStream, &prop, 1); +} + + +STDMETHODIMP CFastEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) +{ + CHECK_H(_encoder.Begin()); + size_t inSize; + unsigned long dSize; + do + { + BYTE* dict = _encoder.GetAvailableBuffer(dSize); + + inSize = dSize; + HRESULT err = ReadStream(inStream, dict, &inSize); + if (err != S_OK) { + _encoder.Cancel(); + return err; + } + CHECK_H(_encoder.AddByteCount(inSize, outStream, progress)); + + } while (inSize == dSize); + + CHECK_H(_encoder.End(outStream, progress)); + + return S_OK; +} + +}} diff --git a/CPP/7zip/Compress/FastLzma2Encoder.h b/CPP/7zip/Compress/FastLzma2Encoder.h new file mode 100644 index 000000000..541076c47 --- /dev/null +++ b/CPP/7zip/Compress/FastLzma2Encoder.h @@ -0,0 +1,66 @@ +// FastLzma2Encoder.h + +#ifndef __FASTLZMA2_ENCODER_H +#define __FASTLZMA2_ENCODER_H + +#include "../../../Codecs/fast-lzma2/fast-lzma2.h" + +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +namespace NCompress { +namespace NLzma2 { + +class CFastEncoder : + public ICompressCoder, + public ICompressSetCoderProperties, + public ICompressWriteCoderProperties, + public CMyUnknownImp +{ + class FastLzma2 + { + public: + FastLzma2(); + ~FastLzma2(); + HRESULT SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + size_t GetDictSize() const; + HRESULT Begin(); + BYTE* GetAvailableBuffer(unsigned long& size); + HRESULT AddByteCount(size_t count, ISequentialOutStream *outStream, ICompressProgressInfo *progress); + HRESULT End(ISequentialOutStream *outStream, ICompressProgressInfo *progress); + void Cancel(); + + private: + bool UpdateProgress(ICompressProgressInfo *progress); + HRESULT WaitAndReport(size_t& res, ICompressProgressInfo *progress); + HRESULT WriteBuffers(ISequentialOutStream *outStream); + + FL2_CStream* fcs; + FL2_dictBuffer dict; + size_t dict_pos; + + FastLzma2(const FastLzma2&) = delete; + FastLzma2& operator=(const FastLzma2&) = delete; + }; + + FastLzma2 _encoder; + +public: + MY_UNKNOWN_IMP3( + ICompressCoder, + ICompressSetCoderProperties, + ICompressWriteCoderProperties) + + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream); + + CFastEncoder(); + virtual ~CFastEncoder(); +}; + +}} + +#endif diff --git a/CPP/7zip/Compress/FastLzma2Register.cpp b/CPP/7zip/Compress/FastLzma2Register.cpp new file mode 100644 index 000000000..e05f1a8f6 --- /dev/null +++ b/CPP/7zip/Compress/FastLzma2Register.cpp @@ -0,0 +1,18 @@ +// FastLzma2Register.cpp + +#include "StdAfx.h" + +#include "../Common/RegisterCodec.h" + +#include "Lzma2Decoder.h" + +#ifndef EXTRACT_ONLY +#include "FastLzma2Encoder.h" +#endif + +REGISTER_CODEC_E( + FLZMA2, + NCompress::NLzma2::CDecoder(), + NCompress::NLzma2::CFastEncoder(), + 0x21, + "FLZMA2") diff --git a/Codecs/fast-lzma2 b/Codecs/fast-lzma2 new file mode 160000 index 000000000..9d13017df --- /dev/null +++ b/Codecs/fast-lzma2 @@ -0,0 +1 @@ +Subproject commit 9d13017df02e3a0c6cdd1dfd87e141a6a5e2ba30