@@ -110,45 +110,67 @@ 1.8.2 Manual
Same compression function, just using an externally allocated memory space to store compression state.
Use LZ4_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
- Then, provide it as 'void* state' to compression function.
+ Then, provide this buffer as 'void* state' to compression function.
int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
- Reverse the logic : compresses as much data as possible from 'src' buffer
- into already allocated buffer 'dst' of size 'targetDestSize'.
- This function either compresses the entire 'src' content into 'dst' if it's large enough,
- or fill 'dst' buffer completely with as much data as possible from 'src'.
- *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
- New value is necessarily <= old value.
- return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
- or 0 if compression fails
+
Reverse the logic : compresses as much data as possible from 'src' buffer
+ into already allocated buffer 'dst', of size >= 'targetDestSize'.
+ This function either compresses the entire 'src' content into 'dst' if it's large enough,
+ or fill 'dst' buffer completely with as much data as possible from 'src'.
+ note: acceleration parameter is fixed to "default".
+
+ *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
+ New value is necessarily <= input value.
+ @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
+ or 0 if compression fails.
int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-This function is a bit faster than LZ4_decompress_safe(),
-but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
- originalSize : is the uncompressed size to regenerate
- Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
- return : number of bytes read from source buffer (== compressed size).
- If the source stream is detected malformed, the function stops decoding and return a negative result.
- note : This function is only usable if the originalSize of uncompressed data is known in advance.
- The caller should also check that all the compressed input has been consumed properly,
- i.e. that the return value matches the size of the buffer with compressed input.
- The function never writes past the output buffer. However, since it doesn't know its 'src' size,
- it may read past the intended input. Also, because match offsets are not validated during decoding,
- reads from 'src' may underflow. Use this function in trusted environment **only**.
+
This function used to be a bit faster than LZ4_decompress_safe(),
+ though situation has changed in recent versions,
+ and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`.
+ Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data.
+ As a consequence, this function is no longer recommended, and may be deprecated in future versions.
+ It's only remaining specificity is that it can decompress data without knowing its compressed size.
+
+ originalSize : is the uncompressed size to regenerate.
+ `dst` must be already allocated, its size must be >= 'originalSize' bytes.
+ @return : number of bytes read from source buffer (== compressed size).
+ If the source stream is detected malformed, the function stops decoding and returns a negative result.
+ note : This function requires uncompressed originalSize to be known in advance.
+ The function never writes past the output buffer.
+ However, since it doesn't know its 'src' size, it may read past the intended input.
+ Also, because match offsets are not validated during decoding,
+ reads from 'src' may underflow.
+ Use this function in trusted environment **only**.
+
int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
- This function decompress a compressed block of size 'srcSize' at position 'src'
- into destination buffer 'dst' of size 'dstCapacity'.
- The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
- However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity).
- @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
- Note : this number can also be < targetOutputSize, if compressed block contains less data.
- Therefore, always control how many bytes were decoded.
- If source stream is detected malformed, function returns a negative result.
- This function is protected against malicious data packets.
+
Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
+ into destination buffer 'dst' of size 'dstCapacity'.
+ Up to 'targetOutputSize' bytes will be decoded.
+ The function stops decoding on reaching this objective,
+ which can boost performance when only the beginning of a block is required.
+
+ @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity)
+ If source stream is detected malformed, function returns a negative result.
+
+ Note : @return can be < targetOutputSize, if compressed block contains less data.
+
+ Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity,
+ and expects targetOutputSize <= dstCapacity.
+ It effectively stops decoding on reaching targetOutputSize,
+ so dstCapacity is kind of redundant.
+ This is because in a previous version of this function,
+ decoding operation would not "break" a sequence in the middle.
+ As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize,
+ it could write more bytes, though only up to dstCapacity.
+ Some "margin" used to be required for this operation to work properly.
+ This is no longer necessary.
+ The function nonetheless keeps its signature, in an effort to not break API.
+
Streaming Compression Functions
@@ -179,16 +201,23 @@ 1.8.2 Manual
'dst' buffer must be already allocated.
If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory!
-
- Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
- Make sure that buffers are separated by at least one byte.
- This way, each block only depends on previous block.
- Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
-
@return : size of compressed block
or 0 if there is an error (typically, cannot fit into 'dst').
- After an error, the stream status is invalid, it can only be reset or freed.
+
+ Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.
+ Each block has precise boundaries.
+ It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.
+ Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata.
+
+ Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory!
+
+ Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.
+ Make sure that buffers are separated, by at least one byte.
+ This construction ensures that each block only depends on previous block.
+
+ Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+
+ Note 5 : After an error, the stream status is invalid, it can only be reset or freed.
@@ -214,7 +243,7 @@ 1.8.2 Manual
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
Use this function to start decompression of a new stream of blocks.
- A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+ A dictionary can optionally be set. Use NULL or size 0 for a reset order.
Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
@return : 1 if OK, 0 if error
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 53ea7eb19..fb8e0ceb3 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
-1.8.2 Manual
+1.8.3 Manual
-1.8.2 Manual
+1.8.3 Manual
Contents
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 9bfea483f..a0c5d3d80 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -31,12 +31,13 @@ static const LZ4F_preferences_t kPrefs = {
static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
{
size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
- size_t const expectedSize = eltSize * nbElt; /* note : should check for overflow */
+ size_t const expectedSize = eltSize * nbElt;
+ assert(expectedSize / nbElt == eltSize); /* check overflow */
if (writtenSize < expectedSize) {
if (ferror(f)) /* note : ferror() must follow fwrite */
- printf("Write failed\n");
+ fprintf(stderr, "Write failed \n");
else
- printf("Short write\n");
+ fprintf(stderr, "Short write \n");
exit(1);
}
}
@@ -54,9 +55,9 @@ typedef struct {
static compressResult_t
compress_file_internal(FILE* f_in, FILE* f_out,
- LZ4F_compressionContext_t ctx,
- void* inBuff, size_t inChunkSize,
- void* outBuff, size_t outCapacity)
+ LZ4F_compressionContext_t ctx,
+ void* inBuff, size_t inChunkSize,
+ void* outBuff, size_t outCapacity)
{
compressResult_t result = { 1, 0, 0 }; /* result for an error */
unsigned long long count_in = 0, count_out;
@@ -167,9 +168,9 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
/* @return : 1==error, 0==success */
static int
decompress_file_internal(FILE* f_in, FILE* f_out,
- LZ4F_dctx* dctx,
- void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
- void* dst, size_t dstCapacity)
+ LZ4F_dctx* dctx,
+ void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
+ void* dst, size_t dstCapacity)
{
int firstChunk = 1;
size_t ret = 1;
@@ -194,7 +195,7 @@ decompress_file_internal(FILE* f_in, FILE* f_out,
* Continue while there is more input to read (srcPtr != srcEnd)
* and the frame isn't over (ret != 0)
*/
- while (srcPtr != srcEnd && ret != 0) {
+ while (srcPtr < srcEnd && ret != 0) {
/* Any data within dst has been flushed at this stage */
size_t dstSize = dstCapacity;
size_t srcSize = srcEnd - srcPtr;
@@ -208,9 +209,20 @@ decompress_file_internal(FILE* f_in, FILE* f_out,
/* Update input */
srcPtr += srcSize;
}
+
+ assert(srcPtr <= srcEnd);
+
+ /* Ensure all input data has been consumed.
+ * It is valid to have multiple frames in the same file,
+ * but this example only supports one frame.
+ */
+ if (srcPtr < srcEnd) {
+ printf("Decompress: Trailing data left in file after frame\n");
+ return 1;
+ }
}
- /* Check that there isn't trailing input data after the frame.
+ /* Check that there isn't trailing data in the file after the frame.
* It is valid to have multiple frames in the same file,
* but this example only supports one frame.
*/
@@ -260,7 +272,7 @@ decompress_file_allocDst(FILE* f_in, FILE* f_out,
int const decompressionResult = decompress_file_internal(
f_in, f_out,
dctx,
- src, srcCapacity, readSize, consumedSize,
+ src, srcCapacity, readSize-consumedSize, consumedSize,
dst, dstCapacity);
free(dst);
@@ -278,7 +290,7 @@ static int decompress_file(FILE* f_in, FILE* f_out)
if (!src) { perror("decompress_file(src)"); return 1; }
LZ4F_dctx* dctx;
- { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
+ { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
if (LZ4F_isError(dctxStatus)) {
printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
} }
diff --git a/lib/Makefile b/lib/Makefile
index abb6c075a..88d9b4f28 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -45,6 +45,7 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
BUILD_SHARED:=yes
BUILD_STATIC:=yes
+OS ?= $(shell uname)
CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
CFLAGS ?= -O3
DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
@@ -58,7 +59,7 @@ SRCFILES := $(sort $(wildcard *.c))
# OS X linker doesn't support -soname, and use different extension
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
-ifeq ($(shell uname), Darwin)
+ifeq ($(OS), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
@@ -123,7 +124,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
@@ -142,14 +143,14 @@ libdir ?= $(LIBDIR)
INCLUDEDIR ?= $(prefix)/include
includedir ?= $(INCLUDEDIR)
-ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
+ifneq (,$(filter $(OS),OpenBSD FreeBSD NetBSD DragonFly))
PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig
else
PKGCONFIGDIR ?= $(libdir)/pkgconfig
endif
pkgconfigdir ?= $(PKGCONFIGDIR)
-ifneq (,$(filter $(shell uname),SunOS))
+ifneq (,$(filter $(OS),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
diff --git a/lib/lz4.c b/lib/lz4.c
index e51a3e0a4..4046102e6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2017, Yann Collet.
+ Copyright (C) 2011-present, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -297,8 +297,9 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
#define MINMATCH 4
#define WILDCOPYLENGTH 8
-#define LASTLITERALS 5
-#define MFLIMIT (WILDCOPYLENGTH+MINMATCH)
+#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
+#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
+#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
static const int LZ4_minLength = (MFLIMIT+1);
#define KB *(1 <<10)
@@ -483,9 +484,6 @@ typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive;
typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
-typedef enum { full = 0, partial = 1 } earlyEnd_directive;
-
/*-************************************
* Local Utils
@@ -496,6 +494,21 @@ int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
+/*-************************************
+* Internal Definitions used in Tests
+**************************************/
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
+
+int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
+
+#if defined (__cplusplus)
+}
+#endif
+
/*-******************************
* Compression functions
********************************/
@@ -669,9 +682,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
/* the dictCtx currentOffset is indexed on the start of the dictionary,
* while a dictionary in the current context precedes the currentOffset */
- const BYTE* dictBase = dictDirective == usingDictCtx ?
- dictionary + dictSize - dictCtx->currentOffset :
- dictionary + dictSize - startIndex;
+ const BYTE* dictBase = (dictDirective == usingDictCtx) ?
+ dictionary + dictSize - dictCtx->currentOffset :
+ dictionary + dictSize - startIndex;
BYTE* op = (BYTE*) dest;
BYTE* const olimit = op + maxOutputSize;
@@ -699,7 +712,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
cctx->dictSize += (U32)inputSize;
}
cctx->currentOffset += (U32)inputSize;
- cctx->tableType = tableType;
+ cctx->tableType = (U16)tableType;
if (inputSize oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */
+ assert(lowPrefix <= op);
+ assert(src != NULL);
if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
- if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
+ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1);
if ((endOnInput) && unlikely(srcSize==0)) return -1;
/* Main Loop : decode sequences */
@@ -1428,7 +1447,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
size_t offset;
unsigned const token = *ip++;
- size_t length = token >> ML_BITS; /* literal length */
+ size_t length = token >> ML_BITS; /* literal length */
assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
@@ -1453,6 +1472,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
length = token & ML_MASK; /* match length */
offset = LZ4_readLE16(ip); ip += 2;
match = op - offset;
+ assert(match <= op); /* check overflow */
/* Do not deal with overlapping matches. */
if ( (length != ML_MASK)
@@ -1486,11 +1506,12 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
/* copy literals */
cpy = op+length;
- if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
+ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
+ if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
+ || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
{
if (partialDecoding) {
- if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */
+ if (cpy > oend) { cpy = oend; length = oend-op; } /* Partial decoding : stop in the middle of literal segment */
if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
} else {
if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
@@ -1499,10 +1520,15 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
memcpy(op, ip, length);
ip += length;
op += length;
- break; /* Necessarily EOF, due to parsing restrictions */
+ if (!partialDecoding || (cpy == oend)) {
+ /* Necessarily EOF, due to parsing restrictions */
+ break;
+ }
+
+ } else {
+ LZ4_wildCopy(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
+ ip += length; op = cpy;
}
- LZ4_wildCopy(op, ip, cpy);
- ip += length; op = cpy;
/* get offset */
offset = LZ4_readLE16(ip); ip+=2;
@@ -1513,7 +1539,11 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
_copy_match:
if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
- LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */
+ if (!partialDecoding) {
+ assert(oend > op);
+ assert(oend - op >= 4);
+ LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */
+ } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */
if (length == ML_MASK) {
unsigned s;
@@ -1526,21 +1556,24 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
}
length += MINMATCH;
- /* check external dictionary */
+ /* match starting within external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
- if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */
+ if (unlikely(op+length > oend-LASTLITERALS)) {
+ if (partialDecoding) length = MIN(length, (size_t)(oend-op));
+ else goto _output_error; /* doesn't respect parsing restriction */
+ }
if (length <= (size_t)(lowPrefix-match)) {
- /* match can be copied as a single segment from external dictionary */
+ /* match fits entirely within external dictionary : just copy */
memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
- /* match encompass external dictionary and current block */
- size_t const copySize = (size_t)(lowPrefix-match);
+ /* match stretches into both external dictionary and current block */
+ size_t const copySize = (size_t)(lowPrefix - match);
size_t const restSize = length - copySize;
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
- if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */
+ if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
BYTE* const endOfMatch = op + restSize;
const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
@@ -1553,6 +1586,23 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
/* copy match within block */
cpy = op + length;
+
+ /* partialDecoding : may not respect endBlock parsing restrictions */
+ assert(op<=oend);
+ if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
+ size_t const mlen = MIN(length, (size_t)(oend-op));
+ const BYTE* const matchEnd = match + mlen;
+ BYTE* const copyEnd = op + mlen;
+ if (matchEnd > op) { /* overlap copy */
+ while (op < copyEnd) *op++ = *match++;
+ } else {
+ memcpy(op, match, mlen);
+ }
+ op = copyEnd;
+ if (op==oend) break;
+ continue;
+ }
+
if (unlikely(offset<8)) {
op[0] = match[0];
op[1] = match[1];
@@ -1561,23 +1611,26 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
match += inc32table[offset];
memcpy(op+4, match, 4);
match -= dec64table[offset];
- } else { memcpy(op, match, 8); match+=8; }
+ } else {
+ memcpy(op, match, 8);
+ match += 8;
+ }
op += 8;
- if (unlikely(cpy>oend-12)) {
- BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1);
+ if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
+ BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);
if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
if (op < oCopyLimit) {
LZ4_wildCopy(op, match, oCopyLimit);
match += oCopyLimit - op;
op = oCopyLimit;
}
- while (op16) LZ4_wildCopy(op+8, match+8, cpy);
+ if (length > 16) LZ4_wildCopy(op+8, match+8, cpy);
}
- op = cpy; /* correction */
+ op = cpy; /* wildcopy correction */
}
/* end of decoding */
@@ -1598,23 +1651,24 @@ LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
- endOnInputSize, full, 0, noDict,
+ endOnInputSize, decode_full_block, noDict,
(BYTE*)dest, NULL, 0);
}
LZ4_FORCE_O2_GCC_PPC64LE
-int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
+int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
- endOnInputSize, partial, targetOutputSize,
- noDict, (BYTE*)dest, NULL, 0);
+ dstCapacity = MIN(targetOutputSize, dstCapacity);
+ return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity,
+ endOnInputSize, partial_decode,
+ noDict, (BYTE*)dst, NULL, 0);
}
LZ4_FORCE_O2_GCC_PPC64LE
int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, full, 0, withPrefix64k,
+ endOnOutputSize, decode_full_block, withPrefix64k,
(BYTE*)dest - 64 KB, NULL, 0);
}
@@ -1624,7 +1678,7 @@ LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */
int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, full, 0, withPrefix64k,
+ endOnInputSize, decode_full_block, withPrefix64k,
(BYTE*)dest - 64 KB, NULL, 0);
}
@@ -1641,17 +1695,17 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i
size_t prefixSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, full, 0, noDict,
+ endOnInputSize, decode_full_block, noDict,
(BYTE*)dest-prefixSize, NULL, 0);
}
-LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */
-#define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict
-int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
- const void* dictStart, size_t dictSize)
+LZ4_FORCE_O2_GCC_PPC64LE
+int LZ4_decompress_safe_forceExtDict(const char* source, char* dest,
+ int compressedSize, int maxOutputSize,
+ const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, full, 0, usingExtDict,
+ endOnInputSize, decode_full_block, usingExtDict,
(BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
@@ -1660,7 +1714,7 @@ static int LZ4_decompress_fast_extDict(const char* source, char* dest, int origi
const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, full, 0, usingExtDict,
+ endOnOutputSize, decode_full_block, usingExtDict,
(BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
@@ -1673,7 +1727,7 @@ int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compresse
size_t prefixSize, const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, full, 0, usingExtDict,
+ endOnInputSize, decode_full_block, usingExtDict,
(BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
}
@@ -1682,7 +1736,7 @@ int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalS
size_t prefixSize, const void* dictStart, size_t dictSize)
{
return LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, full, 0, usingExtDict,
+ endOnOutputSize, decode_full_block, usingExtDict,
(BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
}
@@ -1773,8 +1827,8 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
/* The buffer wraps around, or they're switching to another buffer. */
lz4sd->extDictSize = lz4sd->prefixSize;
lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
- result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize,
- lz4sd->externalDict, lz4sd->extDictSize);
+ result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize,
+ lz4sd->externalDict, lz4sd->extDictSize);
if (result <= 0) return result;
lz4sd->prefixSize = result;
lz4sd->prefixEnd = (BYTE*)dest + result;
@@ -1834,7 +1888,7 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
}
- return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
+ return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
}
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index 7d1312219..059ef7c1b 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -1,7 +1,7 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
- * Copyright (C) 2011-2017, Yann Collet.
+ * Copyright (C) 2011-present, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -46,7 +46,7 @@ extern "C" {
/**
Introduction
- LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core,
+ LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core,
scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
@@ -62,8 +62,8 @@ extern "C" {
An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md),
take care of encoding standard metadata alongside LZ4-compressed blocks.
- If your application requires interoperability, it's recommended to use it.
- A library is provided to take care of it, see lz4frame.h.
+ Frame format is required for interoperability.
+ It is delivered through a companion API, declared in lz4frame.h.
*/
/*^***************************************************************
@@ -93,7 +93,7 @@ extern "C" {
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@@ -183,55 +183,72 @@ LZ4_compress_fast_extState() :
Same compression function, just using an externally allocated memory space to store compression state.
Use LZ4_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
- Then, provide it as 'void* state' to compression function.
+ Then, provide this buffer as 'void* state' to compression function.
*/
LZ4LIB_API int LZ4_sizeofState(void);
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-/*!
-LZ4_compress_destSize() :
- Reverse the logic : compresses as much data as possible from 'src' buffer
- into already allocated buffer 'dst' of size 'targetDestSize'.
- This function either compresses the entire 'src' content into 'dst' if it's large enough,
- or fill 'dst' buffer completely with as much data as possible from 'src'.
- *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
- New value is necessarily <= old value.
- return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
- or 0 if compression fails
+/*! LZ4_compress_destSize() :
+ * Reverse the logic : compresses as much data as possible from 'src' buffer
+ * into already allocated buffer 'dst', of size >= 'targetDestSize'.
+ * This function either compresses the entire 'src' content into 'dst' if it's large enough,
+ * or fill 'dst' buffer completely with as much data as possible from 'src'.
+ * note: acceleration parameter is fixed to "default".
+ *
+ * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
+ * New value is necessarily <= input value.
+ * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
+ * or 0 if compression fails.
*/
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
-/*!
-LZ4_decompress_fast() : **unsafe!**
-This function is a bit faster than LZ4_decompress_safe(),
-but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
- originalSize : is the uncompressed size to regenerate
- Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
- return : number of bytes read from source buffer (== compressed size).
- If the source stream is detected malformed, the function stops decoding and return a negative result.
- note : This function is only usable if the originalSize of uncompressed data is known in advance.
- The caller should also check that all the compressed input has been consumed properly,
- i.e. that the return value matches the size of the buffer with compressed input.
- The function never writes past the output buffer. However, since it doesn't know its 'src' size,
- it may read past the intended input. Also, because match offsets are not validated during decoding,
- reads from 'src' may underflow. Use this function in trusted environment **only**.
-*/
+/*! LZ4_decompress_fast() : **unsafe!**
+ * This function used to be a bit faster than LZ4_decompress_safe(),
+ * though situation has changed in recent versions,
+ * and now `LZ4_decompress_safe()` can be as fast and sometimes faster than `LZ4_decompress_fast()`.
+ * Moreover, LZ4_decompress_fast() is not protected vs malformed input, as it doesn't perform full validation of compressed data.
+ * As a consequence, this function is no longer recommended, and may be deprecated in future versions.
+ * It's only remaining specificity is that it can decompress data without knowing its compressed size.
+ *
+ * originalSize : is the uncompressed size to regenerate.
+ * `dst` must be already allocated, its size must be >= 'originalSize' bytes.
+ * @return : number of bytes read from source buffer (== compressed size).
+ * If the source stream is detected malformed, the function stops decoding and returns a negative result.
+ * note : This function requires uncompressed originalSize to be known in advance.
+ * The function never writes past the output buffer.
+ * However, since it doesn't know its 'src' size, it may read past the intended input.
+ * Also, because match offsets are not validated during decoding,
+ * reads from 'src' may underflow.
+ * Use this function in trusted environment **only**.
+ */
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-/*!
-LZ4_decompress_safe_partial() :
- This function decompress a compressed block of size 'srcSize' at position 'src'
- into destination buffer 'dst' of size 'dstCapacity'.
- The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
- However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity).
- @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
- Note : this number can also be < targetOutputSize, if compressed block contains less data.
- Therefore, always control how many bytes were decoded.
- If source stream is detected malformed, function returns a negative result.
- This function is protected against malicious data packets.
-*/
+/*! LZ4_decompress_safe_partial() :
+ * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
+ * into destination buffer 'dst' of size 'dstCapacity'.
+ * Up to 'targetOutputSize' bytes will be decoded.
+ * The function stops decoding on reaching this objective,
+ * which can boost performance when only the beginning of a block is required.
+ *
+ * @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity)
+ * If source stream is detected malformed, function returns a negative result.
+ *
+ * Note : @return can be < targetOutputSize, if compressed block contains less data.
+ *
+ * Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity,
+ * and expects targetOutputSize <= dstCapacity.
+ * It effectively stops decoding on reaching targetOutputSize,
+ * so dstCapacity is kind of redundant.
+ * This is because in a previous version of this function,
+ * decoding operation would not "break" a sequence in the middle.
+ * As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize,
+ * it could write more bytes, though only up to dstCapacity.
+ * Some "margin" used to be required for this operation to work properly.
+ * This is no longer necessary.
+ * The function nonetheless keeps its signature, in an effort to not break API.
+ */
LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
@@ -266,16 +283,23 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
* 'dst' buffer must be already allocated.
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
*
- * Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory!
- *
- * Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
- * Make sure that buffers are separated by at least one byte.
- * This way, each block only depends on previous block.
- * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
- *
* @return : size of compressed block
* or 0 if there is an error (typically, cannot fit into 'dst').
- * After an error, the stream status is invalid, it can only be reset or freed.
+ *
+ * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.
+ * Each block has precise boundaries.
+ * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.
+ * Each block must be decompressed separately, calling LZ4_decompress_*() with associated metadata.
+ *
+ * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory!
+ *
+ * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.
+ * Make sure that buffers are separated, by at least one byte.
+ * This construction ensures that each block only depends on previous block.
+ *
+ * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+ *
+ * Note 5 : After an error, the stream status is invalid, it can only be reset or freed.
*/
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
@@ -305,7 +329,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
/*! LZ4_setStreamDecode() :
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
* Use this function to start decompression of a new stream of blocks.
- * A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+ * A dictionary can optionally be set. Use NULL or size 0 for a reset order.
* Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
* @return : 1 if OK, 0 if error
*/
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e1d0b1d02..08bf0faee 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -738,7 +738,7 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- int const acceleration = (level < -1) ? -level : 1;
+ int const acceleration = (level < 0) ? -level + 1 : 1;
LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
if (cdict) {
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
@@ -749,7 +749,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
{
- int const acceleration = (level < -1) ? -level : 1;
+ int const acceleration = (level < 0) ? -level + 1 : 1;
(void)cdict; /* init once at beginning of frame */
return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
}
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index fb434ff76..75f1fd91b 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -33,9 +33,10 @@
*/
/* LZ4F is a stand-alone API to create LZ4-compressed frames
- * conformant with specification v1.5.1.
+ * conformant with specification v1.6.1.
* It also offers streaming capabilities.
- * lz4.h is not required when using lz4frame.h.
+ * lz4.h is not required when using lz4frame.h,
+ * except to get constant such as LZ4_VERSION_NUMBER.
* */
#ifndef LZ4F_H_09782039843
@@ -159,8 +160,9 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
/*! LZ4F_frameInfo_t :
* makes it possible to set or read frame parameters.
- * It's not required to set all fields, as long as the structure was initially memset() to zero.
- * For all fields, 0 sets it to default value */
+ * Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO,
+ * setting all parameters to default.
+ * It's then possible to update selectively some parameters */
typedef struct {
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */
LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
@@ -171,24 +173,30 @@ typedef struct {
LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
} LZ4F_frameInfo_t;
+#define LZ4F_INIT_FRAMEINFO { 0, 0, 0, 0, 0, 0, 0 } /* v1.8.3+ */
+
/*! LZ4F_preferences_t :
- * makes it possible to supply detailed compression parameters to the stream interface.
- * Structure is presumed initially memset() to zero, representing default settings.
+ * makes it possible to supply advanced compression instructions to streaming interface.
+ * Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES,
+ * setting all parameters to default.
* All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
- unsigned autoFlush; /* 1: always flush, to reduce usage of internal buffers */
- unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */ /* >= v1.8.2 */
+ unsigned autoFlush; /* 1: always flush; reduces usage of internal buffers */
+ unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */
unsigned reserved[3]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
-LZ4FLIB_API int LZ4F_compressionLevel_max(void);
+#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0, 0, { 0, 0, 0 } } /* v1.8.3+ */
/*-*********************************
* Simple compression function
***********************************/
+
+LZ4FLIB_API int LZ4F_compressionLevel_max(void);
+
/*! LZ4F_compressFrameBound() :
* Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
* `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
@@ -222,8 +230,9 @@ typedef struct {
/*--- Resource Management ---*/
-#define LZ4F_VERSION 100
+#define LZ4F_VERSION 100 /* This number can be used to check for an incompatible API breaking change */
LZ4FLIB_API unsigned LZ4F_getVersion(void);
+
/*! LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 8108ea011..e913ee7b3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -327,6 +327,8 @@ LZ4HC_InsertAndGetWiderMatch (
if (lookBackLength==0) { /* no back possible */
size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
if ((size_t)longest < maxML) {
+ assert(base + matchIndex < ip);
+ if (ip - (base+matchIndex) > MAX_DISTANCE) break;
assert(maxML < 2 GB);
longest = (int)maxML;
*matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
@@ -450,6 +452,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
*op += length;
/* Encode Offset */
+ assert( (*ip - match) <= MAX_DISTANCE ); /* note : consider providing offset as a value, rather than as a pointer difference */
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
/* Encode MatchLength */
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index bb5e07373..970fa3966 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -246,6 +246,10 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API int LZ4_resetStr
#ifndef LZ4_HC_SLO_098092834
#define LZ4_HC_SLO_098092834
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental)
* Will try to compress as much data from `src` as possible
* that can fit into `targetDstSize` budget.
@@ -343,5 +347,9 @@ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* ds
*/
LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream);
+#if defined (__cplusplus)
+}
+#endif
+
#endif /* LZ4_HC_SLO_098092834 */
#endif /* LZ4_HC_STATIC_LINKING_ONLY */
diff --git a/programs/Makefile b/programs/Makefile
index 72bdcaac1..bd33d9be0 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -82,7 +82,7 @@ lz4-release: DEBUGFLAGS=
lz4-release: lz4
lz4c: lz4
- ln -s lz4 lz4c
+ ln -s lz4$(EXT) lz4c$(EXT)
lz4c32: CFLAGS += -m32
lz4c32 : $(SRCFILES)
@@ -102,20 +102,20 @@ preview-man: clean-man man
clean:
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
@$(RM) core *.o *.test tmp* \
- lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
+ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4$(EXT) lz4cat$(EXT)
@echo Cleaning completed
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
unlz4: lz4
- ln -s lz4 unlz4
+ ln -s lz4$(EXT) unlz4$(EXT)
lz4cat: lz4
- ln -s lz4 lz4cat
+ ln -s lz4$(EXT) lz4cat$(EXT)
DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
@@ -147,10 +147,10 @@ INSTALL_DATA ?= $(INSTALL) -m 644
install: lz4
@echo Installing binaries
@$(INSTALL) -d -m 755 $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
- @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(bindir)/lz4
- @ln -sf lz4 $(DESTDIR)$(bindir)/lz4c
- @ln -sf lz4 $(DESTDIR)$(bindir)/lz4cat
- @ln -sf lz4 $(DESTDIR)$(bindir)/unlz4
+ @$(INSTALL_PROGRAM) lz4$(EXT) $(DESTDIR)$(bindir)/lz4$(EXT)
+ @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4c$(EXT)
+ @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ @ln -sf lz4$(EXT) $(DESTDIR)$(bindir)/unlz4$(EXT)
@echo Installing man pages
@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
@ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
@@ -159,10 +159,10 @@ install: lz4
@echo lz4 installation completed
uninstall:
- @$(RM) $(DESTDIR)$(bindir)/lz4cat
- @$(RM) $(DESTDIR)$(bindir)/unlz4
- @$(RM) $(DESTDIR)$(bindir)/lz4
- @$(RM) $(DESTDIR)$(bindir)/lz4c
+ @$(RM) $(DESTDIR)$(bindir)/lz4cat$(EXT)
+ @$(RM) $(DESTDIR)$(bindir)/unlz4$(EXT)
+ @$(RM) $(DESTDIR)$(bindir)/lz4$(EXT)
+ @$(RM) $(DESTDIR)$(bindir)/lz4c$(EXT)
@$(RM) $(DESTDIR)$(man1dir)/lz4.1
@$(RM) $(DESTDIR)$(man1dir)/lz4c.1
@$(RM) $(DESTDIR)$(man1dir)/lz4cat.1
diff --git a/programs/bench.c b/programs/bench.c
index 770191cfa..11bf0440c 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -49,7 +49,10 @@
#include "lz4.h"
#define COMPRESSOR0 LZ4_compress_local
-static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); }
+static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) {
+ int const acceleration = (clevel < 0) ? -clevel + 1 : 1;
+ return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
+}
#include "lz4hc.h"
#define COMPRESSOR1 LZ4_compress_HC
#define DEFAULTCOMPRESSOR COMPRESSOR0
@@ -326,7 +329,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
- DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
+ DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u= LZ4_compressBound(srcSize)`
+ * for compression to be guaranteed to work */
+static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
+ void* compressedBuff, size_t compressedBuffCapacity,
+ const void* srcBuff, size_t srcSize,
+ int clevel)
+{
+ int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize);
+ int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */
+ compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast;
+ int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel);
+ CONTROL_MSG(cSize == 0, "Compression error !");
+
+ { int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity);
+ CONTROL_MSG(dSize < 0, "Decompression detected an error !");
+ CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !");
+ }
+
+ /* check potential content corruption error */
+ assert(resultBuffCapacity >= srcSize);
+ { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize);
+ CONTROL_MSG(errorPos != srcSize,
+ "Silent decoding corruption, at pos %u !!!",
+ (unsigned)errorPos);
+ }
+
+}
+
+static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel)
+{
+ size_t const cBuffSize = LZ4_compressBound((int)srcSize);
+ void* const cBuff = malloc(cBuffSize);
+ void* const rBuff = malloc(cBuffSize);
+
+ if (!cBuff || !rBuff) {
+ fprintf(stderr, "not enough memory ! \n");
+ exit(1);
+ }
+
+ roundTripTest(rBuff, cBuffSize,
+ cBuff, cBuffSize,
+ srcBuff, srcSize,
+ clevel);
+
+ free(rBuff);
+ free(cBuff);
+}
+
+
+static size_t getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (size_t)statbuf.st_size;
+}
+
+
+static int isDirectory(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+/** loadFile() :
+ * requirement : `buffer` size >= `fileSize` */
+static void loadFile(void* buffer, const char* fileName, size_t fileSize)
+{
+ FILE* const f = fopen(fileName, "rb");
+ if (isDirectory(fileName)) {
+ MSG("Ignoring %s directory \n", fileName);
+ exit(2);
+ }
+ if (f==NULL) {
+ MSG("Impossible to open %s \n", fileName);
+ exit(3);
+ }
+ { size_t const readSize = fread(buffer, 1, fileSize, f);
+ if (readSize != fileSize) {
+ MSG("Error reading %s \n", fileName);
+ exit(5);
+ } }
+ fclose(f);
+}
+
+
+static void fileCheck(const char* fileName, int clevel)
+{
+ size_t const fileSize = getFileSize(fileName);
+ void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
+ if (!buffer) {
+ MSG("not enough memory \n");
+ exit(4);
+ }
+ loadFile(buffer, fileName, fileSize);
+ roundTripCheck(buffer, fileSize, clevel);
+ free (buffer);
+}
+
+
+int bad_usage(const char* exeName)
+{
+ MSG(" \n");
+ MSG("bad usage: \n");
+ MSG(" \n");
+ MSG("%s [Options] fileName \n", exeName);
+ MSG(" \n");
+ MSG("Options: \n");
+ MSG("-# : use #=[0-9] compression level (default:0 == random) \n");
+ return 1;
+}
+
+
+int main(int argCount, const char** argv)
+{
+ const char* const exeName = argv[0];
+ int argNb = 1;
+ int clevel = 0;
+
+ assert(argCount >= 1);
+ if (argCount < 2) return bad_usage(exeName);
+
+ if (argv[1][0] == '-') {
+ clevel = argv[1][1] - '0';
+ argNb = 2;
+ }
+
+ if (argNb >= argCount) return bad_usage(exeName);
+
+ fileCheck(argv[argNb], clevel);
+ MSG("no pb detected \n");
+ return 0;
+}