From b077a99347a384225d03f5458c2b57bedb134c62 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 14 Jan 2018 16:57:25 -0800
Subject: [PATCH 001/257] added checkTag

checkTag verifies that provided tag and library version match.
It's started automatically in circleCI when a new tag is created.
---
 .travis.yml      |  8 +++++
 tests/.gitignore |  1 +
 tests/Makefile   |  5 ++-
 tests/checkTag.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 tests/checkTag.c

diff --git a/.travis.yml b/.travis.yml
index 3353dee74..9dea485e5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -145,7 +145,15 @@ matrix:
             - gcc-multilib
             - gcc-4.4
 
+    # tag-specific test
+    - if: tag =~ ^v[0-9]\.[0-9]
+      os: linux
+      sudo: false
+      env: Cmd="make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH " COMPILER=cc
+
+
 script:
+  - uname -a
   - echo Cmd=$Cmd
   - $COMPILER -v
   - sh -c "$Cmd"
diff --git a/tests/.gitignore b/tests/.gitignore
index 4c0f31141..36dff4207 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -8,6 +8,7 @@ fullbench32
 fuzzer
 fuzzer32
 fasttest
+checkTag
 
 # test artefacts
 tmp*
diff --git a/tests/Makefile b/tests/Makefile
index 819ba43ce..ddc0d2e8c 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -111,7 +111,7 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        fasttest$(EXT) datagen$(EXT)
+        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 
@@ -119,6 +119,9 @@ clean:
 versionsTest:
 	$(PYTHON) test-lz4-versions.py
 
+checkTag: checkTag.c $(LZ4DIR)/lz4.h
+	$(CC) $(FLAGS) $< -o $@$(EXT)
+
 
 #-----------------------------------------------------------------------------
 # validated only for Linux, OSX, BSD, Hurd and Solaris targets
diff --git a/tests/checkTag.c b/tests/checkTag.c
new file mode 100644
index 000000000..4a3341536
--- /dev/null
+++ b/tests/checkTag.c
@@ -0,0 +1,79 @@
+/*
+    checkTag.c - Version validation tool for LZ4
+    Copyright (C) Yann Collet 2018 - present
+
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+    - LZ4 homepage : http://www.lz4.org
+    - LZ4 source repo : https://github.com/lz4/lz4
+*/
+
+/* checkTag command :
+ * $ ./checkTag tag
+ * checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any}
+ * The tag is then compared to LZ4 version number.
+ * They are compatible if first 3 digits are identical.
+ * Anything beyond that is free, and doesn't impact validation.
+ * Example : tag v1.8.1.2 is compatible with version 1.8.1
+ * When tag and version are not compatible, program exits with error code 1.
+ * When they are compatible, it exists with a code 0.
+ * checkTag is intended to be used in automated testing environment.
+ */
+
+#include <stdio.h>   /* printf */
+#include <string.h>  /* strlen, strncmp */
+#include "lz4.h"     /* LZ4_VERSION_STRING */
+
+
+/*  validate() :
+ * @return 1 if tag is compatible, 0 if not.
+ */
+static int validate(const char* const tag)
+{
+    size_t const tagLength = strlen(tag);
+    size_t const verLength = strlen(LZ4_VERSION_STRING);
+
+    if (tagLength < 2) return 0;
+    if (tag[0] != 'v') return 0;
+    if (tagLength <= verLength) return 0;
+
+    if (strncmp(LZ4_VERSION_STRING, tag+1, verLength)) return 0;
+
+    return 1;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+    const char* const tag = argv[1];
+    if (argc!=2) {
+        printf("incorrect usage : %s tag \n", exeName);
+        return 2;
+    }
+
+    printf("Version : %s \n", LZ4_VERSION_STRING);
+    printf("Tag     : %s \n", tag);
+
+    if (validate(tag)) {
+        printf("OK : tag is compatible with lz4 version \n");
+        return 0;
+    }
+
+    printf("!! error : tag and versions are not compatible !! \n");
+    return 1;
+}

From 7dba09af47dd3daa1562a6332a643a1a59dba4a8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 16 Jan 2018 10:21:37 -0800
Subject: [PATCH 002/257] use more restrictive conditions for clock_gettime()

---
 programs/util.h | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/programs/util.h b/programs/util.h
index fc7f63e81..a3576d7e3 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -141,6 +141,7 @@ extern "C" {
 *  Time functions
 ******************************************/
 #if defined(_WIN32)   /* Windows */
+
     typedef LARGE_INTEGER UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
     UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
@@ -165,7 +166,9 @@ extern "C" {
         }
         return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
     }
+
 #elif defined(__APPLE__) && defined(__MACH__)
+
     #include <mach/mach_time.h>
     typedef U64 UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
@@ -189,7 +192,9 @@ extern "C" {
         }
         return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
     }
-#elif (PLATFORM_POSIX_VERSION >= 200112L)
+
+#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2))
+
     #include <time.h>
     typedef struct timespec UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void)
@@ -227,7 +232,9 @@ extern "C" {
         nano += diff.tv_nsec;
         return nano;
     }
+
 #else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+
     typedef clock_t UTIL_time_t;
     UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
     UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }

From 75b81bbbf09f7d76f0e439ff29d12d922e74c0bb Mon Sep 17 00:00:00 2001
From: Po-Chuan Hsieh <sunpoet@sunpoet.net>
Date: Thu, 18 Jan 2018 03:13:05 +0800
Subject: [PATCH 003/257] Change file format back to ASCII (from UTF-8)

- Replace U+00A0 by space
- Fix build failure of archivers/py-borgbackup in FreeBSD

Reference: https://bugs.FreeBSD.org/225235
---
 lib/lz4.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index a06b8a46c..7710b11d6 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -264,8 +264,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
  *
  *  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
- *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
- *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+ *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+ *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
  * @return : size of compressed block
  *           or 0 if there is an error (typically, compressed data cannot fit into 'dst')

From 30e92f320cc355c91a9b1840fd14026566ba9477 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 22 Jan 2018 12:50:06 -0800
Subject: [PATCH 004/257] [lz4hc] level == 0 means default, not level 1

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f2c2566f4..cface811a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -728,7 +728,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
-    if (compressionLevel < 1) compressionLevel = 1;
+    if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
     if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;
     LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
 }

From fd0c125ff1c7962f6ec2567718ffa576aa4248a8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 30 Jan 2018 17:53:00 -0800
Subject: [PATCH 005/257] proposed a minor change to LZ4 Frame format
 specification

add new terms "LZ4 Frame Header" and "LZ4 Frame Footer"
---
 doc/lz4_Frame_format.md | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index 77454b2d4..359e56372 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-1.6.0 (08/08/2017)
+1.6.1 (30/01/2018)
 
 
 Introduction
@@ -72,12 +72,15 @@ Value : 0x184D2204
 
 __Frame Descriptor__
 
-3 to 15 Bytes, to be detailed in the next part.
-Most important part of the spec.
+3 to 15 Bytes, to be detailed in its own paragraph,
+as it is the most important part of the spec.
+
+The combined __Magic Number__ and __Frame Descriptor__ fields are sometimes
+called ___LZ4 Frame Header___. Its size vary between 7 and 19 bytes.
 
 __Data Blocks__
 
-To be detailed later on.
+To be detailed in its own paragraph.
 That’s where compressed data is stored.
 
 __EndMark__
@@ -98,6 +101,9 @@ that all blocks were fully transmitted in the correct order and without error,
 and also that the encoding/decoding process itself generated no distortion.
 Its usage is recommended.
 
+The combined __EndMark__ and __Content Checksum__ fields might sometimes be
+referred as ___LZ4 Frame Footer___. Its size vary between 4 and 8 bytes.
+
 __Frame Concatenation__
 
 In some circumstances, it may be preferable to append multiple frames,
@@ -380,6 +386,8 @@ and trigger an error if it does not fit within acceptable range.
 Version changes
 ---------------
 
+1.6.1 : introduced terms "LZ4 Frame Header" and "LZ4 Frame Footer"
+
 1.6.0 : restored Dictionary ID field in Frame header
 
 1.5.1 : changed document format to MarkDown

From 865bd83e13590dc4a4c82f559f454cefcd86fd69 Mon Sep 17 00:00:00 2001
From: Asger Hautop Drewsen <asgerdrewsen@gmail.com>
Date: Wed, 31 Jan 2018 13:33:07 +0100
Subject: [PATCH 006/257] Ensure LZ4_DEPRECATED("...") is before LZ4LIB_API

When using clang++ with std c++14 or c++17 you would get the error "an attribute list cannot appear here" when including "lz4.h" as the visibility attribute is before the c++ attribute.
This ensures that the [[deprecated]] c++ attribute is before everything
else in the function declarations.
---
 lib/lz4.h   | 28 ++++++++++++++--------------
 lib/lz4hc.h | 34 +++++++++++++++++-----------------
 2 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 7710b11d6..df854726d 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -450,26 +450,26 @@ union LZ4_streamDecode_u {
 #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
 
 /* Obsolete compression functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress               (const char* source, char* dest, int sourceSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress               (const char* source, char* dest, int sourceSize);
+LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /* Obsolete decompression functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
+LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
 /* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int   LZ4_sizeofStreamState(void);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead")  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead")     char* LZ4_slideInputBuffer (void* state);
+LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
 
 #endif /* LZ4_H_2983827168210 */
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index d41bf4200..a7f77f987 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -195,25 +195,25 @@ union LZ4_streamHC_u {
 /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */
 
 /* deprecated compression functions */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC               (const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC               (const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /* Deprecated Streaming functions using older model; should no longer be used */
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead")     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead")   int   LZ4_freeHC (void* LZ4HC_Data);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int   LZ4_sizeofStreamStateHC(void);
-LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API   int   LZ4_freeHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int   LZ4_sizeofStreamStateHC(void);
+LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
 
 
 #if defined (__cplusplus)

From d03224b63345d4ace45c56773f456da16e6c0acd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 09:54:30 -0800
Subject: [PATCH 007/257] fix typos as suggested by @psteinb

---
 doc/lz4_Frame_format.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index 359e56372..0c98df14d 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -76,7 +76,7 @@ __Frame Descriptor__
 as it is the most important part of the spec.
 
 The combined __Magic Number__ and __Frame Descriptor__ fields are sometimes
-called ___LZ4 Frame Header___. Its size vary between 7 and 19 bytes.
+called ___LZ4 Frame Header___. Its size varies between 7 and 19 bytes.
 
 __Data Blocks__
 
@@ -102,7 +102,7 @@ and also that the encoding/decoding process itself generated no distortion.
 Its usage is recommended.
 
 The combined __EndMark__ and __Content Checksum__ fields might sometimes be
-referred as ___LZ4 Frame Footer___. Its size vary between 4 and 8 bytes.
+referred to as ___LZ4 Frame Footer___. Its size varies between 4 and 8 bytes.
 
 __Frame Concatenation__
 

From c129f480e777d795f9945bc9ae9dafe01f3772c4 Mon Sep 17 00:00:00 2001
From: Asger Hautop Drewsen <asgerdrewsen@gmail.com>
Date: Wed, 31 Jan 2018 20:23:20 +0100
Subject: [PATCH 008/257] Always prefer c++14 attributes if available

---
 lib/lz4.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index df854726d..6179c05f3 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -433,11 +433,9 @@ union LZ4_streamDecode_u {
 #  define LZ4_DEPRECATED(message)   /* disable deprecation warnings */
 #else
 #  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */
-#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
-#  elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
 #    define LZ4_DEPRECATED(message) [[deprecated(message)]]
-#  elif (LZ4_GCC_VERSION >= 405)
+#  elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (LZ4_GCC_VERSION >= 301)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated))

From 87fb7a1d031a7e6df72d5ca50ab543a57ddca21f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 14:33:16 -0800
Subject: [PATCH 009/257] refactored frameCompress example

to better reflect LZ4F API usage.
---
 Makefile                 |   3 +-
 doc/lz4_manual.html      |   4 +-
 doc/lz4frame_manual.html |  44 ++++++------
 examples/frameCompress.c | 150 +++++++++++++++++++++------------------
 lib/lz4frame.c           |   3 +-
 lib/lz4frame.h           |  12 ++--
 6 files changed, 120 insertions(+), 96 deletions(-)

diff --git a/Makefile b/Makefile
index 0aa1b9de9..d17b4a766 100644
--- a/Makefile
+++ b/Makefile
@@ -70,7 +70,7 @@ lz4 lz4-release :
 
 .PHONY: examples
 examples: lib lz4
-	$(MAKE) -C $(EXDIR) test
+	$(MAKE) -C $(EXDIR) all
 
 .PHONY: manuals
 manuals:
@@ -125,6 +125,7 @@ list:
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
+	$(MAKE) -C $(EXDIR) $@
 
 clangtest: clean
 	clang -v
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 6b7935d5b..46e6c3d09 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -176,8 +176,8 @@ <h1>1.8.1 Manual</h1>
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
 
   Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
-  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
-  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
  @return : size of compressed block
            or 0 if there is an error (typically, compressed data cannot fit into 'dst')
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 590c63287..1ddf6d877 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -101,8 +101,10 @@ <h1>1.8.1 Manual</h1>
 <a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>
 
 <pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-</b><p>  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
-  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+</b><p>  Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
+ `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
+  Note : this result is only usable with LZ4F_compressFrame().
+         It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
  
 </p></pre><BR>
 
@@ -151,19 +153,21 @@ <h1>1.8.1 Manual</h1>
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-</b><p> Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
- prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
- Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
- When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
+</b><p>  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
+  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
+  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
+  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
+  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
+  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
-</b><p> LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
- This value is provided by LZ4F_compressBound().
- If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+</b><p>  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+  This value is provided by LZ4F_compressBound().
+  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
+  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
@@ -171,8 +175,8 @@ <h1>1.8.1 Manual</h1>
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-</b><p> When data must be generated and sent immediately, without waiting for a block to be completely filled,
- it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
+</b><p>  When data must be generated and sent immediately, without waiting for a block to be completely filled,
+  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  `dstCapacity` must be large enough to ensure the operation will be successful.
  `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
  @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
@@ -200,13 +204,13 @@ <h1>1.8.1 Manual</h1>
 </b></pre><BR>
 <pre><b>LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
 LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
-</b><p> Create an LZ4F_dctx object, to track all decompression operations.
- The version provided MUST be LZ4F_VERSION.
- The function provides a pointer to an allocated and initialized LZ4F_dctx object.
- The result is an errorCode, which can be tested using LZ4F_isError().
- dctx memory can be released using LZ4F_freeDecompressionContext();
- The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
- That is, it should be == 0 if decompression has been completed fully and correctly.
+</b><p>  Create an LZ4F_dctx object, to track all decompression operations.
+  The version provided MUST be LZ4F_VERSION.
+  The function provides a pointer to an allocated and initialized LZ4F_dctx object.
+  The result is an errorCode, which can be tested using LZ4F_isError().
+  dctx memory can be released using LZ4F_freeDecompressionContext();
+  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+  That is, it should be == 0 if decompression has been completed fully and correctly.
  
 </p></pre><BR>
 
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index d66a8dc98..35dbde23b 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -1,122 +1,133 @@
-// LZ4frame API example : compress a file
-// Based on sample code from Zbigniew Jędrzejewski-Szmek
+/* LZ4frame API example : compress a file
+ * Based on sample code from Zbigniew Jędrzejewski-Szmek
+ *
+ * This example streams an input file into an output file
+ * using a bounded memory budget.
+ * Input is read in chunks of IN_CHUNK_SIZE */
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
+#include <assert.h>
 
 #include <lz4frame.h>
 
-#define BUF_SIZE 16*1024
-#define LZ4_HEADER_SIZE 19
-#define LZ4_FOOTER_SIZE 4
+
+#define IN_CHUNK_SIZE  (16*1024)
 
 static const LZ4F_preferences_t lz4_preferences = {
     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
-      0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
-    0,   /* compression level */
+      0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
+    0,   /* compression level; 0 == default */
     0,   /* autoflush */
     { 0, 0, 0, 0 },  /* reserved, must be set to 0 */
 };
 
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
-    size_t r=1;  /* function result; 1 == error, default (early exit) */
-    LZ4F_compressionContext_t ctx;
-    char *src, *buf = NULL;
-    size_t size, count_in = 0, count_out, offset = 0, frame_size;
 
+/* safe_fwrite() :
+ * performs fwrite(), ensure operation success, or immediately exit() */
+static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
+{
+    size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
+    size_t const expectedSize = eltSize * nbElt;   /* note : should check for overflow */
+    if (writtenSize < expectedSize) {
+        if (ferror(f))  /* note : ferror() must follow fwrite */
+            printf("Write failed\n");
+        else
+            printf("Short write\n");
+        exit(1);
+    }
+}
+
+
+static size_t
+compress_file(FILE* in, FILE* out,
+              unsigned long long* size_in,
+              unsigned long long* size_out)
+{
+    size_t result = 1;  /* function result; 1 == error, default (early exit) */
+    unsigned long long count_in = 0, count_out;
+
+    /* init */
+    LZ4F_compressionContext_t ctx;
     if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
-        printf("Failed to create context: error %zu\n", r);
-        return 1;
+        printf("error: failed to create context \n");
+        return result;
     }
 
-    src = malloc(BUF_SIZE);
+    char* outbuff = NULL;
+    void* const src = malloc(IN_CHUNK_SIZE);
     if (!src) {
         printf("Not enough memory\n");
         goto cleanup;
     }
 
-    frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
-    size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
-    buf = malloc(size);
-    if (!buf) {
+    size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &lz4_preferences);   /* large enough for any input <= IN_CHUNK_SIZE */
+    outbuff = malloc(outbufCapacity);
+    if (!outbuff) {
         printf("Not enough memory\n");
         goto cleanup;
     }
 
-    {   size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
+    /* write frame header */
+    assert(outbufCapacity >= LZ4F_HEADER_SIZE_MAX);
+    {   size_t const headerSize = LZ4F_compressBegin(ctx, outbuff, outbufCapacity, &lz4_preferences);
         if (LZ4F_isError(headerSize)) {
             printf("Failed to start compression: error %zu\n", headerSize);
             goto cleanup;
         }
-        offset = count_out = headerSize;
-        printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize);
+        count_out = headerSize;
+        printf("Buffer size is %zu bytes, header size %zu bytes\n", outbufCapacity, headerSize);
+        safe_fwrite(outbuff, 1, headerSize, out);
     }
 
-
+    /* stream file */
     for (;;) {
-        size_t const readSize = fread(src, 1, BUF_SIZE, in);
-        if (readSize == 0)
-            break;
+        size_t const readSize = fread(src, 1, outbufCapacity, in);
+        if (readSize == 0) break;
         count_in += readSize;
 
-        {   size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL);
-            if (LZ4F_isError(compressedSize)) {
-                printf("Compression failed: error %zu\n", compressedSize);
-                goto cleanup;
-            }
-            offset += compressedSize;
-            count_out += compressedSize;
+        size_t const compressedSize = LZ4F_compressUpdate(ctx,
+                                                outbuff, outbufCapacity,
+                                                src, readSize,
+                                                NULL);
+        if (LZ4F_isError(compressedSize)) {
+            printf("Compression failed: error %zu\n", compressedSize);
+            goto cleanup;
         }
 
-        if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
-            size_t writtenSize;
-            printf("Writing %zu bytes\n", offset);
-
-            writtenSize = fwrite(buf, 1, offset, out);
-            if (writtenSize < offset) {
-                if (ferror(out))  /* note : ferror() must follow fwrite */
-                    printf("Write failed\n");
-                else
-                    printf("Short write\n");
-                goto cleanup;
-            }
-
-            offset = 0;
-        }
+        printf("Writing %zu bytes\n", compressedSize);
+        safe_fwrite(outbuff, 1, compressedSize, out);
+        count_out += compressedSize;
     }
 
-    {   size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
+    /* flush whatever remains within internal buffers */
+    {   size_t const compressedSize = LZ4F_compressEnd(ctx,
+                                            outbuff, outbufCapacity,
+                                            NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Failed to end compression: error %zu\n", compressedSize);
             goto cleanup;
         }
-        offset += compressedSize;
+
+        printf("Writing %zu bytes\n", compressedSize);
+        safe_fwrite(outbuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
-    printf("Writing %zu bytes\n", offset);
-    {   size_t const writtenSize = fwrite(buf, 1, offset, out);
-        if (writtenSize < offset) {
-            if (ferror(out))
-                printf("Write failed\n");
-            else
-                printf("Short write\n");
-            goto cleanup;
-    }   }
-
     *size_in = count_in;
     *size_out = count_out;
-    r = 0;   /* success */
+    result = 0;   /* success */
 
  cleanup:
     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
     free(src);
-    free(buf);
-    return r;
+    free(outbuff);
+    return result;
 }
 
+
 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
     switch (info->blockSizeID) {
         case LZ4F_default:
@@ -131,7 +142,7 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
 }
 
 static size_t decompress_file(FILE* in, FILE* out) {
-    void* const src = malloc(BUF_SIZE);
+    void* const src = malloc(IN_CHUNK_SIZE);
     void* dst = NULL;
     size_t dstCapacity = 0;
     LZ4F_dctx* dctx = NULL;
@@ -148,7 +159,7 @@ static size_t decompress_file(FILE* in, FILE* out) {
     /* Decompression */
     while (ret != 0) {
         /* Load more input */
-        size_t srcSize = fread(src, 1, BUF_SIZE, in);
+        size_t srcSize = fread(src, 1, IN_CHUNK_SIZE, in);
         const void* srcPtr = src;
         const void* const srcEnd = srcPtr + srcSize;
         if (srcSize == 0 || ferror(in)) {
@@ -215,6 +226,7 @@ static size_t decompress_file(FILE* in, FILE* out) {
     return LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
 }
 
+
 int compare(FILE* fp0, FILE* fp1)
 {
     int result = 0;
@@ -238,6 +250,7 @@ int compare(FILE* fp0, FILE* fp1)
     return result;
 }
 
+
 int main(int argc, const char **argv) {
     char inpFilename[256] = { 0 };
     char lz4Filename[256] = { 0 };
@@ -259,8 +272,8 @@ int main(int argc, const char **argv) {
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
-        size_t sizeIn = 0;
-        size_t sizeOut = 0;
+        unsigned long long sizeIn = 0;
+        unsigned long long sizeOut = 0;
         size_t ret;
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
@@ -270,7 +283,8 @@ int main(int argc, const char **argv) {
             return (int)ret;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
-            inpFilename, sizeIn, sizeOut,
+            inpFilename,
+            (size_t)sizeIn, (size_t)sizeOut,  /* might overflow */
             (double)sizeOut / sizeIn * 100);
         printf("compress : done\n");
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 0b26f75a6..a394d1f18 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -726,7 +726,8 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
 
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
-    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
+        return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     memset(&cOptionsNull, 0, sizeof(cOptionsNull));
     if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index eb55e45e5..15484d7e3 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -189,8 +189,10 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void);
 *  Simple compression function
 ***********************************/
 /*! LZ4F_compressFrameBound() :
- *  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- *  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+ *  Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
+ * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
+ *  Note : this result is only usable with LZ4F_compressFrame().
+ *         It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
  */
 LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
@@ -235,7 +237,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 /*----    Compression    ----*/
 
-#define LZ4F_HEADER_SIZE_MAX 19
+#define LZ4F_HEADER_SIZE_MAX 19   /* LZ4 Frame header size can vary from 7 to 19 bytes */
 /*! LZ4F_compressBegin() :
  *  will write the frame header into dstBuffer.
  *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
@@ -248,7 +250,9 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
                                       const LZ4F_preferences_t* prefsPtr);
 
 /*! LZ4F_compressBound() :
- *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case situations.
+ *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
+ *  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
+ *  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
  *  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
  *  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
  *  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.

From 1acca240a9ce824612619dde36de44723785ebe3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 16:11:45 -0800
Subject: [PATCH 010/257] ensure proper dependencies are built for /examples

also : use liblz4.a static lib to share compilation time
---
 examples/Makefile | 44 +++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 9321c2496..c56d45538 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -52,34 +52,40 @@ default: all
 all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
      lineCompress frameCompress simpleBuffer
 
-printVersion: $(LZ4DIR)/lz4.c printVersion.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4opt.h $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
+	$(MAKE) -C $(LZ4DIR) liblz4.a
 
-doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+printVersion: $(LZ4DIR)/liblz4.a printVersion.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-dictionaryRandomAccess: $(LZ4DIR)/lz4.c dictionaryRandomAccess.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+doubleBuffer: $(LZ4DIR)/liblz4.a blockStreaming_doubleBuffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBuffer  : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+dictionaryRandomAccess: $(LZ4DIR)/liblz4.a dictionaryRandomAccess.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+ringBuffer  : $(LZ4DIR)/liblz4.a blockStreaming_ringBuffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+ringBufferHC: $(LZ4DIR)/liblz4.a HCStreaming_ringBuffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-frameCompress: frameCompress.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a
+lineCompress: $(LZ4DIR)/liblz4.a blockStreaming_lineByLine.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-compressFunctions: $(LZ4DIR)/lz4.c compress_functions.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT) -lrt
+frameCompress: $(LZ4DIR)/liblz4.a frameCompress.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+compressFunctions: $(LZ4DIR)/liblz4.a compress_functions.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -lrt
 
-test : all
+simpleBuffer: $(LZ4DIR)/liblz4.a simple_buffer.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
+$(LZ4) :
+	$(MAKE) -C ../programs lz4
+
+test : all $(LZ4)
 	@echo "\n=== Print Version ==="
 	./printVersion$(EXT)
 	@echo "\n=== Simple compression example ==="

From ff3c67fdb2de43cc5c27b48cf15d73e4686c74da Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 16:13:02 -0800
Subject: [PATCH 011/257] fixed read size, as noticed by @terrelln

---
 examples/frameCompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 35dbde23b..5d78ef0c8 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -84,7 +84,7 @@ compress_file(FILE* in, FILE* out,
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(src, 1, outbufCapacity, in);
+        size_t const readSize = fread(src, 1, IN_CHUNK_SIZE, in);
         if (readSize == 0) break;
         count_in += readSize;
 

From b515ae9c9938700150bb4efba1eb78c6166293e0 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 16:39:37 -0800
Subject: [PATCH 012/257] refactored frameCompress.c example code

compression function returns a struct.
Also : nested structure ressources->computation
to make it easier to manage multiple exit points.
---
 examples/frameCompress.c | 132 ++++++++++++++++++++++++---------------
 1 file changed, 81 insertions(+), 51 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 5d78ef0c8..5275ffef7 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -16,7 +16,7 @@
 
 #define IN_CHUNK_SIZE  (16*1024)
 
-static const LZ4F_preferences_t lz4_preferences = {
+static const LZ4F_preferences_t kPrefs = {
     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
       0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
     0,   /* compression level; 0 == default */
@@ -41,84 +41,113 @@ static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
 }
 
 
-static size_t
-compress_file(FILE* in, FILE* out,
-              unsigned long long* size_in,
-              unsigned long long* size_out)
-{
-    size_t result = 1;  /* function result; 1 == error, default (early exit) */
-    unsigned long long count_in = 0, count_out;
+/* ================================================= */
+/*     Streaming Compression example               */
+/* ================================================= */
 
-    /* init */
-    LZ4F_compressionContext_t ctx;
-    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
-        printf("error: failed to create context \n");
-        return result;
-    }
+typedef struct {
+    int error;
+    unsigned long long size_in;
+    unsigned long long size_out;
+} compressResult_t;
 
-    char* outbuff = NULL;
-    void* const src = malloc(IN_CHUNK_SIZE);
-    if (!src) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
+static compressResult_t
+compress_file_internal(FILE* in, FILE* out,
+                    LZ4F_compressionContext_t ctx,
+                    void* inBuff, size_t inChunkSize,
+                    void* outBuff, size_t outCapacity)
+{
+    compressResult_t result = { 1, 0, 0 };  /* result for an error */
+    unsigned long long count_in = 0, count_out;
 
-    size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &lz4_preferences);   /* large enough for any input <= IN_CHUNK_SIZE */
-    outbuff = malloc(outbufCapacity);
-    if (!outbuff) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
+    assert(ctx != NULL);
+    assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
+    assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
 
     /* write frame header */
-    assert(outbufCapacity >= LZ4F_HEADER_SIZE_MAX);
-    {   size_t const headerSize = LZ4F_compressBegin(ctx, outbuff, outbufCapacity, &lz4_preferences);
+    {   size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs);
         if (LZ4F_isError(headerSize)) {
             printf("Failed to start compression: error %zu\n", headerSize);
-            goto cleanup;
+            return result;
         }
         count_out = headerSize;
-        printf("Buffer size is %zu bytes, header size %zu bytes\n", outbufCapacity, headerSize);
-        safe_fwrite(outbuff, 1, headerSize, out);
+        printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize);
+        safe_fwrite(outBuff, 1, headerSize, out);
     }
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(src, 1, IN_CHUNK_SIZE, in);
+        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, in);
         if (readSize == 0) break;
         count_in += readSize;
 
         size_t const compressedSize = LZ4F_compressUpdate(ctx,
-                                                outbuff, outbufCapacity,
-                                                src, readSize,
+                                                outBuff, outCapacity,
+                                                inBuff, readSize,
                                                 NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Compression failed: error %zu\n", compressedSize);
-            goto cleanup;
+            return result;
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outbuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
     /* flush whatever remains within internal buffers */
     {   size_t const compressedSize = LZ4F_compressEnd(ctx,
-                                            outbuff, outbufCapacity,
+                                            outBuff, outCapacity,
                                             NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Failed to end compression: error %zu\n", compressedSize);
-            goto cleanup;
+            return result;
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outbuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
-    *size_in = count_in;
-    *size_out = count_out;
-    result = 0;   /* success */
+    result.size_in = count_in;
+    result.size_out = count_out;
+    result.error = 0;
+    return result;
+}
+
+static compressResult_t
+compress_file(FILE* in, FILE* out)
+{
+    compressResult_t result = { 1, 0, 0 };  /* == error, default (early exit) */
+
+    assert(in != NULL);
+    assert(out != NULL);
+
+    /* allocate ressources */
+    LZ4F_compressionContext_t ctx;
+    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
+        printf("error: failed to create context \n");
+        return result;
+    }
+
+    char* outbuff = NULL;
+    void* const src = malloc(IN_CHUNK_SIZE);
+    if (!src) {
+        printf("Not enough memory\n");
+        goto cleanup;
+    }
+
+    size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
+    outbuff = malloc(outbufCapacity);
+    if (!outbuff) {
+        printf("Not enough memory\n");
+        goto cleanup;
+    }
+
+    result = compress_file_internal(in, out,
+                                        ctx,
+                                        src, IN_CHUNK_SIZE,
+                                        outbuff, outbufCapacity);
 
  cleanup:
     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
@@ -128,6 +157,10 @@ compress_file(FILE* in, FILE* out,
 }
 
 
+/* ================================================= */
+/*     Streaming decompression example               */
+/* ================================================= */
+
 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
     switch (info->blockSizeID) {
         case LZ4F_default:
@@ -272,20 +305,17 @@ int main(int argc, const char **argv) {
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
-        unsigned long long sizeIn = 0;
-        unsigned long long sizeOut = 0;
-        size_t ret;
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
-        ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-        if (ret) {
-            printf("compress : failed with code %zu\n", ret);
-            return (int)ret;
+        compressResult_t const ret = compress_file(inpFp, outFp);
+        if (ret.error) {
+            printf("compress : failed with code %i\n", ret.error);
+            return ret.error;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
             inpFilename,
-            (size_t)sizeIn, (size_t)sizeOut,  /* might overflow */
-            (double)sizeOut / sizeIn * 100);
+            (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow */
+            (double)ret.size_out / ret.size_in * 100);
         printf("compress : done\n");
 
         fclose(outFp);

From 3b751a50c506224e64f39dae7e5fdc61339cf08b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 17:15:02 -0800
Subject: [PATCH 013/257] modified gpptest recipe

---
 Makefile | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 0aa1b9de9..786f8d48e 100644
--- a/Makefile
+++ b/Makefile
@@ -159,17 +159,14 @@ platformTest: clean
 versionsTest: clean
 	$(MAKE) -C $(TESTDIR) $@
 
-gpptest: clean
-	g++ -v
-	CC=g++ $(MAKE) -C $(LZ4DIR)  all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(PRGDIR)  all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(TESTDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-
-gpptest32: clean
-	g++ -v
-	CC=g++ $(MAKE) -C $(LZ4DIR)  all    CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(PRGDIR)  native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
-	CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+gpptest gpptest32: CC = "$(CXX) -Wno-deprecated"
+gpptest gpptest32: CFLAGS = -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
+gpptest32: CFLAGS += -m32
+gpptest gpptest32: clean
+	$(CXX) -v
+	CC=$(CC) $(MAKE) -C $(LZ4DIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(PRGDIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
 
 c_standards: clean
 	# note : lz4 is not C90 compatible, because it requires long long support

From d1ccd620d62acb1863357aeb1bb64fe374b0d055 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 17:16:48 -0800
Subject: [PATCH 014/257] travisci : ensure "clean" betweeb 2 tests

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 9dea485e5..466d55e6e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,7 +19,7 @@ matrix:
 
     - os: linux
       sudo: false
-      env: Ubu=12.04cont Cmd="make gpptest && make clean examples && make clean cmake && make clean travis-install && make clean clangtest" COMPILER=cc
+      env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean cmake && make clean travis-install && make clean clangtest" COMPILER=cc
 
 
     # 14.04 LTS Server Edition 64 bit

From 886a4858451800f6ebd621a2816a4698486d7ff1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 31 Jan 2018 23:17:52 -0800
Subject: [PATCH 015/257] examples/Makefile : changed dependency order

static library *.a must come after source files *.c on linux
---
 Makefile          |  2 +-
 examples/Makefile | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index b48cfdd50..83320fa61 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,7 @@ lz4 lz4-release :
 	@cp $(PRGDIR)/lz4$(EXT) .
 
 .PHONY: examples
-examples: lib lz4
+examples:
 	$(MAKE) -C $(EXDIR) all
 
 .PHONY: manuals
diff --git a/examples/Makefile b/examples/Makefile
index c56d45538..f9e9e7a37 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -55,31 +55,31 @@ all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
 $(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4opt.h $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
 	$(MAKE) -C $(LZ4DIR) liblz4.a
 
-printVersion: $(LZ4DIR)/liblz4.a printVersion.c
+printVersion: printVersion.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-doubleBuffer: $(LZ4DIR)/liblz4.a blockStreaming_doubleBuffer.c
+doubleBuffer: blockStreaming_doubleBuffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-dictionaryRandomAccess: $(LZ4DIR)/liblz4.a dictionaryRandomAccess.c
+dictionaryRandomAccess: dictionaryRandomAccess.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBuffer  : $(LZ4DIR)/liblz4.a blockStreaming_ringBuffer.c
+ringBuffer  : blockStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-ringBufferHC: $(LZ4DIR)/liblz4.a HCStreaming_ringBuffer.c
+ringBufferHC: HCStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-lineCompress: $(LZ4DIR)/liblz4.a blockStreaming_lineByLine.c
+lineCompress: blockStreaming_lineByLine.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-frameCompress: $(LZ4DIR)/liblz4.a frameCompress.c
+frameCompress: frameCompress.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-compressFunctions: $(LZ4DIR)/liblz4.a compress_functions.c
+compressFunctions: compress_functions.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -lrt
 
-simpleBuffer: $(LZ4DIR)/liblz4.a simple_buffer.c
+simpleBuffer: simple_buffer.c $(LZ4DIR)/liblz4.a
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 $(LZ4) :

From 25efdd80c5aee15a425f7e6bea00b4c34bfb8db2 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 1 Feb 2018 01:36:38 -0800
Subject: [PATCH 016/257] refactored ressource allocation to avoid goto

---
 examples/frameCompress.c | 35 ++++++++++++-----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 5275ffef7..d62053fac 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -116,40 +116,29 @@ compress_file_internal(FILE* in, FILE* out,
 }
 
 static compressResult_t
-compress_file(FILE* in, FILE* out)
+compress_file(FILE* f_in, FILE* f_out)
 {
-    compressResult_t result = { 1, 0, 0 };  /* == error, default (early exit) */
+    compressResult_t result = { 1, 0, 0 };  /* == error (default) */
 
-    assert(in != NULL);
-    assert(out != NULL);
+    assert(f_in != NULL);
+    assert(f_out != NULL);
 
-    /* allocate ressources */
+    /* ressource allocation */
     LZ4F_compressionContext_t ctx;
-    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
-        printf("error: failed to create context \n");
-        return result;
-    }
-
-    char* outbuff = NULL;
+    size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
     void* const src = malloc(IN_CHUNK_SIZE);
-    if (!src) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
-
     size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
-    outbuff = malloc(outbufCapacity);
-    if (!outbuff) {
-        printf("Not enough memory\n");
-        goto cleanup;
-    }
+    void* const outbuff = malloc(outbufCapacity);
 
-    result = compress_file_internal(in, out,
+    if (!LZ4F_isError(ctxCreation) && src && outbuff) {
+        result = compress_file_internal(f_in, f_out,
                                         ctx,
                                         src, IN_CHUNK_SIZE,
                                         outbuff, outbufCapacity);
+    } else {
+        printf("error : ressource allocation failed \n");
+    }
 
- cleanup:
     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
     free(src);
     free(outbuff);

From 3ce289bcce012b8173edd4c1040208da2692d060 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 1 Feb 2018 02:48:20 -0800
Subject: [PATCH 017/257] modified decompression part of frameCompress.c

using same logic as prior modifications for compression part.
---
 examples/frameCompress.c | 237 +++++++++++++++++++++++----------------
 1 file changed, 138 insertions(+), 99 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index d62053fac..972f716f8 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -1,5 +1,5 @@
 /* LZ4frame API example : compress a file
- * Based on sample code from Zbigniew Jędrzejewski-Szmek
+ * Modified from an example code by Zbigniew Jędrzejewski-Szmek
  *
  * This example streams an input file into an output file
  * using a bounded memory budget.
@@ -52,7 +52,7 @@ typedef struct {
 } compressResult_t;
 
 static compressResult_t
-compress_file_internal(FILE* in, FILE* out,
+compress_file_internal(FILE* f_in, FILE* f_out,
                     LZ4F_compressionContext_t ctx,
                     void* inBuff, size_t inChunkSize,
                     void* outBuff, size_t outCapacity)
@@ -60,6 +60,7 @@ compress_file_internal(FILE* in, FILE* out,
     compressResult_t result = { 1, 0, 0 };  /* result for an error */
     unsigned long long count_in = 0, count_out;
 
+    assert(f_in != NULL); assert(f_out != NULL);
     assert(ctx != NULL);
     assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
     assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
@@ -72,13 +73,13 @@ compress_file_internal(FILE* in, FILE* out,
         }
         count_out = headerSize;
         printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize);
-        safe_fwrite(outBuff, 1, headerSize, out);
+        safe_fwrite(outBuff, 1, headerSize, f_out);
     }
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, in);
-        if (readSize == 0) break;
+        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
+        if (readSize == 0) break; /* nothing left to read from input file */
         count_in += readSize;
 
         size_t const compressedSize = LZ4F_compressUpdate(ctx,
@@ -91,21 +92,21 @@ compress_file_internal(FILE* in, FILE* out,
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outBuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, f_out);
         count_out += compressedSize;
     }
 
     /* flush whatever remains within internal buffers */
     {   size_t const compressedSize = LZ4F_compressEnd(ctx,
-                                            outBuff, outCapacity,
-                                            NULL);
+                                                outBuff, outCapacity,
+                                                NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Failed to end compression: error %zu\n", compressedSize);
             return result;
         }
 
         printf("Writing %zu bytes\n", compressedSize);
-        safe_fwrite(outBuff, 1, compressedSize, out);
+        safe_fwrite(outBuff, 1, compressedSize, f_out);
         count_out += compressedSize;
     }
 
@@ -118,8 +119,6 @@ compress_file_internal(FILE* in, FILE* out,
 static compressResult_t
 compress_file(FILE* f_in, FILE* f_out)
 {
-    compressResult_t result = { 1, 0, 0 };  /* == error (default) */
-
     assert(f_in != NULL);
     assert(f_out != NULL);
 
@@ -130,6 +129,7 @@ compress_file(FILE* f_in, FILE* f_out)
     size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
     void* const outbuff = malloc(outbufCapacity);
 
+    compressResult_t result = { 1, 0, 0 };  /* == error (default) */
     if (!LZ4F_isError(ctxCreation) && src && outbuff) {
         result = compress_file_internal(f_in, f_out,
                                         ctx,
@@ -158,115 +158,152 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
         case LZ4F_max1MB:   return 1 << 20;
         case LZ4F_max4MB:   return 1 << 22;
         default:
-            printf("Impossible unless more block sizes are allowed\n");
+            printf("Impossible with expected frame specification (<=v1.6.1)\n");
             exit(1);
     }
 }
 
-static size_t decompress_file(FILE* in, FILE* out) {
-    void* const src = malloc(IN_CHUNK_SIZE);
-    void* dst = NULL;
-    size_t dstCapacity = 0;
-    LZ4F_dctx* dctx = NULL;
+/* @return : 1==error, 0==success */
+static int
+decompress_file_internal(FILE* f_in, FILE* f_out,
+                        LZ4F_dctx* dctx,
+                        void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
+                        void* dst, size_t dstCapacity)
+{
+    int firstChunk = 1;
     size_t ret = 1;
 
-    /* Initialization */
-    if (!src) { perror("decompress_file(src)"); goto cleanup; }
-    {   size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
-        if (LZ4F_isError(dctxStatus)) {
-            printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
-            goto cleanup;
-    }   }
+    assert(f_in != NULL); assert(f_out != NULL);
+    assert(dctx != NULL);
+    assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled);
+    assert(dst != NULL); assert(dstCapacity > 0);
 
     /* Decompression */
     while (ret != 0) {
         /* Load more input */
-        size_t srcSize = fread(src, 1, IN_CHUNK_SIZE, in);
-        const void* srcPtr = src;
-        const void* const srcEnd = srcPtr + srcSize;
-        if (srcSize == 0 || ferror(in)) {
+        size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0;
+        const void* srcPtr = src + alreadyConsumed; alreadyConsumed=0;
+        const void* const srcEnd = srcPtr + readSize;
+        if (readSize == 0 || ferror(f_in)) {
             printf("Decompress: not enough input or error reading file\n");
-            goto cleanup;
-        }
-        /* Allocate destination buffer if it isn't already */
-        if (!dst) {
-            LZ4F_frameInfo_t info;
-            ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
-            if (LZ4F_isError(ret)) {
-                printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
-                goto cleanup;
-            }
-            /* Allocating enough space for an entire block isn't necessary for
-             * correctness, but it allows some memcpy's to be elided.
-             */
-            dstCapacity = get_block_size(&info);
-            dst = malloc(dstCapacity);
-            if (!dst) { perror("decompress_file(dst)"); goto cleanup; }
-            srcPtr += srcSize;
-            srcSize = srcEnd - srcPtr;
+            return 1;
         }
+
         /* Decompress:
-         * Continue while there is more input to read and the frame isn't over.
-         * If srcPtr == srcEnd then we know that there is no more output left in the
-         * internal buffer left to flush.
+         * Continue while there is more input to read (srcPtr != srcEnd)
+         * and the frame isn't over (ret != 0)
          */
         while (srcPtr != srcEnd && ret != 0) {
-            /* INVARIANT: Any data left in dst has already been written */
+            /* Any data within dst has been flushed at this stage */
             size_t dstSize = dstCapacity;
+            size_t srcSize = srcEnd - srcPtr;
             ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
             if (LZ4F_isError(ret)) {
                 printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
-                goto cleanup;
+                return 1;
             }
             /* Flush output */
-            if (dstSize != 0){
-                size_t written = fwrite(dst, 1, dstSize, out);
-                printf("Writing %zu bytes\n", dstSize);
-                if (written != dstSize) {
-                    printf("Decompress: Failed to write to file\n");
-                    goto cleanup;
-                }
-            }
+            if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out);
             /* Update input */
             srcPtr += srcSize;
-            srcSize = srcEnd - srcPtr;
         }
     }
+
     /* Check that there isn't trailing input data after the frame.
-     * It is valid to have multiple frames in the same file, but this example
-     * doesn't support it.
+     * It is valid to have multiple frames in the same file,
+     * but this example only supports one frame.
      */
-    ret = fread(src, 1, 1, in);
-    if (ret != 0 || !feof(in)) {
-        printf("Decompress: Trailing data left in file after frame\n");
-        goto cleanup;
+    {   size_t const readSize = fread(src, 1, 1, f_in);
+        if (readSize != 0 || !feof(f_in)) {
+            printf("Decompress: Trailing data left in file after frame\n");
+            return 1;
+    }   }
+
+    return 0;
+}
+
+
+/* @return : 1==error, 0==completed */
+static int
+decompress_file_allocDst(FILE* f_in, FILE* f_out,
+                        LZ4F_dctx* dctx,
+                        void* src, size_t srcCapacity)
+{
+    assert(f_in != NULL); assert(f_out != NULL);
+    assert(dctx != NULL);
+    assert(src != NULL);
+    assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX);  /* ensure LZ4F_getFrameInfo() can read enough data */
+
+    /* Read Frame header */
+    size_t const readSize = fread(src, 1, srcCapacity, f_in);
+    if (readSize == 0 || ferror(f_in)) {
+        printf("Decompress: not enough input or error reading file\n");
+        return 1;
     }
 
-cleanup:
-    free(src);
+    LZ4F_frameInfo_t info;
+    size_t consumedSize = readSize;
+    {   size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize);
+        if (LZ4F_isError(fires)) {
+            printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires));
+            return 1;
+    }   }
+
+    /* Allocating enough space for an entire block isn't necessary for
+     * correctness, but it allows some memcpy's to be elided.
+     */
+    size_t const dstCapacity = get_block_size(&info);
+    void* const dst = malloc(dstCapacity);
+    if (!dst) { perror("decompress_file(dst)"); return 1; }
+
+    int const decompressionResult = decompress_file_internal(
+                        f_in, f_out,
+                        dctx,
+                        src, srcCapacity, readSize, consumedSize,
+                        dst, dstCapacity);
+
     free(dst);
-    return LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
+    return decompressionResult;
+}
+
+
+/* @result : 1==error, 0==success */
+static int decompress_file(FILE* f_in, FILE* f_out)
+{
+    assert(f_in != NULL); assert(f_out != NULL);
+
+    /* Ressource allocation */
+    void* const src = malloc(IN_CHUNK_SIZE);
+    if (!src) { perror("decompress_file(src)"); return 1; }
+
+    LZ4F_dctx* dctx;
+    {   size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
+        if (LZ4F_isError(dctxStatus)) {
+            printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
+    }   }
+
+    int const result = !dctx ? 1 /* error */ :
+                       decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE);
+
+    free(src);
+    LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
+    return result;
 }
 
 
-int compare(FILE* fp0, FILE* fp1)
+int compareFiles(FILE* fp0, FILE* fp1)
 {
     int result = 0;
 
-    while(0 == result) {
+    while (result==0) {
         char b0[1024];
         char b1[1024];
-        const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-        const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+        size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
+        size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
 
-        result = (int) r0 - (int) r1;
-
-        if (0 == r0 || 0 == r1) {
-            break;
-        }
-        if (0 == result) {
-            result = memcmp(b0, b1, r0);
-        }
+        result = (r0 != r1);
+        if (!r0 || !r1) break;
+        if (!result) result = memcmp(b0, b1, r0);
     }
 
     return result;
@@ -278,7 +315,7 @@ int main(int argc, const char **argv) {
     char lz4Filename[256] = { 0 };
     char decFilename[256] = { 0 };
 
-    if(argc < 2) {
+    if (argc < 2) {
         printf("Please specify input filename\n");
         return 0;
     }
@@ -297,35 +334,36 @@ int main(int argc, const char **argv) {
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
         compressResult_t const ret = compress_file(inpFp, outFp);
+
+        fclose(outFp);
+        fclose(inpFp);
+
         if (ret.error) {
             printf("compress : failed with code %i\n", ret.error);
             return ret.error;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
             inpFilename,
-            (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow */
+            (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
             (double)ret.size_out / ret.size_in * 100);
         printf("compress : done\n");
-
-        fclose(outFp);
-        fclose(inpFp);
     }
 
     /* decompress */
     {   FILE* const inpFp = fopen(lz4Filename, "rb");
         FILE* const outFp = fopen(decFilename, "wb");
-        size_t ret;
 
         printf("decompress : %s -> %s\n", lz4Filename, decFilename);
-        ret = decompress_file(inpFp, outFp);
-        if (ret) {
-            printf("decompress : failed with code %zu\n", ret);
-            return (int)ret;
-        }
-        printf("decompress : done\n");
+        int const ret = decompress_file(inpFp, outFp);
 
         fclose(outFp);
         fclose(inpFp);
+
+        if (ret) {
+            printf("decompress : failed with code %i\n", ret);
+            return ret;
+        }
+        printf("decompress : done\n");
     }
 
     /* verify */
@@ -333,15 +371,16 @@ int main(int argc, const char **argv) {
         FILE* const decFp = fopen(decFilename, "rb");
 
         printf("verify : %s <-> %s\n", inpFilename, decFilename);
-        const int cmp = compare(inpFp, decFp);
-        if(0 == cmp) {
-            printf("verify : OK\n");
-        } else {
-            printf("verify : NG\n");
-        }
+        int const cmp = compareFiles(inpFp, decFp);
 
         fclose(decFp);
         fclose(inpFp);
+
+        if (cmp) {
+            printf("corruption detected : decompressed file differs from original\n");
+            return cmp;
+        }
+        printf("verify : OK\n");
     }
 
     return 0;

From e832a3d87aeb5a8b0c250393e46c7df214ff840b Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Thu, 1 Feb 2018 16:08:59 -0800
Subject: [PATCH 018/257] Clarify the requirements of the LZ4 streaming API

---
 lib/lz4.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 6179c05f3..84f352435 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -263,7 +263,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  *  'dst' buffer must be already allocated.
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
  *
- *  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
+ *  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
+ *              If less than 64KB has been compressed all the data must be present.
  *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
  *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
@@ -306,7 +307,8 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const
  *  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
  *  A block is an unsplittable entity, it must be presented entirely to a decompression function.
  *  Decompression functions only accept one block at a time.
- *  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
+ *  The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
+ *  If less than 64KB of data has been decoded all the data must be present.
  *
  *  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
  *  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)

From 20e969e5793aa6773593df8768d068a1ae13f746 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 5 Feb 2018 15:18:00 -0800
Subject: [PATCH 019/257] fuzzer: added low address compression test

is expected to work on linux+gcc only.
---
 lib/lz4hc.c    |  9 +++++----
 lib/lz4hc.h    |  8 ++++----
 tests/fuzzer.c | 46 +++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index cface811a..79cf651bf 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -856,16 +856,17 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
     LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1;   /* Error : pointer is not aligned for pointer (32 or 64 bits) */
     LZ4HC_init(ctx, (const BYTE*)inputBuffer);
-    ctx->inputBuffer = (BYTE*)inputBuffer;
+    ctx->inputBuffer = inputBuffer;
     return 0;
 }
 
-void* LZ4_createHC (char* inputBuffer)
+void* LZ4_createHC (const char* inputBuffer)
 {
     LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
-    hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer;
+    assert(sizeof(size_t) == sizeof(void*));
+    hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer;   /* ugly hack, circumvent -Wcast-qual */
     return hc4;
 }
 
@@ -889,5 +890,5 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
 {
     LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
     int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
-    return (char*)(hc4->inputBuffer + dictSize);
+    return (char*)(hc4->inputBuffer) + dictSize;
 }
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index a7f77f987..7a25bee6e 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -148,7 +148,7 @@ typedef struct
     const uint8_t* end;         /* next block here to continue on current prefix */
     const uint8_t* base;        /* All index relative to this position */
     const uint8_t* dictBase;    /* alternate base for extDict */
-    uint8_t* inputBuffer;       /* deprecated */
+    void*      inputBuffer;     /* deprecated */
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
@@ -164,7 +164,7 @@ typedef struct
     const unsigned char* end;        /* next block here to continue on current prefix */
     const unsigned char* base;       /* All index relative to this position */
     const unsigned char* dictBase;   /* alternate base for extDict */
-    unsigned char* inputBuffer;      /* deprecated */
+    void*          inputBuffer;      /* deprecated */
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
@@ -206,8 +206,8 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
-/* Deprecated Streaming functions using older model; should no longer be used */
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (char* inputBuffer);
+/* Deprecated Streaming functions; should no longer be used */
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
 LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
 LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API   int   LZ4_freeHC (void* LZ4HC_Data);
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c134fe3e1..9415e942f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -240,6 +240,42 @@ static int FUZ_AddressOverflow(void)
 }
 
 
+#ifdef __unix__   /* is expected to be triggered on linux+gcc */
+
+#include <sys/mman.h>   /* mmap */
+
+static void* FUZ_createLowAddr(size_t size)
+{
+    void* const lowBuff = mmap((void*)(0x1000), size,
+                    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
+                    -1, 0);
+    DISPLAYLEVEL(2, "generating low buffer at address %p \n", lowBuff);
+    return lowBuff;
+}
+
+static void FUZ_freeLowAddr(void* buffer, size_t size)
+{
+    if (munmap(buffer, size)) {
+        perror("fuzzer: freeing low address buffer");
+        abort();
+    }
+}
+
+#else
+
+static void* FUZ_createLowAddr(size_t size)
+{
+    return malloc(size);
+}
+
+static void FUZ_freeLowAddr(void* buffer, size_t size)
+{
+    (void)size;
+    free(buffer);
+}
+
+#endif
+
 /*! FUZ_findDiff() :
 *   find the first different byte between buff1 and buff2.
 *   presumes buff1 != buff2.
@@ -266,6 +302,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
     size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE);
     char* const compressedBuffer = (char*)malloc(compressedBufferSize);
     char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
+    size_t const labSize = 96 KB;
+    void* const lowAddrBuffer = FUZ_createLowAddr(labSize);
     void* const stateLZ4   = malloc(LZ4_sizeofState());
     void* const stateLZ4HC = malloc(LZ4_sizeofStateHC());
     LZ4_stream_t LZ4dict;
@@ -306,7 +344,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
         int const dictSize = MIN(dictSizeRand, blockStart);
         int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
-        char* const block = ((char*)CNBuffer) + blockStart;
+        const char* block = ((char*)CNBuffer) + blockStart;
         const char* dict = block - dictSize;
         int compressedSize, HCcompressedSize;
         int blockContinueCompressedSize;
@@ -317,6 +355,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_displayUpdate(cycleNb);
 
         /* Compression tests */
+        if ( ((FUZ_rand(&randState) & 63) == 2)
+          && ((size_t)blockSize < labSize) ) {
+            memcpy(lowAddrBuffer, block, blockSize);
+            block = lowAddrBuffer;
+        }
 
         /* Test compression destSize */
         FUZ_DISPLAYTEST;
@@ -705,6 +748,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         free(CNBuffer);
         free(compressedBuffer);
         free(decodedBuffer);
+        FUZ_freeLowAddr(lowAddrBuffer, labSize);
         free(stateLZ4);
         free(stateLZ4HC);
         return result;

From ea25250c990769a01606dacb4149d52466b1638e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 7 Feb 2018 02:21:25 -0800
Subject: [PATCH 020/257] fixed code comment as detected in #466

Also clarified a few API code comments
and updated associated html documentation
---
 doc/lz4_manual.html | 65 +++++++++++++++++++++++----------------------
 lib/lz4.h           | 55 +++++++++++++++++++-------------------
 2 files changed, 61 insertions(+), 59 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 46e6c3d09..ef715f874 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -42,9 +42,9 @@ <h1>1.8.1 Manual</h1>
 
 <a name="Chapter2"></a><h2>Version</h2><pre></pre>
 
-<pre><b>int LZ4_versionNumber (void);  </b>/**< library version number; to be used when checking dll version */<b>
+<pre><b>int LZ4_versionNumber (void);  </b>/**< library version number; useful to check dll version */<b>
 </b></pre><BR>
-<pre><b>const char* LZ4_versionString (void);   </b>/**< library version string; to be used when checking dll version */<b>
+<pre><b>const char* LZ4_versionString (void);   </b>/**< library version string; unseful to check dll version */<b>
 </b></pre><BR>
 <a name="Chapter3"></a><h2>Tuning parameter</h2><pre></pre>
 
@@ -53,7 +53,7 @@ <h1>1.8.1 Manual</h1>
 #endif
 </b><p> Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
  Increasing memory usage improves compression ratio
- Reduced memory usage can improve speed, due to cache effect
+ Reduced memory usage may improve speed, thanks to cache effect
  Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
  
 </p></pre><BR>
@@ -65,12 +65,12 @@ <h1>1.8.1 Manual</h1>
     into already allocated 'dst' buffer of size 'dstCapacity'.
     Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
     It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'src' into a limited 'dst' budget,
+    If the function cannot compress 'src' into a more limited 'dst' budget,
     compression stops *immediately*, and the function result is zero.
-    As a consequence, 'dst' content is not valid.
-    This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
-        srcSize : supported max value is LZ4_MAX_INPUT_VALUE
-        dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
+    Note : as a consequence, 'dst' content is not valid.
+    Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+        srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
+        dstCapacity : size of buffer 'dst' (which must be already allocated)
         return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
                   or 0 if compression fails 
 </p></pre><BR>
@@ -81,8 +81,7 @@ <h1>1.8.1 Manual</h1>
     return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
              If destination buffer is not large enough, decoding will stop and output an error code (negative value).
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function is protected against buffer overflow exploits, including malicious data packets.
-             It never writes outside output buffer, nor reads outside input buffer.
+             This function is protected against malicious data packets.
 </p></pre><BR>
 
 <a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre>
@@ -91,18 +90,18 @@ <h1>1.8.1 Manual</h1>
 </b><p>    Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
     This function is primarily useful for memory allocation purposes (destination buffer size).
     Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
-    Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
+    Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
         inputSize  : max supported value is LZ4_MAX_INPUT_SIZE
         return : maximum output size in a "worst case" scenario
-              or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+              or 0, if input size is incorrect (too large or negative)
 </p></pre><BR>
 
 <pre><b>int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-</b><p>    Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
+</b><p>    Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
     The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
     It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
     An acceleration value of "1" is the same as regular LZ4_compress_default()
-    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).
 </p></pre><BR>
 
 <pre><b>int LZ4_sizeofState(void);
@@ -125,26 +124,28 @@ <h1>1.8.1 Manual</h1>
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-</b><p>    originalSize : is the original uncompressed size
-    return : the number of bytes read from the source buffer (in other words, the compressed size)
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
+</b><p>This function is a bit faster than LZ4_decompress_safe(),
+but doesn't provide any security guarantee.
+    originalSize : is the uncompressed size to regenerate
+                   Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
+    return : number of bytes read from source buffer (== compressed size).
+             If the source stream is detected malformed, the function stops decoding and return a negative result.
     note : This function respects memory boundaries for *properly formed* compressed data.
-           It is a bit faster than LZ4_decompress_safe().
-           However, it does not provide any protection against intentionally modified data stream (malicious input).
-           Use this function in trusted environment only (data to decode comes from a trusted source).
+           However, it does not provide any protection against malicious input.
+           It also doesn't know 'src' size, and implies it's >= compressed size.
+           Use this function in trusted environment **only**.
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
 </b><p>    This function decompress a compressed block of size 'srcSize' at position 'src'
     into destination buffer 'dst' of size 'dstCapacity'.
     The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
-    However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
+    However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity).
    @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
-       Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
-             Always control how many bytes were decoded.
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
+        Note : this number can also be < targetOutputSize, if compressed block contains less data.
+            Therefore, always control how many bytes were decoded.
+            If source stream is detected malformed, function returns a negative result.
+            This function is protected against malicious data packets.
 </p></pre><BR>
 
 <a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre>
@@ -175,7 +176,8 @@ <h1>1.8.1 Manual</h1>
   'dst' buffer must be already allocated.
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
 
-  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
+  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
+              If less than 64KB has been compressed all the data must be present.
   Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
   Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
@@ -215,7 +217,8 @@ <h1>1.8.1 Manual</h1>
 </b><p>  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
   A block is an unsplittable entity, it must be presented entirely to a decompression function.
   Decompression functions only accept one block at a time.
-  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
+  The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
+  If less than 64KB of data has been decoded all the data must be present.
 
   Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
   - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
@@ -311,11 +314,9 @@ <h1>1.8.1 Manual</h1>
 #  define LZ4_DEPRECATED(message)   </b>/* disable deprecation warnings */<b>
 #else
 #  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  if defined(__clang__) </b>/* clang doesn't handle mixed C++11 and CNU attributes */<b>
-#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
-#  elif defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b>
+#  if defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b>
 #    define LZ4_DEPRECATED(message) [[deprecated(message)]]
-#  elif (LZ4_GCC_VERSION >= 405)
+#  elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (LZ4_GCC_VERSION >= 301)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated))
diff --git a/lib/lz4.h b/lib/lz4.h
index 84f352435..c5d50e2ce 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -102,8 +102,8 @@ extern "C" {
 #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
 #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
 
-LZ4LIB_API int LZ4_versionNumber (void);  /**< library version number; to be used when checking dll version */
-LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string; to be used when checking dll version */
+LZ4LIB_API int LZ4_versionNumber (void);  /**< library version number; useful to check dll version */
+LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string; unseful to check dll version */
 
 
 /*-************************************
@@ -113,7 +113,7 @@ LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string;
  * LZ4_MEMORY_USAGE :
  * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
  * Increasing memory usage improves compression ratio
- * Reduced memory usage can improve speed, due to cache effect
+ * Reduced memory usage may improve speed, thanks to cache effect
  * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
  */
 #ifndef LZ4_MEMORY_USAGE
@@ -128,12 +128,12 @@ LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string;
     into already allocated 'dst' buffer of size 'dstCapacity'.
     Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
     It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'src' into a limited 'dst' budget,
+    If the function cannot compress 'src' into a more limited 'dst' budget,
     compression stops *immediately*, and the function result is zero.
-    As a consequence, 'dst' content is not valid.
-    This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
-        srcSize : supported max value is LZ4_MAX_INPUT_VALUE
-        dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
+    Note : as a consequence, 'dst' content is not valid.
+    Note 2 : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
+        srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
+        dstCapacity : size of buffer 'dst' (which must be already allocated)
         return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
                   or 0 if compression fails */
 LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
@@ -144,8 +144,7 @@ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int
     return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
              If destination buffer is not large enough, decoding will stop and output an error code (negative value).
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function is protected against buffer overflow exploits, including malicious data packets.
-             It never writes outside output buffer, nor reads outside input buffer.
+             This function is protected against malicious data packets.
 */
 LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
 
@@ -161,20 +160,20 @@ LZ4_compressBound() :
     Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
     This function is primarily useful for memory allocation purposes (destination buffer size).
     Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
-    Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
+    Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
         inputSize  : max supported value is LZ4_MAX_INPUT_SIZE
         return : maximum output size in a "worst case" scenario
-              or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+              or 0, if input size is incorrect (too large or negative)
 */
 LZ4LIB_API int LZ4_compressBound(int inputSize);
 
 /*!
 LZ4_compress_fast() :
-    Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
+    Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
     The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
     It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
     An acceleration value of "1" is the same as regular LZ4_compress_default()
-    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).
 */
 LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
@@ -205,15 +204,17 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt
 
 
 /*!
-LZ4_decompress_fast() : (unsafe!!)
-    originalSize : is the original uncompressed size
-    return : the number of bytes read from the source buffer (in other words, the compressed size)
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
+LZ4_decompress_fast() : **unsafe!**
+This function is a bit faster than LZ4_decompress_safe(),
+but doesn't provide any security guarantee.
+    originalSize : is the uncompressed size to regenerate
+                   Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
+    return : number of bytes read from source buffer (== compressed size).
+             If the source stream is detected malformed, the function stops decoding and return a negative result.
     note : This function respects memory boundaries for *properly formed* compressed data.
-           It is a bit faster than LZ4_decompress_safe().
-           However, it does not provide any protection against intentionally modified data stream (malicious input).
-           Use this function in trusted environment only (data to decode comes from a trusted source).
+           However, it does not provide any protection against malicious input.
+           It also doesn't know 'src' size, and implies it's >= compressed size.
+           Use this function in trusted environment **only**.
 */
 LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 
@@ -222,12 +223,12 @@ LZ4_decompress_safe_partial() :
     This function decompress a compressed block of size 'srcSize' at position 'src'
     into destination buffer 'dst' of size 'dstCapacity'.
     The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
-    However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
+    However, it's not accurate, and may write more than 'targetOutputSize' (but always <= dstCapacity).
    @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
-       Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
-             Always control how many bytes were decoded.
-             If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
+        Note : this number can also be < targetOutputSize, if compressed block contains less data.
+            Therefore, always control how many bytes were decoded.
+            If source stream is detected malformed, function returns a negative result.
+            This function is protected against malicious data packets.
 */
 LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
 

From c4671be550fa65241c46e9e3a9214d227fbfe27f Mon Sep 17 00:00:00 2001
From: Ben Boeckel <ben.boeckel@kitware.com>
Date: Thu, 8 Feb 2018 09:15:07 -0500
Subject: [PATCH 021/257] intel: do not use __attribute__((packed)) on Windows

On Windows, the Intel compiler is closer to MSVC rather than GCC and
does not support the GCC attribute syntax.

Fixes #468
---
 lib/lz4.c    | 2 +-
 lib/xxhash.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 213b08516..7bf8677a1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -71,7 +71,7 @@
 #ifndef LZ4_FORCE_MEMORY_ACCESS   /* can be defined externally */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define LZ4_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || defined(__GNUC__)
+#  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
 #    define LZ4_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
diff --git a/lib/xxhash.c b/lib/xxhash.c
index bcf1f1dbe..3fc97fd2a 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -52,7 +52,7 @@
 #ifndef XXH_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define XXH_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
+#  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \
   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 #    define XXH_FORCE_MEMORY_ACCESS 1
 #  endif

From 3ad3b0f850ed3bcf5f3faafa414f21f84706461a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 01:42:12 -0800
Subject: [PATCH 022/257] slightly improved decompression speed (~+1-2%)

by making shortcut slightly more common
---
 lib/lz4.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7bf8677a1..6e5e9f58f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1220,10 +1220,13 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             size_t const ll = token >> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-            if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
+            if ((off >= 8) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
                 size_t const ml = (token & ML_MASK) + MINMATCH;
                 memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
-                memcpy(op, matchPtr, 18); op += ml;
+                memcpy(op+ 0, matchPtr+ 0, 8);
+                memcpy(op+ 8, matchPtr+ 8, 8);
+                memcpy(op+16, matchPtr+16, 2);
+                op += ml;
                 continue;
             }
         }

From 2b674bf02f698b565b4525a36cdf4651899b88d1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 02:45:36 -0800
Subject: [PATCH 023/257] slightly improved hc compression speed (+~1-2%)

by removing bad candidates faster.
---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index cface811a..9f59e80b6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -220,7 +220,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
-            if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
+            if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
     #if 0

From 219abab74bf8914d3f8761db5b398103cbbd77d7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 22:20:09 -0800
Subject: [PATCH 024/257] removed LZ4_copy8

better use memcpy() directly
---
 lib/lz4.c | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6e5e9f58f..5d4bb21cc 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -270,11 +270,6 @@ static void LZ4_writeLE16(void* memPtr, U16 value)
     }
 }
 
-static void LZ4_copy8(void* dst, const void* src)
-{
-    memcpy(dst,src,8);
-}
-
 /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
 LZ4_FORCE_O2_INLINE_GCC_PPC64LE
 void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
@@ -283,7 +278,7 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
     const BYTE* s = (const BYTE*)srcPtr;
     BYTE* const e = (BYTE*)dstEnd;
 
-    do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e);
+    do { memcpy(d,s,8); d+=8; s+=8; } while (d<e);
 }
 
 
@@ -1223,9 +1218,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             if ((off >= 8) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
                 size_t const ml = (token & ML_MASK) + MINMATCH;
                 memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
-                memcpy(op+ 0, matchPtr+ 0, 8);
-                memcpy(op+ 8, matchPtr+ 8, 8);
-                memcpy(op+16, matchPtr+16, 2);
+                memcpy(op + 0, matchPtr + 0, 8);
+                memcpy(op + 8, matchPtr + 8, 8);
+                memcpy(op +16, matchPtr +16, 2);
                 op += ml;
                 continue;
             }
@@ -1316,7 +1311,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             match += inc32table[offset];
             memcpy(op+4, match, 4);
             match -= dec64table[offset];
-        } else { LZ4_copy8(op, match); match+=8; }
+        } else { memcpy(op, match, 8); match+=8; }
         op += 8;
 
         if (unlikely(cpy>oend-12)) {
@@ -1329,7 +1324,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             }
             while (op<cpy) *op++ = *match++;
         } else {
-            LZ4_copy8(op, match);
+            memcpy(op, match, 8);
             if (length>16) LZ4_wildCopy(op+8, match+8, cpy);
         }
         op = cpy;   /* correction */

From d3a13397d921d9d2ebae56d0503d23c73fee39b1 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 11 Feb 2018 21:03:39 -0800
Subject: [PATCH 025/257] slight hc speed benefit (~+1%)

by optimizing countback
---
 lib/lz4hc.c | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9f59e80b6..c888ee189 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -123,10 +123,12 @@ LZ4_FORCE_INLINE
 int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
                     const BYTE* const iMin, const BYTE* const mMin)
 {
-    int back=0;
-    while ( (ip+back > iMin)
-         && (match+back > mMin)
-         && (ip[back-1] == match[back-1]))
+    int back = 0;
+    int const min = (int)MAX(iMin - ip, mMin - match);
+    assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31));
+    assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31));
+    while ( (back > min)
+         && (ip[back-1] == match[back-1]) )
             back--;
     return back;
 }
@@ -223,17 +225,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-    #if 0
-                    /* more generic but unfortunately slower on clang */
-                    int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr);
-    #else
-                    int back = 0;
-                    while ( (ip+back > iLowLimit)
-                         && (matchPtr+back > lowPrefixPtr)
-                         && (ip[back-1] == matchPtr[back-1])) {
-                            back--;
-                    }
-    #endif
+                    int const back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
                     mlt -= back;
 
                     if (mlt > longest) {
@@ -252,10 +244,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
                     mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
-                while ( (ip+back > iLowLimit)
-                     && (matchIndex+back > lowLimit)
-                     && (ip[back-1] == matchPtr[back-1]))
-                        back--;
+                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictBase+lowLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;
@@ -333,7 +322,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     size_t length;
     BYTE* const token = (*op)++;
 
-#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6)
     static const BYTE* start = NULL;
     static U32 totalCost = 0;
     U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start);
@@ -343,7 +332,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = *anchor;  /* only works for single segment */
     //g_debuglog_enable = (pos >= 2228) & (pos <= 2262);
-    DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
+    DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
                 pos,
                 (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
                 cost, totalCost);

From b202c672344e9676ebcadeee8506f767795611be Mon Sep 17 00:00:00 2001
From: hobomind <ak160226@yandex.ru>
Date: Wed, 14 Feb 2018 18:47:56 +0300
Subject: [PATCH 026/257] fix: missed semicolon at programs/lz4io.c:954

---
 programs/lz4io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 927928ad1..c712fe1e4 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -951,7 +951,7 @@ static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigne
         total += readBytes;
         storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips);
     }
-    if (ferror(finput)) EXM_THROW(51, "Read Error")
+    if (ferror(finput)) EXM_THROW(51, "Read Error");
 
     LZ4IO_fwriteSparseEnd(foutput, storedSkips);
     return total;

From d74f07974892aa5e8d1adc28b8ad9058e06fcadf Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 18 Feb 2018 11:00:33 -0800
Subject: [PATCH 027/257] update API doc regarding double-buffer strategy

answering question #473
---
 doc/lz4_manual.html | 4 +++-
 lib/lz4.h           | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index ef715f874..b5a40426d 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -178,7 +178,9 @@ <h1>1.8.1 Manual</h1>
 
   Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
               If less than 64KB has been compressed all the data must be present.
-  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+  Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
+              Make sure that buffers are separated by at least one byte.
+              This way, rule becomes simple : each block depends on previous block only.
   Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
  @return : size of compressed block
diff --git a/lib/lz4.h b/lib/lz4.h
index c5d50e2ce..08f06c71d 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -266,7 +266,9 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  *
  *  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
  *              If less than 64KB has been compressed all the data must be present.
- *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+ *  Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
+ *              Make sure that buffers are separated by at least one byte.
+ *              This way, rule becomes simple : each block depends on previous block only.
  *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
  * @return : size of compressed block

From 1a233c5f0fb665fa6b65c856b150bdef92654b42 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 11:37:19 -0800
Subject: [PATCH 028/257] update bench.c to use less time invocations

translating into more accurate speed measurements for small sources
---
 programs/bench.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index fac2a8778..91292bc8f 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -41,6 +41,7 @@
 #include <string.h>      /* memset */
 #include <stdio.h>       /* fprintf, fopen, ftello */
 #include <time.h>        /* clock_t, clock, CLOCKS_PER_SEC */
+#include <assert.h>      /* assert */
 
 #include "datagen.h"     /* RDG_genBuffer */
 #include "xxhash.h"
@@ -219,6 +220,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
         U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
+        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative decode speed estimate */
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4
@@ -279,13 +281,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             /* Decompression */
             if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
 
-            UTIL_sleepMilli(1); /* give processor time to other processes */
+            UTIL_sleepMilli(5); /* give processor time to other processes */
             UTIL_waitForNextTick();
-            clockStart = UTIL_getTime();
 
             if (!dCompleted) {
-                U32 nbLoops = 0;
-                do {
+                U32 nbLoops;
+                clockStart = UTIL_getTime();
+                for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
@@ -294,15 +296,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                             clockLoop = 0;   /* force immediate test end */
                             break;
                         }
-
                         blockTable[blockNb].resSize = regenSize;
-                    }
-                    nbLoops++;
-                } while (UTIL_clockSpanMicro(clockStart) < DECOMP_MULT*clockLoop);
+                }   }
                 {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
                     if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
+                    nbDecodeLoops = (U32)(1000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
                     totalDTime += clockSpan;
-                    dCompleted = totalDTime>(DECOMP_MULT*maxTime);
+                    dCompleted = totalDTime > (DECOMP_MULT*maxTime);
             }   }
 
             markNb = (markNb+1) % NB_MARKS;

From ae3dededed1c6c1a9b4c631d9edba690c4abb59c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 13:05:22 -0800
Subject: [PATCH 029/257] ensure bench speed measurement is more accurate for
 small inputs

Previous method would produce too many time() invocations,
becoming a significant fraction of workload measured.

The new strategy is to use time() only once per batch,
and dynamically resize batch size so that each round lasts approximately 1 second.

This only matters for small inputs.
Measurement for large files (such as silesia.tar) are much less impacted
(though decoding speed is so fast that even medium-size files will notice an improvement).
---
 programs/bench.c | 57 +++++++++++++++++++++++++++++-------------------
 programs/util.h  |  8 ++++++-
 2 files changed, 41 insertions(+), 24 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 91292bc8f..0b1cf5c21 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -67,6 +67,7 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
 
 #define NBSECONDS             3
 #define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
+#define TIMELOOP_NANOSEC      1*1000000000ULL /* 1 second */
 #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
 #define COOLPERIOD_SEC        10
 #define DECOMP_MULT           1 /* test decompression DECOMP_MULT times longer than compression */
@@ -219,8 +220,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     {   U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
-        U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
-        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative decode speed estimate */
+        U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
+        U32 nbCompressionLoops = ((5 MB) / (srcSize+1)) + 1;  /* conservative initial compression speed estimate */
+        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative initial decode speed estimate */
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4
@@ -232,9 +234,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         coolTime = UTIL_getTime();
         DISPLAYLEVEL(2, "\r%79s\r", "");
         while (!cCompleted || !dCompleted) {
-            UTIL_time_t clockStart;
-            U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
-
             /* overheat protection */
             if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
                 DISPLAYLEVEL(2, "\rcooling down ...    \r");
@@ -248,21 +247,27 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
 
             UTIL_sleepMilli(1);  /* give processor time to other processes */
             UTIL_waitForNextTick();
-            clockStart = UTIL_getTime();
 
             if (!cCompleted) {   /* still some time to do compression tests */
-                U32 nbLoops = 0;
-                do {
+                UTIL_time_t const clockStart = UTIL_getTime();
+                U32 nbLoops;
+                for (nbLoops=0; nbLoops < nbCompressionLoops; nbLoops++) {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         size_t const rSize = compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
                         if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4_compress() failed");
                         blockTable[blockNb].cSize = rSize;
+                }   }
+                {   U64 const clockSpan = UTIL_clockSpanNano(clockStart);
+                    if (clockSpan > 0) {
+                        if (clockSpan < fastestC * nbCompressionLoops)
+                            fastestC = clockSpan / nbCompressionLoops;
+                        assert(fastestC > 0);
+                        nbCompressionLoops = (U32)(1000000000/*1sec*/ / fastestC) + 1;  /* aim for ~1sec */
+                    } else {
+                        assert(nbCompressionLoops < 40000000);   /* avoid overflow */
+                        nbCompressionLoops *= 100;
                     }
-                    nbLoops++;
-                } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
-                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
-                    if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
                     totalCTime += clockSpan;
                     cCompleted = totalCTime>maxTime;
             }   }
@@ -274,7 +279,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
                     marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
-                    (double)srcSize / fastestC );
+                    ((double)srcSize / fastestC) * 1000 );
 
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
 #if 1
@@ -285,22 +290,28 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             UTIL_waitForNextTick();
 
             if (!dCompleted) {
+                UTIL_time_t const clockStart = UTIL_getTime();
                 U32 nbLoops;
-                clockStart = UTIL_getTime();
                 for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
                         if (LZ4_isError(regenSize)) {
-                            DISPLAY("LZ4_decompress_safe() failed on block %u  \n", blockNb);
-                            clockLoop = 0;   /* force immediate test end */
+                            DISPLAY("LZ4_decompress_safe() failed on block %u \n", blockNb);
                             break;
                         }
                         blockTable[blockNb].resSize = regenSize;
                 }   }
-                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
-                    if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
-                    nbDecodeLoops = (U32)(1000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
+                {   U64 const clockSpan = UTIL_clockSpanNano(clockStart);
+                    if (clockSpan > 0) {
+                        if (clockSpan < fastestD * nbDecodeLoops)
+                            fastestD = clockSpan / nbDecodeLoops;
+                        assert(fastestD > 0);
+                        nbDecodeLoops = (U32)(1000000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
+                    } else {
+                        assert(nbDecodeLoops < 40000000);   /* avoid overflow */
+                        nbDecodeLoops *= 100;
+                    }
                     totalDTime += clockSpan;
                     dCompleted = totalDTime > (DECOMP_MULT*maxTime);
             }   }
@@ -308,8 +319,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
                     marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
-                    (double)srcSize / fastestC,
-                    (double)srcSize / fastestD );
+                    ((double)srcSize / fastestC) * 1000,
+                    ((double)srcSize / fastestD) * 1000);
 
             /* CRC Checking */
             {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -339,8 +350,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         }   /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
 
         if (g_displayLevel == 1) {
-            double cSpeed = (double)srcSize / fastestC;
-            double dSpeed = (double)srcSize / fastestD;
+            double const cSpeed = ((double)srcSize / fastestC) * 1000;
+            double const dSpeed = ((double)srcSize / fastestD) * 1000;
             if (g_additionalParam)
                 DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
             else
diff --git a/programs/util.h b/programs/util.h
index a3576d7e3..ff2510625 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -180,7 +180,7 @@ extern "C" {
             mach_timebase_info(&rate);
             init = 1;
         }
-        return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
+        return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom)) / 1000ULL;
     }
     UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
     {
@@ -249,6 +249,12 @@ UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart)
     return UTIL_getSpanTimeMicro(clockStart, clockEnd);
 }
 
+/* returns time span in nanoseconds */
+UTIL_STATIC U64 UTIL_clockSpanNano(UTIL_time_t clockStart)
+{
+    UTIL_time_t const clockEnd = UTIL_getTime();
+    return UTIL_getSpanTimeNano(clockStart, clockEnd);
+}
 
 UTIL_STATIC void UTIL_waitForNextTick(void)
 {

From 34c1634d4bb8058909a736c284e889e5c0d7f146 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 14:13:13 -0800
Subject: [PATCH 030/257] fixed minor conversion warning

---
 programs/bench.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 0b1cf5c21..14b6dd18f 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -221,8 +221,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
         U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
-        U32 nbCompressionLoops = ((5 MB) / (srcSize+1)) + 1;  /* conservative initial compression speed estimate */
-        U32 nbDecodeLoops = ((200 MB) / (srcSize+1)) + 1;  /* conservative initial decode speed estimate */
+        U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1;  /* conservative initial compression speed estimate */
+        U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1;  /* conservative initial decode speed estimate */
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4

From 25b16e8a2e51c41f5864ce5cc94ce65ab6fd52b9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 15:25:45 -0800
Subject: [PATCH 031/257] added one assert()

suggested by @terrelln
---
 lib/lz4hc.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c888ee189..f3631c596 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -135,7 +135,8 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
 
 /* LZ4HC_countPattern() :
  * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */
-static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
+static unsigned
+LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
 {
     const BYTE* const iStart = ip;
     reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32;
@@ -167,7 +168,8 @@ static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 c
 /* LZ4HC_reverseCountPattern() :
  * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!)
  * read using natural platform endianess */
-static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
+static unsigned
+LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
 {
     const BYTE* const iStart = ip;
 
@@ -185,7 +187,8 @@ static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow
 
 typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
 
-LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
+LZ4_FORCE_INLINE int
+LZ4HC_InsertAndGetWiderMatch (
     LZ4HC_CCtx_internal* hc4,
     const BYTE* const ip,
     const BYTE* const iLowLimit,
@@ -222,6 +225,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
+            assert(longest >= 1);
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);

From 179670f32f4960f00a89c7ef08382b06dd024873 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 20 Feb 2018 15:26:59 -0800
Subject: [PATCH 032/257] use TIMELOOP_NANOSEC,

as suggested by @terrelln
---
 programs/bench.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 14b6dd18f..002eac990 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -263,7 +263,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         if (clockSpan < fastestC * nbCompressionLoops)
                             fastestC = clockSpan / nbCompressionLoops;
                         assert(fastestC > 0);
-                        nbCompressionLoops = (U32)(1000000000/*1sec*/ / fastestC) + 1;  /* aim for ~1sec */
+                        nbCompressionLoops = (U32)(TIMELOOP_NANOSEC / fastestC) + 1;  /* aim for ~1sec */
                     } else {
                         assert(nbCompressionLoops < 40000000);   /* avoid overflow */
                         nbCompressionLoops *= 100;
@@ -307,7 +307,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         if (clockSpan < fastestD * nbDecodeLoops)
                             fastestD = clockSpan / nbDecodeLoops;
                         assert(fastestD > 0);
-                        nbDecodeLoops = (U32)(1000000000/*1sec*/ / fastestD) + 1;  /* aim for ~1sec */
+                        nbDecodeLoops = (U32)(TIMELOOP_NANOSEC / fastestD) + 1;  /* aim for ~1sec */
                     } else {
                         assert(nbDecodeLoops < 40000000);   /* avoid overflow */
                         nbDecodeLoops *= 100;

From 7173a631db61ab9535bd0d6e5e00e9dc081d4df3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 24 Feb 2018 11:47:53 -0800
Subject: [PATCH 033/257] edge case : compress up to end-mflimit (12 bytes)

The LZ4 block format specification
states that the last match must start
at a minimum distance of 12 bytes from the end of the block.

However, out of an abundance of caution,
the reference implementation would actually stop searching matches
at 13 bytes from the end of the block.

This patch fixes this small detail.
The new version is now able to properly compress a limit case
such as `aaaaaaaabaaa\n`
as reported by Gao Xiang (@hsiangkao).

Obviously, it doesn't change a lot of things.
This is just one additional match candidate per block, with a maximum match length of 7 (since last 5 bytes must remain literals).

With default policy, blocks are 4 MB long, so it doesn't happen too often
Compressing silesia.tar at default level 1 saves 5 bytes (100930101 -> 100930096).
At max level 12, it saves a grand 16 bytes (77389871 -> 77389855).

The impact is a bit more visible when blocks are smaller, hence more numerous.
For example, compressing silesia with blocks of 64 KB (using -12 -B4D) saves 543 bytes (77304583 -> 77304040).
So the smaller the packet size, the more visible the impact.

And it happens we have a ton of scenarios with little blocks using LZ4 compression ...

And a useless "hooray" sidenote :
the patch improves the LZ4 compression record of silesia (using -12 -B7D --no-frame-crc) by 16 bytes (77270672 -> 77270656)
and the record on enwik9 by 44 bytes (371680396 -> 371680352) (previously claimed by [smallz4](http://create.stephan-brumme.com/smallz4/) ).
---
 lib/lz4.c    | 7 ++++---
 lib/lz4hc.c  | 6 +++---
 lib/lz4opt.h | 6 +++---
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 5d4bb21cc..38a865f32 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -545,7 +545,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
-    const BYTE* const mflimit = iend - MFLIMIT;
+    const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1;
     const BYTE* const matchlimit = iend - LASTLITERALS;
 
     BYTE* op = (BYTE*) dest;
@@ -594,7 +594,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardIp += step;
                 step = (searchMatchNb++ >> LZ4_skipTrigger);
 
-                if (unlikely(forwardIp > mflimit)) goto _last_literals;
+                if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
+                assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
                 if (dict==usingExtDict) {
@@ -680,7 +681,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         anchor = ip;
 
         /* Test end of chunk */
-        if (ip > mflimit) break;
+        if (ip >= mflimitPlusOne) break;
 
         /* Fill table */
         LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f3631c596..726cfaa52 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -425,7 +425,7 @@ static int LZ4HC_compress_hashChain (
     if (inputSize < LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* Main Loop */
-    while (ip < mflimit) {
+    while (ip <= mflimit) {
         ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
         if (ml<MINMATCH) { ip++; continue; }
 
@@ -435,7 +435,7 @@ static int LZ4HC_compress_hashChain (
         ml0 = ml;
 
 _Search2:
-        if (ip+ml < mflimit)
+        if (ip+ml <= mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
                             maxNbAttempts, patternAnalysis);
@@ -482,7 +482,7 @@ static int LZ4HC_compress_hashChain (
         }
         /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
 
-        if (start2 + ml2 < mflimit)
+        if (start2 + ml2 <= mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
                             maxNbAttempts, patternAnalysis);
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 5a8438c1e..9bd482246 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -127,7 +127,7 @@ static int LZ4HC_compress_optimal (
 
     /* Main Loop */
     assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
-    while (ip < mflimit) {
+    while (ip <= mflimit) {
         int const llen = (int)(ip - anchor);
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
@@ -186,7 +186,7 @@ static int LZ4HC_compress_optimal (
             const BYTE* const curPtr = ip + cur;
             LZ4HC_match_t newMatch;
 
-            if (curPtr >= mflimit) break;
+            if (curPtr > mflimit) break;
             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
                     cur, opt[cur].price, opt[cur+1].price, cur+1);
             if (fullUpdate) {
@@ -314,7 +314,7 @@ static int LZ4HC_compress_optimal (
                 if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
                     goto _dest_overflow;
         }   }
-    }  /* while (ip < mflimit) */
+    }  /* while (ip <= mflimit) */
 
 _last_literals:
     /* Encode Last Literals */

From 550b40849f6b77850453a1190b97430b121802bc Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 25 Feb 2018 00:32:09 -0800
Subject: [PATCH 034/257] merge lz4opt.h into lz4hc.c

Having a dedicated file for optimal parser
made sense during its creation,
it allowed Przemyslaw to work more freely on lz4opt, with less dependency on lz4hc,
moreover, the optimal parser was more complex, with its own search functions.

Since the optimal was rewritten last year, it's now a lot lighter.
It makes more sense now to integrate it directly inside lz4hc.c,
making it easier to edit (editors are a bit "lost" inside a `*.h` dependent on its #include position),
it also reduces the number of files in the project,
which fits pretty well with lz4 objectives.
(adding lz4hc requires "just" lz4hc.h and lz4hc.c).
---
 examples/Makefile |   2 +-
 lib/README.md     |   2 +-
 lib/lz4hc.c       | 331 +++++++++++++++++++++++++++++++++++++++++-
 lib/lz4opt.h      | 356 ----------------------------------------------
 4 files changed, 329 insertions(+), 362 deletions(-)
 delete mode 100644 lib/lz4opt.h

diff --git a/examples/Makefile b/examples/Makefile
index f9e9e7a37..103e7eccf 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -52,7 +52,7 @@ default: all
 all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
      lineCompress frameCompress simpleBuffer
 
-$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4opt.h $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
+$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
 	$(MAKE) -C $(LZ4DIR) liblz4.a
 
 printVersion: printVersion.c $(LZ4DIR)/liblz4.a
diff --git a/lib/README.md b/lib/README.md
index fc5d4e9b6..7082fe3ee 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -15,7 +15,7 @@ They generate and decode data using [LZ4 block format].
 
 For more compression ratio at the cost of compression speed,
 the High Compression variant called **lz4hc** is available.
-Add files **`lz4hc.c`**, **`lz4hc.h`** and **`lz4opt.h`**.
+Add files **`lz4hc.c`** and **`lz4hc.h`**.
 The variant still depends on regular `lib/lz4.*` source files.
 
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 726cfaa52..2a6e080ba 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -67,6 +67,7 @@
 
 /*===   Constants   ===*/
 #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
+#define LZ4_OPT_NUM   (1<<12)
 
 
 /*===   Macros   ===*/
@@ -383,10 +384,6 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     return 0;
 }
 
-/* btopt */
-#include "lz4opt.h"
-
-
 static int LZ4HC_compress_hashChain (
     LZ4HC_CCtx_internal* const ctx,
     const char* const source,
@@ -605,6 +602,12 @@ static int LZ4HC_compress_hashChain (
     return 0;
 }
 
+static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
+    const char* const source, char* dst,
+    int* srcSizePtr, int dstCapacity,
+    int const nbSearches, size_t sufficient_len,
+    limitedOutput_directive limit, int const fullUpdate);
+
 
 static int LZ4HC_compress_generic (
     LZ4HC_CCtx_internal* const ctx,
@@ -884,3 +887,323 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
     int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
     return (char*)(hc4->inputBuffer + dictSize);
 }
+
+
+/* ================================================
+ * LZ4 Optimal parser (levels 10-12)
+ * ===============================================*/
+typedef struct {
+    int price;
+    int off;
+    int mlen;
+    int litlen;
+} LZ4HC_optimal_t;
+
+/* price in bytes */
+LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
+{
+    int price = litlen;
+    if (litlen >= (int)RUN_MASK)
+        price += 1 + (litlen-RUN_MASK)/255;
+    return price;
+}
+
+
+/* requires mlen >= MINMATCH */
+LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
+{
+    int price = 1 + 2 ; /* token + 16-bit offset */
+
+    price += LZ4HC_literalsPrice(litlen);
+
+    if (mlen >= (int)(ML_MASK+MINMATCH))
+        price += 1 + (mlen-(ML_MASK+MINMATCH))/255;
+
+    return price;
+}
+
+
+typedef struct {
+    int off;
+    int len;
+} LZ4HC_match_t;
+
+LZ4_FORCE_INLINE LZ4HC_match_t
+LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
+                      const BYTE* ip, const BYTE* const iHighLimit,
+                      int minLen, int nbSearches)
+{
+    LZ4HC_match_t match = { 0 , 0 };
+    const BYTE* matchPtr = NULL;
+    /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
+     * but this won't be the case here, as we define iLowLimit==ip,
+     * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
+    int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
+                                ip, ip, iHighLimit, minLen, &matchPtr, &ip,
+                                nbSearches, 1 /* patternAnalysis */);
+    if (matchLength <= minLen) return match;
+    match.len = matchLength;
+    match.off = (int)(ip-matchPtr);
+    return match;
+}
+
+
+static int LZ4HC_compress_optimal (
+    LZ4HC_CCtx_internal* ctx,
+    const char* const source,
+    char* dst,
+    int* srcSizePtr,
+    int dstCapacity,
+    int const nbSearches,
+    size_t sufficient_len,
+    limitedOutput_directive limit,
+    int const fullUpdate
+    )
+{
+#define TRAILING_LITERALS 3
+    LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS];   /* ~64 KB, which is a bit large for stack... */
+
+    const BYTE* ip = (const BYTE*) source;
+    const BYTE* anchor = ip;
+    const BYTE* const iend = ip + *srcSizePtr;
+    const BYTE* const mflimit = iend - MFLIMIT;
+    const BYTE* const matchlimit = iend - LASTLITERALS;
+    BYTE* op = (BYTE*) dst;
+    BYTE* opSaved = (BYTE*) dst;
+    BYTE* oend = op + dstCapacity;
+
+    /* init */
+    DEBUGLOG(5, "LZ4HC_compress_optimal");
+    *srcSizePtr = 0;
+    if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
+    if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
+
+    /* Main Loop */
+    assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
+    while (ip <= mflimit) {
+         int const llen = (int)(ip - anchor);
+         int best_mlen, best_off;
+         int cur, last_match_pos = 0;
+
+         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
+         if (firstMatch.len==0) { ip++; continue; }
+
+         if ((size_t)firstMatch.len > sufficient_len) {
+             /* good enough solution : immediate encoding */
+             int const firstML = firstMatch.len;
+             const BYTE* const matchPos = ip - firstMatch.off;
+             opSaved = op;
+             if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
+                 goto _dest_overflow;
+             continue;
+         }
+
+         /* set prices for first positions (literals) */
+         {   int rPos;
+             for (rPos = 0 ; rPos < MINMATCH ; rPos++) {
+                 int const cost = LZ4HC_literalsPrice(llen + rPos);
+                 opt[rPos].mlen = 1;
+                 opt[rPos].off = 0;
+                 opt[rPos].litlen = llen + rPos;
+                 opt[rPos].price = cost;
+                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
+                             rPos, cost, opt[rPos].litlen);
+         }   }
+         /* set prices using initial match */
+         {   int mlen = MINMATCH;
+             int const matchML = firstMatch.len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
+             int const offset = firstMatch.off;
+             assert(matchML < LZ4_OPT_NUM);
+             for ( ; mlen <= matchML ; mlen++) {
+                 int const cost = LZ4HC_sequencePrice(llen, mlen);
+                 opt[mlen].mlen = mlen;
+                 opt[mlen].off = offset;
+                 opt[mlen].litlen = llen;
+                 opt[mlen].price = cost;
+                 DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
+                             mlen, cost, mlen);
+         }   }
+         last_match_pos = firstMatch.len;
+         {   int addLit;
+             for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
+                 opt[last_match_pos+addLit].mlen = 1; /* literal */
+                 opt[last_match_pos+addLit].off = 0;
+                 opt[last_match_pos+addLit].litlen = addLit;
+                 opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
+                             last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
+         }   }
+
+         /* check further positions */
+         for (cur = 1; cur < last_match_pos; cur++) {
+             const BYTE* const curPtr = ip + cur;
+             LZ4HC_match_t newMatch;
+
+             if (curPtr > mflimit) break;
+             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
+                     cur, opt[cur].price, opt[cur+1].price, cur+1);
+             if (fullUpdate) {
+                 /* not useful to search here if next position has same (or lower) cost */
+                 if ( (opt[cur+1].price <= opt[cur].price)
+                   /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */
+                   && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) )
+                     continue;
+             } else {
+                 /* not useful to search here if next position has same (or lower) cost */
+                 if (opt[cur+1].price <= opt[cur].price) continue;
+             }
+
+             DEBUGLOG(7, "search at rPos:%u", cur);
+             if (fullUpdate)
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
+             else
+                 /* only test matches of minimum length; slightly faster, but misses a few bytes */
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
+             if (!newMatch.len) continue;
+
+             if ( ((size_t)newMatch.len > sufficient_len)
+               || (newMatch.len + cur >= LZ4_OPT_NUM) ) {
+                 /* immediate encoding */
+                 best_mlen = newMatch.len;
+                 best_off = newMatch.off;
+                 last_match_pos = cur + 1;
+                 goto encode;
+             }
+
+             /* before match : set price with literals at beginning */
+             {   int const baseLitlen = opt[cur].litlen;
+                 int litlen;
+                 for (litlen = 1; litlen < MINMATCH; litlen++) {
+                     int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen);
+                     int const pos = cur + litlen;
+                     if (price < opt[pos].price) {
+                         opt[pos].mlen = 1; /* literal */
+                         opt[pos].off = 0;
+                         opt[pos].litlen = baseLitlen+litlen;
+                         opt[pos].price = price;
+                         DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
+                                     pos, price, opt[pos].litlen);
+             }   }   }
+
+             /* set prices using match at position = cur */
+             {   int const matchML = newMatch.len;
+                 int ml = MINMATCH;
+
+                 assert(cur + newMatch.len < LZ4_OPT_NUM);
+                 for ( ; ml <= matchML ; ml++) {
+                     int const pos = cur + ml;
+                     int const offset = newMatch.off;
+                     int price;
+                     int ll;
+                     DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
+                                 pos, last_match_pos);
+                     if (opt[cur].mlen == 1) {
+                         ll = opt[cur].litlen;
+                         price = ((cur > ll) ? opt[cur - ll].price : 0)
+                               + LZ4HC_sequencePrice(ll, ml);
+                     } else {
+                         ll = 0;
+                         price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
+                     }
+
+                     if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
+                         DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
+                                     pos, price, ml);
+                         assert(pos < LZ4_OPT_NUM);
+                         if ( (ml == matchML)  /* last pos of last match */
+                           && (last_match_pos < pos) )
+                             last_match_pos = pos;
+                         opt[pos].mlen = ml;
+                         opt[pos].off = offset;
+                         opt[pos].litlen = ll;
+                         opt[pos].price = price;
+             }   }   }
+             /* complete following positions with literals */
+             {   int addLit;
+                 for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
+                     opt[last_match_pos+addLit].mlen = 1; /* literal */
+                     opt[last_match_pos+addLit].off = 0;
+                     opt[last_match_pos+addLit].litlen = addLit;
+                     opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                     DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
+             }   }
+         }  /* for (cur = 1; cur <= last_match_pos; cur++) */
+
+         best_mlen = opt[last_match_pos].mlen;
+         best_off = opt[last_match_pos].off;
+         cur = last_match_pos - best_mlen;
+
+ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
+         assert(cur < LZ4_OPT_NUM);
+         assert(last_match_pos >= 1);  /* == 1 when only one candidate */
+         DEBUGLOG(6, "reverse traversal, looking for shortest path")
+         DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
+         {   int candidate_pos = cur;
+             int selected_matchLength = best_mlen;
+             int selected_offset = best_off;
+             while (1) {  /* from end to beginning */
+                 int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
+                 int const next_offset = opt[candidate_pos].off;
+                 DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
+                 opt[candidate_pos].mlen = selected_matchLength;
+                 opt[candidate_pos].off = selected_offset;
+                 selected_matchLength = next_matchLength;
+                 selected_offset = next_offset;
+                 if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
+                 assert(next_matchLength > 0);  /* can be 1, means literal */
+                 candidate_pos -= next_matchLength;
+         }   }
+
+         /* encode all recorded sequences in order */
+         {   int rPos = 0;  /* relative position (to ip) */
+             while (rPos < last_match_pos) {
+                 int const ml = opt[rPos].mlen;
+                 int const offset = opt[rPos].off;
+                 if (ml == 1) { ip++; rPos++; continue; }  /* literal; note: can end up with several literals, in which case, skip them */
+                 rPos += ml;
+                 assert(ml >= MINMATCH);
+                 assert((offset >= 1) && (offset <= MAX_DISTANCE));
+                 opSaved = op;
+                 if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
+                     goto _dest_overflow;
+         }   }
+     }  /* while (ip <= mflimit) */
+
+ _last_literals:
+     /* Encode Last Literals */
+     {   size_t lastRunSize = (size_t)(iend - anchor);  /* literals */
+         size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+         size_t const totalSize = 1 + litLength + lastRunSize;
+         if (limit == limitedDestSize) oend += LASTLITERALS;  /* restore correct value */
+         if (limit && (op + totalSize > oend)) {
+             if (limit == limitedOutput) return 0;  /* Check output limit */
+             /* adapt lastRunSize to fill 'dst' */
+             lastRunSize  = (size_t)(oend - op) - 1;
+             litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+             lastRunSize -= litLength;
+         }
+         ip = anchor + lastRunSize;
+
+         if (lastRunSize >= RUN_MASK) {
+             size_t accumulator = lastRunSize - RUN_MASK;
+             *op++ = (RUN_MASK << ML_BITS);
+             for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
+             *op++ = (BYTE) accumulator;
+         } else {
+             *op++ = (BYTE)(lastRunSize << ML_BITS);
+         }
+         memcpy(op, anchor, lastRunSize);
+         op += lastRunSize;
+     }
+
+     /* End */
+     *srcSizePtr = (int) (((const char*)ip) - source);
+     return (int) ((char*)op-dst);
+
+ _dest_overflow:
+     if (limit == limitedDestSize) {
+         op = opSaved;  /* restore correct out pointer */
+         goto _last_literals;
+     }
+     return 0;
+ }
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
deleted file mode 100644
index 9bd482246..000000000
--- a/lib/lz4opt.h
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
-    lz4opt.h - Optimal Mode of LZ4
-    Copyright (C) 2015-2017, Przemyslaw Skibinski <inikep@gmail.com>
-    Note : this file is intended to be included within lz4hc.c
-
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-       - LZ4 source repository : https://github.com/lz4/lz4
-       - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-
-#define LZ4_OPT_NUM   (1<<12)
-
-typedef struct {
-    int price;
-    int off;
-    int mlen;
-    int litlen;
-} LZ4HC_optimal_t;
-
-
-/* price in bytes */
-LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
-{
-    int price = litlen;
-    if (litlen >= (int)RUN_MASK)
-        price += 1 + (litlen-RUN_MASK)/255;
-    return price;
-}
-
-
-/* requires mlen >= MINMATCH */
-LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
-{
-    int price = 1 + 2 ; /* token + 16-bit offset */
-
-    price += LZ4HC_literalsPrice(litlen);
-
-    if (mlen >= (int)(ML_MASK+MINMATCH))
-        price += 1 + (mlen-(ML_MASK+MINMATCH))/255;
-
-    return price;
-}
-
-
-/*-*************************************
-*  Match finder
-***************************************/
-typedef struct {
-    int off;
-    int len;
-} LZ4HC_match_t;
-
-LZ4_FORCE_INLINE
-LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
-                        const BYTE* ip, const BYTE* const iHighLimit,
-                        int minLen, int nbSearches)
-{
-    LZ4HC_match_t match = { 0 , 0 };
-    const BYTE* matchPtr = NULL;
-    /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
-     * but this won't be the case here, as we define iLowLimit==ip,
-     * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
-                                ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */);
-    if (matchLength <= minLen) return match;
-    match.len = matchLength;
-    match.off = (int)(ip-matchPtr);
-    return match;
-}
-
-
-static int LZ4HC_compress_optimal (
-    LZ4HC_CCtx_internal* ctx,
-    const char* const source,
-    char* dst,
-    int* srcSizePtr,
-    int dstCapacity,
-    int const nbSearches,
-    size_t sufficient_len,
-    limitedOutput_directive limit,
-    int const fullUpdate
-    )
-{
-#define TRAILING_LITERALS 3
-    LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS];   /* this uses a bit too much stack memory to my taste ... */
-
-    const BYTE* ip = (const BYTE*) source;
-    const BYTE* anchor = ip;
-    const BYTE* const iend = ip + *srcSizePtr;
-    const BYTE* const mflimit = iend - MFLIMIT;
-    const BYTE* const matchlimit = iend - LASTLITERALS;
-    BYTE* op = (BYTE*) dst;
-    BYTE* opSaved = (BYTE*) dst;
-    BYTE* oend = op + dstCapacity;
-
-    /* init */
-    DEBUGLOG(5, "LZ4HC_compress_optimal");
-    *srcSizePtr = 0;
-    if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
-    if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
-
-    /* Main Loop */
-    assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
-    while (ip <= mflimit) {
-        int const llen = (int)(ip - anchor);
-        int best_mlen, best_off;
-        int cur, last_match_pos = 0;
-
-        LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
-        if (firstMatch.len==0) { ip++; continue; }
-
-        if ((size_t)firstMatch.len > sufficient_len) {
-            /* good enough solution : immediate encoding */
-            int const firstML = firstMatch.len;
-            const BYTE* const matchPos = ip - firstMatch.off;
-            opSaved = op;
-            if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
-                goto _dest_overflow;
-            continue;
-        }
-
-        /* set prices for first positions (literals) */
-        {   int rPos;
-            for (rPos = 0 ; rPos < MINMATCH ; rPos++) {
-                int const cost = LZ4HC_literalsPrice(llen + rPos);
-                opt[rPos].mlen = 1;
-                opt[rPos].off = 0;
-                opt[rPos].litlen = llen + rPos;
-                opt[rPos].price = cost;
-                DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
-                            rPos, cost, opt[rPos].litlen);
-        }   }
-        /* set prices using initial match */
-        {   int mlen = MINMATCH;
-            int const matchML = firstMatch.len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
-            int const offset = firstMatch.off;
-            assert(matchML < LZ4_OPT_NUM);
-            for ( ; mlen <= matchML ; mlen++) {
-                int const cost = LZ4HC_sequencePrice(llen, mlen);
-                opt[mlen].mlen = mlen;
-                opt[mlen].off = offset;
-                opt[mlen].litlen = llen;
-                opt[mlen].price = cost;
-                DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
-                            mlen, cost, mlen);
-        }   }
-        last_match_pos = firstMatch.len;
-        {   int addLit;
-            for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
-                opt[last_match_pos+addLit].mlen = 1; /* literal */
-                opt[last_match_pos+addLit].off = 0;
-                opt[last_match_pos+addLit].litlen = addLit;
-                opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
-                DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
-                            last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
-        }   }
-
-        /* check further positions */
-        for (cur = 1; cur < last_match_pos; cur++) {
-            const BYTE* const curPtr = ip + cur;
-            LZ4HC_match_t newMatch;
-
-            if (curPtr > mflimit) break;
-            DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
-                    cur, opt[cur].price, opt[cur+1].price, cur+1);
-            if (fullUpdate) {
-                /* not useful to search here if next position has same (or lower) cost */
-                if ( (opt[cur+1].price <= opt[cur].price)
-                  /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */
-                  && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) )
-                    continue;
-            } else {
-                /* not useful to search here if next position has same (or lower) cost */
-                if (opt[cur+1].price <= opt[cur].price) continue;
-            }
-
-            DEBUGLOG(7, "search at rPos:%u", cur);
-            if (fullUpdate)
-                newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
-            else
-                /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
-            if (!newMatch.len) continue;
-
-            if ( ((size_t)newMatch.len > sufficient_len)
-              || (newMatch.len + cur >= LZ4_OPT_NUM) ) {
-                /* immediate encoding */
-                best_mlen = newMatch.len;
-                best_off = newMatch.off;
-                last_match_pos = cur + 1;
-                goto encode;
-            }
-
-            /* before match : set price with literals at beginning */
-            {   int const baseLitlen = opt[cur].litlen;
-                int litlen;
-                for (litlen = 1; litlen < MINMATCH; litlen++) {
-                    int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen);
-                    int const pos = cur + litlen;
-                    if (price < opt[pos].price) {
-                        opt[pos].mlen = 1; /* literal */
-                        opt[pos].off = 0;
-                        opt[pos].litlen = baseLitlen+litlen;
-                        opt[pos].price = price;
-                        DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
-                                    pos, price, opt[pos].litlen);
-            }   }   }
-
-            /* set prices using match at position = cur */
-            {   int const matchML = newMatch.len;
-                int ml = MINMATCH;
-
-                assert(cur + newMatch.len < LZ4_OPT_NUM);
-                for ( ; ml <= matchML ; ml++) {
-                    int const pos = cur + ml;
-                    int const offset = newMatch.off;
-                    int price;
-                    int ll;
-                    DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
-                                pos, last_match_pos);
-                    if (opt[cur].mlen == 1) {
-                        ll = opt[cur].litlen;
-                        price = ((cur > ll) ? opt[cur - ll].price : 0)
-                              + LZ4HC_sequencePrice(ll, ml);
-                    } else {
-                        ll = 0;
-                        price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
-                    }
-
-                    if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
-                        DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
-                                    pos, price, ml);
-                        assert(pos < LZ4_OPT_NUM);
-                        if ( (ml == matchML)  /* last pos of last match */
-                          && (last_match_pos < pos) )
-                            last_match_pos = pos;
-                        opt[pos].mlen = ml;
-                        opt[pos].off = offset;
-                        opt[pos].litlen = ll;
-                        opt[pos].price = price;
-            }   }   }
-            /* complete following positions with literals */
-            {   int addLit;
-                for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
-                    opt[last_match_pos+addLit].mlen = 1; /* literal */
-                    opt[last_match_pos+addLit].off = 0;
-                    opt[last_match_pos+addLit].litlen = addLit;
-                    opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
-                    DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
-            }   }
-        }  /* for (cur = 1; cur <= last_match_pos; cur++) */
-
-        best_mlen = opt[last_match_pos].mlen;
-        best_off = opt[last_match_pos].off;
-        cur = last_match_pos - best_mlen;
-
-encode: /* cur, last_match_pos, best_mlen, best_off must be set */
-        assert(cur < LZ4_OPT_NUM);
-        assert(last_match_pos >= 1);  /* == 1 when only one candidate */
-        DEBUGLOG(6, "reverse traversal, looking for shortest path")
-        DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
-        {   int candidate_pos = cur;
-            int selected_matchLength = best_mlen;
-            int selected_offset = best_off;
-            while (1) {  /* from end to beginning */
-                int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
-                int const next_offset = opt[candidate_pos].off;
-                DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
-                opt[candidate_pos].mlen = selected_matchLength;
-                opt[candidate_pos].off = selected_offset;
-                selected_matchLength = next_matchLength;
-                selected_offset = next_offset;
-                if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
-                assert(next_matchLength > 0);  /* can be 1, means literal */
-                candidate_pos -= next_matchLength;
-        }   }
-
-        /* encode all recorded sequences in order */
-        {   int rPos = 0;  /* relative position (to ip) */
-            while (rPos < last_match_pos) {
-                int const ml = opt[rPos].mlen;
-                int const offset = opt[rPos].off;
-                if (ml == 1) { ip++; rPos++; continue; }  /* literal; note: can end up with several literals, in which case, skip them */
-                rPos += ml;
-                assert(ml >= MINMATCH);
-                assert((offset >= 1) && (offset <= MAX_DISTANCE));
-                opSaved = op;
-                if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
-                    goto _dest_overflow;
-        }   }
-    }  /* while (ip <= mflimit) */
-
-_last_literals:
-    /* Encode Last Literals */
-    {   size_t lastRunSize = (size_t)(iend - anchor);  /* literals */
-        size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
-        size_t const totalSize = 1 + litLength + lastRunSize;
-        if (limit == limitedDestSize) oend += LASTLITERALS;  /* restore correct value */
-        if (limit && (op + totalSize > oend)) {
-            if (limit == limitedOutput) return 0;  /* Check output limit */
-            /* adapt lastRunSize to fill 'dst' */
-            lastRunSize  = (size_t)(oend - op) - 1;
-            litLength = (lastRunSize + 255 - RUN_MASK) / 255;
-            lastRunSize -= litLength;
-        }
-        ip = anchor + lastRunSize;
-
-        if (lastRunSize >= RUN_MASK) {
-            size_t accumulator = lastRunSize - RUN_MASK;
-            *op++ = (RUN_MASK << ML_BITS);
-            for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
-            *op++ = (BYTE) accumulator;
-        } else {
-            *op++ = (BYTE)(lastRunSize << ML_BITS);
-        }
-        memcpy(op, anchor, lastRunSize);
-        op += lastRunSize;
-    }
-
-    /* End */
-    *srcSizePtr = (int) (((const char*)ip) - source);
-    return (int) ((char*)op-dst);
-
-_dest_overflow:
-    if (limit == limitedDestSize) {
-        op = opSaved;  /* restore correct out pointer */
-        goto _last_literals;
-    }
-    return 0;
-}

From ba115386fa69c148cd1b27790a8b95cd817ff172 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 26 Feb 2018 13:31:18 -0800
Subject: [PATCH 035/257] update code comment on LZ4 streaming interface

notably regarding LZ4_saveDict() speed advantage,
answering #477.
---
 doc/lz4_manual.html | 21 +++++++++++----------
 lib/lz4.h           | 23 ++++++++++++-----------
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index b5a40426d..43107c6e3 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -172,28 +172,29 @@ <h1>1.8.1 Manual</h1>
 </p></pre><BR>
 
 <pre><b>int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-</b><p>  Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
+</b><p>  Compress 'src' content using data from previously compressed blocks, for better compression ratio.
   'dst' buffer must be already allocated.
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
 
-  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
-              If less than 64KB has been compressed all the data must be present.
+  Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory!
+
   Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
               Make sure that buffers are separated by at least one byte.
-              This way, rule becomes simple : each block depends on previous block only.
+              This way, each block only depends on previous block.
   Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
  @return : size of compressed block
-           or 0 if there is an error (typically, compressed data cannot fit into 'dst')
+           or 0 if there is an error (typically, cannot fit into 'dst').
   After an error, the stream status is invalid, it can only be reset or freed.
  
 </p></pre><BR>
 
-<pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
-</b><p>  If previously compressed data block is not guaranteed to remain available at its current memory location,
+<pre><b>int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
+</b><p>  If last 64KB data cannot be guaranteed to remain available at its current memory location,
   save it into a safer place (char* safeBuffer).
-  Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
-  @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+  This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
+  but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
+ @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
  
 </p></pre><BR>
 
@@ -209,7 +210,7 @@ <h1>1.8.1 Manual</h1>
 <pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
 </b><p>  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
   Use this function to start decompression of a new stream of blocks.
-  A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
+  A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
  @return : 1 if OK, 0 if error
  
 </p></pre><BR>
diff --git a/lib/lz4.h b/lib/lz4.h
index 08f06c71d..2dd1ff54b 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -236,7 +236,7 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS
 /*-*********************************************
 *  Streaming Compression Functions
 ***********************************************/
-typedef union LZ4_stream_u LZ4_stream_t;   /* incomplete type (defined later) */
+typedef union LZ4_stream_u LZ4_stream_t;  /* incomplete type (defined later) */
 
 /*! LZ4_createStream() and LZ4_freeStream() :
  *  LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure.
@@ -260,30 +260,31 @@ LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
 LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
 
 /*! LZ4_compress_fast_continue() :
- *  Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
+ *  Compress 'src' content using data from previously compressed blocks, for better compression ratio.
  *  'dst' buffer must be already allocated.
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
  *
- *  Important : The previous 64KB of compressed data is assumed to remain preset and unmodified in memory!
- *              If less than 64KB has been compressed all the data must be present.
+ *  Important : The previous 64KB of compressed data is assumed to remain present and unmodified in memory!
+ *
  *  Special 1 : When input is a double-buffer, they can have any size, including < 64 KB.
  *              Make sure that buffers are separated by at least one byte.
- *              This way, rule becomes simple : each block depends on previous block only.
+ *              This way, each block only depends on previous block.
  *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
  *
  * @return : size of compressed block
- *           or 0 if there is an error (typically, compressed data cannot fit into 'dst')
+ *           or 0 if there is an error (typically, cannot fit into 'dst').
  *  After an error, the stream status is invalid, it can only be reset or freed.
  */
 LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 /*! LZ4_saveDict() :
- *  If previously compressed data block is not guaranteed to remain available at its current memory location,
+ *  If last 64KB data cannot be guaranteed to remain available at its current memory location,
  *  save it into a safer place (char* safeBuffer).
- *  Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
- *  @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ *  This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
+ *  but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
+ * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
  */
-LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
+LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
 
 
 /*-**********************************************
@@ -301,7 +302,7 @@ LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
 /*! LZ4_setStreamDecode() :
  *  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
  *  Use this function to start decompression of a new stream of blocks.
- *  A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
+ *  A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
  * @return : 1 if OK, 0 if error
  */
 LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);

From 39fda9a447d4dd51f395c5d94f9b63c7aec9b68b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 26 Feb 2018 13:50:04 -0800
Subject: [PATCH 036/257] bumped version number to v1.8.2

updated NEWS was current progresses
---
 NEWS                     | 5 +++++
 doc/lz4_manual.html      | 4 ++--
 doc/lz4frame_manual.html | 4 ++--
 lib/lz4.h                | 2 +-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/NEWS b/NEWS
index f9c503ff8..9ac41f298 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+v1.8.2
+perf: slightly faster HC compression and decompression speed
+perf: very small compression ratio improvement
+cli : benchmark mode more accurate for small inputs
+
 v1.8.1
 perf : faster and stronger ultra modes (levels 10+)
 perf : slightly faster compression and decompression speed
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 43107c6e3..d14467f6f 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.8.1 Manual</title>
+<title>1.8.2 Manual</title>
 </head>
 <body>
-<h1>1.8.1 Manual</h1>
+<h1>1.8.2 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 1ddf6d877..db004abff 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.8.1 Manual</title>
+<title>1.8.2 Manual</title>
 </head>
 <body>
-<h1>1.8.1 Manual</h1>
+<h1>1.8.2 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
diff --git a/lib/lz4.h b/lib/lz4.h
index 2dd1ff54b..911ea79df 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -93,7 +93,7 @@ extern "C" {
 /*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ4_VERSION_MINOR    8    /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
 
 #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
 

From 0ddd1ceb1db0f3a40a5231d52f59d25ceacd6480 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 26 Feb 2018 14:09:46 -0800
Subject: [PATCH 037/257] added target make check

according to GNU Makefile conventions,
the Makefile should feature a make check target
to self-test the generated program:
https://www.gnu.org/prep/standards/html_node/Standard-Targets.html .

this is much less thorough and less taxing than `make test`,
and can be run on any target in a reasonable timeframe (several seconds).
---
 Makefile       |  8 +++++---
 tests/Makefile | 29 ++++++++++++++++-------------
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index 83320fa61..8a7d15516 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,8 @@
 # ################################################################
 # LZ4 - Makefile
-# Copyright (C) Yann Collet 2011-2016
+# Copyright (C) Yann Collet 2011-present
 # All rights reserved.
 #
-# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
-#
 # BSD license
 # Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
@@ -122,6 +120,10 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 list:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
 
+.PHONY: check
+check:
+	$(MAKE) -C $(TESTDIR) test-lz4-essentials
+
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
diff --git a/tests/Makefile b/tests/Makefile
index ddc0d2e8c..5954370b3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,8 +1,6 @@
 # ##########################################################################
 # LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-2017
-#
-# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
+# Copyright (C) Yann Collet 2011-present
 #
 # GPL v2 License
 #
@@ -53,7 +51,7 @@ else
 EXT  =
 VOID = /dev/null
 endif
-LZ4     := $(PRGDIR)/lz4$(EXT)
+LZ4 := $(PRGDIR)/lz4$(EXT)
 
 
 # Default test parameters
@@ -184,12 +182,6 @@ test-lz4-contentSize: lz4 datagen
 	$(LZ4) -v tmplc1 | $(LZ4) -t
 	$(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2
 	$(DIFF) -s tmplc1 tmplc2
-	# test large size [2-4] GB
-	@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmplc1
-	@ls -ls tmplc1
-	@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmplc2
-	@ls -ls tmplc2
-	$(DIFF) -s tmplc1 tmplc2
 	@$(RM) tmplc*
 
 test-lz4-frame-concatenation: lz4 datagen
@@ -293,6 +285,13 @@ test-lz4-hugefile: lz4 datagen
 	@echo "\n ---- test huge files compression/decompression ----"
 	./datagen -g6GB   | $(LZ4) -vB5D  | $(LZ4) -qt
 	./datagen -g6GB   | $(LZ4) -v5BD  | $(LZ4) -qt
+	# test large file size [2-4] GB
+	@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
+	@ls -ls tmphf1
+	@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
+	@ls -ls tmphf2
+	$(DIFF) -s tmphf1 tmphf2
+	@$(RM) tmphf*
 
 test-lz4-testmode: lz4 datagen
 	@echo "\n ---- bench mode ----"
@@ -326,9 +325,13 @@ test-lz4-opt-parser: lz4 datagen
 	./datagen -g16M -P90  | $(LZ4) -11B5    | $(LZ4) -t
 	./datagen -g32M -P10  | $(LZ4) -11B5D   | $(LZ4) -t
 
-test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \
-          test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \
-          test-lz4-contentSize test-lz4-hugefile test-lz4-dict
+test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple \
+                      test-lz4-frame-concatenation test-lz4-testmode \
+                      test-lz4-contentSize
+	@$(RM) tmp*
+
+test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
+          test-lz4-sparse test-lz4-hugefile test-lz4-dict
 	@$(RM) tmp*
 
 test-lz4c: lz4c datagen

From b5233d3726b416b1176c71483d20b4c543851c6f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 27 Feb 2018 23:23:27 -0800
Subject: [PATCH 038/257] updated LZ4F_compressBound() documentation

to clarify it includes potentially buffered data.
---
 doc/lz4frame_manual.html | 9 ++++++---
 lib/lz4frame.h           | 9 ++++++---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index db004abff..f0f7f5afb 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -153,10 +153,13 @@ <h1>1.8.2 Manual</h1>
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-</b><p>  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
-  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
-  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
+</b><p>  Provides minimum dstCapacity required to guarantee compression success
+  given a srcSize and preferences, covering worst case scenario.
   prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
+  Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(),
+  Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
+  It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd().
+  Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin().
   Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
   When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 15484d7e3..9efaca024 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -250,10 +250,13 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
                                       const LZ4F_preferences_t* prefsPtr);
 
 /*! LZ4F_compressBound() :
- *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
- *  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
- *  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
+ *  Provides minimum dstCapacity required to guarantee compression success
+ *  given a srcSize and preferences, covering worst case scenario.
  *  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
+ *  Estimation is valid for either LZ4F_compressUpdate(), LZ4F_flush() or LZ4F_compressEnd(),
+ *  Estimation includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
+ *  It also includes frame footer (ending + checksum), which would have to be generated by LZ4F_compressEnd().
+ *  Estimation doesn't include frame header, as it was already generated by LZ4F_compressBegin().
  *  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
  *  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  */

From 6d4e60e365d1353c9dac46f5bcfa48edfd9ab623 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 9 Mar 2018 09:57:29 -0800
Subject: [PATCH 039/257] fix #481: ensure liblz4.a dependency for `make all`

`make all` will trigger several sub-directory makefiles.
several of them need `liblz4.a`.
When built with `-j#`, there are several concurrent liblz4.a built

Make liblz4.a a dependency, which is built once,
before moving to sub-directory Makefiles
---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 8a7d15516..ff84d6d42 100644
--- a/Makefile
+++ b/Makefile
@@ -56,6 +56,7 @@ all: allmost manuals
 allmost: lib lz4 examples
 
 .PHONY: lib lib-release liblz4.a
+lib: liblz4.a
 lib lib-release liblz4.a:
 	@$(MAKE) -C $(LZ4DIR) $@
 
@@ -67,7 +68,7 @@ lz4 lz4-release :
 	@cp $(PRGDIR)/lz4$(EXT) .
 
 .PHONY: examples
-examples:
+examples: liblz4.a
 	$(MAKE) -C $(EXDIR) all
 
 .PHONY: manuals

From 6c23f03b93608a444f8747940bca26708a58fa78 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 9 Mar 2018 11:54:32 -0800
Subject: [PATCH 040/257] fix #482: change CFLAGS to CXXFLAGS

as they are associated with $(CXX)
---
 contrib/gen_manual/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/contrib/gen_manual/Makefile b/contrib/gen_manual/Makefile
index 9fbe858a0..95abe2e8d 100644
--- a/contrib/gen_manual/Makefile
+++ b/contrib/gen_manual/Makefile
@@ -30,10 +30,10 @@
 # ################################################################
 
 
-CFLAGS ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
-CFLAGS += $(MOREFLAGS)
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CXXFLAGS ?= -O3
+CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
+CXXFLAGS += $(MOREFLAGS)
+FLAGS   = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
 
 LZ4API = ../../lib/lz4.h
 LZ4MANUAL = ../../doc/lz4_manual.html

From 8c006b19bbed0b3bb2466dc8af2a603ca0ba7fab Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 29 Jan 2018 13:20:16 -0500
Subject: [PATCH 041/257] Add a Benchmarking Tool For Compression with Context
 Re-Use

---
 tests/Makefile     |   8 +-
 tests/framebench.c | 293 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 299 insertions(+), 2 deletions(-)
 create mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index 5954370b3..77d9e36cd 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -62,7 +62,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest datagen framebench
 
 all32: CFLAGS+=-m32
 all32: all
@@ -98,6 +98,9 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -109,7 +112,8 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
+        framebench$(EXT) \
+        fasttest$(EXT) datagen$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 
diff --git a/tests/framebench.c b/tests/framebench.c
new file mode 100644
index 000000000..8dcfa417a
--- /dev/null
+++ b/tests/framebench.c
@@ -0,0 +1,293 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
+
+typedef struct {
+  size_t iter;
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  char *obuf;
+  size_t osize;
+  const char* ibuf;
+  size_t isize;
+  size_t num_ibuf;
+  const LZ4F_CDict* cdict;
+  LZ4F_preferences_t* prefs;
+  const LZ4F_compressOptions_t* options;
+} bench_params_t;
+
+size_t compress_frame(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressFrame_usingCDict(
+    obuf,
+    osize,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    cdict,
+    prefs);
+  LZ4F_CHECK(oused);
+
+  return oused;
+}
+
+size_t compress_begin(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+  const LZ4F_compressOptions_t* options = p->options;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressUpdate(
+    cctx,
+    obuf,
+    oend - obuf,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    options);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
+  LZ4F_CHECK(oused);
+
+  return obuf - p->obuf;
+}
+
+size_t compress_default(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_stream_t *ctx = p->ctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_fast_extState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+uint64_t bench(
+    size_t (*fun)(bench_params_t *),
+    uint64_t repetitions,
+    bench_params_t *params,
+    size_t *osizePtr
+) {
+  struct timespec start, end;
+  size_t i, osize = 0;
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
+
+  for (i = 0; i < repetitions; i++) {
+    size_t o;
+    params->iter = i;
+    o = fun(params);
+    if (!o) return 0;
+    osize += o;
+  }
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+  *osizePtr = osize / repetitions;
+  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+}
+
+int main(int argc, char *argv[]) {
+
+
+  struct stat st;
+  size_t bytes_read;
+
+  const char *dict_fn;
+  size_t dict_size;
+  char *dict_buf;
+  FILE *dict_file;
+
+  const char *in_fn;
+  size_t in_size;
+  size_t num_in_buf;
+  size_t cur_in_buf;
+  char *in_buf;
+  FILE *in_file;
+
+  size_t out_size;
+  char *out_buf;
+
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  LZ4F_CDict *cdict;
+  LZ4F_preferences_t prefs;
+  LZ4F_compressOptions_t options;
+
+  size_t out_used;
+
+  uint64_t time_taken;
+  uint64_t repetitions;
+
+  bench_params_t params;
+
+  if (argc != 3) return 1;
+  dict_fn = argv[1];
+  in_fn = argv[2];
+
+  if (stat(dict_fn, &st)) return 1;
+  dict_size = st.st_size;
+  dict_buf = (char *)malloc(dict_size);
+  if (!dict_buf) return 1;
+  dict_file = fopen(dict_fn, "r");
+  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
+  if (bytes_read != dict_size) return 1;
+
+  if (stat(in_fn, &st)) return 1;
+  in_size = st.st_size;
+  num_in_buf = 256 * 1024 * 1024 / in_size;
+  if (num_in_buf == 0) {
+    num_in_buf = 1;
+  }
+
+  in_buf = (char *)malloc(in_size * num_in_buf);
+  if (!in_buf) return 1;
+  in_file = fopen(in_fn, "r");
+  bytes_read = fread(in_buf, 1, in_size, in_file);
+  if (bytes_read != in_size) return 1;
+
+  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
+    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
+  }
+
+  if (in_size <= 1024) {
+    repetitions = 100000;
+  } else
+  if (in_size <= 16384) {
+    repetitions = 10000;
+  } else
+  if (in_size <= 131072) {
+    repetitions = 1000;
+  } else
+  if (in_size <= 1048576) {
+    repetitions = 100;
+  } else {
+    repetitions = 50;
+  }
+
+  memset(&prefs, 0, sizeof(prefs));
+  prefs.autoFlush = 1;
+  if (in_size < 64 * 1024)
+    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+  prefs.frameInfo.contentSize = in_size;
+
+  memset(&options, 0, sizeof(options));
+  options.stableSrc = 1;
+
+  out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  out_buf = (char *)malloc(out_size);
+  if (!out_buf) return 1;
+
+  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
+  ctx = LZ4_createStream();
+  if (ctx == NULL) return 1;
+
+  cdict = LZ4F_createCDict(dict_buf, dict_size);
+  if (!cdict) return 1;
+
+  fprintf(stderr, "dict  size: %zd\n", dict_size);
+  fprintf(stderr, "input size: %zd\n", in_size);
+
+  params.ctx = ctx;
+  params.cctx = cctx;
+  params.obuf = out_buf;
+  params.osize = out_size;
+  params.ibuf = in_buf;
+  params.isize = in_size;
+  params.num_ibuf = num_in_buf;
+  params.cdict = NULL;
+  params.prefs = &prefs;
+  params.options = &options;
+
+  time_taken = bench(compress_default, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  return 0;
+}
\ No newline at end of file

From 6d156fea56debbde1699b54a52c4907ff0b96017 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Oct 2017 16:13:33 -0400
Subject: [PATCH 042/257] Allow Empty Dictionaries

---
 lib/lz4.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 38a865f32..898b82aa1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1018,12 +1018,6 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     if ((dict->initCheck) || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
-    if (dictSize < (int)HASH_UNIT) {
-        dict->dictionary = NULL;
-        dict->dictSize = 0;
-        return 0;
-    }
-
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
     dict->currentOffset += 64 KB;
     base = p - dict->currentOffset;
@@ -1031,6 +1025,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
 
+    if (dictSize < (int)HASH_UNIT) {
+        return 0;
+    }
+
     while (p <= dictEnd-HASH_UNIT) {
         LZ4_putPosition(p, dict->hashTable, byU32, base);
         p+=3;

From 6933f5ad9ca57caa8a76103f1382560fba0f9691 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Oct 2017 15:23:02 -0400
Subject: [PATCH 043/257] Remove Obsolete Stream Functions to Free Space in
 LZ4_stream_t

---
 lib/lz4.c | 31 -------------------------------
 lib/lz4.h |  6 ------
 2 files changed, 37 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 898b82aa1..6168a2a6c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1515,37 +1515,6 @@ int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
 
-/* Obsolete Streaming functions */
-
-int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
-
-static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base)
-{
-    MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t));
-    lz4ds->internal_donotuse.bufferStart = base;
-}
-
-int LZ4_resetStreamState(void* state, char* inputBuffer)
-{
-    if ((((uptrval)state) & 3) != 0) return 1;   /* Error : pointer is not aligned on 4-bytes boundary */
-    LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer);
-    return 0;
-}
-
-void* LZ4_create (char* inputBuffer)
-{
-    LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t));
-    LZ4_init (lz4ds, (BYTE*)inputBuffer);
-    return lz4ds;
-}
-
-char* LZ4_slideInputBuffer (void* LZ4_Data)
-{
-    LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse;
-    int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB);
-    return (char*)(ctx->bufferStart + dictSize);
-}
-
 /* Obsolete streaming decompression functions */
 
 int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index 911ea79df..e5d403973 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -465,12 +465,6 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
-
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
 LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);

From 5709891de6513ee8253f27e8d3207ec1fdcff14b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:11:35 -0500
Subject: [PATCH 044/257] Add a Table Type Field to LZ4_stream_t

---
 lib/lz4.c | 24 ++++++++++++++++++------
 lib/lz4.h |  6 ++++--
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6168a2a6c..7b8de778b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -446,7 +446,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 *  Local Structures and types
 **************************************/
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
-typedef enum { byPtr, byU32, byU16 } tableType_t;
+typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
 
 typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
@@ -496,6 +496,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
 {
     switch (tableType)
     {
+    case unusedTable: { /* illegal! */ assert(0); return; }
     case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }
     case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }
     case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }
@@ -551,6 +552,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
+    ptrdiff_t retval = 0;
+
     U32 forwardH;
 
     /* Init conditions */
@@ -622,7 +625,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             token = op++;
             if ((outputLimited) &&  /* Check output buffer overflow */
                 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
-                return 0;
+                goto _clean_up;
             if (litLength >= RUN_MASK) {
                 int len = (int)litLength-RUN_MASK;
                 *token = (RUN_MASK<<ML_BITS);
@@ -662,7 +665,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
-                return 0;
+                goto _clean_up;
             if (matchCode >= ML_MASK) {
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
@@ -711,7 +714,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     {   size_t const lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
             ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
-            return 0;
+            goto _clean_up;
         if (lastRun >= RUN_MASK) {
             size_t accumulator = lastRun - RUN_MASK;
             *op++ = RUN_MASK << ML_BITS;
@@ -724,8 +727,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         op += lastRun;
     }
 
+    retval = (((char*)op)-dest);
+
+_clean_up:
+    cctx->tableType = tableType;
+
     /* End */
-    return (int) (((char*)op)-dest);
+    return (int)retval;
 }
 
 
@@ -997,6 +1005,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(4, "LZ4_resetStream");
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
+    LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
@@ -1015,7 +1024,9 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    if ((dict->initCheck) || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
+    if ((dict->initCheck)
+      || (dict->tableType != byU32 && dict->tableType != unusedTable)
+      || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
@@ -1024,6 +1035,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
+    dict->tableType = byU32;
 
     if (dictSize < (int)HASH_UNIT) {
         return 0;
diff --git a/lib/lz4.h b/lib/lz4.h
index e5d403973..d2e2103e9 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -360,7 +360,8 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 typedef struct {
     uint32_t hashTable[LZ4_HASH_SIZE_U32];
     uint32_t currentOffset;
-    uint32_t initCheck;
+    uint16_t initCheck;
+    uint16_t tableType;
     const uint8_t* dictionary;
     uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
     uint32_t dictSize;
@@ -378,7 +379,8 @@ typedef struct {
 typedef struct {
     unsigned int hashTable[LZ4_HASH_SIZE_U32];
     unsigned int currentOffset;
-    unsigned int initCheck;
+    unsigned short initCheck;
+    unsigned short tableType;
     const unsigned char* dictionary;
     unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
     unsigned int dictSize;

From f34fb3c42dbb824af97800eefdb487dff31e0c13 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:25:29 -0500
Subject: [PATCH 045/257] Add Bounds Check to locateBuffDiff

---
 tests/frametest.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index 88d0afd4d..986a16d8d 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -730,15 +730,17 @@ int basicTests(U32 seed, double compressibility)
 
 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
 {
-    int p=0;
+    size_t p=0;
     const BYTE* b1=(const BYTE*)buff1;
     const BYTE* b2=(const BYTE*)buff2;
     if (nonContiguous) {
         DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
         return;
     }
-    while (b1[p]==b2[p]) p++;
-    DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
+    while (p < size && b1[p]==b2[p]) p++;
+    if (p != size) {
+        DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
+    }
 }
 
 

From d6a3024dbb6375190e6e02ad8b06aae28e76b946 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:41:11 -0500
Subject: [PATCH 046/257] Add LZ4_compress_fast_safeExtState Function

---
 lib/lz4.c | 10 +++++++++-
 lib/lz4.h |  8 ++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7b8de778b..45cc7c9a4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -738,10 +738,18 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
 
 int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+{
+  LZ4_resetStream((LZ4_stream_t*)state);
+  return LZ4_compress_fast_safeExtState(state, source, dest, inputSize, maxOutputSize, acceleration);
+}
+
+
+int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
-    LZ4_resetStream((LZ4_stream_t*)state);
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
+    ctx->dictionary = NULL;
+    ctx->dictSize = 0;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit)
diff --git a/lib/lz4.h b/lib/lz4.h
index d2e2103e9..0013ffe22 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,13 +179,21 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
+LZ4_compress_fast_safeExtState() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
+
+    Use _safeExtState variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to this function leave
+    the state in a safe state, so zeroing is not required between calls).
+    Otherwise, using legacy _extState requires LZ4 to reinitialize the state
+    internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
+LZ4LIB_API int LZ4_compress_fast_safeExtState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 

From aa36e118f10d825ceb3899d3e137d643ed469331 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 2 Feb 2018 11:45:59 -0500
Subject: [PATCH 047/257] Const-ify Table Arg to LZ4_getPosition(OnHash)

---
 lib/lz4.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 45cc7c9a4..46935f8a2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -509,14 +509,14 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }
 
-static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
-    if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
-    if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
-    { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
+    if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; }
+    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; }
+    { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);

From b3628cb0c51bb93f681707b2d0cdbce5ccdae818 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:09:38 -0500
Subject: [PATCH 048/257] Avoid Resetting the Context When Possible

---
 lib/lz4.c          | 61 ++++++++++++++++++++++++++++++++--------------
 lib/lz4frame.c     | 21 ++++++++++++++--
 tests/framebench.c |  2 +-
 3 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 46935f8a2..d558b3b66 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -538,7 +538,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const U32 acceleration)
 {
     const BYTE* ip = (const BYTE*) source;
-    const BYTE* base;
+
+    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+     * therefore safe to use no matter what mode we're in. Otherwise, we figure
+     * out if it's safe to leave as is or whether it needs to be reset.
+     */
+    const int resetTable = cctx->tableType != unusedTable && (
+                           cctx->tableType != tableType ||
+                            (tableType == byU16 &&
+                             cctx->currentOffset + inputSize >= 0xFFFFU) ||
+                           tableType == byPtr);
+
+    size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
+                            !resetTable) ? cctx->currentOffset : 0;
+    const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
     const BYTE* const lowRefLimit = ip - cctx->dictSize;
     const BYTE* const dictionary = cctx->dictionary;
@@ -556,17 +569,18 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     U32 forwardH;
 
+    /* TODO: resurrect dictIssue check */
+    (void)dictIssue;
+
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     switch(dict)
     {
     case noDict:
     default:
-        base = (const BYTE*)source;
         lowLimit = (const BYTE*)source;
         break;
     case withPrefix64k:
-        base = (const BYTE*)source - cctx->currentOffset;
         lowLimit = (const BYTE*)source - cctx->dictSize;
         break;
     case usingExtDict:
@@ -575,6 +589,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         break;
     }
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+
+    if (resetTable) {
+      DEBUGLOG(4, "Resetting table in %p", cctx);
+      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+      cctx->currentOffset = 0;
+    }
+
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -612,7 +633,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
+            } while ( match < lowRefLimit
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
@@ -700,7 +721,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 lowLimit = (const BYTE*)source;
         }   }
         LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
+        if ( match >= lowRefLimit
             && (match+MAX_DISTANCE>=ip)
             && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
         { token=op++; *token=0; goto _next_match; }
@@ -730,6 +751,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     retval = (((char*)op)-dest);
 
 _clean_up:
+    cctx->currentOffset += (U32)inputSize;
+    cctx->dictSize += (U32)inputSize;
     cctx->tableType = tableType;
 
     /* End */
@@ -767,14 +790,16 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
 
 int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+    int result;
 #if (LZ4_HEAPMODE)
-    void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    ctxPtr->internal_donotuse.currentOffset = 0;
 #else
     LZ4_stream_t ctx;
-    void* const ctxPtr = &ctx;
+    LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-
-    int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+    LZ4_resetStream(ctxPtr);
+    result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
     FREEMEM(ctxPtr);
@@ -1005,20 +1030,23 @@ LZ4_stream_t* LZ4_createStream(void)
 {
     LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64);
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
+    DEBUGLOG(4, "LZ4_createStream %p", lz4s);
     LZ4_resetStream(lz4s);
     return lz4s;
 }
 
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
-    DEBUGLOG(4, "LZ4_resetStream");
+    DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
+    LZ4_stream->internal_donotuse.currentOffset = 0;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 {
     if (!LZ4_stream) return 0;   /* support free on NULL */
+    DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream);
     FREEMEM(LZ4_stream);
     return (0);
 }
@@ -1032,6 +1060,8 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
+    DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
+
     if ((dict->initCheck)
       || (dict->tableType != byU32 && dict->tableType != unusedTable)
       || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
@@ -1066,6 +1096,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
         U32 const delta = LZ4_dict->currentOffset - 64 KB;
         const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
         int i;
+        DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict);
         for (i=0; i<LZ4_HASH_SIZE_U32; i++) {
             if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
             else LZ4_dict->hashTable[i] -= delta;
@@ -1100,14 +1131,10 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
         else
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
-        streamPtr->dictSize += (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
@@ -1118,7 +1145,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
             result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
         streamPtr->dictionary = (const BYTE*)source;
         streamPtr->dictSize = (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
         return result;
     }
 }
@@ -1139,7 +1165,6 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 
     streamPtr->dictionary = (const BYTE*)source;
     streamPtr->dictSize = (U32)inputSize;
-    streamPtr->currentOffset += (U32)inputSize;
 
     return result;
 }
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a394d1f18..bf70b5c63 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -354,6 +354,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
     if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+        LZ4_resetStream(&lz4ctx);
         cctxI.lz4CtxPtr = &lz4ctx;
         cctxI.lz4CtxLevel = 1;
     }  /* fast compression context pre-created on stack */
@@ -521,7 +522,14 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->lz4CtxLevel = ctxTypeID;
-    }   }
+        } else if (cctxPtr->lz4CtxLevel != ctxTypeID) {
+            /* otherwise, we must be transitioning from HC -> LZ4.
+             * In that case, avoid reallocating, since a LZ4 ctx
+             * fits in an HC ctx. Just reset. */
+            LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+            cctxPtr->lz4CtxLevel = ctxTypeID;
+        }
+    }
 
     /* Buffer Management */
     if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
@@ -654,11 +662,20 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
+    LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)ctx)->internal_donotuse;
+    assert(!internal_ctx->initCheck);
+    if (internal_ctx->currentOffset > 1 GB) {
+        /* Init the context */
+        LZ4_resetStream((LZ4_stream_t*)ctx);
+    }
+    /* Clear any local dictionary */
+    internal_ctx->dictionary = NULL;
+    internal_ctx->dictSize = 0;
     if (cdict) {
         memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
-    return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
+    return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
diff --git a/tests/framebench.c b/tests/framebench.c
index 8dcfa417a..21c37049a 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -118,7 +118,7 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_safeExtState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
   obuf += oused;
 
   return obuf - p->obuf;

From 7060bcabf041395d6249a701729d02e28529bf82 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 14:44:53 -0500
Subject: [PATCH 049/257] Only Re-Alloc / Reset When Needed When Switching
 Between Regular and High Compression Modes

---
 lib/lz4frame.c | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index bf70b5c63..cd619256a 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -188,7 +188,15 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    U32    lz4CtxLevel;   /* 0: unallocated;  1: LZ4_stream_t;  3: LZ4_streamHC_t */
+    /* lz4CtxLevel records both what size of memory has been allocated into the
+     * ctx pointer field and what it's being used as.
+     * - The low bit (& 1) value indicates whether the ctx is being used for
+     *   hc (1) or not (0).
+     * - The next bit (& 2) indicates whether the allocated memory is big
+     *   enough for a non-hc context.
+     * - The next bit (& 4) indicates whether the allocated memory is big
+     *   enough for an hc context. */
+    U32    lz4CtxLevel;
 } LZ4F_cctx_t;
 
 
@@ -356,7 +364,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream(&lz4ctx);
         cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 1;
+        cctxI.lz4CtxLevel = 2;
     }  /* fast compression context pre-created on stack */
 
     memset(&options, 0, sizeof(options));
@@ -513,21 +521,27 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->prefs = *preferencesPtr;
 
     /* Ctx Management */
-    {   U32 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;  /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
-        if (cctxPtr->lz4CtxLevel < ctxTypeID) {
+    {   U32 const ctxSufficientAllocBits = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 2 : 6;
+        U32 const ctxTypeBit = cctxPtr->prefs.compressionLevel >= LZ4HC_CLEVEL_MIN;
+        if ((cctxPtr->lz4CtxLevel & ctxSufficientAllocBits) != ctxSufficientAllocBits) {
             FREEMEM(cctxPtr->lz4CtxPtr);
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
-            else
+            } else {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
+            }
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            cctxPtr->lz4CtxLevel = ctxTypeID;
-        } else if (cctxPtr->lz4CtxLevel != ctxTypeID) {
-            /* otherwise, we must be transitioning from HC -> LZ4.
-             * In that case, avoid reallocating, since a LZ4 ctx
-             * fits in an HC ctx. Just reset. */
-            LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
-            cctxPtr->lz4CtxLevel = ctxTypeID;
+            cctxPtr->lz4CtxLevel = ctxSufficientAllocBits | ctxTypeBit;
+        } else if ((cctxPtr->lz4CtxLevel & 1) != ctxTypeBit) {
+            /* otherwise, a sufficient buffer is allocated, but we need to
+             * reset it to the correct context type */
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+                cctxPtr->lz4CtxLevel &= ~1;
+            } else {
+                LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+                cctxPtr->lz4CtxLevel |= 1;
+            }
         }
     }
 

From 62cb52b3410bdfd696669910de010ab03666edc8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:10:15 -0500
Subject: [PATCH 050/257] Initialize Current Offset to 1

---
 lib/lz4.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d558b3b66..dde27e9d8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -550,7 +550,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                            tableType == byPtr);
 
     size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
-                            !resetTable) ? cctx->currentOffset : 0;
+                            !resetTable) ? cctx->currentOffset : 1;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
     const BYTE* const lowRefLimit = ip - cctx->dictSize;
@@ -593,7 +593,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     if (resetTable) {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 0;
+      cctx->currentOffset = 1;
     }
 
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
@@ -793,7 +793,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
-    ctxPtr->internal_donotuse.currentOffset = 0;
+    ctxPtr->internal_donotuse.currentOffset = 1;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1039,7 +1039,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.currentOffset = 0;
+    LZ4_stream->internal_donotuse.currentOffset = 1;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 

From 73cc39327e3abc28a360323c4f26c3c34d87ff07 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:18:24 -0500
Subject: [PATCH 051/257] Lookup Matches in Separate Dictionary Context

---
 lib/lz4.c | 70 ++++++++++++++++++++++++++++++++++++++++++++-----------
 lib/lz4.h | 14 ++++++-----
 2 files changed, 65 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index dde27e9d8..40cac94ce 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -448,7 +448,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
 typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
 
-typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingExtDictCtx } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
 
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
@@ -533,7 +533,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const int maxOutputSize,
                  const limitedOutput_directive outputLimited,
                  const tableType_t tableType,
-                 const dict_directive dict,
+                 const dict_directive dictDirective,
                  const dictIssue_directive dictIssue,
                  const U32 acceleration)
 {
@@ -553,15 +553,27 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                             !resetTable) ? cctx->currentOffset : 1;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
-    const BYTE* const lowRefLimit = ip - cctx->dictSize;
-    const BYTE* const dictionary = cctx->dictionary;
-    const BYTE* const dictEnd = dictionary + cctx->dictSize;
-    const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source;
+
+    const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
+    const BYTE* const dictionary =
+        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictionary;
+    const U32 dictSize =
+        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictSize;
+
+    const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
+    const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
     const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1;
     const BYTE* const matchlimit = iend - LASTLITERALS;
 
+    /* the dictCtx currentOffset is indexed on the start of the dictionary,
+     * while a dictionary in the current context precedes the currentOffset */
+    const BYTE* dictBase = dictDirective == usingExtDictCtx ?
+        (const BYTE*) source - dictCtx->currentOffset :
+        (const BYTE*) source - dictSize - currentOffset;
+    const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
@@ -574,17 +586,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
-    switch(dict)
+    switch(dictDirective)
     {
     case noDict:
     default:
         lowLimit = (const BYTE*)source;
         break;
     case withPrefix64k:
-        lowLimit = (const BYTE*)source - cctx->dictSize;
+        lowLimit = (const BYTE*)source - dictSize;
         break;
     case usingExtDict:
-        base = (const BYTE*)source - cctx->currentOffset;
+        lowLimit = (const BYTE*)source;
+        break;
+    case usingExtDictCtx:
         lowLimit = (const BYTE*)source;
         break;
     }
@@ -622,8 +636,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dict==usingExtDict) {
+                if (dictDirective == usingExtDictCtx) {
                     if (match < (const BYTE*)source) {
+                        /* there was no match, try the dictionary */
+                        /* TODO: use precalc-ed hash? */
+                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                        refDelta = dictDelta;
+                        lowLimit = dictionary;
+                    } else {
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                } else if (dictDirective==usingExtDict) {
+                    if (match < (const BYTE*)source && dictionary != NULL) {
                         refDelta = dictDelta;
                         lowLimit = dictionary;
                     } else {
@@ -667,7 +692,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dict==usingExtDict) && (lowLimit==dictionary)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictionary)) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -712,8 +737,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
         /* Test next position */
         match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dict==usingExtDict) {
+        if (dictDirective == usingExtDictCtx) {
             if (match < (const BYTE*)source) {
+                /* there was no match, try the dictionary */
+                /* TODO: use precalc-ed hash? */
+                match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                refDelta = dictDelta;
+                lowLimit = dictionary;
+            } else {
+                refDelta = 0;
+                lowLimit = (const BYTE*)source;
+            }
+        } else if (dictDirective==usingExtDict) {
+            if (match < (const BYTE*)source && dictionary != NULL) {
                 refDelta = dictDelta;
                 lowLimit = dictionary;
             } else {
@@ -751,8 +787,15 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     retval = (((char*)op)-dest);
 
 _clean_up:
+    if (dictDirective == usingExtDictCtx) {
+      /* Subsequent linked blocks can't use the dictionary. */
+      /* Instead, they use the block we just compressed. */
+      cctx->dictCtx = NULL;
+      cctx->dictSize = (U32)inputSize;
+    } else {
+      cctx->dictSize += (U32)inputSize;
+    }
     cctx->currentOffset += (U32)inputSize;
-    cctx->dictSize += (U32)inputSize;
     cctx->tableType = tableType;
 
     /* End */
@@ -773,6 +816,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
     ctx->dictionary = NULL;
     ctx->dictSize = 0;
+    ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit)
diff --git a/lib/lz4.h b/lib/lz4.h
index 0013ffe22..58a1e39aa 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -365,15 +365,16 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #include <stdint.h>
 
-typedef struct {
+typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
+struct LZ4_stream_t_internal {
     uint32_t hashTable[LZ4_HASH_SIZE_U32];
     uint32_t currentOffset;
     uint16_t initCheck;
     uint16_t tableType;
     const uint8_t* dictionary;
-    uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
+    const LZ4_stream_t_internal* dictCtx;
     uint32_t dictSize;
-} LZ4_stream_t_internal;
+};
 
 typedef struct {
     const uint8_t* externalDict;
@@ -384,15 +385,16 @@ typedef struct {
 
 #else
 
-typedef struct {
+typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
+struct LZ4_stream_t_internal {
     unsigned int hashTable[LZ4_HASH_SIZE_U32];
     unsigned int currentOffset;
     unsigned short initCheck;
     unsigned short tableType;
     const unsigned char* dictionary;
-    unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
+    const LZ4_stream_t_internal* dictCtx;
     unsigned int dictSize;
-} LZ4_stream_t_internal;
+};
 
 typedef struct {
     const unsigned char* externalDict;

From 68c6bd17b82a0be501237af682f6472b0d39114f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Feb 2018 12:19:13 -0500
Subject: [PATCH 052/257] Set Dictionary Context Pointer Rather than Copying
 the Context In

---
 lib/lz4.c      | 17 +++++++++++++----
 lib/lz4frame.c | 35 +++++++++++++++++++++++++++++------
 2 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 40cac94ce..7d1d65bb6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1183,10 +1183,19 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
-        else
-            result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+            if (streamPtr->dictCtx) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, dictSmall, acceleration);
+            } else {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
+            }
+        } else {
+            if (streamPtr->dictCtx) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, noDictIssue, acceleration);
+            } else {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+            }
+        }
         streamPtr->dictionary = (const BYTE*)source;
         streamPtr->dictSize = (U32)inputSize;
         return result;
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index cd619256a..9ff77666f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -571,16 +571,36 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         /* frame init only for blockLinked : blockIndependent will be init at each block */
         if (cdict) {
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
+                assert(!internal_ctx->initCheck);
+                if (internal_ctx->currentOffset > 1 GB) {
+                    /* Init the context */
+                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
+                }
+                /* Clear any local dictionary */
+                internal_ctx->dictionary = NULL;
+                internal_ctx->dictSize = 0;
+                /* Point to the dictionary context */
+                internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
             } else {
                 memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
                 LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
             }
         } else {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-                LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
-            else
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
+                assert(!internal_ctx->initCheck);
+                if (internal_ctx->currentOffset > 1 GB) {
+                    /* Init the context */
+                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
+                }
+                /* Clear any local dictionary */
+                internal_ctx->dictionary = NULL;
+                internal_ctx->dictSize = 0;
+                internal_ctx->dictCtx = NULL;
+            } else {
                 LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+            }
         }
     }
 
@@ -686,10 +706,13 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     internal_ctx->dictionary = NULL;
     internal_ctx->dictSize = 0;
     if (cdict) {
-        memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+        /* Point to the dictionary context */
+        internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+    } else {
+        internal_ctx->dictCtx = NULL;
+        return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
-    return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)

From cea09d67a9deca8370dc634fd6c57437114cacbf Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 29 Jan 2018 17:09:52 -0500
Subject: [PATCH 053/257] Hoist Table Reset One Level Up

---
 lib/lz4.c | 86 ++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 53 insertions(+), 33 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7d1d65bb6..1f0450e9a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -523,6 +523,29 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 }
 
 
+LZ4_FORCE_INLINE void LZ4_resetTable(
+        LZ4_stream_t_internal* const cctx,
+        const int inputSize,
+        const tableType_t tableType) {
+  /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+   * therefore safe to use no matter what mode we're in. Otherwise, we figure
+   * out if it's safe to leave as is or whether it needs to be reset.
+   */
+  if (cctx->tableType != unusedTable && (
+      cctx->tableType != tableType ||
+      (tableType == byU16 &&
+       cctx->currentOffset + inputSize >= 0xFFFFU) ||
+      (tableType == byU32 &&
+       cctx->currentOffset > 1 GB) ||
+      tableType == byPtr))
+  {
+      DEBUGLOG(4, "Resetting table in %p", cctx);
+      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+      cctx->currentOffset = 1;
+      cctx->tableType = unusedTable;
+  }
+}
+
 /** LZ4_compress_generic() :
     inlined, to ensure branches are decided at compilation time */
 LZ4_FORCE_INLINE int LZ4_compress_generic(
@@ -539,18 +562,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
-     * therefore safe to use no matter what mode we're in. Otherwise, we figure
-     * out if it's safe to leave as is or whether it needs to be reset.
-     */
-    const int resetTable = cctx->tableType != unusedTable && (
-                           cctx->tableType != tableType ||
-                            (tableType == byU16 &&
-                             cctx->currentOffset + inputSize >= 0xFFFFU) ||
-                           tableType == byPtr);
-
-    size_t currentOffset = ((tableType == byU32 || tableType == byU16) &&
-                            !resetTable) ? cctx->currentOffset : 1;
+    size_t currentOffset = cctx->currentOffset;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
 
@@ -604,12 +616,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     }
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
-    if (resetTable) {
-      DEBUGLOG(4, "Resetting table in %p", cctx);
-      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 1;
-    }
-
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -819,15 +825,25 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
-        if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(ctx, source, dest, inputSize,             0,    notLimited,                        byU16, noDict, noDictIssue, acceleration);
-        else
-            return LZ4_compress_generic(ctx, source, dest, inputSize,             0,    notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+        if (inputSize < LZ4_64Klimit) {
+            const tableType_t tableType = byU16;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        }
     } else {
-        if (inputSize < LZ4_64Klimit)
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput,                        byU16, noDict, noDictIssue, acceleration);
-        else
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration);
+        if (inputSize < LZ4_64Klimit) {
+            const tableType_t tableType = byU16;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            LZ4_resetTable(ctx, inputSize, tableType);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        }
     }
 }
 
@@ -842,7 +858,8 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-    LZ4_resetStream(ctxPtr);
+    ctxPtr->internal_donotuse.initCheck = 0;
+    ctxPtr->internal_donotuse.tableType = byPtr; /* always triggers a reset */
     result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
@@ -1154,6 +1171,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
 
 int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
+    const tableType_t tableType = byU32;
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
@@ -1173,27 +1191,29 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
+    LZ4_resetTable(streamPtr, inputSize, tableType);
+
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
     {   int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
             if (streamPtr->dictCtx) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDictCtx, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
         streamPtr->dictionary = (const BYTE*)source;

From 14ce912b705db03bd1d723456dd1a3242b7f7ea7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Feb 2018 13:46:36 -0800
Subject: [PATCH 054/257] Switch Current Offset to 1 Only When in External
 Dictionary Context Mode

---
 lib/lz4.c | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1f0450e9a..cace35fd8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -526,7 +526,8 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 LZ4_FORCE_INLINE void LZ4_resetTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
-        const tableType_t tableType) {
+        const tableType_t tableType,
+        const dict_directive dictDirective) {
   /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
    * therefore safe to use no matter what mode we're in. Otherwise, we figure
    * out if it's safe to leave as is or whether it needs to be reset.
@@ -541,9 +542,12 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 1;
+      cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
+  if (dictDirective == usingExtDictCtx && cctx->currentOffset == 0) {
+      cctx->currentOffset = 1;
+  }
 }
 
 /** LZ4_compress_generic() :
@@ -827,21 +831,21 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType);
+            LZ4_resetTable(ctx, inputSize, tableType, noDict);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -853,7 +857,6 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
-    ctxPtr->internal_donotuse.currentOffset = 1;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1100,7 +1103,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.currentOffset = 1;
     LZ4_stream->internal_donotuse.tableType = unusedTable;
 }
 
@@ -1191,10 +1193,10 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
-    LZ4_resetTable(streamPtr, inputSize, tableType);
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
+        LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1205,14 +1207,18 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     {   int result;
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
             if (streamPtr->dictCtx) {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }

From 9dcd9abc14a33b6ac6c91dd727235db1daabe066 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 11:29:00 -0500
Subject: [PATCH 055/257] Make LZ4F_compressFrame_usingCDict Take a Compression
 Context

---
 lib/lz4frame.c        | 75 ++++++++++++++++++++++++++++++-------------
 lib/lz4frame_static.h |  2 ++
 programs/lz4io.c      |  2 +-
 tests/framebench.c    |  2 ++
 tests/frametest.c     | 19 ++++++-----
 5 files changed, 69 insertions(+), 31 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9ff77666f..e77fd35aa 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -46,6 +46,19 @@ You can contact the author at :
 #endif
 
 
+/*-************************************
+*  Tuning parameters
+**************************************/
+/*
+ * LZ4_HEAPMODE :
+ * Select how default compression functions will allocate memory for their hash table,
+ * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
+ */
+#ifndef LZ4_HEAPMODE
+#  define LZ4_HEAPMODE 0
+#endif
+
+
 /*-************************************
 *  Memory routines
 **************************************/
@@ -332,23 +345,18 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
  * @return : number of bytes written into dstBuffer,
  *           or an error code if it fails (can be tested using LZ4F_isError())
  */
-size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
+size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
+                                     void* dstBuffer, size_t dstCapacity,
                                const void* srcBuffer, size_t srcSize,
                                const LZ4F_CDict* cdict,
                                const LZ4F_preferences_t* preferencesPtr)
 {
-    LZ4F_cctx_t cctxI;
-    LZ4_stream_t lz4ctx;   /* pretty large on stack */
     LZ4F_preferences_t prefs;
     LZ4F_compressOptions_t options;
     BYTE* const dstStart = (BYTE*) dstBuffer;
     BYTE* dstPtr = dstStart;
     BYTE* const dstEnd = dstStart + dstCapacity;
 
-    memset(&cctxI, 0, sizeof(cctxI));
-    cctxI.version = LZ4F_VERSION;
-    cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
-
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
     else
@@ -361,33 +369,24 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
-    if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-        LZ4_resetStream(&lz4ctx);
-        cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 2;
-    }  /* fast compression context pre-created on stack */
-
     memset(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
     if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))  /* condition to guarantee success */
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
 
-    { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs);  /* write header */
+    { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs);  /* write header */
       if (LZ4F_isError(headerSize)) return headerSize;
       dstPtr += headerSize;   /* header size */ }
 
-    { size_t const cSize = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
+    { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
       if (LZ4F_isError(cSize)) return cSize;
       dstPtr += cSize; }
 
-    { size_t const tailSize = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
+    { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options);   /* flush last block, and generate suffix */
       if (LZ4F_isError(tailSize)) return tailSize;
       dstPtr += tailSize; }
 
-    if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN)  /* Ctx allocation only for lz4hc */
-        FREEMEM(cctxI.lz4CtxPtr);
-
     return (dstPtr - dstStart);
 }
 
@@ -403,9 +402,41 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                     const void* srcBuffer, size_t srcSize,
                     const LZ4F_preferences_t* preferencesPtr)
 {
-    return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity,
-                                         srcBuffer, srcSize,
-                                         NULL, preferencesPtr);
+    size_t result;
+#if (LZ4_HEAPMODE)
+    LZ4F_cctx_t *cctxPtr;
+    LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+#else
+    LZ4F_cctx_t cctx;
+    LZ4_stream_t lz4ctx;
+    LZ4F_cctx_t *cctxPtr = &cctx;
+
+    MEM_INIT(&cctx, 0, sizeof(cctx));
+    cctx.version = LZ4F_VERSION;
+    cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
+    if (preferencesPtr == NULL ||
+        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN
+    ) {
+        LZ4_resetStream(&lz4ctx);
+        cctxPtr->lz4CtxPtr = &lz4ctx;
+        cctxPtr->lz4CtxLevel = 2;
+    }
+#endif
+
+    result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
+                                           srcBuffer, srcSize,
+                                           NULL, preferencesPtr);
+
+#if (LZ4_HEAPMODE)
+    LZ4F_freeCompressionContext(cctxPtr);
+#else
+    if (preferencesPtr != NULL &&
+        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN
+    ) {
+        FREEMEM(cctxPtr->lz4CtxPtr);
+    }
+#endif
+    return result;
 }
 
 
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index a59b94b52..be587e6e2 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -107,6 +107,7 @@ LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
 
 /*! LZ4_compressFrame_usingCDict() :
  *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ *  cctx must point to a context created by LZ4F_createCompressionContext().
  *  If cdict==NULL, compress without a dictionary.
  *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
  *  If this condition is not respected, function will fail (@return an errorCode).
@@ -115,6 +116,7 @@ LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
  * @return : number of bytes written into dstBuffer.
  *           or an error code if it fails (can be tested using LZ4F_isError()) */
 LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
     void* dst, size_t dstCapacity,
     const void* src, size_t srcSize,
     const LZ4F_CDict* cdict,
diff --git a/programs/lz4io.c b/programs/lz4io.c
index c712fe1e4..ca13316e8 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -560,7 +560,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     /* single-block file */
     if (readSize < blockSize) {
         /* Compress in single pass */
-        size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
+        size_t cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
         if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
         compressedfilesize = cSize;
         DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ",
diff --git a/tests/framebench.c b/tests/framebench.c
index 21c37049a..a7a270bde 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -30,6 +30,7 @@ typedef struct {
 
 size_t compress_frame(bench_params_t *p) {
   size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
   const char* ibuf = p->ibuf;
@@ -43,6 +44,7 @@ size_t compress_frame(bench_params_t *p) {
   prefs->frameInfo.contentSize = isize;
 
   oused = LZ4F_compressFrame_usingCDict(
+    cctx,
     obuf,
     osize,
     ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
diff --git a/tests/frametest.c b/tests/frametest.c
index 986a16d8d..74d9c88fe 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -164,7 +164,7 @@ static unsigned FUZ_highbit(U32 v32)
 /*-*******************************************************
 *  Tests
 *********************************************************/
-#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error
+#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s\n", LZ4F_getErrorName(v)); goto _output_error; }
 #define CHECK(f)   { LZ4F_errorCode_t const CHECK_V(err_ , f); }
 
 int basicTests(U32 seed, double compressibility)
@@ -509,23 +509,25 @@ int basicTests(U32 seed, double compressibility)
         CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
     }
 
-
     /* Dictionary compression test */
     {   size_t const dictSize = 63 KB;
         size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
         size_t cSizeNoDict, cSizeWithDict;
         LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
         if (cdict == NULL) goto _output_error;
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
         CHECK_V(cSizeNoDict,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               NULL, NULL) );
         DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
 
+        CHECK( LZ4F_freeCompressionContext(cctx) );
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
         CHECK_V(cSizeWithDict,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, NULL) );
         DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -557,7 +559,7 @@ int basicTests(U32 seed, double compressibility)
             memset(&cParams, 0, sizeof(cParams));
             cParams.compressionLevel = -3;
             CHECK_V(cSizeLevelMax,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
@@ -569,7 +571,7 @@ int basicTests(U32 seed, double compressibility)
             memset(&cParams, 0, sizeof(cParams));
             cParams.compressionLevel = LZ4F_compressionLevel_max();
             CHECK_V(cSizeLevelMax,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
@@ -584,7 +586,7 @@ int basicTests(U32 seed, double compressibility)
             cParams.frameInfo.blockMode = LZ4F_blockLinked;
             cParams.frameInfo.blockSizeID = LZ4F_max64KB;
             CHECK_V(cSizeContiguous,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -620,7 +622,7 @@ int basicTests(U32 seed, double compressibility)
             cParams.frameInfo.blockMode = LZ4F_blockIndependent;
             cParams.frameInfo.blockSizeID = LZ4F_max64KB;
             CHECK_V(cSizeIndep,
-                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
             DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
@@ -647,6 +649,7 @@ int basicTests(U32 seed, double compressibility)
         }
 
         LZ4F_freeCDict(cdict);
+        CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
     }
 
 

From 80790c587b55b8e0b932b64c892ba984cde1b70f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 12:06:43 -0500
Subject: [PATCH 056/257] Copy the Dict Table Into the Context for Large
 Compressions

---
 lib/lz4.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index cace35fd8..f7de564d4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1205,20 +1205,25 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-            if (streamPtr->dictCtx) {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+        if (streamPtr->dictCtx && inputSize < 2 KB) {
+            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+            if ((streamPtr->dictCtx->dictSize < 64 KB) && (streamPtr->dictCtx->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
             if (streamPtr->dictCtx) {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
+              /* For compressing large blobs, it is faster to pay the setup
+               * cost to copy the dictionary's tables into the active context,
+               * so that the compression loop is only looking in one table.
+               */
+              memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
+            }
+            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
+            if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }

From efc419a6d4448a5806715e967384ea85f880fe59 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 26 Jan 2018 17:29:50 -0500
Subject: [PATCH 057/257] Replace calloc() Calls With malloc() Where Possible

---
 lib/lz4.c      | 12 ++++++------
 lib/lz4frame.c | 31 ++++++++++++++++---------------
 lib/lz4hc.c    |  2 +-
 3 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f7de564d4..990096cd1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -154,7 +154,8 @@
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOCATOR(n,s) calloc(n,s)
+#define ALLOC(s) malloc(s)
+#define ALLOC_AND_ZERO(s) calloc(1,s)
 #define FREEMEM        free
 #include <string.h>   /* memset, memcpy */
 #define MEM_INIT       memset
@@ -856,7 +857,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
 {
     int result;
 #if (LZ4_HEAPMODE)
-    LZ4_stream_t* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1070,7 +1071,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
 int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
 {
 #if (LZ4_HEAPMODE)
-    LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctxBody;
     LZ4_stream_t* ctx = &ctxBody;
@@ -1092,7 +1093,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
 
 LZ4_stream_t* LZ4_createStream(void)
 {
-    LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64);
+    LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
     DEBUGLOG(4, "LZ4_createStream %p", lz4s);
     LZ4_resetStream(lz4s);
@@ -1193,7 +1194,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
         }
     }
 
-
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
@@ -1493,7 +1493,7 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
 {
-    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
+    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t));
     return lz4s;
 }
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e77fd35aa..8ed89e33f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -63,7 +63,8 @@ You can contact the author at :
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOCATOR(s)   calloc(1,s)
+#define ALLOC(s)   malloc(s)
+#define ALLOC_AND_ZERO(s)   calloc(1,s)
 #define FREEMEM        free
 #include <string.h>   /* memset, memcpy, memmove */
 #define MEM_INIT       memset
@@ -300,7 +301,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
                                           size_t alreadyBuffered)
 {
     LZ4F_preferences_t prefsNull;
-    memset(&prefsNull, 0, sizeof(prefsNull));
+    MEM_INIT(&prefsNull, 0, sizeof(prefsNull));
     prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;   /* worst case */
     {   const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
         U32 const flush = prefsPtr->autoFlush | (srcSize==0);
@@ -329,7 +330,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
     size_t const headerSize = maxFHSize;      /* max header size, including optional fields */
 
     if (preferencesPtr!=NULL) prefs = *preferencesPtr;
-    else memset(&prefs, 0, sizeof(prefs));
+    else MEM_INIT(&prefs, 0, sizeof(prefs));
     prefs.autoFlush = 1;
 
     return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;
@@ -360,7 +361,7 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
     else
-        memset(&prefs, 0, sizeof(prefs));
+        MEM_INIT(&prefs, 0, sizeof(prefs));
     if (prefs.frameInfo.contentSize != 0)
         prefs.frameInfo.contentSize = (U64)srcSize;   /* auto-correct content size if selected (!=0) */
 
@@ -369,7 +370,7 @@ size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
-    memset(&options, 0, sizeof(options));
+    MEM_INIT(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
     if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))  /* condition to guarantee success */
@@ -465,7 +466,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
         dictStart += dictSize - 64 KB;
         dictSize = 64 KB;
     }
-    cdict->dictContent = ALLOCATOR(dictSize);
+    cdict->dictContent = ALLOC(dictSize);
     cdict->fastCtx = LZ4_createStream();
     cdict->HCCtx = LZ4_createStreamHC();
     if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
@@ -504,7 +505,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
  */
 LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
 {
-    LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t));
+    LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
     if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
 
     cctxPtr->version = version;
@@ -547,7 +548,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* headerStart;
 
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
-    memset(&prefNull, 0, sizeof(prefNull));
+    MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
     cctxPtr->prefs = *preferencesPtr;
 
@@ -588,7 +589,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         if (cctxPtr->maxBufferSize < requiredBuffSize) {
             cctxPtr->maxBufferSize = 0;
             FREEMEM(cctxPtr->tmpBuff);
-            cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
+            cctxPtr->tmpBuff = (BYTE*)ALLOC(requiredBuffSize);
             if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->maxBufferSize = requiredBuffSize;
     }   }
@@ -813,7 +814,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
-    memset(&cOptionsNull, 0, sizeof(cOptionsNull));
+    MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
     if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
 
     /* complete tmp buffer */
@@ -1017,7 +1018,7 @@ struct LZ4F_dctx_s {
  */
 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
 {
-    LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
+    LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
     if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
 
     dctx->version = versionNumber;
@@ -1089,7 +1090,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
 
     /* need to decode header to get frameInfo */
     if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete);   /* minimal frame header size */
-    memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
+    MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
 
     /* special case : skippable frames */
     if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
@@ -1316,7 +1317,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
     size_t nextSrcSizeHint = 1;
 
 
-    memset(&optionsNull, 0, sizeof(optionsNull));
+    MEM_INIT(&optionsNull, 0, sizeof(optionsNull));
     if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
     *srcSizePtr = 0;
     *dstSizePtr = 0;
@@ -1365,11 +1366,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 if (bufferNeeded > dctx->maxBufferSize) {   /* tmp buffers too small */
                     dctx->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
                     FREEMEM(dctx->tmpIn);
-                    dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */);
+                    dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */);
                     if (dctx->tmpIn == NULL)
                         return err0r(LZ4F_ERROR_allocation_failed);
                     FREEMEM(dctx->tmpOutBuffer);
-                    dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
+                    dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
                     if (dctx->tmpOutBuffer== NULL)
                         return err0r(LZ4F_ERROR_allocation_failed);
                     dctx->maxBufferSize = bufferNeeded;
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2a6e080ba..0c1b20ae2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -858,7 +858,7 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
 
 void* LZ4_createHC (char* inputBuffer)
 {
-    LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t));
+    LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
     hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer;

From d6ed9a7799c35d9d259af36c3142739216ba8a58 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 30 Jan 2018 15:22:29 -0500
Subject: [PATCH 058/257] Avoid dictionary == NULL Check

---
 lib/lz4.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 990096cd1..3e456ac9f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -590,6 +590,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         (const BYTE*) source - dictCtx->currentOffset :
         (const BYTE*) source - dictSize - currentOffset;
     const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+    const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
@@ -619,6 +620,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         lowLimit = (const BYTE*)source;
         break;
     }
+
+    dictLowLimit = dictionary ? dictionary : lowLimit;
+
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
@@ -653,15 +657,15 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                         /* TODO: use precalc-ed hash? */
                         match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                         refDelta = dictDelta;
-                        lowLimit = dictionary;
+                        lowLimit = dictLowLimit;
                     } else {
                         refDelta = 0;
                         lowLimit = (const BYTE*)source;
                     }
                 } else if (dictDirective==usingExtDict) {
-                    if (match < (const BYTE*)source && dictionary != NULL) {
+                    if (match < (const BYTE*)source) {
                         refDelta = dictDelta;
-                        lowLimit = dictionary;
+                        lowLimit = dictLowLimit;
                     } else {
                         refDelta = 0;
                         lowLimit = (const BYTE*)source;
@@ -703,7 +707,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictionary)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictLowLimit)) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -754,15 +758,15 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 /* TODO: use precalc-ed hash? */
                 match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                 refDelta = dictDelta;
-                lowLimit = dictionary;
+                lowLimit = dictLowLimit;
             } else {
                 refDelta = 0;
                 lowLimit = (const BYTE*)source;
             }
         } else if (dictDirective==usingExtDict) {
-            if (match < (const BYTE*)source && dictionary != NULL) {
+            if (match < (const BYTE*)source) {
                 refDelta = dictDelta;
-                lowLimit = dictionary;
+                lowLimit = dictLowLimit;
             } else {
                 refDelta = 0;
                 lowLimit = (const BYTE*)source;

From 1c4601d64377cdd4d5fc17ccd1f6e1426c7ddd04 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 16 Feb 2018 17:33:51 -0800
Subject: [PATCH 059/257] Restore DictIssue Check

---
 lib/lz4.c | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3e456ac9f..a401ee280 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -599,9 +599,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     U32 forwardH;
 
-    /* TODO: resurrect dictIssue check */
-    (void)dictIssue;
-
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     switch(dictDirective)
@@ -673,7 +670,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( match < lowRefLimit
+            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
@@ -772,7 +769,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 lowLimit = (const BYTE*)source;
         }   }
         LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( match >= lowRefLimit
+        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
             && (match+MAX_DISTANCE>=ip)
             && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
         { token=op++; *token=0; goto _next_match; }
@@ -837,21 +834,37 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            }
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            if (ctx->currentOffset) {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+            } else {
+              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            }
         }
     }
 }
@@ -1249,7 +1262,11 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
     LZ4_renormDictT(streamPtr, smallest);
 
-    result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+    if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
+      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+    } else {
+      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+    }
 
     streamPtr->dictionary = (const BYTE*)source;
     streamPtr->dictSize = (U32)inputSize;

From d571d0cdbaa7032fff48543f272f6d5472041aa7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 5 Mar 2018 11:59:22 -0500
Subject: [PATCH 060/257] Avoid DictSmall Checks By Strategically Bumping
 CurrentOffset

---
 lib/lz4.c | 45 +++++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index a401ee280..7c5b4449a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -546,7 +546,10 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
-  if (dictDirective == usingExtDictCtx && cctx->currentOffset == 0) {
+  if (dictDirective == usingExtDictCtx &&
+      tableType != byPtr &&
+      cctx->currentOffset == 0)
+  {
       cctx->currentOffset = 1;
   }
 }
@@ -842,11 +845,8 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
-            } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
-            }
+            ctx->currentOffset += 64 KB;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
@@ -860,11 +860,8 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
-            } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
-            }
+            ctx->currentOffset += 64 KB;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1222,21 +1219,25 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* external dictionary mode */
     {   int result;
-        if (streamPtr->dictCtx && inputSize < 2 KB) {
-            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-            if ((streamPtr->dictCtx->dictSize < 64 KB) && (streamPtr->dictCtx->dictSize < streamPtr->currentOffset)) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, dictSmall, acceleration);
+        if (streamPtr->dictCtx) {
+            /* We depend here on the fact that dictCtx'es (produced by
+             * LZ4_loadDict) guarantee that their tables contain no references
+             * to offsets between dictCtx->currentOffset - 64 KB and
+             * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe
+             * to use noDictIssue even when the dict isn't a full 64 KB.
+             */
+            if (inputSize > 2 KB) {
+                /* For compressing large blobs, it is faster to pay the setup
+                 * cost to copy the dictionary's tables into the active context,
+                 * so that the compression loop is only looking in one table.
+                 */
+                memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
+                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
-            if (streamPtr->dictCtx) {
-              /* For compressing large blobs, it is faster to pay the setup
-               * cost to copy the dictionary's tables into the active context,
-               * so that the compression loop is only looking in one table.
-               */
-              memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
-            }
             LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);

From 00eadadbfc37587e88733942a252c2f106c67ef2 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Feb 2018 17:06:24 -0800
Subject: [PATCH 061/257] Reset Table on Inputs Larger than 2KB

---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 7c5b4449a..481091047 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -539,7 +539,8 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
        cctx->currentOffset + inputSize >= 0xFFFFU) ||
       (tableType == byU32 &&
        cctx->currentOffset > 1 GB) ||
-      tableType == byPtr))
+      tableType == byPtr ||
+      inputSize >= 2 KB))
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);

From b78cf67c9624a6a826ceaa2a9d65f5feb1dd0cfd Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 6 Mar 2018 11:52:02 -0500
Subject: [PATCH 062/257] Move to 4KB Cut-Off

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 481091047..d147681e3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -540,7 +540,7 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       (tableType == byU32 &&
        cctx->currentOffset > 1 GB) ||
       tableType == byPtr ||
-      inputSize >= 2 KB))
+      inputSize >= 4 KB))
   {
       DEBUGLOG(4, "Resetting table in %p", cctx);
       MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
@@ -1227,7 +1227,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
              * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe
              * to use noDictIssue even when the dict isn't a full 64 KB.
              */
-            if (inputSize > 2 KB) {
+            if (inputSize > 4 KB) {
                 /* For compressing large blobs, it is faster to pay the setup
                  * cost to copy the dictionary's tables into the active context,
                  * so that the compression loop is only looking in one table.

From 64bcbf400ea6c955b029f74e1214dcd03b90f934 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 6 Mar 2018 15:53:22 -0500
Subject: [PATCH 063/257] Optimize Dict Check Condition

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d147681e3..4f98efa38 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -708,7 +708,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && (dictionary != NULL) && (lowLimit==dictLowLimit)) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && ((lowLimit==dictLowLimit) & (dictionary != NULL))) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);

From b4335a6585fe55aa81bcadaa29e265ffe9550ff2 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 12:29:45 -0500
Subject: [PATCH 064/257] Further Avoid a dictionary==NULL Check

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4f98efa38..da68ddd78 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -708,7 +708,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && ((lowLimit==dictLowLimit) & (dictionary != NULL))) {
+            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && lowLimit==dictionary) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);

From 6716325ae8db3f177630c875622401e35e4a84e5 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 12:30:34 -0500
Subject: [PATCH 065/257] Remove Switch In Favor of Ternary Statement

---
 lib/lz4.c | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index da68ddd78..809faf9e2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -605,27 +605,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
-    switch(dictDirective)
-    {
-    case noDict:
-    default:
-        lowLimit = (const BYTE*)source;
-        break;
-    case withPrefix64k:
-        lowLimit = (const BYTE*)source - dictSize;
-        break;
-    case usingExtDict:
-        lowLimit = (const BYTE*)source;
-        break;
-    case usingExtDictCtx:
-        lowLimit = (const BYTE*)source;
-        break;
-    }
 
+    lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
     dictLowLimit = dictionary ? dictionary : lowLimit;
 
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
-
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */

From 5f8967b98364629310d0013b356070ab445f9e66 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 9 Mar 2018 12:14:42 -0500
Subject: [PATCH 066/257] Specialize _extState() for Clean Ctx Rather Than
 Calling _safeExtState()

---
 lib/lz4.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 809faf9e2..82c0ae341 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -805,8 +805,24 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
 int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
-  LZ4_resetStream((LZ4_stream_t*)state);
-  return LZ4_compress_fast_safeExtState(state, source, dest, inputSize, maxOutputSize, acceleration);
+    LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
+    if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
+    LZ4_resetStream((LZ4_stream_t*)state);
+    if (maxOutputSize >= LZ4_compressBound(inputSize)) {
+        if (inputSize < LZ4_64Klimit) {
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, byU16, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+        }
+    } else {
+        if (inputSize < LZ4_64Klimit) {;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+        } else {
+            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+        }
+    }
 }
 
 
@@ -861,9 +877,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
 #endif
-    ctxPtr->internal_donotuse.initCheck = 0;
-    ctxPtr->internal_donotuse.tableType = byPtr; /* always triggers a reset */
-    result = LZ4_compress_fast_safeExtState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
+    result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
 #if (LZ4_HEAPMODE)
     FREEMEM(ctxPtr);

From e34716cc3f54d3fca1cbd797e8af003cd3a63bc8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 8 Mar 2018 14:09:06 -0500
Subject: [PATCH 067/257] Preserve currentOffset==0 When Possible

---
 lib/lz4.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 82c0ae341..1ea8ba136 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -846,7 +846,9 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            ctx->currentOffset += 64 KB;
+            if (ctx->currentOffset) {
+              ctx->currentOffset += 64 KB;
+            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
@@ -861,7 +863,9 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_resetTable(ctx, inputSize, tableType, noDict);
-            ctx->currentOffset += 64 KB;
+            if (ctx->currentOffset) {
+              ctx->currentOffset += 64 KB;
+            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }

From 3ecc1d7a5b0f911c57851cc8a925d7be2518d358 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 16:11:44 -0400
Subject: [PATCH 068/257] Minor Style Fixes

---
 lib/lz4frame.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 8ed89e33f..16769febf 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -50,12 +50,12 @@ You can contact the author at :
 *  Tuning parameters
 **************************************/
 /*
- * LZ4_HEAPMODE :
+ * LZ4F_HEAPMODE :
  * Select how default compression functions will allocate memory for their hash table,
  * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
  */
-#ifndef LZ4_HEAPMODE
-#  define LZ4_HEAPMODE 0
+#ifndef LZ4F_HEAPMODE
+#  define LZ4F_HEAPMODE 0
 #endif
 
 
@@ -404,7 +404,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                     const LZ4F_preferences_t* preferencesPtr)
 {
     size_t result;
-#if (LZ4_HEAPMODE)
+#if (LZ4F_HEAPMODE)
     LZ4F_cctx_t *cctxPtr;
     LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
 #else
@@ -416,8 +416,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     cctx.version = LZ4F_VERSION;
     cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
     if (preferencesPtr == NULL ||
-        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN
-    ) {
+        preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
+    {
         LZ4_resetStream(&lz4ctx);
         cctxPtr->lz4CtxPtr = &lz4ctx;
         cctxPtr->lz4CtxLevel = 2;
@@ -428,12 +428,12 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
                                            srcBuffer, srcSize,
                                            NULL, preferencesPtr);
 
-#if (LZ4_HEAPMODE)
+#if (LZ4F_HEAPMODE)
     LZ4F_freeCompressionContext(cctxPtr);
 #else
     if (preferencesPtr != NULL &&
-        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN
-    ) {
+        preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
+    {
         FREEMEM(cctxPtr->lz4CtxPtr);
     }
 #endif

From 1df5d911aa23e645b873164382a40186ed704bac Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 16:11:55 -0400
Subject: [PATCH 069/257] Hoist LZ4F Dictionary Setup into Helper
 LZ4F_applyCDict()

---
 lib/lz4frame.c | 72 ++++++++++++++++++--------------------------------
 1 file changed, 25 insertions(+), 47 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 16769febf..4cc2ef31e 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -531,6 +531,28 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 }
 
 
+static void LZ4F_applyCDict(void* ctx,
+                            const LZ4F_CDict* cdict,
+                            int level) {
+    if (level < LZ4HC_CLEVEL_MIN) {
+        LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
+        assert(!internal_ctx->initCheck);
+        /* Clear any local dictionary */
+        internal_ctx->dictionary = NULL;
+        internal_ctx->dictSize = 0;
+        /* Point to the dictionary context */
+        internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
+    } else {
+        if (cdict) {
+            memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+            LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        } else {
+            LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level);
+        }
+    }
+}
+
+
 /*! LZ4F_compressBegin_usingCDict() :
  *  init streaming compression and writes frame header into dstBuffer.
  *  dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
@@ -601,39 +623,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->cdict = cdict;
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
         /* frame init only for blockLinked : blockIndependent will be init at each block */
-        if (cdict) {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
-                assert(!internal_ctx->initCheck);
-                if (internal_ctx->currentOffset > 1 GB) {
-                    /* Init the context */
-                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
-                }
-                /* Clear any local dictionary */
-                internal_ctx->dictionary = NULL;
-                internal_ctx->dictSize = 0;
-                /* Point to the dictionary context */
-                internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
-            } else {
-                memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
-                LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
-            }
-        } else {
-            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)cctxPtr->lz4CtxPtr)->internal_donotuse;
-                assert(!internal_ctx->initCheck);
-                if (internal_ctx->currentOffset > 1 GB) {
-                    /* Init the context */
-                    LZ4_resetStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr);
-                }
-                /* Clear any local dictionary */
-                internal_ctx->dictionary = NULL;
-                internal_ctx->dictSize = 0;
-                internal_ctx->dictCtx = NULL;
-            } else {
-                LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
-            }
-        }
+        LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
 
     /* Magic Number */
@@ -728,21 +718,10 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
-    LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t*)ctx)->internal_donotuse;
-    assert(!internal_ctx->initCheck);
-    if (internal_ctx->currentOffset > 1 GB) {
-        /* Init the context */
-        LZ4_resetStream((LZ4_stream_t*)ctx);
-    }
-    /* Clear any local dictionary */
-    internal_ctx->dictionary = NULL;
-    internal_ctx->dictSize = 0;
+    LZ4F_applyCDict(ctx, cdict, level);
     if (cdict) {
-        /* Point to the dictionary context */
-        internal_ctx->dictCtx = &(cdict->fastCtx->internal_donotuse);
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        internal_ctx->dictCtx = NULL;
         return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
@@ -757,8 +736,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     if (cdict) {
-        memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
-        LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        LZ4F_applyCDict(ctx, cdict, level);
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
     return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);

From f9fef255a12470b7beeaa8635b65fc837d835c84 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:13:24 -0400
Subject: [PATCH 070/257] Renames and Comment Fixes

---
 lib/lz4.c          | 32 ++++++++++++++++++++++----------
 lib/lz4.h          | 14 +++++++-------
 lib/lz4frame.c     |  2 +-
 tests/framebench.c |  2 +-
 4 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1ea8ba136..94248bebe 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -524,7 +524,7 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
 }
 
 
-LZ4_FORCE_INLINE void LZ4_resetTable(
+LZ4_FORCE_INLINE void LZ4_prepareTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
         const tableType_t tableType,
@@ -547,6 +547,11 @@ LZ4_FORCE_INLINE void LZ4_resetTable(
       cctx->currentOffset = 0;
       cctx->tableType = unusedTable;
   }
+  /* If the current offset is zero, we will never look in the external
+   * dictionary context, since there is no value a table entry can take that
+   * indicates a miss. In that case, we need to bump the offset to something
+   * non-zero.
+   */
   if (dictDirective == usingExtDictCtx &&
       tableType != byPtr &&
       cctx->currentOffset == 0)
@@ -825,8 +830,15 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     }
 }
 
-
-int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+/**
+ * LZ4_compress_fast_extState_noReset is a variant of LZ4_compress_fast_extState
+ * that can be used when the state is known to have already been initialized
+ * (via LZ4_resetStream or an earlier call to LZ4_compress_fast_extState /
+ * LZ4_compress_fast_extState_noReset). This can provide significantly better
+ * performance when the context reset would otherwise be a significant part of
+ * the cost of the compression, e.g., when the data to be compressed is small.
+ */
+int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -837,7 +849,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -845,7 +857,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               ctx->currentOffset += 64 KB;
             }
@@ -854,7 +866,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -862,7 +874,7 @@ int LZ4_compress_fast_safeExtState(void* state, const char* source, char* dest,
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_resetTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
               ctx->currentOffset += 64 KB;
             }
@@ -1213,7 +1225,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        LZ4_resetTable(streamPtr, inputSize, tableType, withPrefix64k);
+        LZ4_prepareTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1237,11 +1249,11 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDictCtx);
+                LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
             }
         } else {
-            LZ4_resetTable(streamPtr, inputSize, tableType, usingExtDict);
+            LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
diff --git a/lib/lz4.h b/lib/lz4.h
index 58a1e39aa..ca9d55203 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,21 +179,21 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
-LZ4_compress_fast_safeExtState() :
+LZ4_compress_fast_extState_noReset() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
 
-    Use _safeExtState variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to this function leave
-    the state in a safe state, so zeroing is not required between calls).
-    Otherwise, using legacy _extState requires LZ4 to reinitialize the state
-    internally for every call.
+    Use the _noReset variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to both extState
+    functions leave the state in a safe state, so zeroing is not required
+    between calls). Otherwise, using the legacy _extState requires LZ4 to
+    reinitialize the state internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
-LZ4LIB_API int LZ4_compress_fast_safeExtState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4cc2ef31e..7acd7cf69 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -722,7 +722,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        return LZ4_compress_fast_safeExtState(ctx, src, dst, srcSize, dstCapacity, acceleration);
+        return LZ4_compress_fast_extState_noReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
 
diff --git a/tests/framebench.c b/tests/framebench.c
index a7a270bde..fb4f38cc4 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -120,7 +120,7 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_safeExtState(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_extState_noReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
   obuf += oused;
 
   return obuf - p->obuf;

From 299f34909a2da7dc7096747e1c62cfe3994ec467 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 9 Mar 2018 12:05:31 -0500
Subject: [PATCH 071/257] Simpler Ternary Statements

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 94248bebe..6a680e676 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -582,9 +582,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
     const BYTE* const dictionary =
-        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictionary;
+        dictDirective == usingExtDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
-        (dictDirective == usingExtDictCtx ? dictCtx : cctx)->dictSize;
+        dictDirective == usingExtDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
     const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
     const BYTE* const dictEnd = dictionary + dictSize;

From 5149767a1b3caba4c17c0e79247c6092f7ae23ef Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:32:24 -0400
Subject: [PATCH 072/257] Add NULL Checks

---
 lib/lz4.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 6a680e676..822d57258 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -889,6 +889,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
     int result;
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    if (ctxPtr == NULL) return 0;
 #else
     LZ4_stream_t ctx;
     LZ4_stream_t* const ctxPtr = &ctx;
@@ -1101,6 +1102,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
 {
 #if (LZ4_HEAPMODE)
     LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
+    if (ctx == NULL) return 0;
 #else
     LZ4_stream_t ctxBody;
     LZ4_stream_t* ctx = &ctxBody;
@@ -1125,6 +1127,7 @@ LZ4_stream_t* LZ4_createStream(void)
     LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t));
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
     DEBUGLOG(4, "LZ4_createStream %p", lz4s);
+    if (lz4s == NULL) return NULL;
     LZ4_resetStream(lz4s);
     return lz4s;
 }

From b8e9c7785559b01f1f5ae0db6700e2673f4823db Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 12 Mar 2018 18:46:54 -0400
Subject: [PATCH 073/257] Whitespace Fixes

---
 lib/lz4.c | 89 +++++++++++++++++++++++++++----------------------------
 1 file changed, 44 insertions(+), 45 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 822d57258..0209beb85 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -529,35 +529,34 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
         const int inputSize,
         const tableType_t tableType,
         const dict_directive dictDirective) {
-  /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
-   * therefore safe to use no matter what mode we're in. Otherwise, we figure
-   * out if it's safe to leave as is or whether it needs to be reset.
-   */
-  if (cctx->tableType != unusedTable && (
-      cctx->tableType != tableType ||
-      (tableType == byU16 &&
-       cctx->currentOffset + inputSize >= 0xFFFFU) ||
-      (tableType == byU32 &&
-       cctx->currentOffset > 1 GB) ||
-      tableType == byPtr ||
-      inputSize >= 4 KB))
-  {
-      DEBUGLOG(4, "Resetting table in %p", cctx);
-      MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
-      cctx->currentOffset = 0;
-      cctx->tableType = unusedTable;
-  }
-  /* If the current offset is zero, we will never look in the external
-   * dictionary context, since there is no value a table entry can take that
-   * indicates a miss. In that case, we need to bump the offset to something
-   * non-zero.
-   */
-  if (dictDirective == usingExtDictCtx &&
-      tableType != byPtr &&
-      cctx->currentOffset == 0)
-  {
-      cctx->currentOffset = 1;
-  }
+    /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
+     * therefore safe to use no matter what mode we're in. Otherwise, we figure
+     * out if it's safe to leave as is or whether it needs to be reset.
+     */
+    if (cctx->tableType != unusedTable) {
+        if (cctx->tableType != tableType
+          || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
+          || (tableType == byU32 && cctx->currentOffset > 1 GB)
+          || tableType == byPtr
+          || inputSize >= 4 KB)
+        {
+            DEBUGLOG(4, "Resetting table in %p", cctx);
+            MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
+            cctx->currentOffset = 0;
+            cctx->tableType = unusedTable;
+        }
+    }
+    /* If the current offset is zero, we will never look in the external
+     * dictionary context, since there is no value a table entry can take that
+     * indicates a miss. In that case, we need to bump the offset to something
+     * non-zero.
+     */
+    if (dictDirective == usingExtDictCtx &&
+        tableType != byPtr &&
+        cctx->currentOffset == 0)
+    {
+        cctx->currentOffset = 1;
+    }
 }
 
 /** LZ4_compress_generic() :
@@ -793,12 +792,12 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
 _clean_up:
     if (dictDirective == usingExtDictCtx) {
-      /* Subsequent linked blocks can't use the dictionary. */
-      /* Instead, they use the block we just compressed. */
-      cctx->dictCtx = NULL;
-      cctx->dictSize = (U32)inputSize;
+        /* Subsequent linked blocks can't use the dictionary. */
+        /* Instead, they use the block we just compressed. */
+        cctx->dictCtx = NULL;
+        cctx->dictSize = (U32)inputSize;
     } else {
-      cctx->dictSize += (U32)inputSize;
+        cctx->dictSize += (U32)inputSize;
     }
     cctx->currentOffset += (U32)inputSize;
     cctx->tableType = tableType;
@@ -815,10 +814,10 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     LZ4_resetStream((LZ4_stream_t*)state);
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, byU16, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {;
@@ -851,32 +850,32 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              ctx->currentOffset += 64 KB;
+                ctx->currentOffset += 64 KB;
             }
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0,    notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
-              return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
             if (ctx->currentOffset) {
-              ctx->currentOffset += 64 KB;
+                ctx->currentOffset += 64 KB;
             }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
@@ -1282,9 +1281,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     LZ4_renormDictT(streamPtr, smallest);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-      result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;

From 2be38a742917651359cdfc737b74bda8b50ece73 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:07:19 -0400
Subject: [PATCH 074/257] Rename Enums and Add Comment

---
 lib/lz4.c | 59 +++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 19 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0209beb85..f86d3ae56 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -447,9 +447,32 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 *  Local Structures and types
 **************************************/
 typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
-typedef enum { unusedTable = 0, byPtr = 1, byU32 = 2, byU16 = 3 } tableType_t;
+typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
 
-typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingExtDictCtx } dict_directive;
+/**
+ * This enum distinguishes several different modes of accessing previous
+ * content in the stream.
+ *
+ * - noDict        : There is no preceding content.
+ * - withPrefix64k : Table entries up to ctx->dictSize before the current blob
+ *                   blob being compressed are valid and refer to the preceding
+ *                   content (of length ctx->dictSize), which is available
+ *                   contiguously preceding in memory the content currently
+ *                   being compressed.
+ * - usingExtDict  : Like withPrefix64k, but the preceding content is somewhere
+ *                   else in memory, starting at ctx->dictionary with length
+ *                   ctx->dictSize.
+ * - usingDictCtx  : Like usingExtDict, but everything concerning the preceding
+ *                   content is in a separate context, pointed to by
+ *                   ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table
+ *                   entries in the current context that refer to positions
+ *                   preceding the beginning of the current compression are
+ *                   ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx
+ *                   ->dictSize describe the location and size of the preceding
+ *                   content, and matches are found by looking in the ctx
+ *                   ->dictCtx->hashTable.
+ */
+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive;
 typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
 
 typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
@@ -497,7 +520,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
 {
     switch (tableType)
     {
-    case unusedTable: { /* illegal! */ assert(0); return; }
+    case clearedTable: { /* illegal! */ assert(0); return; }
     case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }
     case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }
     case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }
@@ -533,7 +556,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * therefore safe to use no matter what mode we're in. Otherwise, we figure
      * out if it's safe to leave as is or whether it needs to be reset.
      */
-    if (cctx->tableType != unusedTable) {
+    if (cctx->tableType != clearedTable) {
         if (cctx->tableType != tableType
           || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU)
           || (tableType == byU32 && cctx->currentOffset > 1 GB)
@@ -543,7 +566,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             DEBUGLOG(4, "Resetting table in %p", cctx);
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
-            cctx->tableType = unusedTable;
+            cctx->tableType = clearedTable;
         }
     }
     /* If the current offset is zero, we will never look in the external
@@ -551,7 +574,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * indicates a miss. In that case, we need to bump the offset to something
      * non-zero.
      */
-    if (dictDirective == usingExtDictCtx &&
+    if (dictDirective == usingDictCtx &&
         tableType != byPtr &&
         cctx->currentOffset == 0)
     {
@@ -581,9 +604,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
     const BYTE* const dictionary =
-        dictDirective == usingExtDictCtx ? dictCtx->dictionary : cctx->dictionary;
+        dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
-        dictDirective == usingExtDictCtx ? dictCtx->dictSize : cctx->dictSize;
+        dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
     const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
     const BYTE* const dictEnd = dictionary + dictSize;
@@ -594,7 +617,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
-    const BYTE* dictBase = dictDirective == usingExtDictCtx ?
+    const BYTE* dictBase = dictDirective == usingDictCtx ?
         (const BYTE*) source - dictCtx->currentOffset :
         (const BYTE*) source - dictSize - currentOffset;
     const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
@@ -640,10 +663,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dictDirective == usingExtDictCtx) {
+                if (dictDirective == usingDictCtx) {
                     if (match < (const BYTE*)source) {
                         /* there was no match, try the dictionary */
-                        /* TODO: use precalc-ed hash? */
                         match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                         refDelta = dictDelta;
                         lowLimit = dictLowLimit;
@@ -696,7 +718,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingExtDictCtx) && lowLimit==dictionary) {
+            if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
@@ -741,10 +763,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
         /* Test next position */
         match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dictDirective == usingExtDictCtx) {
+        if (dictDirective == usingDictCtx) {
             if (match < (const BYTE*)source) {
                 /* there was no match, try the dictionary */
-                /* TODO: use precalc-ed hash? */
                 match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
                 refDelta = dictDelta;
                 lowLimit = dictLowLimit;
@@ -791,7 +812,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     retval = (((char*)op)-dest);
 
 _clean_up:
-    if (dictDirective == usingExtDictCtx) {
+    if (dictDirective == usingDictCtx) {
         /* Subsequent linked blocks can't use the dictionary. */
         /* Instead, they use the block we just compressed. */
         cctx->dictCtx = NULL;
@@ -1135,7 +1156,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.tableType = unusedTable;
+    LZ4_stream->internal_donotuse.tableType = clearedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
@@ -1158,7 +1179,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
 
     if ((dict->initCheck)
-      || (dict->tableType != byU32 && dict->tableType != unusedTable)
+      || (dict->tableType != byU32 && dict->tableType != clearedTable)
       || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ4_resetStream(LZ4_dict);
 
@@ -1251,8 +1272,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDictCtx);
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDictCtx, noDictIssue, acceleration);
+                LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
             LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);

From c4aef7cd62cff0b71464dc58f40efc284bcb5a7b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:18:08 -0400
Subject: [PATCH 075/257] Restore checkTag Cleaning

---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 77d9e36cd..a351d35e2 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -113,7 +113,7 @@ clean:
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
         framebench$(EXT) \
-        fasttest$(EXT) datagen$(EXT)
+        fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
 

From 146e67653171b9b43322e2ab3d8be7b085b03bf7 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 15:42:03 -0400
Subject: [PATCH 076/257] Restore LZ4_sizeofStreamState, We Didn't Actually
 Need to Delete It

---
 lib/lz4.c | 3 +++
 lib/lz4.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index f86d3ae56..48cdad190 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1703,6 +1703,9 @@ They are only provided here for compatibility with older user programs.
 int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
 int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
+/* Obsolete Streaming functions */
+
+int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
 
 /* Obsolete streaming decompression functions */
 
diff --git a/lib/lz4.h b/lib/lz4.h
index ca9d55203..e4a257b88 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -477,6 +477,9 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
 LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);

From 640db34e43481fdd7c69462ca8a84c9d964cdfd6 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:35:44 -0400
Subject: [PATCH 077/257] Another Allocation Fail Check

---
 lib/lz4frame.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7acd7cf69..d0c69d15e 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -406,7 +406,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     size_t result;
 #if (LZ4F_HEAPMODE)
     LZ4F_cctx_t *cctxPtr;
-    LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+    result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
+    if (LZ4F_isError(result)) return result;
 #else
     LZ4F_cctx_t cctx;
     LZ4_stream_t lz4ctx;

From 995756f218e4f144953ca4dfc8df37493e8f775b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:45:09 -0400
Subject: [PATCH 078/257] Split lz4CtxLevel into Two Fields

---
 lib/lz4frame.c | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d0c69d15e..87e209f3a 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -202,15 +202,8 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    /* lz4CtxLevel records both what size of memory has been allocated into the
-     * ctx pointer field and what it's being used as.
-     * - The low bit (& 1) value indicates whether the ctx is being used for
-     *   hc (1) or not (0).
-     * - The next bit (& 2) indicates whether the allocated memory is big
-     *   enough for a non-hc context.
-     * - The next bit (& 4) indicates whether the allocated memory is big
-     *   enough for an hc context. */
-    U32    lz4CtxLevel;
+    U16    lz4CtxAlloc; // sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
+    U16    lz4CtxState; // in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
 } LZ4F_cctx_t;
 
 
@@ -421,7 +414,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     {
         LZ4_resetStream(&lz4ctx);
         cctxPtr->lz4CtxPtr = &lz4ctx;
-        cctxPtr->lz4CtxLevel = 2;
+        cctxPtr->lz4CtxAlloc = 1;
+        cctxPtr->lz4CtxState = 1;
     }
 #endif
 
@@ -576,9 +570,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->prefs = *preferencesPtr;
 
     /* Ctx Management */
-    {   U32 const ctxSufficientAllocBits = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 2 : 6;
-        U32 const ctxTypeBit = cctxPtr->prefs.compressionLevel >= LZ4HC_CLEVEL_MIN;
-        if ((cctxPtr->lz4CtxLevel & ctxSufficientAllocBits) != ctxSufficientAllocBits) {
+    {   U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
+        if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
             FREEMEM(cctxPtr->lz4CtxPtr);
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
@@ -586,17 +579,17 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             }
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            cctxPtr->lz4CtxLevel = ctxSufficientAllocBits | ctxTypeBit;
-        } else if ((cctxPtr->lz4CtxLevel & 1) != ctxTypeBit) {
+            cctxPtr->lz4CtxAlloc = ctxTypeID;
+            cctxPtr->lz4CtxState = ctxTypeID;
+        } else if (cctxPtr->lz4CtxState != ctxTypeID) {
             /* otherwise, a sufficient buffer is allocated, but we need to
              * reset it to the correct context type */
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
                 LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
-                cctxPtr->lz4CtxLevel &= ~1;
             } else {
                 LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
-                cctxPtr->lz4CtxLevel |= 1;
             }
+            cctxPtr->lz4CtxState = ctxTypeID;
         }
     }
 

From c852f20c39e877b877da49dea52dd4e36a5d6cb9 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 13 Mar 2018 17:47:34 -0400
Subject: [PATCH 079/257] Switch ALLOC() to ALLOC_AND_ZERO() to Paper Over
 Existing Uninitialized Read

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 87e209f3a..b91cb7c31 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -605,7 +605,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         if (cctxPtr->maxBufferSize < requiredBuffSize) {
             cctxPtr->maxBufferSize = 0;
             FREEMEM(cctxPtr->tmpBuff);
-            cctxPtr->tmpBuff = (BYTE*)ALLOC(requiredBuffSize);
+            cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
             if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->maxBufferSize = requiredBuffSize;
     }   }

From 66b6fbfe6fa9d7e8d4686e6525883ff4aef31e02 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 14 Mar 2018 15:51:59 -0400
Subject: [PATCH 080/257] Restore the Other Old Streaming Functions in a
 Degraded Fashion

---
 lib/lz4.c | 19 +++++++++++++++++++
 lib/lz4.h | 11 +++++++++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 48cdad190..37782f5d8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1707,6 +1707,25 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize,
 
 int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
 
+int LZ4_resetStreamState(void* state, char* inputBuffer)
+{
+    (void)inputBuffer;
+    LZ4_resetStream((LZ4_stream_t*)state);
+    return 0;
+}
+
+void* LZ4_create (char* inputBuffer)
+{
+    (void)inputBuffer;
+    return LZ4_createStream();
+}
+
+char* LZ4_slideInputBuffer (void* state)
+{
+    // avoid const char * -> char * conversion warning
+    return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
+}
+
 /* Obsolete streaming decompression functions */
 
 int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
diff --git a/lib/lz4.h b/lib/lz4.h
index e4a257b88..6cb6589d6 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -477,8 +477,15 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+/* Broken, obsolete streaming functions; do not use!
+ *
+ * These functions depended on data that is no longer tracked in the state. They
+ * are therefore broken--they don't retain any history across compressions.
+ */
+LZ4_DEPRECATED("Broken!!! Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("Broken!!! Use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("Broken!!! Use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);

From b0a18896fe53cf2b69142c092f955e57b2b0d8df Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 14 Mar 2018 15:58:38 -0400
Subject: [PATCH 081/257] Move LZ4_compress_fast_extState_noReset Declaration
 to Unstable Section

---
 lib/lz4.h | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 6cb6589d6..d0ec20470 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -179,21 +179,13 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
 
 
 /*!
-LZ4_compress_fast_extState_noReset() :
 LZ4_compress_fast_extState() :
     Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
-
-    Use the _noReset variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to both extState
-    functions leave the state in a safe state, so zeroing is not required
-    between calls). Otherwise, using the legacy _extState requires LZ4 to
-    reinitialize the state internally for every call.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
-LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
@@ -351,6 +343,29 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 /*^**********************************************
  * !!!!!!   STATIC LINKING ONLY   !!!!!!
  ***********************************************/
+
+/*-************************************
+ *  Unstable declarations
+ **************************************
+ * Declarations in this section should be considered unstable.
+ * Use at your own peril, etc., etc.
+ * They may be removed in the future.
+ * Their signatures may change.
+ **************************************/
+
+/*!
+LZ4_compress_fast_extState_noReset() :
+    A variant of LZ4_compress_fast_extState().
+
+    Use the _noReset variant if LZ4_resetStream() was called on the state
+    buffer before being used for the first time (calls to both extState
+    functions leave the state in a safe state, so zeroing is not required
+    between calls). Otherwise, using the plain _extState requires LZ4 to
+    reinitialize the state internally for every call.
+*/
+LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
+
 /*-************************************
  *  Private definitions
  **************************************

From 70f14823a46719e81e808d9ed9df90f478bcfd3f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 31 Jan 2018 18:11:37 -0500
Subject: [PATCH 082/257] Remove Framebench Tool

---
 tests/Makefile     |   6 +-
 tests/framebench.c | 295 ---------------------------------------------
 2 files changed, 1 insertion(+), 300 deletions(-)
 delete mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index a351d35e2..5954370b3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -62,7 +62,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen framebench
+all: fullbench fuzzer frametest datagen
 
 all32: CFLAGS+=-m32
 all32: all
@@ -98,9 +98,6 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -112,7 +109,6 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
deleted file mode 100644
index fb4f38cc4..000000000
--- a/tests/framebench.c
+++ /dev/null
@@ -1,295 +0,0 @@
-#include <assert.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-
-#include "lz4.h"
-#include "lz4frame.h"
-#include "lz4frame_static.h"
-
-#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
-
-typedef struct {
-  size_t iter;
-  LZ4_stream_t *ctx;
-  LZ4F_cctx *cctx;
-  char *obuf;
-  size_t osize;
-  const char* ibuf;
-  size_t isize;
-  size_t num_ibuf;
-  const LZ4F_CDict* cdict;
-  LZ4F_preferences_t* prefs;
-  const LZ4F_compressOptions_t* options;
-} bench_params_t;
-
-size_t compress_frame(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressFrame_usingCDict(
-    cctx,
-    obuf,
-    osize,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
-    isize,
-    cdict,
-    prefs);
-  LZ4F_CHECK(oused);
-
-  return oused;
-}
-
-size_t compress_begin(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-  const LZ4F_compressOptions_t* options = p->options;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressUpdate(
-    cctx,
-    obuf,
-    oend - obuf,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
-    isize,
-    options);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
-  LZ4F_CHECK(oused);
-
-  return obuf - p->obuf;
-}
-
-size_t compress_default(bench_params_t *p) {
-  size_t iter = p->iter;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_extState(bench_params_t *p) {
-  size_t iter = p->iter;
-  LZ4_stream_t *ctx = p->ctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
-  size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_fast_extState_noReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-uint64_t bench(
-    size_t (*fun)(bench_params_t *),
-    uint64_t repetitions,
-    bench_params_t *params,
-    size_t *osizePtr
-) {
-  struct timespec start, end;
-  size_t i, osize = 0;
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
-
-  for (i = 0; i < repetitions; i++) {
-    size_t o;
-    params->iter = i;
-    o = fun(params);
-    if (!o) return 0;
-    osize += o;
-  }
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
-
-  *osizePtr = osize / repetitions;
-  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-}
-
-int main(int argc, char *argv[]) {
-
-
-  struct stat st;
-  size_t bytes_read;
-
-  const char *dict_fn;
-  size_t dict_size;
-  char *dict_buf;
-  FILE *dict_file;
-
-  const char *in_fn;
-  size_t in_size;
-  size_t num_in_buf;
-  size_t cur_in_buf;
-  char *in_buf;
-  FILE *in_file;
-
-  size_t out_size;
-  char *out_buf;
-
-  LZ4_stream_t *ctx;
-  LZ4F_cctx *cctx;
-  LZ4F_CDict *cdict;
-  LZ4F_preferences_t prefs;
-  LZ4F_compressOptions_t options;
-
-  size_t out_used;
-
-  uint64_t time_taken;
-  uint64_t repetitions;
-
-  bench_params_t params;
-
-  if (argc != 3) return 1;
-  dict_fn = argv[1];
-  in_fn = argv[2];
-
-  if (stat(dict_fn, &st)) return 1;
-  dict_size = st.st_size;
-  dict_buf = (char *)malloc(dict_size);
-  if (!dict_buf) return 1;
-  dict_file = fopen(dict_fn, "r");
-  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
-  if (bytes_read != dict_size) return 1;
-
-  if (stat(in_fn, &st)) return 1;
-  in_size = st.st_size;
-  num_in_buf = 256 * 1024 * 1024 / in_size;
-  if (num_in_buf == 0) {
-    num_in_buf = 1;
-  }
-
-  in_buf = (char *)malloc(in_size * num_in_buf);
-  if (!in_buf) return 1;
-  in_file = fopen(in_fn, "r");
-  bytes_read = fread(in_buf, 1, in_size, in_file);
-  if (bytes_read != in_size) return 1;
-
-  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
-    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
-  }
-
-  if (in_size <= 1024) {
-    repetitions = 100000;
-  } else
-  if (in_size <= 16384) {
-    repetitions = 10000;
-  } else
-  if (in_size <= 131072) {
-    repetitions = 1000;
-  } else
-  if (in_size <= 1048576) {
-    repetitions = 100;
-  } else {
-    repetitions = 50;
-  }
-
-  memset(&prefs, 0, sizeof(prefs));
-  prefs.autoFlush = 1;
-  if (in_size < 64 * 1024)
-    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
-  prefs.frameInfo.contentSize = in_size;
-
-  memset(&options, 0, sizeof(options));
-  options.stableSrc = 1;
-
-  out_size = LZ4F_compressFrameBound(in_size, &prefs);
-  out_buf = (char *)malloc(out_size);
-  if (!out_buf) return 1;
-
-  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  ctx = LZ4_createStream();
-  if (ctx == NULL) return 1;
-
-  cdict = LZ4F_createCDict(dict_buf, dict_size);
-  if (!cdict) return 1;
-
-  fprintf(stderr, "dict  size: %zd\n", dict_size);
-  fprintf(stderr, "input size: %zd\n", in_size);
-
-  params.ctx = ctx;
-  params.cctx = cctx;
-  params.obuf = out_buf;
-  params.osize = out_size;
-  params.ibuf = in_buf;
-  params.isize = in_size;
-  params.num_ibuf = num_in_buf;
-  params.cdict = NULL;
-  params.prefs = &prefs;
-  params.options = &options;
-
-  time_taken = bench(compress_default, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_extState, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  return 0;
-}
\ No newline at end of file

From d6711a7cffe3b9f7104ea48c193be7d7fc9e4d69 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 19 Mar 2018 16:18:10 -0700
Subject: [PATCH 083/257] minor man fix on clevels

---
 programs/lz4.1.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index b7b857037..a5168e99d 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -117,9 +117,9 @@ only the latest one will be applied.
 ### Operation modifiers
 
 * `-#`:
-  Compression level, with # being any value from 1 to 16.
+  Compression level, with # being any value from 1 to 12.
   Higher values trade compression speed for compression ratio.
-  Values above 16 are considered the same as 16.
+  Values above 12 are considered the same as 12.
   Recommended values are 1 for fast compression (default),
   and 9 for high compression.
   Speed/compression trade-off will vary depending on data to compress.
@@ -206,7 +206,7 @@ only the latest one will be applied.
   Benchmark multiple compression levels, from b# to e# (included)
 
 * `-i#`:
-  Minimum evaluation in seconds \[1-9\] (default : 3)
+  Minimum evaluation time in seconds \[1-9\] (default : 3)
 
 
 BUGS

From 1faa7e2698e73864aa20729caba8071148af5b4c Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 19 Mar 2018 17:19:25 -0700
Subject: [PATCH 084/257] bench: introduced hidden command -S

to benchmark multiple files with separate results
---
 programs/bench.c  | 34 +++++++++++++++++++++++++++-------
 programs/bench.h  |  5 +++--
 programs/lz4cli.c | 10 +++++++---
 3 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 002eac990..770191cfa 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -119,21 +119,21 @@ static clock_t g_time = 0;
 static U32 g_nbSeconds = NBSECONDS;
 static size_t g_blockSize = 0;
 int g_additionalParam = 0;
+int g_benchSeparately = 0;
 
 void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
 void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
 
-void BMK_SetNbSeconds(unsigned nbSeconds)
+void BMK_setNbSeconds(unsigned nbSeconds)
 {
     g_nbSeconds = nbSeconds;
     DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
 }
 
-void BMK_SetBlockSize(size_t blockSize)
-{
-    g_blockSize = blockSize;
-}
+void BMK_setBlockSize(size_t blockSize) { g_blockSize = blockSize; }
+
+void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); }
 
 
 /* ********************************************************
@@ -515,6 +515,22 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
 }
 
 
+int BMK_benchFilesSeparately(const char** fileNamesTable, unsigned nbFiles,
+                   int cLevel, int cLevelLast)
+{
+    unsigned fileNb;
+    if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
+    if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
+    if (cLevelLast < cLevel) cLevelLast = cLevel;
+    if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
+
+    for (fileNb=0; fileNb<nbFiles; fileNb++)
+        BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast);
+
+    return 0;
+}
+
+
 int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
                    int cLevel, int cLevelLast)
 {
@@ -527,7 +543,11 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
 
     if (nbFiles == 0)
         BMK_syntheticTest(cLevel, cLevelLast, compressibility);
-    else
-        BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
+    else {
+        if (g_benchSeparately)
+            BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast);
+        else
+            BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
+    }
     return 0;
 }
diff --git a/programs/bench.h b/programs/bench.h
index 15def93ff..bb67bee38 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -29,9 +29,10 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
                    int cLevel, int cLevelLast);
 
 /* Set Parameters */
-void BMK_SetNbSeconds(unsigned nbLoops);
-void BMK_SetBlockSize(size_t blockSize);
+void BMK_setNbSeconds(unsigned nbLoops);
+void BMK_setBlockSize(size_t blockSize);
 void BMK_setAdditionalParam(int additionalParam);
 void BMK_setNotificationLevel(unsigned level);
+void BMK_setBenchSeparately(int separate);
 
 #endif   /* BENCH_H_125623623633 */
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 362ba49fc..42392ebcf 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -458,11 +458,11 @@ int main(int argc, const char** argv)
                                 if (B < 4) badusage(exeName);
                                 if (B <= 7) {
                                     blockSize = LZ4IO_setBlockSizeID(B);
-                                    BMK_SetBlockSize(blockSize);
+                                    BMK_setBlockSize(blockSize);
                                     DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
                                 } else {
                                     if (B < 32) badusage(exeName);
-                                    BMK_SetBlockSize(B);
+                                    BMK_setBlockSize(B);
                                     if (B >= 1024) {
                                         DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10));
                                     } else {
@@ -480,6 +480,10 @@ int main(int argc, const char** argv)
                 case 'b': mode = om_bench; multiple_inputs=1;
                     break;
 
+                    /* hidden command : benchmark files, but do not fuse result */
+                case 'S': BMK_setBenchSeparately(1);
+                    break;
+
 #ifdef UTIL_HAS_CREATEFILELIST
                     /* recursive */
                 case 'r': recursive=1;
@@ -496,7 +500,7 @@ int main(int argc, const char** argv)
                         iters = readU32FromChar(&argument);
                         argument--;
                         BMK_setNotificationLevel(displayLevel);
-                        BMK_SetNbSeconds(iters);   /* notification if displayLevel >= 3 */
+                        BMK_setNbSeconds(iters);   /* notification if displayLevel >= 3 */
                     }
                     break;
 

From 863e24892d853ea8100c9224c8f3a0c917e055c7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 21 Mar 2018 07:07:24 -0700
Subject: [PATCH 085/257] fix comment style

---
 lib/lz4.c      | 2 +-
 lib/lz4frame.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 37782f5d8..0fdbe5e40 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1722,7 +1722,7 @@ void* LZ4_create (char* inputBuffer)
 
 char* LZ4_slideInputBuffer (void* state)
 {
-    // avoid const char * -> char * conversion warning
+    /* avoid const char * -> char * conversion warning */
     return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
 }
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index b91cb7c31..a080fa656 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -202,8 +202,8 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    U16    lz4CtxAlloc; // sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
-    U16    lz4CtxState; // in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx
+    U16    lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+    U16    lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
 } LZ4F_cctx_t;
 
 

From c3f0ed28ffa66fd7e28ec3b6dbbe95eb0974bfef Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Wed, 21 Mar 2018 07:14:13 -0700
Subject: [PATCH 086/257] added low address fuzzer tests

---
 tests/Makefile |   3 +-
 tests/fuzzer.c | 113 ++++++++++++++++++++++++++-----------------------
 2 files changed, 63 insertions(+), 53 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index ddc0d2e8c..34b8b247b 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,7 +35,8 @@ PRGDIR  := ../programs
 TESTDIR := versionsTest
 PYTHON  ?= python3
 
-DEBUGFLAGS = -g -DLZ4_DEBUG=1
+DEBUGLEVEL?= 1
+DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
 CFLAGS  ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make
 CFLAGS  += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 9415e942f..2b9b9263a 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -34,9 +34,13 @@
 
 #define LZ4_DISABLE_DEPRECATE_WARNINGS
 
+
 /*-************************************
 *  Dependencies
 **************************************/
+#ifdef __unix__   /* must be included before platform.h for MAP_ANONYMOUS */
+#  include <sys/mman.h>   /* mmap */
+#endif
 #include "platform.h"   /* _CRT_SECURE_NO_WARNINGS */
 #include "util.h"       /* U32 */
 #include <stdlib.h>
@@ -242,8 +246,6 @@ static int FUZ_AddressOverflow(void)
 
 #ifdef __unix__   /* is expected to be triggered on linux+gcc */
 
-#include <sys/mman.h>   /* mmap */
-
 static void* FUZ_createLowAddr(size_t size)
 {
     void* const lowBuff = mmap((void*)(0x1000), size,
@@ -276,6 +278,7 @@ static void FUZ_freeLowAddr(void* buffer, size_t size)
 
 #endif
 
+
 /*! FUZ_findDiff() :
 *   find the first different byte between buff1 and buff2.
 *   presumes buff1 != buff2.
@@ -316,10 +319,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
 #   define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
                                                 printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
-#   define FUZ_DISPLAYTEST          { testNb++; g_displayLevel>=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; }
+#   define FUZ_DISPLAYTEST(...) {                 \
+                testNb++;                         \
+                if (g_displayLevel>=4) {          \
+                    printf("\r%4u - %2u ", seed, testNb);  \
+                    printf(" " __VA_ARGS__);      \
+                    printf("   ");                \
+                    fflush(stdout);               \
+            }   }
 
 
     /* init */
+    DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel);
     if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
         DISPLAY("Not enough memory to start fuzzer tests");
         goto _output_error;
@@ -362,7 +373,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Test compression destSize */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_destSize()");
         {   int srcSize = blockSize;
             int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
             char endCheck = FUZ_rand(&randState) & 255;
@@ -377,7 +388,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 U32 const crcBase = XXH32(block, srcSize, 0);
                 char const canary = FUZ_rand(&randState) & 255;
                 FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
-                FUZ_DISPLAYTEST;
+                FUZ_DISPLAYTEST();
                 compressedSize = ret;
                 decodedBuffer[srcSize] = canary;
                 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
@@ -393,7 +404,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Test compression HC destSize */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()");
         {   int srcSize = blockSize;
             int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
             char const endCheck = FUZ_rand(&randState) & 255;
@@ -407,14 +418,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
             FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
             FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
-            DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
-                            compressionLevel, ret, targetSize, srcSize, blockSize);
             if (targetSize>0) {
                 /* check correctness */
                 U32 const crcBase = XXH32(block, srcSize, 0);
                 char const canary = FUZ_rand(&randState) & 255;
                 FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
-                FUZ_DISPLAYTEST;
+                FUZ_DISPLAYTEST();
                 compressedSize = ret;
                 decodedBuffer[srcSize] = canary;
                 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
@@ -430,31 +439,31 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Test compression HC */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC()");
         ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
         HCcompressedSize = ret;
 
         /* Test compression HC using external state */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
 
         /* Test compression using external state */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
         /* Test compression */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_default()");
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed");
         compressedSize = ret;
 
         /* Decompression tests */
 
-        /* Test decoding with output size being exactly what's necessary => must work */
-        FUZ_DISPLAYTEST;
+        /* Test decoding with output size exactly correct => must work */
+        FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
         FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
@@ -462,19 +471,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
 
         /* Test decoding with one byte missing => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
 
         /* Test decoding with one byte too much => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
 
         /* Test decoding with output size exactly what's necessary => must work */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
@@ -484,7 +493,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
 
         // Test decoding with more than enough output size => must work
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         decodedBuffer[blockSize+1] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
@@ -496,14 +505,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
 
         // Test decoding with output size being one byte too short => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
 
         // Test decoding with output size being 10 bytes too short => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         if (blockSize>10) {
             decodedBuffer[blockSize-10] = 0;
             ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
@@ -512,51 +521,51 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         // Test decoding with input size being one byte too short => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
 
         // Test decoding with input size being one byte too large => must fail
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
 
         // Test partial decoding with target output size being max/2 => must work
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
 
         // Test partial decoding with target output size being just below max => must work
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
 
         /* Test Compression with limited output size */
 
         /* Test compression with output size being exactly what's necessary (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer just the right size");
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space");
 
         /* Test compression with output size being exactly what's necessary and external state (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with output buffer just the right size");
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space");
 
         /* Test HC compression with output size being exactly what's necessary (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer just the right size");
         ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space");
 
         /* Test HC compression with output size being exactly what's necessary (should work) */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC() with output buffer just the right size");
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space");
 
         /* Test compression with missing bytes into output buffer => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer a bit too short");
         {   int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
             if (missingBytes >= compressedSize) missingBytes = compressedSize-1;
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
@@ -567,7 +576,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Test HC compression with missing bytes into output buffer => must fail */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer a bit too short");
         {   int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
             if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
@@ -583,7 +592,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /*-******************/
 
         /* Compress using dictionary */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize);
         {   LZ4_stream_t LZ4_stream;
             LZ4_resetStream(&LZ4_stream);
             LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1);   /* Just to fill hash tables */
@@ -592,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Decompress with dictionary as prefix */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as prefix");
         memcpy(decodedBuffer, dict, dictSize);
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
@@ -605,33 +614,33 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
         /* Compress using External dictionary */
-        FUZ_DISPLAYTEST;
-        dict -= (FUZ_rand(&randState) & 0xF) + 1;   /* Separation, so it is an ExtDict */
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
+        dict -= (FUZ_rand(&randState) & 0xF) + 1;   /* create space, so now dictionary is an ExtDict */
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte");
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
@@ -640,7 +649,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -648,19 +657,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         crcCheck = XXH32(decodedBuffer, blockSize, 0);
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
             if ((U32)blockSize > missingBytes) {
                 decodedBuffer[blockSize-missingBytes] = 0;
@@ -670,7 +679,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress HC using External dictionary */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         dict -= (FUZ_rand(&randState) & 7);    /* even bigger separation */
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
         LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
@@ -679,18 +688,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
         ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
         ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -701,10 +710,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
         /* Compress HC continue destSize */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {   int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
             int consumedSize = blockSize;
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
             LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
             blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
@@ -713,7 +722,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
             FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             decodedBuffer[consumedSize] = 0;
             ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize);
             FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
@@ -1118,13 +1127,13 @@ int main(int argc, const char** argv)
                     return FUZ_usage(programName);
 
                 case 'v':   /* verbose mode */
-                    argument++;
                     g_displayLevel++;
+                    argument++;
                     break;
 
                 case 'p':   /* pause at the end */
-                    argument++;
                     use_pause=1;
+                    argument++;
                     break;
 
                 case 'i':

From 7b4c448571b678978bf8fc77e7ba89759086b672 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 21 Mar 2018 07:19:48 -0700
Subject: [PATCH 087/257] added c90 test to c_standards

to catch `//` comments
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ff84d6d42..3a958080f 100644
--- a/Makefile
+++ b/Makefile
@@ -173,7 +173,7 @@ gpptest gpptest32: clean
 	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
 
 c_standards: clean
-	# note : lz4 is not C90 compatible, because it requires long long support
+	CFLAGS="-std=c90   -Werror" $(MAKE) clean allmost
 	CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost
 	CFLAGS="-std=c99   -Werror" $(MAKE) clean allmost
 	CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost

From ebdcbc359baa5a836a3c9e647f083fe092456dd4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:28:51 -0400
Subject: [PATCH 088/257] Add Dependency to Fix Parallel `make test` Runs

When run with `-jN`, the `rm tmp*` can run in the middle of the `test-lz4-dict`
job, which will then fail, finding its files to have been axed. This adds a
dependency between the two.
---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 5954370b3..77f5d02df 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -327,7 +327,7 @@ test-lz4-opt-parser: lz4 datagen
 
 test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple \
                       test-lz4-frame-concatenation test-lz4-testmode \
-                      test-lz4-contentSize
+                      test-lz4-contentSize test-lz4-dict
 	@$(RM) tmp*
 
 test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \

From a3a9b80dffc4d2c8e2496b851f78b78f0948c4bf Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:39:41 -0400
Subject: [PATCH 089/257] Better Describe Functionality of Obsolete Streaming
 Functions

---
 lib/lz4.h | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index d0ec20470..80f040f7c 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -492,15 +492,19 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
 LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
-/* Broken, obsolete streaming functions; do not use!
+/* Obsolete streaming functions; degraded functionality; do not use!
  *
- * These functions depended on data that is no longer tracked in the state. They
- * are therefore broken--they don't retain any history across compressions.
+ * In order to perform streaming compression, these functions depended on data
+ * that is no longer tracked in the state. They have been preserved as well as
+ * possible: using them will still produce a correct output. However, they don't
+ * actually retain any history between compression calls. The compression ratio
+ * achieved will therefore be no better than compressing each chunk
+ * independently.
  */
-LZ4_DEPRECATED("Broken!!! Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
 LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int   LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("Broken!!! Use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("Broken!!! Use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
+LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
 LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);

From 126f18d3e09d92c06d06d33d9cb7a1ec51962525 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:48:35 -0400
Subject: [PATCH 090/257] Also Fix a Comment

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 37782f5d8..0fdbe5e40 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1722,7 +1722,7 @@ void* LZ4_create (char* inputBuffer)
 
 char* LZ4_slideInputBuffer (void* state)
 {
-    // avoid const char * -> char * conversion warning
+    /* avoid const char * -> char * conversion warning */
     return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
 }
 

From 606afdb16428347d0bf216b2c614d962018b3fc3 Mon Sep 17 00:00:00 2001
From: Kenji Chan <kenjichanhkg@gmail.com>
Date: Mon, 2 Apr 2018 10:52:45 +0800
Subject: [PATCH 091/257] added vs2017 projects

---
 visual/VS2017/datagen/datagen.vcxproj         | 173 ++++++++++++++++
 visual/VS2017/frametest/frametest.vcxproj     | 180 +++++++++++++++++
 .../fullbench-dll/fullbench-dll.vcxproj       | 184 ++++++++++++++++++
 visual/VS2017/fullbench/fullbench.vcxproj     | 180 +++++++++++++++++
 visual/VS2017/fuzzer/fuzzer.vcxproj           | 177 +++++++++++++++++
 visual/VS2017/liblz4-dll/liblz4-dll.rc        |  51 +++++
 visual/VS2017/liblz4-dll/liblz4-dll.vcxproj   | 183 +++++++++++++++++
 visual/VS2017/liblz4/liblz4.vcxproj           | 179 +++++++++++++++++
 visual/VS2017/lz4.sln                         |  98 ++++++++++
 9 files changed, 1405 insertions(+)
 create mode 100644 visual/VS2017/datagen/datagen.vcxproj
 create mode 100644 visual/VS2017/frametest/frametest.vcxproj
 create mode 100644 visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
 create mode 100644 visual/VS2017/fullbench/fullbench.vcxproj
 create mode 100644 visual/VS2017/fuzzer/fuzzer.vcxproj
 create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.rc
 create mode 100644 visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
 create mode 100644 visual/VS2017/liblz4/liblz4.vcxproj
 create mode 100644 visual/VS2017/lz4.sln

diff --git a/visual/VS2017/datagen/datagen.vcxproj b/visual/VS2017/datagen/datagen.vcxproj
new file mode 100644
index 000000000..30e159ea5
--- /dev/null
+++ b/visual/VS2017/datagen/datagen.vcxproj
@@ -0,0 +1,173 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{D745AE2F-596A-403A-9B91-81A8C6779243}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>datagen</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\tests\datagencli.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\programs\datagen.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/frametest/frametest.vcxproj b/visual/VS2017/frametest/frametest.vcxproj
new file mode 100644
index 000000000..a3a403d1b
--- /dev/null
+++ b/visual/VS2017/frametest/frametest.vcxproj
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>frametest</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\frametest.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
new file mode 100644
index 000000000..d54a8d728
--- /dev/null
+++ b/visual/VS2017/fullbench-dll/fullbench-dll.vcxproj
@@ -0,0 +1,184 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{13992FD2-077E-4954-B065-A428198201A9}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fullbench-dll</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>liblz4.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\fullbench.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/fullbench/fullbench.vcxproj b/visual/VS2017/fullbench/fullbench.vcxproj
new file mode 100644
index 000000000..54c974390
--- /dev/null
+++ b/visual/VS2017/fullbench/fullbench.vcxproj
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fullbench</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\fullbench.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/fuzzer/fuzzer.vcxproj b/visual/VS2017/fuzzer/fuzzer.vcxproj
new file mode 100644
index 000000000..aa6fe4250
--- /dev/null
+++ b/visual/VS2017/fuzzer/fuzzer.vcxproj
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{18B9F1A7-9C66-4352-898B-30804DADE0FD}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fuzzer</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\tests\fuzzer.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.rc b/visual/VS2017/liblz4-dll/liblz4-dll.rc
new file mode 100644
index 000000000..b1871feae
--- /dev/null
+++ b/visual/VS2017/liblz4-dll/liblz4-dll.rc
@@ -0,0 +1,51 @@
+// Microsoft Visual C++ generated resource script.
+//
+
+#include "lz4.h" /* LZ4_VERSION_STRING */
+#define APSTUDIO_READONLY_SYMBOLS
+#include "verrsrc.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+LANGUAGE 9, 1
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO     VERSIONINFO
+  FILEVERSION       LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+  PRODUCTVERSION    LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
+ FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+ FILEFLAGS VS_FF_DEBUG
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS VOS_NT_WINDOWS32
+ FILETYPE VFT_DLL
+ FILESUBTYPE VFT2_UNKNOWN
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904B0"
+        BEGIN
+            VALUE "CompanyName", "Yann Collet"
+            VALUE "FileDescription", "Extremely fast compression"
+            VALUE "FileVersion", LZ4_VERSION_STRING
+            VALUE "InternalName", "lz4.dll"
+            VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
+            VALUE "OriginalFilename", "lz4.dll"
+            VALUE "ProductName", "LZ4"
+            VALUE "ProductVersion", LZ4_VERSION_STRING
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x0409, 1200
+    END
+END
+
+#endif
diff --git a/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
new file mode 100644
index 000000000..8e7ee3b97
--- /dev/null
+++ b/visual/VS2017/liblz4-dll/liblz4-dll.vcxproj
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{9800039D-4AAA-43A4-BB78-FEF6F4836927}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>liblz4-dll</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+    <ProjectName>liblz4-dll</ProjectName>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ResourceCompile Include="liblz4-dll.rc" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/liblz4/liblz4.vcxproj b/visual/VS2017/liblz4/liblz4.vcxproj
new file mode 100644
index 000000000..948f7db37
--- /dev/null
+++ b/visual/VS2017/liblz4/liblz4.vcxproj
@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>liblz4</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+    <WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <TargetName>liblz4_static</TargetName>
+    <IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+    <RunCodeAnalysis>true</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>true</EnablePREfast>
+      <AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
+      <RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/visual/VS2017/lz4.sln b/visual/VS2017/lz4.sln
new file mode 100644
index 000000000..78f223bf4
--- /dev/null
+++ b/visual/VS2017/lz4.sln
@@ -0,0 +1,98 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Express 2012 for Windows Desktop
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4-dll", "liblz4-dll\liblz4-dll.vcxproj", "{9800039D-4AAA-43A4-BB78-FEF6F4836927}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4", "liblz4\liblz4.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}"
+	ProjectSection(ProjectDependencies) = postProject
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927} = {9800039D-4AAA-43A4-BB78-FEF6F4836927}
+	EndProjectSection
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.ActiveCfg = Debug|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.Build.0 = Debug|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.ActiveCfg = Debug|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.Build.0 = Debug|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.ActiveCfg = Release|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|Win32.Build.0 = Release|Win32
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.ActiveCfg = Release|x64
+		{9800039D-4AAA-43A4-BB78-FEF6F4836927}.Release|x64.Build.0 = Release|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64
+		{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64
+		{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64
+		{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64
+		{D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64
+		{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

From 6d931b6a93100fe2cb4dafc032ff486c814d1fed Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 12:40:33 -0700
Subject: [PATCH 092/257] fixed lz4 compression starting at small address

when using byU32 and byU16 modes
---
 lib/lz4.c      | 70 +++++++++++++++++++++++++++++++++++++++++++++++---
 tests/fuzzer.c |  2 +-
 2 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0fdbe5e40..bcebc92df 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -533,6 +533,20 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
 }
 
+/* LZ4_getIndexOnHash() :
+ * Index of match position registered in hash table.
+ * hash position must be calculated by using base+index, or dictBase+index.
+ * Assumption 1 : only valid if tableType == byU32 or byU16.
+ * Assumption 2 : h is presumed valid (within limits of hash table)
+ */
+static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
+{
+    LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
+    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; }
+    if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; }
+    assert(0); return 0;  /* forbidden case */
+}
+
 static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; }
@@ -598,7 +612,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    size_t currentOffset = cctx->currentOffset;
+    size_t const currentOffset = cctx->currentOffset;
     const BYTE* base = (const BYTE*) source - currentOffset;
     const BYTE* lowLimit;
 
@@ -650,7 +664,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         BYTE* token;
 
         /* Find a match */
-        {   const BYTE* forwardIp = ip;
+        if (tableType == byPtr) {
+            const BYTE* forwardIp = ip;
             unsigned step = 1;
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
@@ -687,6 +702,54 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
                 || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+
+        } else {   /* byU32, byU16 */
+
+            const BYTE* forwardIp = ip;
+            unsigned step = 1;
+            unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
+            do {
+                U32 const h = forwardH;
+                U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+                ip = forwardIp;
+                forwardIp += step;
+                step = (searchMatchNb++ >> LZ4_skipTrigger);
+
+                if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
+                assert(ip < mflimitPlusOne);
+
+                if (dictDirective == usingDictCtx) {
+                    if (matchIndex < currentOffset) {
+                        /* there was no match, try the dictionary */
+                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
+                        refDelta = dictDelta;
+                        lowLimit = dictLowLimit;
+                    } else {
+                        match = base + matchIndex;
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                } else if (dictDirective==usingExtDict) {
+                    if (matchIndex < currentOffset) {
+                        match = dictBase + matchIndex;
+                        refDelta = dictDelta;
+                        lowLimit = dictLowLimit;
+                    } else {
+                        match = base + matchIndex;
+                        refDelta = 0;
+                        lowLimit = (const BYTE*)source;
+                    }
+                } else {   /* single continuous memory segment */
+                    match = base + matchIndex;
+                    refDelta = 0;
+                    lowLimit = (const BYTE*)source;
+                }
+                forwardH = LZ4_hashPosition(forwardIp, tableType);
+                LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
+
+            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
+                || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
+                || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
         }
 
         /* Catch up */
@@ -718,7 +781,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode MatchLength */
         {   unsigned matchCode;
 
-            if ((dictDirective==usingExtDict || dictDirective==usingDictCtx) && lowLimit==dictionary) {
+            if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
+              && (lowLimit==dictionary) ) {
                 const BYTE* limit;
                 match += refDelta;
                 limit = ip + (dictEnd-match);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2b9b9263a..0b7d54e1e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -322,7 +322,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 #   define FUZ_DISPLAYTEST(...) {                 \
                 testNb++;                         \
                 if (g_displayLevel>=4) {          \
-                    printf("\r%4u - %2u ", seed, testNb);  \
+                    printf("\r%4u - %2u ", cycleNb, testNb);  \
                     printf(" " __VA_ARGS__);      \
                     printf("   ");                \
                     fflush(stdout);               \

From 64a3e41acaf9e186937d32c9dd2dc104e5bc4a72 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 16:38:43 -0700
Subject: [PATCH 093/257] changed LZ4_compress_generic() logic

to use indexes (U32) instead of Ptr.

byPtr is still present.
---
 lib/lz4.c | 170 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 100 insertions(+), 70 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bcebc92df..4b219d2cf 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -516,6 +516,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
     return LZ4_hash4(LZ4_read32(p), tableType);
 }
 
+static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType)
+{
+    switch (tableType)
+    {
+    default: /* fallthrough */
+    case clearedTable: /* fallthrough */
+    case byPtr: { /* illegal! */ assert(0); return; }
+    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; }
+    case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; }
+    }
+}
+
 static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
 {
     switch (tableType)
@@ -612,8 +624,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    size_t const currentOffset = cctx->currentOffset;
-    const BYTE* base = (const BYTE*) source - currentOffset;
+    size_t const startIndex = cctx->currentOffset;
+    const BYTE* base = (const BYTE*) source - startIndex;
     const BYTE* lowLimit;
 
     const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx;
@@ -622,7 +634,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
 
-    const BYTE* const lowRefLimit = (const BYTE*) source - dictSize;
+    int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
+    U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */
     const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
     const BYTE* const iend = ip + inputSize;
@@ -633,19 +646,20 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
         (const BYTE*) source - dictCtx->currentOffset :
-        (const BYTE*) source - dictSize - currentOffset;
-    const ptrdiff_t dictDelta = dictionary ? dictEnd - (const BYTE*) source : 0;
+        (const BYTE*) source - dictSize - startIndex;
     const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
+    U32 offset = 0;
     ptrdiff_t retval = 0;
 
     U32 forwardH;
 
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
+    if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
     dictLowLimit = dictionary ? dictionary : lowLimit;
@@ -659,7 +673,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     /* Main Loop */
     for ( ; ; ) {
-        ptrdiff_t refDelta = 0;
         const BYTE* match;
         BYTE* token;
 
@@ -678,82 +691,65 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(ip < mflimitPlusOne);
 
                 match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base);
-                if (dictDirective == usingDictCtx) {
-                    if (match < (const BYTE*)source) {
-                        /* there was no match, try the dictionary */
-                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
-                        refDelta = dictDelta;
-                        lowLimit = dictLowLimit;
-                    } else {
-                        refDelta = 0;
-                        lowLimit = (const BYTE*)source;
-                    }
-                } else if (dictDirective==usingExtDict) {
-                    if (match < (const BYTE*)source) {
-                        refDelta = dictDelta;
-                        lowLimit = dictLowLimit;
-                    } else {
-                        refDelta = 0;
-                        lowLimit = (const BYTE*)source;
-                }   }
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
-                || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
-                || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+            } while ( LZ4_read32(match) != LZ4_read32(ip) );
 
         } else {   /* byU32, byU16 */
 
             const BYTE* forwardIp = ip;
-            unsigned step = 1;
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
                 U32 const h = forwardH;
-                U32 const matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+                U32 const current = forwardIp - base;
+                U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+                assert(matchIndex <= current);
                 ip = forwardIp;
-                forwardIp += step;
-                step = (searchMatchNb++ >> LZ4_skipTrigger);
+                assert(searchMatchNb >= (1<<LZ4_skipTrigger));
+                forwardIp += (searchMatchNb++ >> LZ4_skipTrigger);
 
                 if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
                 assert(ip < mflimitPlusOne);
 
                 if (dictDirective == usingDictCtx) {
-                    if (matchIndex < currentOffset) {
+                    if (matchIndex < startIndex) {
                         /* there was no match, try the dictionary */
-                        match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
-                        refDelta = dictDelta;
+                        matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
+                        match = dictBase + matchIndex;
                         lowLimit = dictLowLimit;
                     } else {
                         match = base + matchIndex;
-                        refDelta = 0;
                         lowLimit = (const BYTE*)source;
                     }
                 } else if (dictDirective==usingExtDict) {
-                    if (matchIndex < currentOffset) {
+                    if (matchIndex < startIndex) {
                         match = dictBase + matchIndex;
-                        refDelta = dictDelta;
                         lowLimit = dictLowLimit;
                     } else {
                         match = base + matchIndex;
-                        refDelta = 0;
                         lowLimit = (const BYTE*)source;
                     }
                 } else {   /* single continuous memory segment */
                     match = base + matchIndex;
-                    refDelta = 0;
-                    lowLimit = (const BYTE*)source;
                 }
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
-                LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
+                LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+
+                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;    /* match outside of valid area */
+                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue;  /* too far */
+                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
 
-            } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
-                || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
-                || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
+                if (LZ4_read32(match) == LZ4_read32(ip)) {
+                    if (maybe_ext_memSegment) offset = current - matchIndex;
+                    break;   /* match found */
+                }
+
+            } while(1);
         }
 
         /* Catch up */
-        while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; }
+        while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
 
         /* Encode Literals */
         {   unsigned const litLength = (unsigned)(ip - anchor);
@@ -776,7 +772,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
 _next_match:
         /* Encode Offset */
-        LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
+        if (maybe_ext_memSegment) {   /* static test */
+            assert(offset <= MAX_DISTANCE && offset > 0);
+            LZ4_writeLE16(op, (U16)offset); op+=2;
+        } else  {
+            assert(ip-match <= MAX_DISTANCE);
+            LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
+        }
 
         /* Encode MatchLength */
         {   unsigned matchCode;
@@ -784,7 +786,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
               && (lowLimit==dictionary) ) {
                 const BYTE* limit;
-                match += refDelta;
                 limit = ip + (dictEnd-match);
                 if (limit > matchlimit) limit = matchlimit;
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
@@ -799,6 +800,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 ip += MINMATCH + matchCode;
             }
 
+            DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH);
+
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
                 goto _clean_up;
@@ -825,34 +828,61 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Fill table */
         LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
 
+#if 1
         /* Test next position */
-        match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
-        if (dictDirective == usingDictCtx) {
-            if (match < (const BYTE*)source) {
-                /* there was no match, try the dictionary */
-                match = LZ4_getPosition(ip, dictCtx->hashTable, byU32, dictBase);
-                refDelta = dictDelta;
-                lowLimit = dictLowLimit;
-            } else {
-                refDelta = 0;
-                lowLimit = (const BYTE*)source;
+        if (tableType == byPtr) {
+
+            match = LZ4_getPosition(ip, cctx->hashTable, tableType, base);
+            LZ4_putPosition(ip, cctx->hashTable, tableType, base);
+            if ( (match+MAX_DISTANCE >= ip)
+              && (LZ4_read32(match) == LZ4_read32(ip)) )
+            { token=op++; *token=0; goto _next_match; }
+
+        } else {   /* byU32, byU16 */
+
+            U32 const h = LZ4_hashPosition(ip, tableType);
+            U32 const current = (U32)(ip-base);
+            U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
+            assert(matchIndex < current);
+            if (dictDirective == usingDictCtx) {
+                if (match < (const BYTE*)source) {
+                    /* there was no match, try the dictionary */
+                    matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
+                    match = dictBase + matchIndex;
+                } else {
+                    match = base + matchIndex;
+                }
+            } else if (dictDirective==usingExtDict) {
+                if (match < (const BYTE*)source) {
+                    match = dictBase + matchIndex;
+                } else {
+                    match = base + matchIndex;
+                }
+            } else {   /* single memory segment */
+                match = base + matchIndex;
             }
-        } else if (dictDirective==usingExtDict) {
-            if (match < (const BYTE*)source) {
-                refDelta = dictDelta;
-                lowLimit = dictLowLimit;
-            } else {
-                refDelta = 0;
-                lowLimit = (const BYTE*)source;
-        }   }
-        LZ4_putPosition(ip, cctx->hashTable, tableType, base);
-        if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
-            && (match+MAX_DISTANCE>=ip)
-            && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
-        { token=op++; *token=0; goto _next_match; }
+            LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+            if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
+              && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+              && (LZ4_read32(match) == LZ4_read32(ip)) ) {
+                token=op++;
+                *token=0;
+                if (maybe_ext_memSegment)
+                    offset = current - matchIndex;
+                goto _next_match;
+            }
+        }
 
         /* Prepare next loop */
         forwardH = LZ4_hashPosition(++ip, tableType);
+
+#else
+
+        /* Prepare next loop */
+        forwardH = LZ4_hashPosition(ip, tableType);
+
+#endif
+
     }
 
 _last_literals:

From f2a4d6ef37f653c21627274634d171af66126d5e Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 17:16:33 -0700
Subject: [PATCH 094/257] fixed immediate match search

---
 lib/lz4.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4b219d2cf..8d8c1e865 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -624,7 +624,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 {
     const BYTE* ip = (const BYTE*) source;
 
-    size_t const startIndex = cctx->currentOffset;
+    U32 const startIndex = cctx->currentOffset;
     const BYTE* base = (const BYTE*) source - startIndex;
     const BYTE* lowLimit;
 
@@ -645,8 +645,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        (const BYTE*) source - dictCtx->currentOffset :
-        (const BYTE*) source - dictSize - startIndex;
+        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
+        dictionary + dictSize - startIndex;
     const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
@@ -800,8 +800,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 ip += MINMATCH + matchCode;
             }
 
-            DEBUGLOG(2,"matchLength:%7u ", matchCode+MINMATCH);
-
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
                 goto _clean_up;
@@ -845,7 +843,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
             assert(matchIndex < current);
             if (dictDirective == usingDictCtx) {
-                if (match < (const BYTE*)source) {
+                if (matchIndex < startIndex) {
                     /* there was no match, try the dictionary */
                     matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                     match = dictBase + matchIndex;
@@ -853,7 +851,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                     match = base + matchIndex;
                 }
             } else if (dictDirective==usingExtDict) {
-                if (match < (const BYTE*)source) {
+                if (matchIndex < startIndex) {
                     match = dictBase + matchIndex;
                 } else {
                     match = base + matchIndex;

From b4be1e0a743f2200eaf1c13d322c925b64b872e2 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 17:52:54 -0700
Subject: [PATCH 095/257] fixed byPtr match search

---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 8d8c1e865..5791556f2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -694,7 +694,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base);
 
-            } while ( LZ4_read32(match) != LZ4_read32(ip) );
+            } while ( (match+MAX_DISTANCE < ip)
+                   || (LZ4_read32(match) != LZ4_read32(ip)) );
 
         } else {   /* byU32, byU16 */
 

From f4e06e28e6d285f7f145798d7dfe1cbe71ae1efa Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 18:29:42 -0700
Subject: [PATCH 096/257] fixed byPtr mode

switch to byU32 when src address is < 64K

note : byPtr is still useful in 32-bits, as it's about ~10% faster
---
 lib/lz4.c | 47 +++++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 5791556f2..4fb54f8ec 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -930,14 +930,14 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
         if (inputSize < LZ4_64Klimit) {
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {;
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -951,7 +951,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
  * performance when the context reset would otherwise be a significant part of
  * the cost of the compression, e.g., when the data to be compressed is small.
  */
-int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_noReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -959,39 +959,39 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
     ctx->dictSize = 0;
     ctx->dictCtx = NULL;
 
-    if (maxOutputSize >= LZ4_compressBound(inputSize)) {
-        if (inputSize < LZ4_64Klimit) {
+    if (dstCapacity >= LZ4_compressBound(srcSize)) {
+        if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            const tableType_t tableType =  ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
                 ctx->currentOffset += 64 KB;
             }
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
-        if (inputSize < LZ4_64Klimit) {
+        if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+            LZ4_prepareTable(ctx, srcSize, tableType, noDict);
             if (ctx->currentOffset) {
                 ctx->currentOffset += 64 KB;
             }
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1098,6 +1098,8 @@ static int LZ4_compress_destSize_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
 
+                DEBUGLOG(2, "match:%p , ip:%p", match, ip);
+
             } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match) != LZ4_read32(ip)) );
         }
@@ -1203,11 +1205,12 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
     if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) {  /* compression success is guaranteed */
         return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
     } else {
-        if (*srcSizePtr < LZ4_64Klimit)
+        if (*srcSizePtr < LZ4_64Klimit) {
             return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16);
-        else
-            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr);
-    }
+        } else {
+            tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
+            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, tableType);
+    }   }
 }
 
 

From 038a0d95bfe2d3a544aa2f5551998f6fd8bc0722 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 18:39:22 -0700
Subject: [PATCH 097/257] added low-memory address test to travis

requires modification linux configuration (sudo)
---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 466d55e6e..a44642093 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,7 @@ matrix:
 
     - os: linux
       sudo: false
-      env: Ubu=12.04cont Cmd='make -C tests test-frametest test-fuzzer' COMPILER=cc
+      env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc
 
     - os: linux
       sudo: false
@@ -59,7 +59,7 @@ matrix:
             - libc6-dev-i386
             - gcc-multilib
 
-    - env: Ubu=14.04 Cmd='make -C tests test-frametest32 test-fuzzer32' COMPILER=cc
+    - env: Ubu=14.04 Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest32 test-fuzzer32' COMPILER=cc
       dist: trusty
       sudo: required
       addons:

From f9992fa37f1b0810c4d0a3e3e6a0eb4880168c57 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 5 Apr 2018 19:05:49 -0700
Subject: [PATCH 098/257] noticed a bug when re-using hash table

./fuzzer -vv -s4217 -t7518
---
 lib/lz4.c      |  7 +++---
 tests/fuzzer.c | 59 +++++++++++++++++++++++++-------------------------
 2 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4fb54f8ec..54336c7f4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -593,6 +593,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
             cctx->tableType = clearedTable;
+        } else {
+            DEBUGLOG(4, "Re-use hash table (no reset)");
+            //if (tableType == byU32) cctx->currentOffset += 64 KB;
         }
     }
     /* If the current offset is zero, we will never look in the external
@@ -1098,8 +1101,6 @@ static int LZ4_compress_destSize_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
 
-                DEBUGLOG(2, "match:%p , ip:%p", match, ip);
-
             } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
                 || (LZ4_read32(match) != LZ4_read32(ip)) );
         }
@@ -1371,7 +1372,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
-        } else {
+        } else {  /* no dictCtx */
             LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 0b7d54e1e..8ffdb2287 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -291,7 +291,7 @@ static void FUZ_findDiff(const void* buff1, const void* buff2)
     const BYTE* const b2 = (const BYTE*)buff2;
     size_t u = 0;
     while (b1[u]==b2[u]) u++;
-    DISPLAY("Wrong Byte at position %u \n", (unsigned)u);
+    DISPLAY("\nWrong Byte at position %u \n", (unsigned)u);
 }
 
 
@@ -330,7 +330,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
 
     /* init */
-    DISPLAYLEVEL(2, " g_displayLevel = %u \n", g_displayLevel);
     if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
         DISPLAY("Not enough memory to start fuzzer tests");
         goto _output_error;
@@ -360,7 +359,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         int compressedSize, HCcompressedSize;
         int blockContinueCompressedSize;
         U32 const crcOrig = XXH32(block, blockSize, 0);
-        U32 crcCheck;
         int ret;
 
         FUZ_displayUpdate(cycleNb);
@@ -467,8 +465,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
         FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
+        }
 
         /* Test decoding with one byte missing => must fail */
         FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
@@ -489,8 +488,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        }
 
         // Test decoding with more than enough output size => must work
         FUZ_DISPLAYTEST();
@@ -501,8 +501,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
         //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size");   // well, is that an issue ?
         FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
+        }
 
         // Test decoding with output size being one byte too short => must fail
         FUZ_DISPLAYTEST();
@@ -605,20 +606,17 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         memcpy(decodedBuffer, dict, dictSize);
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
-        crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
-        if (crcCheck!=crcOrig) {
-            int i=0;
-            while (block[i]==decodedBuffer[i]) i++;
-            printf("Wrong Byte at position %i/%i\n", i, blockSize);
-
+        {   U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
         }
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
         FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
-        crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
         /* Compress using External dictionary */
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
@@ -640,22 +638,24 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST();
+        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize);
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        }
 
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
@@ -704,10 +704,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        if (crcCheck!=crcOrig)
-            FUZ_findDiff(block, decodedBuffer);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
         /* Compress HC continue destSize */
         FUZ_DISPLAYTEST();
@@ -729,8 +729,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
             {   U32 const crcSrc = XXH32(block, consumedSize, 0);
                 U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0);
-                if (crcSrc!=crcDst)
-                    FUZ_findDiff(block, decodedBuffer);
+                if (crcSrc!=crcDst) FUZ_findDiff(block, decodedBuffer);
                 FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data");
             }
         }

From 133a50b780be302e726da772592e505d5b143f44 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 6 Apr 2018 14:16:23 -0700
Subject: [PATCH 099/257] fixed DISPLAYUPDATE()

wrong comparison, which was always overflowing (hence was always true)
except when it was not (i386, reported by pmc)
in which case it would never show any information.
---
 programs/lz4io.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index ca13316e8..6d0d0d018 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -94,9 +94,12 @@
 static int g_displayLevel = 0;   /* 0 : no display  ; 1: errors  ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \
-            { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stderr); } }
+            if ( ((clock() - g_time) > refreshRate)    \
+              || (g_displayLevel>=4) ) {               \
+                g_time = clock();                      \
+                DISPLAY(__VA_ARGS__);                  \
+                if (g_displayLevel>=4) fflush(stderr); \
+        }   }
 static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
 static clock_t g_time = 0;
 

From 5622c276e114c75f11f6171f9fe2295a8b626e89 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 6 Apr 2018 18:52:55 -0400
Subject: [PATCH 100/257] Return to Allowing Early Returns in
 LZ4_compress_generic()

Or: `goto` Considered Harmful

Or: https://xkcd.com/292/
---
 lib/lz4.c | 38 +++++++++++++++++---------------------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0fdbe5e40..8743cb96b 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -626,8 +626,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
 
-    ptrdiff_t retval = 0;
-
     U32 forwardH;
 
     /* Init conditions */
@@ -637,6 +635,19 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     dictLowLimit = dictionary ? dictionary : lowLimit;
 
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+
+    /* Update context state */
+    if (dictDirective == usingDictCtx) {
+        /* Subsequent linked blocks can't use the dictionary. */
+        /* Instead, they use the block we just compressed. */
+        cctx->dictCtx = NULL;
+        cctx->dictSize = (U32)inputSize;
+    } else {
+        cctx->dictSize += (U32)inputSize;
+    }
+    cctx->currentOffset += (U32)inputSize;
+    cctx->tableType = tableType;
+
     if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* First Byte */
@@ -697,7 +708,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             token = op++;
             if ((outputLimited) &&  /* Check output buffer overflow */
                 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
-                goto _clean_up;
+                return 0;
             if (litLength >= RUN_MASK) {
                 int len = (int)litLength-RUN_MASK;
                 *token = (RUN_MASK<<ML_BITS);
@@ -737,7 +748,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
             if ( outputLimited &&    /* Check output buffer overflow */
                 (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
-                goto _clean_up;
+                return 0;
             if (matchCode >= ML_MASK) {
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
@@ -796,7 +807,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     {   size_t const lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
             ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
-            goto _clean_up;
+            return 0;
         if (lastRun >= RUN_MASK) {
             size_t accumulator = lastRun - RUN_MASK;
             *op++ = RUN_MASK << ML_BITS;
@@ -809,22 +820,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         op += lastRun;
     }
 
-    retval = (((char*)op)-dest);
-
-_clean_up:
-    if (dictDirective == usingDictCtx) {
-        /* Subsequent linked blocks can't use the dictionary. */
-        /* Instead, they use the block we just compressed. */
-        cctx->dictCtx = NULL;
-        cctx->dictSize = (U32)inputSize;
-    } else {
-        cctx->dictSize += (U32)inputSize;
-    }
-    cctx->currentOffset += (U32)inputSize;
-    cctx->tableType = tableType;
-
-    /* End */
-    return (int)retval;
+    return (int)(((char*)op) - dest);
 }
 
 

From f88dc900553b3a029e9cd114800016f9364ffc7b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 6 Apr 2018 16:52:29 -0400
Subject: [PATCH 101/257] Avoid Calling LZ4_prepareTable() in
 LZ4_compress_fast_continue()

---
 lib/lz4.c | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 8743cb96b..4f353c595 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -574,11 +574,16 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * indicates a miss. In that case, we need to bump the offset to something
      * non-zero.
      */
-    if (dictDirective == usingDictCtx &&
-        tableType != byPtr &&
-        cctx->currentOffset == 0)
-    {
-        cctx->currentOffset = 1;
+    if (cctx->currentOffset == 0) {
+        if (dictDirective == usingDictCtx) {
+            if (tableType == byU16) {
+                cctx->currentOffset = 1;
+            } else if (tableType == byU32) {
+                cctx->currentOffset = 64 KB;
+            }
+        }
+    } else if (tableType == byU32) {
+        cctx->currentOffset += 64 KB;
     }
 }
 
@@ -874,9 +879,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-                ctx->currentOffset += 64 KB;
-            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
@@ -891,9 +893,6 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
             LZ4_prepareTable(ctx, inputSize, tableType, noDict);
-            if (ctx->currentOffset) {
-                ctx->currentOffset += 64 KB;
-            }
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -1168,31 +1167,28 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 {
     LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse;
+    const tableType_t tableType = byU32;
     const BYTE* p = (const BYTE*)dictionary;
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
     DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
 
-    if ((dict->initCheck)
-      || (dict->tableType != byU32 && dict->tableType != clearedTable)
-      || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
-        LZ4_resetStream(LZ4_dict);
+    LZ4_prepareTable(dict, 0, tableType, usingExtDict);
 
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
-    dict->currentOffset += 64 KB;
     base = p - dict->currentOffset;
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
     dict->currentOffset += dict->dictSize;
-    dict->tableType = byU32;
+    dict->tableType = tableType;
 
     if (dictSize < (int)HASH_UNIT) {
         return 0;
     }
 
     while (p <= dictEnd-HASH_UNIT) {
-        LZ4_putPosition(p, dict->hashTable, byU32, base);
+        LZ4_putPosition(p, dict->hashTable, tableType, base);
         p+=3;
     }
 
@@ -1244,7 +1240,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
-        LZ4_prepareTable(streamPtr, inputSize, tableType, withPrefix64k);
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
             return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
@@ -1272,7 +1267,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
-            LZ4_prepareTable(streamPtr, inputSize, tableType, usingExtDict);
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {

From cf2f06a6c5046ef7d576bc8ad210c3f2efe1f401 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Mon, 9 Apr 2018 17:08:17 -0700
Subject: [PATCH 102/257] fixed minor conversion warning

ptr diff -> U32
---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3db37b0ef..a3c2860c5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -717,9 +717,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
                 U32 const h = forwardH;
-                U32 const current = forwardIp - base;
+                U32 const current = (U32)(forwardIp - base);
                 U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType);
                 assert(matchIndex <= current);
+                assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));
                 ip = forwardIp;
                 assert(searchMatchNb >= (1<<LZ4_skipTrigger));
                 forwardIp += (searchMatchNb++ >> LZ4_skipTrigger);

From ad7e040384d1835c097b0f727aeaa3d598a401b6 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Mon, 9 Apr 2018 20:38:00 -0700
Subject: [PATCH 103/257] fix minor conversion warning

cast from void not implicit for C++
---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8ffdb2287..2e3ee92a3 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -367,7 +367,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         if ( ((FUZ_rand(&randState) & 63) == 2)
           && ((size_t)blockSize < labSize) ) {
             memcpy(lowAddrBuffer, block, blockSize);
-            block = lowAddrBuffer;
+            block = (const char*)lowAddrBuffer;
         }
 
         /* Test compression destSize */

From 59c7d951213bf8593438ae811549655d5a78d2fc Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 10 Apr 2018 13:12:30 -0400
Subject: [PATCH 104/257] Expose a Faster Stream Reset Function

---
 lib/lz4.c      | 53 ++++++++++++++++++++++++++------------------------
 lib/lz4.h      |  8 ++++++++
 lib/lz4frame.c |  4 +---
 3 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4f353c595..30146bba1 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -546,12 +546,10 @@ LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBas
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
 }
 
-
 LZ4_FORCE_INLINE void LZ4_prepareTable(
         LZ4_stream_t_internal* const cctx,
         const int inputSize,
-        const tableType_t tableType,
-        const dict_directive dictDirective) {
+        const tableType_t tableType) {
     /* If the table hasn't been used, it's guaranteed to be zeroed out, and is
      * therefore safe to use no matter what mode we're in. Otherwise, we figure
      * out if it's safe to leave as is or whether it needs to be reset.
@@ -569,22 +567,23 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             cctx->tableType = clearedTable;
         }
     }
-    /* If the current offset is zero, we will never look in the external
-     * dictionary context, since there is no value a table entry can take that
-     * indicates a miss. In that case, we need to bump the offset to something
-     * non-zero.
+
+    /* Adding a gap, so all previous entries are > MAX_DISTANCE back, is faster
+     * than compressing without a gap. However, compressing with
+     * currentOffset == 0 is faster still, so we preserve that case.
      */
-    if (cctx->currentOffset == 0) {
-        if (dictDirective == usingDictCtx) {
-            if (tableType == byU16) {
-                cctx->currentOffset = 1;
-            } else if (tableType == byU32) {
-                cctx->currentOffset = 64 KB;
-            }
-        }
-    } else if (tableType == byU32) {
+    if (cctx->currentOffset != 0 && tableType == byU32) {
         cctx->currentOffset += 64 KB;
     }
+
+    /* Finally, clear history */
+    cctx->dictCtx = NULL;
+    cctx->dictionary = NULL;
+    cctx->dictSize = 0;
+}
+
+void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+    LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
 }
 
 /** LZ4_compress_generic() :
@@ -863,14 +862,11 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
-    ctx->dictionary = NULL;
-    ctx->dictSize = 0;
-    ctx->dictCtx = NULL;
 
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -878,13 +874,13 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -892,7 +888,7 @@ int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* de
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType, noDict);
+            LZ4_prepareTable(ctx, inputSize, tableType);
             return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
@@ -1174,7 +1170,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 
     DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
 
-    LZ4_prepareTable(dict, 0, tableType, usingExtDict);
+    LZ4_prepareTable(dict, 0, tableType);
 
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
     base = p - dict->currentOffset;
@@ -1263,7 +1259,14 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                LZ4_prepareTable(streamPtr, inputSize, tableType, usingDictCtx);
+                /* If the current offset is zero, we will never look in the
+                 * external dictionary context, since there is no value a table
+                 * entry can take that indicate a miss. In that case, we need
+                 * to bump the offset to something non-zero.
+                 */
+                if (streamPtr->currentOffset == 0) {
+                    streamPtr->currentOffset = 64 KB;
+                }
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
diff --git a/lib/lz4.h b/lib/lz4.h
index 80f040f7c..9c1b6945f 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -366,6 +366,14 @@ LZ4_compress_fast_extState_noReset() :
 LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
+
+/*! LZ4_resetStream_fast() :
+ *  An LZ4_stream_t structure can be allocated once and re-used multiple times.
+ *  Use this function to start compressing a new stream.
+ */
+LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+
+
 /*-************************************
  *  Private definitions
  **************************************
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a080fa656..afa5af3fa 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -532,9 +532,7 @@ static void LZ4F_applyCDict(void* ctx,
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
         assert(!internal_ctx->initCheck);
-        /* Clear any local dictionary */
-        internal_ctx->dictionary = NULL;
-        internal_ctx->dictSize = 0;
+        LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         /* Point to the dictionary context */
         internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
     } else {

From c18bff933b18106ea000868ce13f37f0589cb58e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 15:12:34 -0400
Subject: [PATCH 105/257] Remove Extraneous Assignment (clearedTable == 0)

---
 lib/lz4.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 30146bba1..f79ee2c30 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1147,7 +1147,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-    LZ4_stream->internal_donotuse.tableType = clearedTable;
 }
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)

From 21f0c9700b28301bdc8b49fcbe4ccdddf4cf5dea Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 15:13:01 -0400
Subject: [PATCH 106/257] Rename _extState_noReset -> _extState_fastReset and
 Edit Comments

---
 lib/lz4.c      | 23 ++++++++++++-----------
 lib/lz4.h      | 43 ++++++++++++++++++++++++++++---------------
 lib/lz4frame.c |  2 +-
 3 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f79ee2c30..c2d1c327c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -582,10 +582,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
     cctx->dictSize = 0;
 }
 
-void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
-    LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
-}
-
 /** LZ4_compress_generic() :
     inlined, to ensure branches are decided at compilation time */
 LZ4_FORCE_INLINE int LZ4_compress_generic(
@@ -851,14 +847,15 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
 }
 
 /**
- * LZ4_compress_fast_extState_noReset is a variant of LZ4_compress_fast_extState
- * that can be used when the state is known to have already been initialized
- * (via LZ4_resetStream or an earlier call to LZ4_compress_fast_extState /
- * LZ4_compress_fast_extState_noReset). This can provide significantly better
- * performance when the context reset would otherwise be a significant part of
- * the cost of the compression, e.g., when the data to be compressed is small.
+ * LZ4_compress_fast_extState_fastReset() :
+ * A variant of LZ4_compress_fast_extState().
+ *
+ * Using this variant avoids an expensive initialization step. It is only safe
+ * to call if the state buffer is known to be correctly initialized already
+ * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of
+ * "correctly initialized").
  */
-int LZ4_compress_fast_extState_noReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -1149,6 +1146,10 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
+void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+    LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
+}
+
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 {
     if (!LZ4_stream) return 0;   /* support free on NULL */
diff --git a/lib/lz4.h b/lib/lz4.h
index 9c1b6945f..427beaf42 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -353,26 +353,39 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
  * Their signatures may change.
  **************************************/
 
-/*!
-LZ4_compress_fast_extState_noReset() :
-    A variant of LZ4_compress_fast_extState().
-
-    Use the _noReset variant if LZ4_resetStream() was called on the state
-    buffer before being used for the first time (calls to both extState
-    functions leave the state in a safe state, so zeroing is not required
-    between calls). Otherwise, using the plain _extState requires LZ4 to
-    reinitialize the state internally for every call.
-*/
-LZ4LIB_API int LZ4_compress_fast_extState_noReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-
-
 
 /*! LZ4_resetStream_fast() :
- *  An LZ4_stream_t structure can be allocated once and re-used multiple times.
- *  Use this function to start compressing a new stream.
+ *  When an LZ4_stream_t is known to be in a internally coherent state,
+ *  it can often be prepared for a new compression with almost no work, only
+ *  sometimes falling back to the full, expensive reset that is always required
+ *  when the stream is in an indeterminate state (i.e., the reset performed by
+ *  LZ4_resetStream()).
+ *
+ *  LZ4_streams are guaranteed to be in a valid state when:
+ *  - returned from LZ4_createStream()
+ *  - reset by LZ4_resetStream()
+ *  - memset(stream, 0, sizeof(LZ4_stream_t))
+ *  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
+ *  - the stream was in a valid state and was then used in any compression call
+ *    that returned success
+ *  - the stream was in an indeterminate state and was used in a compression
+ *    call that fully reset the state (LZ4_compress_fast_extState()) and that
+ *    returned success
  */
 LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
 
+/*! LZ4_compress_fast_extState_fastReset() :
+ *  A variant of LZ4_compress_fast_extState().
+ *
+ *  Using this variant avoids an expensive initialization step. It is only safe
+ *  to call if the state buffer is known to be correctly initialized already
+ *  (see above comment on LZ4_resetStream_fast() for a definition of "correctly
+ *  initialized"). From a high level, the difference is that this function
+ *  initializes the provided state with a call to LZ4_resetStream_fast() while
+ *  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ */
+LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+
 
 /*-************************************
  *  Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index afa5af3fa..5e11ba213 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -714,7 +714,7 @@ static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
-        return LZ4_compress_fast_extState_noReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
+        return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
     }
 }
 

From afa52c9b95dd1f83d80cd9a783a408822700e2d3 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 16:04:24 -0400
Subject: [PATCH 107/257] Expose dictCtx Functionality in LZ4

---
 lib/lz4.c      |  4 ++++
 lib/lz4.h      | 28 ++++++++++++++++++++++++++++
 lib/lz4frame.c |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c2d1c327c..cee85ec89 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1191,6 +1191,10 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     return dict->dictSize;
 }
 
+void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
+    working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+}
+
 
 static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
 {
diff --git a/lib/lz4.h b/lib/lz4.h
index 427beaf42..f90120e8e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -386,6 +386,34 @@ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
  */
 LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
+/*! LZ4_attach_dictionary() :
+ *  This is an experimental API that allows for the efficient use of a
+ *  static dictionary many times.
+ *
+ *  Rather than re-loading the dictionary buffer into a working context before
+ *  each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
+ *  working LZ4_stream_t, this function introduces a no-copy setup mechanism,
+ *  in which the working stream references the dictionary stream in-place.
+ *
+ *  Several assumptions are made about the state of the dictionary stream.
+ *  Currently, only streams which have been prepared by LZ4_loadDict() should
+ *  be expected to work.
+ *
+ *  Alternatively, the provided dictionary stream pointer may be NULL, in which
+ *  case any existing dictionary stream is unset.
+ *
+ *  If a dictionary is provided, it replaces any pre-existing stream history.
+ *  The dictionary contents are the only history that can be referenced and
+ *  logically immediately precede the data compressed in the first subsequent
+ *  compression call.
+ *
+ *  The dictionary will only remain attached to the working stream through the
+ *  first compression call, at the end of which it is cleared. The dictionary
+ *  stream (and source buffer) must remain in-place / accessible / unchanged
+ *  through the completion of the first compression call on the stream.
+ */
+LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+
 
 /*-************************************
  *  Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 5e11ba213..d973b5fb4 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -533,8 +533,7 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
         assert(!internal_ctx->initCheck);
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
-        /* Point to the dictionary context */
-        internal_ctx->dictCtx = cdict ? &(cdict->fastCtx->internal_donotuse) : NULL;
+        LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
         if (cdict) {
             memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));

From 3a0c571272731166774405e1f96b3161827ca09f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 16:31:52 -0400
Subject: [PATCH 108/257] Add a LZ4_STATIC_LINKING_ONLY Macro to Guard
 Experimental APIs

---
 lib/lz4.c      | 1 +
 lib/lz4.h      | 2 ++
 lib/lz4frame.c | 1 +
 3 files changed, 4 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index cee85ec89..9e6b1e221 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -89,6 +89,7 @@
 /*-************************************
 *  Dependency
 **************************************/
+#define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
 /* see also "memory routines" below */
 
diff --git a/lib/lz4.h b/lib/lz4.h
index f90120e8e..0dfa19e00 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -353,6 +353,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
  * Their signatures may change.
  **************************************/
 
+#ifdef LZ4_STATIC_LINKING_ONLY
 
 /*! LZ4_resetStream_fast() :
  *  When an LZ4_stream_t is known to be in a internally coherent state,
@@ -414,6 +415,7 @@ LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* sr
  */
 LZ4LIB_API void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
 
+#endif
 
 /*-************************************
  *  Private definitions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d973b5fb4..7608d90ca 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -74,6 +74,7 @@ You can contact the author at :
 *  Includes
 **************************************/
 #include "lz4frame_static.h"
+#define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
 #define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"

From 51a56c47c0d34b6255bdd8e7b33f0ccb44b80351 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 16:55:12 -0400
Subject: [PATCH 109/257] Minor Fixes

---
 lib/lz4.c      | 22 +++++++++++++---------
 lib/lz4frame.c |  2 --
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 9e6b1e221..111085a80 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1193,7 +1193,19 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 }
 
 void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) {
-    working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+    if (dictionary_stream != NULL) {
+        /* If the current offset is zero, we will never look in the
+         * external dictionary context, since there is no value a table
+         * entry can take that indicate a miss. In that case, we need
+         * to bump the offset to something non-zero.
+         */
+        if (working_stream->internal_donotuse.currentOffset == 0) {
+            working_stream->internal_donotuse.currentOffset = 64 KB;
+        }
+        working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse);
+    } else {
+        working_stream->internal_donotuse.dictCtx = NULL;
+    }
 }
 
 
@@ -1264,14 +1276,6 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                /* If the current offset is zero, we will never look in the
-                 * external dictionary context, since there is no value a table
-                 * entry can take that indicate a miss. In that case, we need
-                 * to bump the offset to something non-zero.
-                 */
-                if (streamPtr->currentOffset == 0) {
-                    streamPtr->currentOffset = 64 KB;
-                }
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7608d90ca..507e4feed 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -531,8 +531,6 @@ static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
     if (level < LZ4HC_CLEVEL_MIN) {
-        LZ4_stream_t_internal* internal_ctx = &((LZ4_stream_t *)ctx)->internal_donotuse;
-        assert(!internal_ctx->initCheck);
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {

From 056ea63215800c9c16e81db92eacc9a9c87548ae Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 18:42:09 -0400
Subject: [PATCH 110/257] Fix Silly Warning (const-ness in declaration has no
 effect on value types!)

---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 111085a80..4b0efb143 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1147,7 +1147,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
-void LZ4_resetStream_fast(LZ4_stream_t* const ctx) {
+void LZ4_resetStream_fast(LZ4_stream_t* ctx) {
     LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32);
 }
 

From 7b3cd10579fdc3d62bbf5e00f07ca19068a4403b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 11 Apr 2018 16:31:43 -0700
Subject: [PATCH 111/257] reduced test time on circle-ci

---
 Makefile       |  4 ++--
 circle.yml     | 14 +++++++-------
 tests/Makefile | 13 ++++++-------
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index 3a958080f..86613fd71 100644
--- a/Makefile
+++ b/Makefile
@@ -143,10 +143,10 @@ clangtest-native: clean
 	@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang
 
 usan: clean
-	CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
+	CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
 
 usan32: clean
-	CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1
+	CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
 
 staticAnalyze: clean
 	CFLAGS=-g scan-build --status-bugs -v $(MAKE) all
diff --git a/circle.yml b/circle.yml
index 9a0d1ec86..aeb52e992 100644
--- a/circle.yml
+++ b/circle.yml
@@ -11,10 +11,10 @@ test:
     - clang -v; make clangtest && make clean
     - g++ -v; make gpptest     && make clean
     - gcc -v; make c_standards && make clean
-    - gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean
-    - gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean
-    - gcc-6 -v; make c_standards CC=gcc-6 && make clean
-    - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
+    - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
+    - gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
+    - gcc-6 -v; CC=gcc-6 make c_standards && make clean
+    - gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check  && make clean
 # Shorter tests
     - make cmake               && make clean
     - make -C tests test-lz4
@@ -22,11 +22,11 @@ test:
     - make -C tests test-frametest
     - make -C tests test-fullbench
     - make -C tests test-fuzzer && make clean
-    - make -C lib all && make clean
-    - pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
+    - make -C lib all          && make clean
+    - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
     - make travis-install      && make clean
   # Longer tests
-    - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+    - gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
     - make usan                && make clean
     - clang -v; make staticAnalyze && make clean
   # Valgrind tests
diff --git a/tests/Makefile b/tests/Makefile
index 77f5d02df..d11b7fead 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -56,7 +56,7 @@ LZ4 := $(PRGDIR)/lz4$(EXT)
 
 # Default test parameters
 TEST_FILES   := COPYING
-FUZZER_TIME  := -T3mn
+FUZZER_TIME  := -T90s
 NB_LOOPS     ?= -i1
 
 
@@ -393,13 +393,12 @@ test-mem: lz4 datagen fuzzer frametest fullbench
 	./datagen -g16KB -s2 > ftmdg16K2
 	./datagen -g16KB -s3 > ftmdg16K3
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
-	./datagen -g16MB > ftmdg16M
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2
+	./datagen -g7MB > ftmdg7M
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
 	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M
-	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2
-	./datagen -g256MB > ftmdg256M
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID)
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
+	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
 	$(RM) ftm*
 	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
 	valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256

From ca388790e689ddb09e5e5f7f86a5bdb17dee67d7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 11 Apr 2018 16:41:25 -0700
Subject: [PATCH 112/257] allow system-defined CPPFLAGS in /tests

---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index d11b7fead..25f00b814 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -39,7 +39,7 @@ CFLAGS  += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
+CPPFLAGS+= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
 FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 

From 1838803948ceaf211b9e79fb405ef9b7340762ce Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Wed, 11 Apr 2018 16:49:40 -0700
Subject: [PATCH 113/257] fixed LZ4_compress_fast_extState_fastReset()

---
 lib/lz4.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d79b2cc9d..d564ddc83 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -953,7 +953,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
  * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of
  * "correctly initialized").
  */
-int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
+int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
 {
     LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
@@ -961,8 +961,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* source, char*
     if (dstCapacity >= LZ4_compressBound(srcSize)) {
         if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType);
-LZ4_prepareTable>>> dev
+            LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -970,13 +969,13 @@ LZ4_prepareTable>>> dev
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            LZ4_prepareTable(ctx, srcSize, tableType);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
-            LZ4_prepareTable(ctx, inputSize, tableType);
+            LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
                 return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
@@ -984,8 +983,8 @@ LZ4_prepareTable>>> dev
             }
         } else {
             const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
-            LZ4_prepareTable(ctx, inputSize, tableType);
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            LZ4_prepareTable(ctx, srcSize, tableType);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }

From d6d9bccd537e307cc0bf3f8475801dea79e97367 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 12 Apr 2018 06:47:27 -0700
Subject: [PATCH 114/257] modified versionsTest

to use MOREFLAGS rather CPPFLAGS
as some older versions of LZ4 overwrite CPPFLAGS environment variable.
---
 circle.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/circle.yml b/circle.yml
index aeb52e992..fa3759069 100644
--- a/circle.yml
+++ b/circle.yml
@@ -23,7 +23,7 @@ test:
     - make -C tests test-fullbench
     - make -C tests test-fuzzer && make clean
     - make -C lib all          && make clean
-    - pyenv global 3.4.4; CPPFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
+    - pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean
     - make travis-install      && make clean
   # Longer tests
     - gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean

From 8af32ce6f7109ecf5cd7d73527e0aba3a63b55e5 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 12 Apr 2018 07:25:40 -0700
Subject: [PATCH 115/257] modified a few traces for debug

---
 lib/lz4.c      | 5 +++--
 lib/lz4hc.c    | 7 +++----
 tests/fuzzer.c | 4 +++-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d564ddc83..e4a68cd76 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -594,7 +594,6 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
             cctx->tableType = clearedTable;
         } else {
             DEBUGLOG(4, "Re-use hash table (no reset)");
-            //if (tableType == byU32) cctx->currentOffset += 64 KB;
         }
     }
 
@@ -602,9 +601,11 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * than compressing without a gap. However, compressing with
      * currentOffset == 0 is faster still, so we preserve that case.
      */
+     DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset);
     if (cctx->currentOffset != 0 && tableType == byU32) {
         cctx->currentOffset += 64 KB;
     }
+    DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset);
 
     /* Finally, clear history */
     cctx->dictCtx = NULL;
@@ -1267,7 +1268,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    DEBUGLOG(4, "LZ4_loadDict %p", LZ4_dict);
+    DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict);
 
     LZ4_prepareTable(dict, 0, tableType);
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d05760b6a..111a09b07 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -336,7 +336,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = *anchor;  /* only works for single segment */
-    //g_debuglog_enable = (pos >= 2228) & (pos <= 2262);
+    /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */
     DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
                 pos,
                 (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
@@ -1137,15 +1137,14 @@ static int LZ4HC_compress_optimal (
  encode: /* cur, last_match_pos, best_mlen, best_off must be set */
          assert(cur < LZ4_OPT_NUM);
          assert(last_match_pos >= 1);  /* == 1 when only one candidate */
-         DEBUGLOG(6, "reverse traversal, looking for shortest path")
-         DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
+         DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos);
          {   int candidate_pos = cur;
              int selected_matchLength = best_mlen;
              int selected_offset = best_off;
              while (1) {  /* from end to beginning */
                  int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
                  int const next_offset = opt[candidate_pos].off;
-                 DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
+                 DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
                  opt[candidate_pos].mlen = selected_matchLength;
                  opt[candidate_pos].off = selected_offset;
                  selected_matchLength = next_matchLength;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2e3ee92a3..a650277f3 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -632,13 +632,15 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
+        DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n", blockSize, block, decodedBuffer, dict, dictSize);
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
         FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict(): decoding %i bytes, dict(%p) of size %i", blockSize, dict, dictSize);
+        FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as extDict");
+        DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n", blockSize, decodedBuffer, dict, dictSize);
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");

From db9aa785c51bbcae50c777e89fb537393bfca856 Mon Sep 17 00:00:00 2001
From: test4973 <Kdo4973@hotmail.com>
Date: Thu, 12 Apr 2018 16:12:21 -0700
Subject: [PATCH 116/257] fixed : counting matches which overlap extDict and
 prefix

---
 lib/lz4.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e4a68cd76..21892d30e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -601,11 +601,9 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * than compressing without a gap. However, compressing with
      * currentOffset == 0 is faster still, so we preserve that case.
      */
-     DEBUGLOG(2, "tableType=%u, currentOffset=%u", cctx->tableType, cctx->currentOffset);
     if (cctx->currentOffset != 0 && tableType == byU32) {
         cctx->currentOffset += 64 KB;
     }
-    DEBUGLOG(2, "currentOffset: %u", cctx->currentOffset);
 
     /* Finally, clear history */
     cctx->dictCtx = NULL;
@@ -652,7 +650,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     const BYTE* dictBase = dictDirective == usingDictCtx ?
         dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
         dictionary + dictSize - startIndex;
-    const BYTE* dictLowLimit;
 
     BYTE* op = (BYTE*) dest;
     BYTE* const olimit = op + maxOutputSize;
@@ -665,7 +662,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
-    dictLowLimit = dictionary ? dictionary : lowLimit;
 
     if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
 
@@ -735,15 +731,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                         /* there was no match, try the dictionary */
                         matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                         match = dictBase + matchIndex;
-                        lowLimit = dictLowLimit;
+                        lowLimit = dictionary;
                     } else {
                         match = base + matchIndex;
                         lowLimit = (const BYTE*)source;
                     }
                 } else if (dictDirective==usingExtDict) {
                     if (matchIndex < startIndex) {
+                        DEBUGLOG(7, "extDict candidate: matchIndex=%5u  <  startIndex=%5u", matchIndex, startIndex);
                         match = dictBase + matchIndex;
-                        lowLimit = dictLowLimit;
+                        lowLimit = dictionary;
                     } else {
                         match = base + matchIndex;
                         lowLimit = (const BYTE*)source;
@@ -786,6 +783,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
+            DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source);
         }
 
 _next_match:
@@ -793,29 +791,33 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
+            DEBUGLOG(6, "                with offset=%u  (ext if > %zi)", offset, ip - (const BYTE*)source);
         } else  {
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
+            DEBUGLOG(6, "                with offset=%u  (same segment)", (U32)(ip - match));
         }
 
         /* Encode MatchLength */
         {   unsigned matchCode;
 
             if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx)
-              && (lowLimit==dictionary) ) {
-                const BYTE* limit;
-                limit = ip + (dictEnd-match);
+              && (lowLimit==dictionary) /* match within extDict */ ) {
+                const BYTE* limit = ip + (dictEnd-match);
+                assert(dictEnd > match);
                 if (limit > matchlimit) limit = matchlimit;
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
                 ip += MINMATCH + matchCode;
                 if (ip==limit) {
-                    unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit);
+                    unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit);
                     matchCode += more;
                     ip += more;
                 }
+                DEBUGLOG(6, "                with matchLength=%u starting in extDict", matchCode+MINMATCH);
             } else {
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
                 ip += MINMATCH + matchCode;
+                DEBUGLOG(6, "                with matchLength=%u", matchCode+MINMATCH);
             }
 
             if ( outputLimited &&    /* Check output buffer overflow */
@@ -865,14 +867,18 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                     /* there was no match, try the dictionary */
                     matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                     match = dictBase + matchIndex;
+                    lowLimit = dictionary;   /* required for match length counter */
                 } else {
                     match = base + matchIndex;
+                    lowLimit = (const BYTE*)source;  /* required for match length counter */
                 }
             } else if (dictDirective==usingExtDict) {
                 if (matchIndex < startIndex) {
                     match = dictBase + matchIndex;
+                    lowLimit = dictionary;   /* required for match length counter */
                 } else {
                     match = base + matchIndex;
+                    lowLimit = (const BYTE*)source;   /* required for match length counter */
                 }
             } else {   /* single memory segment */
                 match = base + matchIndex;
@@ -885,6 +891,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 *token=0;
                 if (maybe_ext_memSegment)
                     offset = current - matchIndex;
+                DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source);
                 goto _next_match;
             }
         }

From 98811d606808da9f59affd990670ba34e5a9bee2 Mon Sep 17 00:00:00 2001
From: Cyan4973 <Kdo4973@hotmail.com>
Date: Fri, 13 Apr 2018 00:59:27 -0700
Subject: [PATCH 117/257] added sudo rights for low-mem-address tests

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index a44642093..0a876f925 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,7 +14,7 @@ matrix:
       env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc
 
     - os: linux
-      sudo: false
+      sudo: required
       env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc
 
     - os: linux

From 57afa36795f478d0f9b069ad19b578761e3fb16a Mon Sep 17 00:00:00 2001
From: Cyan4973 <Kdo4973@hotmail.com>
Date: Fri, 13 Apr 2018 01:01:54 -0700
Subject: [PATCH 118/257] compatibility with gcc-4.4 string.h version

Someone found it would be a great idea to define there a global variable under the very generic name "index".
Cause problem with shadow warnings, so no variable can be named "index" now ...

Also : automatically update API manual
---
 doc/lz4_manual.html | 94 ++++++++++++++++++++++++++++++++++-----------
 lib/lz4.c           |  6 +--
 2 files changed, 75 insertions(+), 25 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index d14467f6f..f8639fe72 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -15,8 +15,9 @@ <h1>1.8.2 Manual</h1>
 <li><a href="#Chapter5">Advanced Functions</a></li>
 <li><a href="#Chapter6">Streaming Compression Functions</a></li>
 <li><a href="#Chapter7">Streaming Decompression Functions</a></li>
-<li><a href="#Chapter8">Private definitions</a></li>
-<li><a href="#Chapter9">Obsolete Functions</a></li>
+<li><a href="#Chapter8">Unstable declarations</a></li>
+<li><a href="#Chapter9">Private definitions</a></li>
+<li><a href="#Chapter10">Obsolete Functions</a></li>
 </ol>
 <hr>
 <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -245,21 +246,79 @@ <h1>1.8.2 Manual</h1>
  
 </p></pre><BR>
 
-<a name="Chapter8"></a><h2>Private definitions</h2><pre>
+<a name="Chapter8"></a><h2>Unstable declarations</h2><pre>
+ Declarations in this section should be considered unstable.
+ Use at your own peril, etc., etc.
+ They may be removed in the future.
+ Their signatures may change.
+<BR></pre>
+
+<pre><b>void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+</b><p>  When an LZ4_stream_t is known to be in a internally coherent state,
+  it can often be prepared for a new compression with almost no work, only
+  sometimes falling back to the full, expensive reset that is always required
+  when the stream is in an indeterminate state (i.e., the reset performed by
+  LZ4_resetStream()).
+
+  LZ4_streams are guaranteed to be in a valid state when:
+  - returned from LZ4_createStream()
+  - reset by LZ4_resetStream()
+  - memset(stream, 0, sizeof(LZ4_stream_t))
+  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
+  - the stream was in a valid state and was then used in any compression call
+    that returned success
+  - the stream was in an indeterminate state and was used in a compression
+    call that fully reset the state (LZ4_compress_fast_extState()) and that
+    returned success
+ 
+</p></pre><BR>
+
+<pre><b>int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+</b><p>  A variant of LZ4_compress_fast_extState().
+
+  Using this variant avoids an expensive initialization step. It is only safe
+  to call if the state buffer is known to be correctly initialized already
+  (see above comment on LZ4_resetStream_fast() for a definition of "correctly
+  initialized"). From a high level, the difference is that this function
+  initializes the provided state with a call to LZ4_resetStream_fast() while
+  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ 
+</p></pre><BR>
+
+<pre><b>void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream);
+</b><p>  This is an experimental API that allows for the efficient use of a
+  static dictionary many times.
+
+  Rather than re-loading the dictionary buffer into a working context before
+  each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
+  working LZ4_stream_t, this function introduces a no-copy setup mechanism,
+  in which the working stream references the dictionary stream in-place.
+
+  Several assumptions are made about the state of the dictionary stream.
+  Currently, only streams which have been prepared by LZ4_loadDict() should
+  be expected to work.
+
+  Alternatively, the provided dictionary stream pointer may be NULL, in which
+  case any existing dictionary stream is unset.
+
+  If a dictionary is provided, it replaces any pre-existing stream history.
+  The dictionary contents are the only history that can be referenced and
+  logically immediately precede the data compressed in the first subsequent
+  compression call.
+
+  The dictionary will only remain attached to the working stream through the
+  first compression call, at the end of which it is cleared. The dictionary
+  stream (and source buffer) must remain in-place / accessible / unchanged
+  through the completion of the first compression call on the stream.
+ 
+</p></pre><BR>
+
+<a name="Chapter9"></a><h2>Private definitions</h2><pre>
  Do not use these definitions.
  They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
  Using these definitions will expose code to API and/or ABI break in future versions of the library.
 <BR></pre>
 
-<pre><b>typedef struct {
-    uint32_t hashTable[LZ4_HASH_SIZE_U32];
-    uint32_t currentOffset;
-    uint32_t initCheck;
-    const uint8_t* dictionary;
-    uint8_t* bufferStart;   </b>/* obsolete, used for slideInputBuffer */<b>
-    uint32_t dictSize;
-} LZ4_stream_t_internal;
-</b></pre><BR>
 <pre><b>typedef struct {
     const uint8_t* externalDict;
     size_t extDictSize;
@@ -267,15 +326,6 @@ <h1>1.8.2 Manual</h1>
     size_t prefixSize;
 } LZ4_streamDecode_t_internal;
 </b></pre><BR>
-<pre><b>typedef struct {
-    unsigned int hashTable[LZ4_HASH_SIZE_U32];
-    unsigned int currentOffset;
-    unsigned int initCheck;
-    const unsigned char* dictionary;
-    unsigned char* bufferStart;   </b>/* obsolete, used for slideInputBuffer */<b>
-    unsigned int dictSize;
-} LZ4_stream_t_internal;
-</b></pre><BR>
 <pre><b>typedef struct {
     const unsigned char* externalDict;
     size_t extDictSize;
@@ -311,7 +361,7 @@ <h1>1.8.2 Manual</h1>
  
 </p></pre><BR>
 
-<a name="Chapter9"></a><h2>Obsolete Functions</h2><pre></pre>
+<a name="Chapter10"></a><h2>Obsolete Functions</h2><pre></pre>
 
 <pre><b>#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
 #  define LZ4_DEPRECATED(message)   </b>/* disable deprecation warnings */<b>
diff --git a/lib/lz4.c b/lib/lz4.c
index 21892d30e..f55e4e196 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -517,15 +517,15 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab
     return LZ4_hash4(LZ4_read32(p), tableType);
 }
 
-static void LZ4_putIndexOnHash(U32 index, U32 h, void* tableBase, tableType_t const tableType)
+static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType)
 {
     switch (tableType)
     {
     default: /* fallthrough */
     case clearedTable: /* fallthrough */
     case byPtr: { /* illegal! */ assert(0); return; }
-    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = index; return; }
-    case byU16: { U16* hashTable = (U16*) tableBase; assert(index < 65536); hashTable[h] = (U16)index; return; }
+    case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; }
+    case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; }
     }
 }
 

From 54ec83ce1f62014398b76a441d1ff1212dad9604 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Apr 2018 02:10:53 -0700
Subject: [PATCH 119/257] fixed potential ptrdiff_t overflow (32-bits mode)

Also removed pointer comparison, which should solve #485
---
 lib/lz4.c | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index f55e4e196..c48aca4df 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1315,15 +1315,14 @@ void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dic
 }
 
 
-static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
+static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize)
 {
-    if ((LZ4_dict->currentOffset > 0x80000000) ||
-        ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) {   /* address space overflow */
+    if (LZ4_dict->currentOffset + nextSize > 0x80000000) {   /* potential ptrdiff_t overflow (32-bits mode) */
         /* rescale hash table */
         U32 const delta = LZ4_dict->currentOffset - 64 KB;
         const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
         int i;
-        DEBUGLOG(4, "LZ4_renormDictT %p", LZ4_dict);
+        DEBUGLOG(4, "LZ4_renormDictT");
         for (i=0; i<LZ4_HASH_SIZE_U32; i++) {
             if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
             else LZ4_dict->hashTable[i] -= delta;
@@ -1341,10 +1340,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-    const BYTE* smallest = (const BYTE*) source;
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
-    if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
-    LZ4_renormDictT(streamPtr, smallest);
+    LZ4_renormDictT(streamPtr, inputSize);   /* avoid index overflow */
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
 
     /* Check overlapping input/dictionary space */
@@ -1377,7 +1374,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
             if (inputSize > 4 KB) {
                 /* For compressing large blobs, it is faster to pay the setup
                  * cost to copy the dictionary's tables into the active context,
-                 * so that the compression loop is only looking in one table.
+                 * so that the compression loop is only looking into one table.
                  */
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
                 result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
@@ -1398,8 +1395,8 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 }
 
 
-/* Hidden debug function, to force external dictionary mode */
-int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
+/* Hidden debug function, to force-test external dictionary mode */
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize)
 {
     LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
     int result;
@@ -1407,16 +1404,16 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 
     const BYTE* smallest = dictEnd;
     if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
-    LZ4_renormDictT(streamPtr, smallest);
+    LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-        result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;
-    streamPtr->dictSize = (U32)inputSize;
+    streamPtr->dictSize = (U32)srcSize;
 
     return result;
 }

From c40bac31d3886c2f69d0364823b6d6aaa972ee5b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Apr 2018 02:26:14 -0700
Subject: [PATCH 120/257] added comment on variables required after _next_match

---
 lib/lz4.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index c48aca4df..54d037ccf 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -787,6 +787,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         }
 
 _next_match:
+        /* at this stage, the following variables must be correctly set :
+         * - ip : at start of LZ operation
+         * - match : at start of previous pattern occurence; can be within current prefix, or within extDict
+         * - offset : if maybe_ext_memSegment==1 (constant)
+         * - lowLimit : must be == dictionary to mean "match is within extDict"; must be == source otherwise
+         * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
+         */
+
         /* Encode Offset */
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);

From d2bcfa31f525aaa11c2d248af0ba487791399c1f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 13 Apr 2018 02:45:32 -0700
Subject: [PATCH 121/257] fixed minor unused variable warning

---
 lib/lz4.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 54d037ccf..e8f9831c9 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1408,10 +1408,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 {
     LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse;
     int result;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-    const BYTE* smallest = dictEnd;
-    if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
     LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {

From 6dd64e0776dccf5c19e8438625657abf0e6f3048 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 12 Apr 2018 18:23:01 -0400
Subject: [PATCH 122/257] Add Tests for LZ4_attach_dictionary and Friends

---
 tests/fuzzer.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c134fe3e1..98b3d9f53 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -43,6 +43,7 @@
 #include <stdio.h>      /* fgets, sscanf */
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC */
+#define LZ4_STATIC_LINKING_ONLY
 #define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
 #define XXH_STATIC_LINKING_ONLY
@@ -402,6 +403,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
+        /* Test compression using fast reset external state*/
+        FUZ_DISPLAYTEST;
+        ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
+
         /* Test compression */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
@@ -626,6 +632,78 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
         }   }
 
+        /* Compress using external dictionary stream */
+        FUZ_DISPLAYTEST;
+        {
+            LZ4_stream_t LZ4_stream;
+
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST;
+            LZ4_resetStream_fast(&LZ4_stream);
+            LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
+            ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+        }
+
+        /* Decompress with dictionary as external */
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize] = 0;
+        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
+        FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+        crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize] = 0;
+        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+        FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+        crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize-1] = 0;
+        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
+        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
+        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+
+        FUZ_DISPLAYTEST;
+        decodedBuffer[blockSize-1] = 0;
+        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
+        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
+        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+
+        FUZ_DISPLAYTEST;
+        {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
+            if ((U32)blockSize > missingBytes) {
+                decodedBuffer[blockSize-missingBytes] = 0;
+                ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
+                FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
+                FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
+        }   }
+
         /* Compress HC using External dictionary */
         FUZ_DISPLAYTEST;
         dict -= (FUZ_rand(&randState) & 7);    /* even bigger separation */

From 9f0f6b89bb27d4d6c5d12a7142fc330f3d1b3a10 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 12 Apr 2018 19:17:53 -0400
Subject: [PATCH 123/257] Further Test that ExtDictCtx Mode Produces the Exact
 Same Output

---
 tests/fuzzer.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 98b3d9f53..db057386d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -303,9 +303,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         U32 testNb = 0;
         U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
         int const blockSize  = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
-        int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
+        int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
         int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
-        int const dictSize = MIN(dictSizeRand, blockStart);
+        int const dictSize = MIN(dictSizeRand, blockStart - 1);
         int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
         char* const block = ((char*)CNBuffer) + blockStart;
         const char* dict = block - dictSize;
@@ -636,6 +636,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_DISPLAYTEST;
         {
             LZ4_stream_t LZ4_stream;
+            int expectedSize;
+            U32 expectedCrc;
+
+            LZ4_loadDict(&LZ4dict, dict, dictSize);
+            expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
+            FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
+            expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
 
             LZ4_loadDict(&LZ4dict, dict, dictSize);
 
@@ -644,6 +651,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
 
+            /* In the future, it might be desirable to let extDictCtx mode's
+             * output diverge from the output generated by regular extDict mode.
+             * Until that time, this comparison serves as a good regression
+             * test.
+             */
+            FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+
             FUZ_DISPLAYTEST;
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
@@ -656,6 +671,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
+            FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
             FUZ_DISPLAYTEST;
             LZ4_resetStream_fast(&LZ4_stream);
@@ -663,6 +680,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
             FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
             FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
+            FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
+            FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
         }
 
         /* Decompress with dictionary as external */

From e9280647976ca468ab67ffbf3dd9c8e533fe23bf Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 15:11:28 -0700
Subject: [PATCH 124/257] fixed gcc performance regression

---
 lib/lz4.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e8f9831c9..c2012f664 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -660,6 +660,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
+    assert(acceleration >= 1);
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
@@ -712,6 +713,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         } else {   /* byU32, byU16 */
 
             const BYTE* forwardIp = ip;
+            unsigned step = 1;
             unsigned searchMatchNb = acceleration << LZ4_skipTrigger;
             do {
                 U32 const h = forwardH;
@@ -720,8 +722,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 assert(matchIndex <= current);
                 assert(forwardIp - base < (ptrdiff_t)(2 GB - 1));
                 ip = forwardIp;
-                assert(searchMatchNb >= (1<<LZ4_skipTrigger));
-                forwardIp += (searchMatchNb++ >> LZ4_skipTrigger);
+                forwardIp += step;
+                step = (searchMatchNb++ >> LZ4_skipTrigger);
 
                 if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals;
                 assert(ip < mflimitPlusOne);

From 4aff9b10b56e44b2ff6326f611c3659565812d80 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 16:14:28 -0700
Subject: [PATCH 125/257] fixed fuzzer tests

which were modified in parallel within branc `dev`
---
 tests/fuzzer.c | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 83ab50b9e..772134559 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -454,7 +454,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
         /* Test compression using fast reset external state*/
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         ret = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
         FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState_fastReset() failed");
 
@@ -687,7 +687,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress using external dictionary stream */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {
             LZ4_stream_t LZ4_stream;
             int expectedSize;
@@ -713,13 +713,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -728,7 +728,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
             FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST;
+            FUZ_DISPLAYTEST();
             LZ4_resetStream_fast(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
@@ -739,36 +739,38 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }
 
         /* Decompress with dictionary as external */
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
+        }
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
         FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
-        crcCheck = XXH32(decodedBuffer, blockSize, 0);
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+            FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+        }
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         decodedBuffer[blockSize-1] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
         FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST();
         {   U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
             if ((U32)blockSize > missingBytes) {
                 decodedBuffer[blockSize-missingBytes] = 0;

From a3aeb34184e20d51616beccfcbbe7aade3cc3a64 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 16:54:03 -0700
Subject: [PATCH 126/257] fixed minor format warnings

---
 lib/lz4.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c2012f664..980a5fdbc 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -785,7 +785,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
-            DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, litLength, ip-(const BYTE*)source);
+            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
         }
 
 _next_match:
@@ -801,7 +801,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         if (maybe_ext_memSegment) {   /* static test */
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (ext if > %zi)", offset, ip - (const BYTE*)source);
+            DEBUGLOG(6, "                with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
         } else  {
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
@@ -901,7 +901,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 *token=0;
                 if (maybe_ext_memSegment)
                     offset = current - matchIndex;
-                DEBUGLOG(6, "seq.start:%zi, literals=%u, match.start:%zi", anchor-(const BYTE*)source, 0, ip-(const BYTE*)source);
+                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
         }

From 444211d2599a2be59e3f50418b46ec2431288e9a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 17:15:02 -0700
Subject: [PATCH 127/257] edited a few traces for debugging

---
 lib/lz4.c      | 14 +++++++-------
 tests/fuzzer.c |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 980a5fdbc..dca4d6973 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -588,12 +588,12 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
           || tableType == byPtr
           || inputSize >= 4 KB)
         {
-            DEBUGLOG(4, "Resetting table in %p", cctx);
+            DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx);
             MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
             cctx->currentOffset = 0;
             cctx->tableType = clearedTable;
         } else {
-            DEBUGLOG(4, "Re-use hash table (no reset)");
+            DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)");
         }
     }
 
@@ -799,13 +799,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
         /* Encode Offset */
         if (maybe_ext_memSegment) {   /* static test */
+            DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
         } else  {
+            DEBUGLOG(6, "             with offset=%u  (same segment)", (U32)(ip - match));
             assert(ip-match <= MAX_DISTANCE);
             LZ4_writeLE16(op, (U16)(ip - match)); op+=2;
-            DEBUGLOG(6, "                with offset=%u  (same segment)", (U32)(ip - match));
         }
 
         /* Encode MatchLength */
@@ -823,11 +823,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                     matchCode += more;
                     ip += more;
                 }
-                DEBUGLOG(6, "                with matchLength=%u starting in extDict", matchCode+MINMATCH);
+                DEBUGLOG(6, "             with matchLength=%u starting in extDict", matchCode+MINMATCH);
             } else {
                 matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
                 ip += MINMATCH + matchCode;
-                DEBUGLOG(6, "                with matchLength=%u", matchCode+MINMATCH);
+                DEBUGLOG(6, "             with matchLength=%u", matchCode+MINMATCH);
             }
 
             if ( outputLimited &&    /* Check output buffer overflow */
@@ -1259,7 +1259,7 @@ LZ4_stream_t* LZ4_createStream(void)
 
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
-    DEBUGLOG(5, "LZ4_resetStream %p", LZ4_stream);
+    DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream);
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 772134559..244cc4fef 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -687,19 +687,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         }   }
 
         /* Compress using external dictionary stream */
-        FUZ_DISPLAYTEST();
         {
             LZ4_stream_t LZ4_stream;
             int expectedSize;
             U32 expectedCrc;
 
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()");
             LZ4_loadDict(&LZ4dict, dict, dictSize);
             expectedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
             expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
 
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
             LZ4_loadDict(&LZ4dict, dict, dictSize);
-
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
@@ -713,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
             FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
 
-            FUZ_DISPLAYTEST();
+            FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
             LZ4_resetStream(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);

From da3b5ba6f0014564b7312511e441067ba9429733 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 16 Apr 2018 23:59:42 -0700
Subject: [PATCH 128/257] fixed dictCtx compression

---
 lib/lz4.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index dca4d6973..0590de44f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -602,6 +602,7 @@ LZ4_FORCE_INLINE void LZ4_prepareTable(
      * currentOffset == 0 is faster still, so we preserve that case.
      */
     if (cctx->currentOffset != 0 && tableType == byU32) {
+        DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset");
         cctx->currentOffset += 64 KB;
     }
 
@@ -636,8 +637,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
+    const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
 
-    int const maybe_ext_memSegment = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
+    int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
     U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */
     const BYTE* const dictEnd = dictionary + dictSize;
     const BYTE* anchor = (const BYTE*) source;
@@ -648,7 +650,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? */
+        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
         dictionary + dictSize - startIndex;
 
     BYTE* op = (BYTE*) dest;
@@ -657,6 +659,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     U32 offset = 0;
     U32 forwardH;
 
+    DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
     /* Init conditions */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
@@ -731,8 +734,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 if (dictDirective == usingDictCtx) {
                     if (matchIndex < startIndex) {
                         /* there was no match, try the dictionary */
+                        assert(tableType == byU32);
                         matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                         match = dictBase + matchIndex;
+                        matchIndex += dictDelta;   /* make dictCtx index comparable with current context */
                         lowLimit = dictionary;
                     } else {
                         match = base + matchIndex;
@@ -758,7 +763,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
-                    if (maybe_ext_memSegment) offset = current - matchIndex;
+                    if (maybe_extMem) offset = current - matchIndex;
                     break;   /* match found */
                 }
 
@@ -798,7 +803,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
          */
 
         /* Encode Offset */
-        if (maybe_ext_memSegment) {   /* static test */
+        if (maybe_extMem) {   /* static test */
             DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
             assert(offset <= MAX_DISTANCE && offset > 0);
             LZ4_writeLE16(op, (U16)offset); op+=2;
@@ -878,6 +883,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                     matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32);
                     match = dictBase + matchIndex;
                     lowLimit = dictionary;   /* required for match length counter */
+                    matchIndex += dictDelta;
                 } else {
                     match = base + matchIndex;
                     lowLimit = (const BYTE*)source;  /* required for match length counter */
@@ -899,8 +905,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
-                if (maybe_ext_memSegment)
-                    offset = current - matchIndex;
+                if (maybe_extMem) offset = current - matchIndex;
                 DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
@@ -1285,7 +1290,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
     const BYTE* const dictEnd = p + dictSize;
     const BYTE* base;
 
-    DEBUGLOG(4, "LZ4_loadDict (%p into %p)", dictionary, LZ4_dict);
+    DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);
 
     LZ4_prepareTable(dict, 0, tableType);
 

From aedc44780468c11d90114bc1d6124af545bfc652 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 17 Apr 2018 14:01:44 -0400
Subject: [PATCH 129/257] Always Bump Offset by 64 KB in LZ4_loadDict()

This actually ensures the guarantee referred to in the comment in
LZ4_compress_fast_continue().
---
 lib/lz4.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4b0efb143..0ce05daee 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1173,11 +1173,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 
     LZ4_prepareTable(dict, 0, tableType);
 
+    /* We always increment the offset by 64 KB, since, if the dict is longer,
+     * we truncate it to the last 64k, and if it's shorter, we still want to
+     * advance by a whole window length so we can provide the guarantee that
+     * there are only valid offsets in the window, which allows an optimization
+     * in LZ4_compress_fast_continue() where it uses noDictIssue even when the
+     * dictionary isn't a full 64k. */
+
     if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
-    base = p - dict->currentOffset;
+    base = dictEnd - 64 KB - dict->currentOffset;
     dict->dictionary = p;
     dict->dictSize = (U32)(dictEnd - p);
-    dict->currentOffset += dict->dictSize;
+    dict->currentOffset += 64 KB;
     dict->tableType = tableType;
 
     if (dictSize < (int)HASH_UNIT) {

From 152064218361e5762fd67b5de425707fdc47095b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 17 Apr 2018 15:29:17 -0700
Subject: [PATCH 130/257] fix matchIndex overflow

can happen with dictCtx
---
 lib/lz4.c | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index b426545c1..c799596ec 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -758,9 +758,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
-                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;    /* match outside of valid area */
-                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue;  /* too far */
-                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
+                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;     /* match outside of valid area */
+                if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */
+                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);      /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
                     if (maybe_extMem) offset = current - matchIndex;
@@ -861,7 +861,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Fill table */
         LZ4_putPosition(ip-2, cctx->hashTable, tableType, base);
 
-#if 1
         /* Test next position */
         if (tableType == byPtr) {
 
@@ -901,7 +900,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             }
             LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
             if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
-              && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
+              && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE))
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
@@ -914,13 +913,6 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Prepare next loop */
         forwardH = LZ4_hashPosition(++ip, tableType);
 
-#else
-
-        /* Prepare next loop */
-        forwardH = LZ4_hashPosition(ip, tableType);
-
-#endif
-
     }
 
 _last_literals:

From 88cca1723e76c8f5031954ba07b28447c0cb55d8 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 17 Apr 2018 16:18:37 -0700
Subject: [PATCH 131/257] fix dictDelta setting error

wrong test
---
 lib/lz4.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c799596ec..e7553edeb 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -637,7 +637,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary;
     const U32 dictSize =
         dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize;
-    const U32 dictDelta = usingDictCtx ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
+    const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0;   /* make indexes in dictCtx comparable with index in current context */
 
     int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx);
     U32 const prefixIdxLimit = startIndex - dictSize;   /* used when dictDirective == dictSmall */

From 5ad4599c5ad18d2408a6ccc545c45a36b99f0c6f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Tue, 17 Apr 2018 16:47:56 -0700
Subject: [PATCH 132/257] fixed LZ4_compress_fast_extState_fastReset() in
 32-bit mode

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e7553edeb..33aa5c7d3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -983,7 +983,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
                 return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
             return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
@@ -997,7 +997,7 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
                 return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
-            const tableType_t tableType = (sizeof(void*)==8) ? byU32 : byPtr;
+            const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
             return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }

From ea6ed46fc273a7b10870caeb515622737eb1c572 Mon Sep 17 00:00:00 2001
From: Dmitrii Rodionov <d.rodionov@samsung.com>
Date: Wed, 18 Apr 2018 12:20:56 +0300
Subject: [PATCH 133/257] Wrap likely/unlikely macroses with #ifndef

It prevent redefine error when project using lz4 has its own likely/unlikely
macroses.
---
 lib/lz4.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0ce05daee..1e969316e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -147,8 +147,12 @@
 #  define expect(expr,value)    (expr)
 #endif
 
+#ifndef likely
 #define likely(expr)     expect((expr) != 0, 1)
+#endif
+#ifndef unlikely
 #define unlikely(expr)   expect((expr) != 0, 0)
+#endif
 
 
 /*-************************************

From 4785bd6a3508c6e10c5cee7126fd112a0ec44599 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 18 Apr 2018 16:49:27 -0700
Subject: [PATCH 134/257] minor length reduction of several large lines

---
 doc/lz4frame_manual.html | 50 +++++++++++++++-----------
 lib/Makefile             |  4 +++
 lib/lz4.c                | 76 ++++++++++++++++++++++++++++------------
 lib/lz4frame.h           | 50 +++++++++++++++-----------
 4 files changed, 115 insertions(+), 65 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index f0f7f5afb..459bac89f 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -30,9 +30,9 @@ <h1>1.8.2 Manual</h1>
 
 <a name="Chapter3"></a><h2>Error management</h2><pre></pre>
 
-<pre><b>unsigned    LZ4F_isError(LZ4F_errorCode_t code);   </b>/**< tells if a `LZ4F_errorCode_t` function result is an error code */<b>
+<pre><b>unsigned    LZ4F_isError(LZ4F_errorCode_t code);   </b>/**< tells when a function result is an error code */<b>
 </b></pre><BR>
-<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   </b>/**< return error code string; useful for debugging */<b>
+<pre><b>const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   </b>/**< return error code string; for debugging */<b>
 </b></pre><BR>
 <a name="Chapter4"></a><h2>Frame compression types</h2><pre></pre>
 
@@ -74,13 +74,13 @@ <h1>1.8.2 Manual</h1>
 } LZ4F_frameType_t;
 </b></pre><BR>
 <pre><b>typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;          </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b>
-  LZ4F_blockMode_t       blockMode;            </b>/* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */<b>
-  LZ4F_contentChecksum_t contentChecksumFlag;  </b>/* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */<b>
-  LZ4F_frameType_t       frameType;            </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b>
-  unsigned long long     contentSize;          </b>/* Size of uncompressed content ; 0 == unknown */<b>
-  unsigned               dictID;               </b>/* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */<b>
-  LZ4F_blockChecksum_t   blockChecksumFlag;    </b>/* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */<b>
+  LZ4F_blockSizeID_t     blockSizeID;         </b>/* max64KB, max256KB, max1MB, max4MB; 0 == default */<b>
+  LZ4F_blockMode_t       blockMode;           </b>/* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */<b>
+  LZ4F_contentChecksum_t contentChecksumFlag; </b>/* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */<b>
+  LZ4F_frameType_t       frameType;           </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b>
+  unsigned long long     contentSize;         </b>/* Size of uncompressed content ; 0 == unknown */<b>
+  unsigned               dictID;              </b>/* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */<b>
+  LZ4F_blockChecksum_t   blockChecksumFlag;   </b>/* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */<b>
 } LZ4F_frameInfo_t;
 </b><p>  makes it possible to set or read frame parameters.
   It's not required to set all fields, as long as the structure was initially memset() to zero.
@@ -89,7 +89,7 @@ <h1>1.8.2 Manual</h1>
 
 <pre><b>typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       </b>/* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */<b>
+  int      compressionLevel;       </b>/* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */<b>
   unsigned autoFlush;              </b>/* 1 == always flush, to reduce usage of internal buffers */<b>
   unsigned reserved[4];            </b>/* must be zero for forward compatibility */<b>
 } LZ4F_preferences_t;
@@ -165,34 +165,42 @@ <h1>1.8.2 Manual</h1>
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+<pre><b>size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                 const void* srcBuffer, size_t srcSize,
+                                 const LZ4F_compressOptions_t* cOptPtr);
 </b><p>  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
-  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+  Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
   This value is provided by LZ4F_compressBound().
   If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
-  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+  LZ4F_compressUpdate() doesn't guarantee error recovery.
+  When an error occurs, compression context must be freed or resized.
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx,
+                              void* dstBuffer, size_t dstCapacity,
+                        const LZ4F_compressOptions_t* cOptPtr);
 </b><p>  When data must be generated and sent immediately, without waiting for a block to be completely filled,
   it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  `dstCapacity` must be large enough to ensure the operation will be successful.
  `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
            or an error code if it fails (which can be tested using LZ4F_isError())
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+<pre><b>size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+                                    void* dstBuffer, size_t dstCapacity,
+                              const LZ4F_compressOptions_t* cOptPtr);
 </b><p>  To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
   It will flush whatever data remained within `cctx` (like LZ4_flush())
   and properly finalize the frame, with an endMark and a checksum.
  `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
            or an error code if it fails (which can be tested using LZ4F_isError())
   A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  
@@ -201,7 +209,7 @@ <h1>1.8.2 Manual</h1>
 <a name="Chapter9"></a><h2>Decompression functions</h2><pre></pre>
 
 <pre><b>typedef struct {
-  unsigned stableDst;    </b>/* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */<b>
+  unsigned stableDst;    </b>/* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */<b>
   unsigned reserved[3];  </b>/* must be set to zero for forward compatibility */<b>
 } LZ4F_decompressOptions_t;
 </b></pre><BR>
@@ -212,7 +220,7 @@ <h1>1.8.2 Manual</h1>
   The function provides a pointer to an allocated and initialized LZ4F_dctx object.
   The result is an errorCode, which can be tested using LZ4F_isError().
   dctx memory can be released using LZ4F_freeDecompressionContext();
-  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+  Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
   That is, it should be == 0 if decompression has been completed fully and correctly.
  
 </p></pre><BR>
@@ -252,8 +260,8 @@ <h1>1.8.2 Manual</h1>
   The function will read up to *srcSizePtr bytes from srcBuffer,
   and decompress data into dstBuffer, of capacity *dstSizePtr.
 
-  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
-  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+  The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+  The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
 
   The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
   Unconsumed source data must be presented again in subsequent invocations.
diff --git a/lib/Makefile b/lib/Makefile
index dd33f5035..7b3123929 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -116,6 +116,10 @@ clean:
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
+.PHONY: listL120
+listL120:  # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
+	find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done
+
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
diff --git a/lib/lz4.c b/lib/lz4.c
index 33aa5c7d3..af3221c3a 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -69,7 +69,9 @@
  * Prefer these methods in priority order (0 > 1 > 2)
  */
 #ifndef LZ4_FORCE_MEMORY_ACCESS   /* can be defined externally */
-#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#  if defined(__GNUC__) && \
+  ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \
+  || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define LZ4_FORCE_MEMORY_ACCESS 2
 #  elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
 #    define LZ4_FORCE_MEMORY_ACCESS 1
@@ -80,7 +82,7 @@
  * LZ4_FORCE_SW_BITCOUNT
  * Define this parameter if your target system or compiler does not support hardware bit count
  */
-#if defined(_MSC_VER) && defined(_WIN32_WCE)   /* Visual Studio for Windows CE does not support Hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE)   /* Visual Studio for WinCE doesn't support Hardware bit count */
 #  define LZ4_FORCE_SW_BITCOUNT
 #endif
 
@@ -318,7 +320,7 @@ static const int LZ4_minLength = (MFLIMIT+1);
 #  endif
 #endif
 
-#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
+#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use after variable declarations */
 
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
 #  include <stdio.h>
@@ -529,7 +531,9 @@ static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t cons
     }
 }
 
-static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
+static void LZ4_putPositionOnHash(const BYTE* p, U32 h,
+                                  void* tableBase, tableType_t const tableType,
+                            const BYTE* srcBase)
 {
     switch (tableType)
     {
@@ -555,8 +559,16 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_
 static U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType)
 {
     LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2);
-    if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-2))); return hashTable[h]; }
-    if (tableType == byU16) { const U16* const hashTable = (const U16*) tableBase; assert(h < (1U << (LZ4_MEMORY_USAGE-1))); return hashTable[h]; }
+    if (tableType == byU32) {
+        const U32* const hashTable = (const U32*) tableBase;
+        assert(h < (1U << (LZ4_MEMORY_USAGE-2)));
+        return hashTable[h];
+    }
+    if (tableType == byU16) {
+        const U16* const hashTable = (const U16*) tableBase;
+        assert(h < (1U << (LZ4_MEMORY_USAGE-1)));
+        return hashTable[h];
+    }
     assert(0); return 0;  /* forbidden case */
 }
 
@@ -567,7 +579,9 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType
     { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, const void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p,
+                                             const void* tableBase, tableType_t tableType,
+                                             const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@@ -650,7 +664,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* the dictCtx currentOffset is indexed on the start of the dictionary,
      * while a dictionary in the current context precedes the currentOffset */
     const BYTE* dictBase = dictDirective == usingDictCtx ?
-        dictionary + dictSize - dictCtx->currentOffset :   /* is it possible that dictCtx->currentOffset != dictCtx->dictSize ? Yes if the dictionary context is not reset */
+        dictionary + dictSize - dictCtx->currentOffset :
         dictionary + dictSize - startIndex;
 
     BYTE* op = (BYTE*) dest;
@@ -667,7 +681,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
-    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
+    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;  /* Size too large (not within 64K limit) */
 
     /* Update context state */
     if (dictDirective == usingDictCtx) {
@@ -681,7 +695,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     cctx->currentOffset += (U32)inputSize;
     cctx->tableType = tableType;
 
-    if (inputSize<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
+    if (inputSize<LZ4_minLength) goto _last_literals;        /* Input too small, no compression (all literals) */
 
     /* First Byte */
     LZ4_putPosition(ip, cctx->hashTable, tableType, base);
@@ -790,7 +804,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             /* Copy Literals */
             LZ4_wildCopy(op, anchor, op+litLength);
             op+=litLength;
-            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
+            DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+                        (int)(anchor-(const BYTE*)source), litLength, (int)(ip-(const BYTE*)source));
         }
 
 _next_match:
@@ -905,7 +920,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 token=op++;
                 *token=0;
                 if (maybe_extMem) offset = current - matchIndex;
-                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
+                DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i",
+                            (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source));
                 goto _next_match;
             }
         }
@@ -1649,19 +1665,25 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+                                  endOnInputSize, full, 0, noDict,
+                                  (BYTE*)dest, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+                                  endOnInputSize, partial, targetOutputSize,
+                                  noDict, (BYTE*)dest, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 {
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
+    return LZ4_decompress_generic(source, dest, 0, originalSize,
+                                  endOnOutputSize, full, 0, withPrefix64k,
+                                  (BYTE*)(dest - 64 KB), NULL, 64 KB);
 }
 
 
@@ -1803,12 +1825,18 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
 *  Obsolete Functions
 ***************************************************/
 /* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
+    return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
+int LZ4_compress(const char* source, char* dest, int inputSize) {
+    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
+    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
+    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
 
 /*
 These function names are deprecated and should no longer be used.
@@ -1816,8 +1844,10 @@ They are only provided here for compatibility with older user programs.
 - LZ4_uncompress is totally equivalent to LZ4_decompress_fast
 - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
 */
-int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize) {
+    return LZ4_decompress_fast(source, dest, outputSize); }
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
+    return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
 
 /* Obsolete Streaming functions */
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 9efaca024..e80b1a1da 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -93,8 +93,8 @@ extern "C" {
  **************************************/
 typedef size_t LZ4F_errorCode_t;
 
-LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
-LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
+LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells when a function result is an error code */
+LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; for debugging */
 
 
 /*-************************************
@@ -162,13 +162,13 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
  *  It's not required to set all fields, as long as the structure was initially memset() to zero.
  *  For all fields, 0 sets it to default value */
 typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;            /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;  /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */
-  LZ4F_frameType_t       frameType;            /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
-  unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
-  unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
-  LZ4F_blockChecksum_t   blockChecksumFlag;    /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */
+  LZ4F_blockSizeID_t     blockSizeID;         /* max64KB, max256KB, max1MB, max4MB; 0 == default */
+  LZ4F_blockMode_t       blockMode;           /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */
+  LZ4F_frameType_t       frameType;           /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
+  unsigned long long     contentSize;         /* Size of uncompressed content ; 0 == unknown */
+  unsigned               dictID;              /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */
+  LZ4F_blockChecksum_t   blockChecksumFlag;   /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
 } LZ4F_frameInfo_t;
 
 /*! LZ4F_preferences_t :
@@ -177,7 +177,7 @@ typedef struct {
  *  All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
+  int      compressionLevel;       /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
   unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
@@ -264,36 +264,44 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
 
 /*! LZ4F_compressUpdate() :
  *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
- *  An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+ *  Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations.
  *  This value is provided by LZ4F_compressBound().
  *  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
- *  LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+ *  LZ4F_compressUpdate() doesn't guarantee error recovery.
+ *  When an error occurs, compression context must be freed or resized.
  * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                 const void* srcBuffer, size_t srcSize,
+                                 const LZ4F_compressOptions_t* cOptPtr);
 
 /*! LZ4F_flush() :
  *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
  *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  * `dstCapacity` must be large enough to ensure the operation will be successful.
  * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+ * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx,
+                              void* dstBuffer, size_t dstCapacity,
+                        const LZ4F_compressOptions_t* cOptPtr);
 
 /*! LZ4F_compressEnd() :
  *  To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
  *  It will flush whatever data remained within `cctx` (like LZ4_flush())
  *  and properly finalize the frame, with an endMark and a checksum.
  * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
- * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark),
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  */
-LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx,
+                                    void* dstBuffer, size_t dstCapacity,
+                              const LZ4F_compressOptions_t* cOptPtr);
 
 
 /*-*********************************
@@ -303,7 +311,7 @@ typedef struct LZ4F_dctx_s LZ4F_dctx;   /* incomplete type */
 typedef LZ4F_dctx* LZ4F_decompressionContext_t;   /* compatibility with previous API versions */
 
 typedef struct {
-  unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
+  unsigned stableDst;    /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */
   unsigned reserved[3];  /* must be set to zero for forward compatibility */
 } LZ4F_decompressOptions_t;
 
@@ -316,7 +324,7 @@ typedef struct {
  *  The function provides a pointer to an allocated and initialized LZ4F_dctx object.
  *  The result is an errorCode, which can be tested using LZ4F_isError().
  *  dctx memory can be released using LZ4F_freeDecompressionContext();
- *  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ *  Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released.
  *  That is, it should be == 0 if decompression has been completed fully and correctly.
  */
 LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
@@ -357,8 +365,8 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
  *  The function will read up to *srcSizePtr bytes from srcBuffer,
  *  and decompress data into dstBuffer, of capacity *dstSizePtr.
  *
- *  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
- *  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
+ *  The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+ *  The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
  *
  *  The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
  *  Unconsumed source data must be presented again in subsequent invocations.

From 95bde2a4ae4a92e984a5783ca1f09f44bf04fadb Mon Sep 17 00:00:00 2001
From: Baruch Siach <baruch@tkos.co.il>
Date: Thu, 19 Apr 2018 12:28:11 +0300
Subject: [PATCH 135/257] lib: allow to disable shared libraries

Just like BUILD_STATIC=no disables static libraries, BUILD_SHARED=no
disabled shared libraries. This is useful to support toolchains that do
not support shared libraries.
---
 lib/Makefile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/Makefile b/lib/Makefile
index dd33f5035..976d57cd7 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -42,6 +42,7 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
 LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER  := $(shell echo $(LIBVER_SCRIPT))
 
+BUILD_SHARED:=yes
 BUILD_STATIC:=yes
 
 CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
@@ -92,6 +93,7 @@ ifeq ($(BUILD_STATIC),yes)  # can be disabled on command line
 endif
 
 $(LIBLZ4): $(SRCFILES)
+ifeq ($(BUILD_SHARED),yes)  # can be disabled on command line
 	@echo compiling dynamic library $(LIBVER)
 ifneq (,$(filter Windows%,$(OS)))
 	@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
@@ -102,6 +104,7 @@ else
 	@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
 	@ln -sf $@ liblz4.$(SHARED_EXT)
 endif
+endif
 
 liblz4: $(LIBLZ4)
 
@@ -159,9 +162,11 @@ ifeq ($(BUILD_STATIC),yes)
 	@$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
 	@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 endif
+ifeq ($(BUILD_SHARED),yes)
 	@$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
 	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
 	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+endif
 	@echo Installing headers in $(INCLUDEDIR)
 	@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
 	@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h

From 46058d71aa97cf29690f7638179104985c28d354 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 19 Apr 2018 10:50:40 -0700
Subject: [PATCH 136/257] modified indentation for consistency

---
 lib/lz4.c | 50 +++++++++++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index af3221c3a..10674a34d 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1825,29 +1825,45 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
 *  Obsolete Functions
 ***************************************************/
 /* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) {
-    return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) {
-    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) {
-    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) {
-    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) {
-    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) {
-    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+    return LZ4_compress_default(source, dest, inputSize, maxOutputSize);
+}
+int LZ4_compress(const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize));
+}
+int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize)
+{
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1);
+}
+int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize)
+{
+    return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1);
+}
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity)
+{
+    return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1);
+}
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1);
+}
 
 /*
-These function names are deprecated and should no longer be used.
+These decompression functions are deprecated and should no longer be used.
 They are only provided here for compatibility with older user programs.
 - LZ4_uncompress is totally equivalent to LZ4_decompress_fast
 - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
 */
-int LZ4_uncompress (const char* source, char* dest, int outputSize) {
-    return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) {
-    return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+int LZ4_uncompress (const char* source, char* dest, int outputSize)
+{
+    return LZ4_decompress_fast(source, dest, outputSize);
+}
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize)
+{
+    return LZ4_decompress_safe(source, dest, isize, maxOutputSize);
+}
 
 /* Obsolete Streaming functions */
 

From b9836b2a030a3756b1f8cf1341875354c84dd207 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 14:01:57 -0400
Subject: [PATCH 137/257] Restore Framebench Tool

This reverts commit 70f14823a46719e81e808d9ed9df90f478bcfd3f.
---
 tests/Makefile     |   6 +-
 tests/framebench.c | 296 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 301 insertions(+), 1 deletion(-)
 create mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index 2b93c9f62..f6a4ff3e3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen
+all: fullbench fuzzer frametest datagen framebench
 
 all32: CFLAGS+=-m32
 all32: all
@@ -99,6 +99,9 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
+
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -110,6 +113,7 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
+        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
new file mode 100644
index 000000000..18b5ff34a
--- /dev/null
+++ b/tests/framebench.c
@@ -0,0 +1,296 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+#define LZ4_STATIC_LINKING_ONLY
+#include "lz4.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
+
+typedef struct {
+  size_t iter;
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  char *obuf;
+  size_t osize;
+  const char* ibuf;
+  size_t isize;
+  size_t num_ibuf;
+  const LZ4F_CDict* cdict;
+  LZ4F_preferences_t* prefs;
+  const LZ4F_compressOptions_t* options;
+} bench_params_t;
+
+size_t compress_frame(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressFrame_usingCDict(
+    cctx,
+    obuf,
+    osize,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    cdict,
+    prefs);
+  LZ4F_CHECK(oused);
+
+  return oused;
+}
+
+size_t compress_begin(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4F_cctx *cctx = p->cctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  const LZ4F_CDict* cdict = p->cdict;
+  LZ4F_preferences_t* prefs = p->prefs;
+  const LZ4F_compressOptions_t* options = p->options;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  prefs->frameInfo.contentSize = isize;
+
+  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressUpdate(
+    cctx,
+    obuf,
+    oend - obuf,
+    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isize,
+    options);
+  LZ4F_CHECK(oused);
+  obuf += oused;
+  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
+  LZ4F_CHECK(oused);
+
+  return obuf - p->obuf;
+}
+
+size_t compress_default(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_stream_t *ctx = p->ctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+uint64_t bench(
+    size_t (*fun)(bench_params_t *),
+    uint64_t repetitions,
+    bench_params_t *params,
+    size_t *osizePtr
+) {
+  struct timespec start, end;
+  size_t i, osize = 0;
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
+
+  for (i = 0; i < repetitions; i++) {
+    size_t o;
+    params->iter = i;
+    o = fun(params);
+    if (!o) return 0;
+    osize += o;
+  }
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+  *osizePtr = osize / repetitions;
+  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+}
+
+int main(int argc, char *argv[]) {
+
+
+  struct stat st;
+  size_t bytes_read;
+
+  const char *dict_fn;
+  size_t dict_size;
+  char *dict_buf;
+  FILE *dict_file;
+
+  const char *in_fn;
+  size_t in_size;
+  size_t num_in_buf;
+  size_t cur_in_buf;
+  char *in_buf;
+  FILE *in_file;
+
+  size_t out_size;
+  char *out_buf;
+
+  LZ4_stream_t *ctx;
+  LZ4F_cctx *cctx;
+  LZ4F_CDict *cdict;
+  LZ4F_preferences_t prefs;
+  LZ4F_compressOptions_t options;
+
+  size_t out_used;
+
+  uint64_t time_taken;
+  uint64_t repetitions;
+
+  bench_params_t params;
+
+  if (argc != 3) return 1;
+  dict_fn = argv[1];
+  in_fn = argv[2];
+
+  if (stat(dict_fn, &st)) return 1;
+  dict_size = st.st_size;
+  dict_buf = (char *)malloc(dict_size);
+  if (!dict_buf) return 1;
+  dict_file = fopen(dict_fn, "r");
+  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
+  if (bytes_read != dict_size) return 1;
+
+  if (stat(in_fn, &st)) return 1;
+  in_size = st.st_size;
+  num_in_buf = 256 * 1024 * 1024 / in_size;
+  if (num_in_buf == 0) {
+    num_in_buf = 1;
+  }
+
+  in_buf = (char *)malloc(in_size * num_in_buf);
+  if (!in_buf) return 1;
+  in_file = fopen(in_fn, "r");
+  bytes_read = fread(in_buf, 1, in_size, in_file);
+  if (bytes_read != in_size) return 1;
+
+  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
+    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
+  }
+
+  if (in_size <= 1024) {
+    repetitions = 100000;
+  } else
+  if (in_size <= 16384) {
+    repetitions = 10000;
+  } else
+  if (in_size <= 131072) {
+    repetitions = 1000;
+  } else
+  if (in_size <= 1048576) {
+    repetitions = 100;
+  } else {
+    repetitions = 50;
+  }
+
+  memset(&prefs, 0, sizeof(prefs));
+  prefs.autoFlush = 1;
+  if (in_size < 64 * 1024)
+    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+  prefs.frameInfo.contentSize = in_size;
+
+  memset(&options, 0, sizeof(options));
+  options.stableSrc = 1;
+
+  out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  out_buf = (char *)malloc(out_size);
+  if (!out_buf) return 1;
+
+  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
+  ctx = LZ4_createStream();
+  if (ctx == NULL) return 1;
+
+  cdict = LZ4F_createCDict(dict_buf, dict_size);
+  if (!cdict) return 1;
+
+  fprintf(stderr, "dict  size: %zd\n", dict_size);
+  fprintf(stderr, "input size: %zd\n", in_size);
+
+  params.ctx = ctx;
+  params.cctx = cctx;
+  params.obuf = out_buf;
+  params.osize = out_size;
+  params.ibuf = in_buf;
+  params.isize = in_size;
+  params.num_ibuf = num_in_buf;
+  params.cdict = NULL;
+  params.prefs = &prefs;
+  params.options = &options;
+
+  time_taken = bench(compress_default, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  return 0;
+}
\ No newline at end of file

From 0bc13ab69c603346ff64050f7e3d3a1b3a5765a4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 15:00:59 -0400
Subject: [PATCH 138/257] Add HC Calls to Framebench

---
 tests/framebench.c | 99 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 82 insertions(+), 17 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 18b5ff34a..e2bcf4001 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -10,6 +10,7 @@
 
 #define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
+#include "lz4hc.h"
 #include "lz4frame.h"
 #include "lz4frame_static.h"
 
@@ -18,12 +19,14 @@
 typedef struct {
   size_t iter;
   LZ4_stream_t *ctx;
+  LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
   char *obuf;
   size_t osize;
   const char* ibuf;
   size_t isize;
   size_t num_ibuf;
+  int clevel;
   const LZ4F_CDict* cdict;
   LZ4F_preferences_t* prefs;
   const LZ4F_compressOptions_t* options;
@@ -117,11 +120,49 @@ size_t compress_extState(bench_params_t *p) {
   const char* ibuf = p->ibuf;
   size_t isize = p->isize;
   size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0);
+  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_hc(bench_params_t *p) {
+  size_t iter = p->iter;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  obuf += oused;
+
+  return obuf - p->obuf;
+}
+
+size_t compress_hc_extState(bench_params_t *p) {
+  size_t iter = p->iter;
+  LZ4_streamHC_t *hcctx = p->hcctx;
+  char *obuf = p->obuf;
+  size_t osize = p->osize;
+  const char* ibuf = p->ibuf;
+  size_t isize = p->isize;
+  size_t num_ibuf = p->num_ibuf;
+  int clevel = p->clevel;
+
+  char *oend = obuf + osize;
+  size_t oused;
+
+  oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -174,6 +215,7 @@ int main(int argc, char *argv[]) {
   char *out_buf;
 
   LZ4_stream_t *ctx;
+  LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
   LZ4F_CDict *cdict;
   LZ4F_preferences_t prefs;
@@ -216,18 +258,18 @@ int main(int argc, char *argv[]) {
   }
 
   if (in_size <= 1024) {
-    repetitions = 100000;
+    repetitions = 10000;
   } else
   if (in_size <= 16384) {
-    repetitions = 10000;
+    repetitions = 1000;
   } else
   if (in_size <= 131072) {
-    repetitions = 1000;
+    repetitions = 100;
   } else
   if (in_size <= 1048576) {
-    repetitions = 100;
+    repetitions = 10;
   } else {
-    repetitions = 50;
+    repetitions = 5;
   }
 
   memset(&prefs, 0, sizeof(prefs));
@@ -249,6 +291,9 @@ int main(int argc, char *argv[]) {
   ctx = LZ4_createStream();
   if (ctx == NULL) return 1;
 
+  hcctx = LZ4_createStreamHC();
+  if (hcctx == NULL) return 1;
+
   cdict = LZ4F_createCDict(dict_buf, dict_size);
   if (!cdict) return 1;
 
@@ -256,41 +301,61 @@ int main(int argc, char *argv[]) {
   fprintf(stderr, "input size: %zd\n", in_size);
 
   params.ctx = ctx;
+  params.hcctx = hcctx;
   params.cctx = cctx;
   params.obuf = out_buf;
   params.osize = out_size;
   params.ibuf = in_buf;
   params.isize = in_size;
   params.num_ibuf = num_in_buf;
+  params.clevel = 1;
   params.cdict = NULL;
   params.prefs = &prefs;
   params.options = &options;
 
   time_taken = bench(compress_default, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_default         : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_extState, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4_compress_fast_extState   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_frame, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressFrame           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_begin, repetitions, &params, &out_used);
-
-  fprintf(stderr, "LZ4F_compressBegin           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
   time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  params.cdict = NULL;
+  params.clevel = LZ4HC_CLEVEL_MIN;
+  params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
+
+  time_taken = bench(compress_hc, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_hc_extState, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  params.cdict = cdict;
+
+  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+
+  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   return 0;
-}
\ No newline at end of file
+}

From ffb2d8b0ed9615694c156ba1a3f51e9e789d3c4e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 16:26:44 -0400
Subject: [PATCH 139/257] Check Compressed Buffer is Correct in Frame Bench

---
 tests/framebench.c | 79 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 65 insertions(+), 14 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index e2bcf4001..a802ce00b 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -21,11 +21,16 @@ typedef struct {
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
+  LZ4F_dctx *dctx;
+  const char *dictbuf;
+  size_t dictsize;
   char *obuf;
   size_t osize;
   const char* ibuf;
   size_t isize;
   size_t num_ibuf;
+  char *checkbuf;
+  size_t checksize;
   int clevel;
   const LZ4F_CDict* cdict;
   LZ4F_preferences_t* prefs;
@@ -168,19 +173,46 @@ size_t compress_hc_extState(bench_params_t *p) {
   return obuf - p->obuf;
 }
 
+size_t check_lz4(bench_params_t *p, size_t csize) {
+  (void)csize;
+  memset(p->checkbuf, 0xFF, p->checksize);
+  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize)
+      && !memcmp(p->ibuf, p->checkbuf, p->isize);
+}
+
+size_t check_lz4f(bench_params_t *p, size_t csize) {
+  size_t cp = 0;
+  size_t dp = 0;
+  size_t dsize = p->checksize;
+  size_t cleft = csize;
+  size_t dleft = dsize;
+  size_t ret;
+  memset(p->checkbuf, 0xFF, p->checksize);
+  LZ4F_resetDecompressionContext(p->dctx);
+  do {
+    ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL);
+    cp += cleft;
+    dp += dleft;
+    cleft = csize - cp;
+    dleft = dsize - dp;
+    if (LZ4F_isError(ret)) return 0;
+  } while (cleft);
+  return !memcmp(p->ibuf, p->checkbuf, p->isize);
+}
+
 uint64_t bench(
     size_t (*fun)(bench_params_t *),
+    size_t (*checkfun)(bench_params_t *, size_t),
     uint64_t repetitions,
     bench_params_t *params,
     size_t *osizePtr
 ) {
   struct timespec start, end;
-  size_t i, osize = 0;
+  size_t i, osize = 0, o = 0;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
   for (i = 0; i < repetitions; i++) {
-    size_t o;
     params->iter = i;
     o = fun(params);
     if (!o) return 0;
@@ -189,6 +221,9 @@ uint64_t bench(
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
 
+  o = checkfun(params, o);
+  if (!o) return 0;
+
   *osizePtr = osize / repetitions;
   return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
 }
@@ -214,9 +249,13 @@ int main(int argc, char *argv[]) {
   size_t out_size;
   char *out_buf;
 
+  size_t check_size;
+  char *check_buf;
+
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
   LZ4F_cctx *cctx;
+  LZ4F_dctx *dctx;
   LZ4F_CDict *cdict;
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
@@ -257,6 +296,10 @@ int main(int argc, char *argv[]) {
     memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
   }
 
+  check_size = in_size;
+  check_buf = (char *)malloc(check_size);
+  if (!check_buf) return 1;
+
   if (in_size <= 1024) {
     repetitions = 10000;
   } else
@@ -288,6 +331,9 @@ int main(int argc, char *argv[]) {
   if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
   if (cctx == NULL) return 1;
 
+  if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1;
+  if (cctx == NULL) return 1;
+
   ctx = LZ4_createStream();
   if (ctx == NULL) return 1;
 
@@ -303,58 +349,63 @@ int main(int argc, char *argv[]) {
   params.ctx = ctx;
   params.hcctx = hcctx;
   params.cctx = cctx;
+  params.dctx = dctx;
+  params.dictbuf = dict_buf;
+  params.dictsize = dict_size;
   params.obuf = out_buf;
   params.osize = out_size;
   params.ibuf = in_buf;
   params.isize = in_size;
   params.num_ibuf = num_in_buf;
+  params.checkbuf = check_buf;
+  params.checksize = check_size;
   params.clevel = 1;
   params.cdict = NULL;
   params.prefs = &prefs;
   params.options = &options;
 
-  time_taken = bench(compress_default, repetitions, &params, &out_used);
+  time_taken = bench(compress_default, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_extState, repetitions, &params, &out_used);
+  time_taken = bench(compress_extState, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = NULL;
   params.clevel = LZ4HC_CLEVEL_MIN;
   params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
 
-  time_taken = bench(compress_hc, repetitions, &params, &out_used);
+  time_taken = bench(compress_hc, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_hc_extState, repetitions, &params, &out_used);
+  time_taken = bench(compress_hc_extState, check_lz4, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   params.cdict = cdict;
 
-  time_taken = bench(compress_frame, repetitions, &params, &out_used);
+  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
-  time_taken = bench(compress_begin, repetitions, &params, &out_used);
+  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
   fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
 
   return 0;

From d4ee7554fc79fecd5db83913536f5c856765bbed Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 16:19:30 -0400
Subject: [PATCH 140/257] Print More Detailed Results Inside bench(), Add
 Compression Levels

---
 tests/framebench.c | 103 ++++++++++++++++++++++-----------------------
 1 file changed, 51 insertions(+), 52 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index a802ce00b..6d99b1eb4 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -111,7 +111,9 @@ size_t compress_default(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf);
+  oused = LZ4_compress_default(
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -130,7 +132,10 @@ size_t compress_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_fast_extState_fastReset(
+      ctx,
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -148,7 +153,9 @@ size_t compress_hc(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_HC(
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -167,7 +174,10 @@ size_t compress_hc_extState(bench_params_t *p) {
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel);
+  oused = LZ4_compress_HC_extStateHC(
+      hcctx,
+      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
@@ -176,7 +186,8 @@ size_t compress_hc_extState(bench_params_t *p) {
 size_t check_lz4(bench_params_t *p, size_t csize) {
   (void)csize;
   memset(p->checkbuf, 0xFF, p->checksize);
-  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize)
+  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
+                                       p->dictbuf, p->dictsize)
       && !memcmp(p->ibuf, p->checkbuf, p->isize);
 }
 
@@ -190,7 +201,9 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
   memset(p->checkbuf, 0xFF, p->checksize);
   LZ4F_resetDecompressionContext(p->dctx);
   do {
-    ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL);
+    ret = LZ4F_decompress_usingDict(
+        p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft,
+        p->dictbuf, p->dictsize, NULL);
     cp += cleft;
     dp += dleft;
     cleft = csize - cp;
@@ -200,15 +213,17 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
   return !memcmp(p->ibuf, p->checkbuf, p->isize);
 }
 
+
 uint64_t bench(
+    char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
     uint64_t repetitions,
-    bench_params_t *params,
-    size_t *osizePtr
+    bench_params_t *params
 ) {
   struct timespec start, end;
   size_t i, osize = 0, o = 0;
+  size_t time_taken;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
@@ -224,8 +239,19 @@ uint64_t bench(
   o = checkfun(params, o);
   if (!o) return 0;
 
-  *osizePtr = osize / repetitions;
-  return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+  time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
+               (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+
+  fprintf(
+      stderr,
+      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n",
+      bench_name, params->clevel,
+      params->isize, osize / repetitions,
+      repetitions, time_taken, time_taken / repetitions,
+      ((double) 1000 * params->isize * repetitions) / time_taken
+  );
+
+  return time_taken;
 }
 
 int main(int argc, char *argv[]) {
@@ -260,9 +286,8 @@ int main(int argc, char *argv[]) {
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
 
-  size_t out_used;
+  int clevels[] = {1, 2, 3, 6, 9, 10, 12};
 
-  uint64_t time_taken;
   uint64_t repetitions;
 
   bench_params_t params;
@@ -364,49 +389,23 @@ int main(int argc, char *argv[]) {
   params.prefs = &prefs;
   params.options = &options;
 
-  time_taken = bench(compress_default, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_default            : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_extState, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_fast_extState      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin              : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+  for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
+    params.clevel = clevels[clevelidx];
+    params.prefs->compressionLevel = clevels[clevelidx];
+    params.cdict = NULL;
 
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict   : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    bench("LZ4_compress_default"         , compress_default    , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , repetitions, &params);
+    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , repetitions, &params);
+    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, repetitions, &params);
 
-  params.cdict = NULL;
-  params.clevel = LZ4HC_CLEVEL_MIN;
-  params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN;
-
-  time_taken = bench(compress_hc, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_HC                 : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_hc_extState, check_lz4, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4_compress_HC_extStateHC      : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    params.cdict = cdict;
 
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_HC           : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  params.cdict = cdict;
-
-  time_taken = bench(compress_frame, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
-
-  time_taken = bench(compress_begin, check_lz4f, repetitions, &params, &out_used);
-  fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken);
+    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, repetitions, &params);
+  }
 
   return 0;
 }

From a9a62321ffd077d1c4694b2762e84b60b2396c2d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 16:36:35 -0400
Subject: [PATCH 141/257] Auto-Calculate Appropriate Repetition Count

---
 tests/framebench.c | 67 ++++++++++++++++++++--------------------------
 1 file changed, 29 insertions(+), 38 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 6d99b1eb4..d5d226877 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -218,33 +218,41 @@ uint64_t bench(
     char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
-    uint64_t repetitions,
     bench_params_t *params
 ) {
   struct timespec start, end;
   size_t i, osize = 0, o = 0;
-  size_t time_taken;
+  size_t time_taken = 0;
+  uint64_t total_repetitions = 0;
+  uint64_t repetitions = 2;
 
   if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
 
-  for (i = 0; i < repetitions; i++) {
-    params->iter = i;
-    o = fun(params);
-    if (!o) return 0;
-    osize += o;
-  }
+  while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms
+    if (total_repetitions) {
+      repetitions = total_repetitions; // double previous
+    }
+
+    for (i = 0; i < repetitions; i++) {
+      params->iter = i;
+      o = fun(params);
+      if (!o) return 0;
+      osize += o;
+    }
 
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+    if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
+
+    time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
+                 (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
+    total_repetitions += repetitions;
+  }
 
   o = checkfun(params, o);
   if (!o) return 0;
 
-  time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
-               (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-
   fprintf(
       stderr,
-      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n",
+      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
       bench_name, params->clevel,
       params->isize, osize / repetitions,
       repetitions, time_taken, time_taken / repetitions,
@@ -288,8 +296,6 @@ int main(int argc, char *argv[]) {
 
   int clevels[] = {1, 2, 3, 6, 9, 10, 12};
 
-  uint64_t repetitions;
-
   bench_params_t params;
 
   if (argc != 3) return 1;
@@ -325,21 +331,6 @@ int main(int argc, char *argv[]) {
   check_buf = (char *)malloc(check_size);
   if (!check_buf) return 1;
 
-  if (in_size <= 1024) {
-    repetitions = 10000;
-  } else
-  if (in_size <= 16384) {
-    repetitions = 1000;
-  } else
-  if (in_size <= 131072) {
-    repetitions = 100;
-  } else
-  if (in_size <= 1048576) {
-    repetitions = 10;
-  } else {
-    repetitions = 5;
-  }
-
   memset(&prefs, 0, sizeof(prefs));
   prefs.autoFlush = 1;
   if (in_size < 64 * 1024)
@@ -394,17 +385,17 @@ int main(int argc, char *argv[]) {
     params.prefs->compressionLevel = clevels[clevelidx];
     params.cdict = NULL;
 
-    bench("LZ4_compress_default"         , compress_default    , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , repetitions, &params);
-    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , repetitions, &params);
-    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, repetitions, &params);
-    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, repetitions, &params);
+    bench("LZ4_compress_default"         , compress_default    , check_lz4 , &params);
+    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , &params);
+    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , &params);
+    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , &params);
+    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, &params);
+    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, &params);
 
     params.cdict = cdict;
 
-    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, repetitions, &params);
-    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, repetitions, &params);
+    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, &params);
+    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, &params);
   }
 
   return 0;

From 09df7a05f9eb4471a5af7d6ca1bb37bea36ccf91 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 17:16:23 -0400
Subject: [PATCH 142/257] Add Run Name to Frame Bench Output

---
 tests/framebench.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index d5d226877..185d186e3 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -17,6 +17,7 @@
 #define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
 
 typedef struct {
+  const char *run_name;
   size_t iter;
   LZ4_stream_t *ctx;
   LZ4_streamHC_t *hcctx;
@@ -252,8 +253,8 @@ uint64_t bench(
 
   fprintf(
       stderr,
-      "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
-      bench_name, params->clevel,
+      "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
+      params->run_name, bench_name, params->clevel,
       params->isize, osize / repetitions,
       repetitions, time_taken, time_taken / repetitions,
       ((double) 1000 * params->isize * repetitions) / time_taken
@@ -263,7 +264,7 @@ uint64_t bench(
 }
 
 int main(int argc, char *argv[]) {
-
+  char *run_name;
 
   struct stat st;
   size_t bytes_read;
@@ -298,9 +299,10 @@ int main(int argc, char *argv[]) {
 
   bench_params_t params;
 
-  if (argc != 3) return 1;
-  dict_fn = argv[1];
-  in_fn = argv[2];
+  if (argc != 4) return 1;
+  run_name = argv[1];
+  dict_fn = argv[2];
+  in_fn = argv[3];
 
   if (stat(dict_fn, &st)) return 1;
   dict_size = st.st_size;
@@ -362,6 +364,7 @@ int main(int argc, char *argv[]) {
   fprintf(stderr, "dict  size: %zd\n", dict_size);
   fprintf(stderr, "input size: %zd\n", in_size);
 
+  params.run_name = run_name;
   params.ctx = ctx;
   params.hcctx = hcctx;
   params.cctx = cctx;

From 66f0c29aa4d81586a4a621a99c763ab17028c270 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 13:12:28 -0400
Subject: [PATCH 143/257] Fix Framebench Statistics

---
 tests/framebench.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 185d186e3..1060e99fb 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -255,9 +255,9 @@ uint64_t bench(
       stderr,
       "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
       params->run_name, bench_name, params->clevel,
-      params->isize, osize / repetitions,
-      repetitions, time_taken, time_taken / repetitions,
-      ((double) 1000 * params->isize * repetitions) / time_taken
+      params->isize, osize / total_repetitions,
+      total_repetitions, time_taken, time_taken / total_repetitions,
+      ((double) 1000 * params->isize * total_repetitions) / time_taken
   );
 
   return time_taken;

From 9d971fd5c3c1e18b225307ef88e529c9b3850fcb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 14:06:23 -0400
Subject: [PATCH 144/257] Switch to Unaligned Samples to Compress Different
 Blobs Each Time

---
 tests/framebench.c | 53 ++++++++++++++++++----------------------------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 1060e99fb..01e30b343 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -28,6 +28,7 @@ typedef struct {
   char *obuf;
   size_t osize;
   const char* ibuf;
+  const char* isample;
   size_t isize;
   size_t num_ibuf;
   char *checkbuf;
@@ -39,13 +40,11 @@ typedef struct {
 } bench_params_t;
 
 size_t compress_frame(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   const LZ4F_CDict* cdict = p->cdict;
   LZ4F_preferences_t* prefs = p->prefs;
 
@@ -57,7 +56,7 @@ size_t compress_frame(bench_params_t *p) {
     cctx,
     obuf,
     osize,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isample,
     isize,
     cdict,
     prefs);
@@ -67,13 +66,11 @@ size_t compress_frame(bench_params_t *p) {
 }
 
 size_t compress_begin(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4F_cctx *cctx = p->cctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   const LZ4F_CDict* cdict = p->cdict;
   LZ4F_preferences_t* prefs = p->prefs;
   const LZ4F_compressOptions_t* options = p->options;
@@ -90,7 +87,7 @@ size_t compress_begin(bench_params_t *p) {
     cctx,
     obuf,
     oend - obuf,
-    ibuf + ((iter * 2654435761U) % num_ibuf) * isize,
+    isample,
     isize,
     options);
   LZ4F_CHECK(oused);
@@ -102,74 +99,61 @@ size_t compress_begin(bench_params_t *p) {
 }
 
 size_t compress_default(bench_params_t *p) {
-  size_t iter = p->iter;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
 
   char *oend = obuf + osize;
   size_t oused;
 
-  oused = LZ4_compress_default(
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf);
+  oused = LZ4_compress_default(isample, obuf, isize, oend - obuf);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_extState(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4_stream_t *ctx = p->ctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
   oused = LZ4_compress_fast_extState_fastReset(
-      ctx,
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf, clevel);
+      ctx, isample, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_hc(bench_params_t *p) {
-  size_t iter = p->iter;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
   size_t oused;
 
   oused = LZ4_compress_HC(
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
-      isize, oend - obuf, clevel);
+      isample, obuf, isize, oend - obuf, clevel);
   obuf += oused;
 
   return obuf - p->obuf;
 }
 
 size_t compress_hc_extState(bench_params_t *p) {
-  size_t iter = p->iter;
   LZ4_streamHC_t *hcctx = p->hcctx;
   char *obuf = p->obuf;
   size_t osize = p->osize;
-  const char* ibuf = p->ibuf;
+  const char* isample = p->isample;
   size_t isize = p->isize;
-  size_t num_ibuf = p->num_ibuf;
   int clevel = p->clevel;
 
   char *oend = obuf + osize;
@@ -177,7 +161,7 @@ size_t compress_hc_extState(bench_params_t *p) {
 
   oused = LZ4_compress_HC_extStateHC(
       hcctx,
-      ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf,
+      isample, obuf,
       isize, oend - obuf, clevel);
   obuf += oused;
 
@@ -189,7 +173,7 @@ size_t check_lz4(bench_params_t *p, size_t csize) {
   memset(p->checkbuf, 0xFF, p->checksize);
   return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
                                        p->dictbuf, p->dictsize)
-      && !memcmp(p->ibuf, p->checkbuf, p->isize);
+      && !memcmp(p->isample, p->checkbuf, p->isize);
 }
 
 size_t check_lz4f(bench_params_t *p, size_t csize) {
@@ -211,7 +195,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
     dleft = dsize - dp;
     if (LZ4F_isError(ret)) return 0;
   } while (cleft);
-  return !memcmp(p->ibuf, p->checkbuf, p->isize);
+  return !memcmp(p->isample, p->checkbuf, p->isize);
 }
 
 
@@ -236,6 +220,11 @@ uint64_t bench(
 
     for (i = 0; i < repetitions; i++) {
       params->iter = i;
+      if (params->num_ibuf == 1) {
+        params->isample = params->ibuf;
+      } else {
+        params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
+      }
       o = fun(params);
       if (!o) return 0;
       osize += o;
@@ -295,7 +284,7 @@ int main(int argc, char *argv[]) {
   LZ4F_preferences_t prefs;
   LZ4F_compressOptions_t options;
 
-  int clevels[] = {1, 2, 3, 6, 9, 10, 12};
+  int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
 
   bench_params_t params;
 

From f646c512e99032cf4545be1ffcfd2a17323fb586 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 18:08:13 -0400
Subject: [PATCH 145/257] Print Failure Message in Framebench

---
 tests/framebench.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 01e30b343..048df347c 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -226,7 +226,14 @@ uint64_t bench(
         params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
       }
       o = fun(params);
-      if (!o) return 0;
+      if (!o) {
+        fprintf(
+            stderr,
+            "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n",
+            params->run_name, bench_name, params->clevel,
+            params->isize);
+        return 0;
+      }
       osize += o;
     }
 
@@ -238,7 +245,14 @@ uint64_t bench(
   }
 
   o = checkfun(params, o);
-  if (!o) return 0;
+  if (!o) {
+    fprintf(
+        stderr,
+        "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n",
+        params->run_name, bench_name, params->clevel,
+        params->isize);
+    return 0;
+  }
 
   fprintf(
       stderr,

From 9dae661b1fcc665a78a9de5341e651310d489f5c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 11 Apr 2018 12:39:31 -0400
Subject: [PATCH 146/257] Fix Cast

---
 tests/framebench.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/framebench.c b/tests/framebench.c
index 048df347c..70cfec5c4 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -200,7 +200,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) {
 
 
 uint64_t bench(
-    char *bench_name,
+    const char *bench_name,
     size_t (*fun)(bench_params_t *),
     size_t (*checkfun)(bench_params_t *, size_t),
     bench_params_t *params
@@ -299,6 +299,7 @@ int main(int argc, char *argv[]) {
   LZ4F_compressOptions_t options;
 
   int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
+  unsigned int clevelidx;
 
   bench_params_t params;
 
@@ -386,7 +387,7 @@ int main(int argc, char *argv[]) {
   params.prefs = &prefs;
   params.options = &options;
 
-  for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
+  for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
     params.clevel = clevels[clevelidx];
     params.prefs->compressionLevel = clevels[clevelidx];
     params.cdict = NULL;

From e0d8add7917f5eafa4894979b5b68b0db1fea6f4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 15:38:41 -0400
Subject: [PATCH 147/257] Fix Framebench Output Buffer Sizing

---
 tests/framebench.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/framebench.c b/tests/framebench.c
index 70cfec5c4..9752f23f3 100644
--- a/tests/framebench.c
+++ b/tests/framebench.c
@@ -347,6 +347,9 @@ int main(int argc, char *argv[]) {
   options.stableSrc = 1;
 
   out_size = LZ4F_compressFrameBound(in_size, &prefs);
+  if ((size_t)LZ4_compressBound(in_size) > out_size) {
+    out_size = LZ4_compressBound(in_size);
+  }
   out_buf = (char *)malloc(out_size);
   if (!out_buf) return 1;
 

From e75153f508073d87e7087e8ef472879773f116f8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 17:47:52 -0400
Subject: [PATCH 148/257] Add Debug Log Statements to HC

---
 lib/lz4frame.c | 21 +++++++++++++++++++++
 lib/lz4hc.c    | 18 ++++++++++++++++--
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 507e4feed..000002382 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -95,6 +95,18 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
+#  include <stdio.h>
+static int g_debuglog_enable = 1;
+#  define DEBUGLOG(l, ...) {                                  \
+                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
+                    fprintf(stderr, __FILE__ ": ");           \
+                    fprintf(stderr, __VA_ARGS__);             \
+                    fprintf(stderr, " \n");                   \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
 
 /*-************************************
 *  Basic Types
@@ -457,6 +469,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+    DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict);
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -479,6 +492,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 
 void LZ4F_freeCDict(LZ4F_CDict* cdict)
 {
+    DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict);
     if (cdict==NULL) return;  /* support free on NULL */
     FREEMEM(cdict->dictContent);
     LZ4_freeStream(cdict->fastCtx);
@@ -530,6 +544,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
+    DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict);
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
@@ -560,6 +575,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     BYTE* headerStart;
 
+    DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict);
+
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
@@ -778,6 +795,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
+    DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
@@ -916,6 +934,9 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
     BYTE* dstPtr = dstStart;
 
     size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
+
+    DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr);
+
     if (LZ4F_isError(flushSize)) return flushSize;
     dstPtr += flushSize;
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 111a09b07..12d26c306 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -86,6 +86,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 **************************************/
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
+    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
@@ -641,6 +642,8 @@ static int LZ4HC_compress_generic (
         { lz4opt,8192, LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
     };
 
+    DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr);
+
     if (limit == limitedDestSize && dstCapacity < 1) return 0;         /* Impossible to store anything */
     if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0;          /* Unsupported input size (too large or negative) */
 
@@ -706,8 +709,14 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 *  Streaming Functions
 **************************************/
 /* allocation */
-LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
-int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+LZ4_streamHC_t* LZ4_createStreamHC(void) {
+    LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
+    DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
+    return LZ4_streamHCPtr;
+}
+
+int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+    DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr);
     if (!LZ4_streamHCPtr) return 0;  /* support free on NULL */
     free(LZ4_streamHCPtr);
     return 0;
@@ -718,6 +727,7 @@ int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
 void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+    DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
@@ -732,6 +742,7 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize);
     if (dictSize > 64 KB) {
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
@@ -747,6 +758,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
 
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
+    DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
     if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
@@ -764,6 +776,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
                                             limitedOutput_directive limit)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr);
     /* auto-init if forgotten */
     if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
 
@@ -812,6 +825,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
 {
     LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
     int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+    DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
     if (dictSize > 64 KB) dictSize = 64 KB;
     if (dictSize < 4) dictSize = 0;
     if (dictSize > prefixSize) dictSize = prefixSize;

From f895b9a6c69a059e62de8baca39b878ed1031418 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 19 Mar 2018 13:16:05 -0400
Subject: [PATCH 149/257] Add a Dictionary Context Pointer to the HC Context

---
 lib/lz4hc.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 7a25bee6e..1b08425e3 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -141,7 +141,8 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
 #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #include <stdint.h>
 
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
 {
     uint32_t   hashTable[LZ4HC_HASHTABLESIZE];
     uint16_t   chainTable[LZ4HC_MAXD];
@@ -153,11 +154,13 @@ typedef struct
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
     int        compressionLevel;
-} LZ4HC_CCtx_internal;
+    const LZ4HC_CCtx_internal* dictCtx;
+};
 
 #else
 
-typedef struct
+typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
+struct LZ4HC_CCtx_internal
 {
     unsigned int   hashTable[LZ4HC_HASHTABLESIZE];
     unsigned short chainTable[LZ4HC_MAXD];
@@ -169,11 +172,12 @@ typedef struct
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
     int            compressionLevel;
-} LZ4HC_CCtx_internal;
+    const LZ4HC_CCtx_internal* dictCtx;
+};
 
 #endif
 
-#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
+#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */
 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
 union LZ4_streamHC_u {
     size_t table[LZ4_STREAMHCSIZE_SIZET];

From a992d11fc2204b7cb0a70962aa5911a58ed4918c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 11:21:07 -0400
Subject: [PATCH 150/257] Fully Bounds Check Hash Table Reads

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 12d26c306..6890613e7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -222,7 +222,7 @@ LZ4HC_InsertAndGetWiderMatch (
     DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
                 matchIndex, lowLimit);
 
-    while ((matchIndex>=lowLimit) && (nbAttempts)) {
+    while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         if (matchIndex >= dictLimit) {
@@ -286,7 +286,7 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
     return longest;
 }

From fdeead0b09188213e51e037edcf9c9daf88d25c4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 14:49:47 -0400
Subject: [PATCH 151/257] Set dictCtx Rather than memcpy'ing Ctx

---
 lib/lz4frame.c | 9 +++------
 lib/lz4hc.c    | 1 +
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 000002382..821cb49b5 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -549,12 +549,9 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
-        if (cdict) {
-            memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
-            LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
-        } else {
-            LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level);
-        }
+        LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
+        LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level);
+        internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
     }
 }
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 6890613e7..16830c29c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -729,6 +729,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 

From 6289ff4fb1a560115ffd2c5dff93389d917a6e4e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 16:43:15 -0400
Subject: [PATCH 152/257] Call LZ4F_applyCDict Even on NULL CDict

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 821cb49b5..ea6a66881 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -739,8 +739,8 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
+    LZ4F_applyCDict(ctx, cdict, level);
     if (cdict) {
-        LZ4F_applyCDict(ctx, cdict, level);
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
     return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);

From 66d217e2400e07fd91753881890722b6dc28ee4b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 21 Mar 2018 16:54:36 -0400
Subject: [PATCH 153/257] Perform Lookups into the Dictionary Context

---
 lib/lz4hc.c | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 16830c29c..2775713dc 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -203,6 +203,7 @@ LZ4HC_InsertAndGetWiderMatch (
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
+    const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx;
     const BYTE* const base = hc4->base;
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
@@ -212,6 +213,7 @@ LZ4HC_InsertAndGetWiderMatch (
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
+    U32 dictMatchIndex;
     repeat_state_e repeat = rep_untested;
     size_t srcPatternLength = 0;
 
@@ -288,6 +290,39 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
+    if (dictCtx != NULL) {
+        ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
+        matchIndex = dictMatchIndex + dictIndexDelta;
+        while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+            const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
+
+            if (LZ4_read32(matchPtr) == pattern) {
+                int mlt;
+                int back = 0;
+                const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
+                if (vLimit > iHighLimit) vLimit = iHighLimit;
+                mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+                // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
+                //     mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
+                // }
+                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+                mlt -= back;
+                if (mlt > longest) {
+                    longest = mlt;
+                    *matchpos = base + matchIndex + back;
+                    *startpos = ip + back;
+                }
+            }
+
+            {
+                U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
+                dictMatchIndex -= nextOffset;
+                matchIndex -= nextOffset;
+            }
+        }
+    }
+
     return longest;
 }
 

From 595ea582890031c30ded483130116ec6a0179a5c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 16:59:50 -0400
Subject: [PATCH 154/257] Avoid Resetting Hash Table

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2775713dc..b6d9bf091 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
     hc4->base = start - 64 KB;

From b6c35ed6422f60a7a542ed237cadfd9dfe76c219 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 22 Mar 2018 17:00:04 -0400
Subject: [PATCH 155/257] Avoid Resetting Chain Table

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b6d9bf091..0caf00196 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
     hc4->nextToUpdate = 64 KB;
     hc4->base = start - 64 KB;
     hc4->end = start;

From b88a0b4e8812777d1e715ab095738197f7f8eb96 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 26 Mar 2018 11:43:52 -0400
Subject: [PATCH 156/257] Only Perform Dict Lookup if Attempts Remain

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0caf00196..e005cd767 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -288,7 +288,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
-    if (dictCtx != NULL) {
+    if (dictCtx != NULL && nbAttempts) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + dictIndexDelta;

From 4f7b7a8ffa3032a64aa173f7d8bfb7c258316154 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 12:25:05 -0400
Subject: [PATCH 157/257] Clear Tables on Dict Load

---
 lib/lz4hc.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e005cd767..85419f3df 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -95,6 +95,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     hc4->lowLimit = 64 KB;
 }
 
+static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
+{
+    MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+    MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
+}
+
 
 /* Update chains up to ip (excluded) */
 LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
@@ -782,6 +788,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictSize = 64 KB;
     }
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;

From 22db704a73e38a869d846fa52ddcbb0a214a6c43 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 28 Mar 2018 12:26:54 -0400
Subject: [PATCH 158/257] Shift Dict Limit Checks out of the Loop

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 85419f3df..8ac650eeb 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -294,11 +294,11 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
 
-    if (dictCtx != NULL && nbAttempts) {
+    if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + dictIndexDelta;
-        while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {

From 895e76cc20f366ea9f2999abc9ff3e0dd11b4222 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 15:20:47 -0400
Subject: [PATCH 159/257] Push Previous Compression Offsets into the Past

---
 lib/lz4hc.c | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 8ac650eeb..c56d976dd 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -84,23 +84,29 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
 /**************************************
 *  HC Compression
 **************************************/
-static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
-{
-    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    hc4->nextToUpdate = 64 KB;
-    hc4->base = start - 64 KB;
-    hc4->end = start;
-    hc4->dictBase = start - 64 KB;
-    hc4->dictLimit = 64 KB;
-    hc4->lowLimit = 64 KB;
-}
-
 static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 {
+    DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
 }
 
+static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
+{
+    U32 startingOffset = hc4->end - hc4->base + 64 KB;
+    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
+    if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
+        LZ4HC_clearTables(hc4);
+        startingOffset = 64 KB;
+    }
+    hc4->nextToUpdate = startingOffset;
+    hc4->base = start - startingOffset;
+    hc4->end = start;
+    hc4->dictBase = start - startingOffset;
+    hc4->dictLimit = startingOffset;
+    hc4->lowLimit = startingOffset;
+}
+
 
 /* Update chains up to ip (excluded) */
 LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
@@ -751,6 +757,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
     DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
+    LZ4_streamHCPtr->internal_donotuse.end = (void *)-1;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
 }
 
@@ -767,6 +776,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);

From bdd7af6f71c235cea53912d0eec3b94555e9ffde Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 4 Apr 2018 15:59:00 -0400
Subject: [PATCH 160/257] Don't Bother Clearing Chain Table for Working
 Contexts

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c56d976dd..673b4b3b2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -96,7 +96,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     U32 startingOffset = hc4->end - hc4->base + 64 KB;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
-        LZ4HC_clearTables(hc4);
+        MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
         startingOffset = 64 KB;
     }
     hc4->nextToUpdate = startingOffset;

From a1beba13f7363df4b5d7ca9afe51963176df068d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 16:32:26 -0400
Subject: [PATCH 161/257] Reset Stream in LZ4_compress_HC

---
 lib/lz4hc.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 673b4b3b2..4a52f0c39 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -732,7 +732,10 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int cSize;
+    statePtr->internal_donotuse.end = (void *)-1;
+    LZ4_resetStreamHC(statePtr, compressionLevel);
+    cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif

From 221211d7d04478d74eec9fd76ccd98a73bd394ee Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 16:32:40 -0400
Subject: [PATCH 162/257] Fix Offset Math

---
 lib/lz4hc.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4a52f0c39..547915927 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -93,12 +93,13 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
-    U32 startingOffset = hc4->end - hc4->base + 64 KB;
+    uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
         MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
-        startingOffset = 64 KB;
+        startingOffset = 0;
     }
+    startingOffset += MAX_DISTANCE;
     hc4->nextToUpdate = startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;

From 8f9a2db0e1ea492e836dd074dd0ea62720fb6169 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 6 Apr 2018 19:24:22 -0400
Subject: [PATCH 163/257] Fix Some Cast/Conversion Warnings

---
 lib/lz4hc.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 547915927..11d820f3f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -100,12 +100,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
         startingOffset = 0;
     }
     startingOffset += MAX_DISTANCE;
-    hc4->nextToUpdate = startingOffset;
+    hc4->nextToUpdate = (U32) startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;
     hc4->dictBase = start - startingOffset;
-    hc4->dictLimit = startingOffset;
-    hc4->lowLimit = startingOffset;
+    hc4->dictLimit = (U32) startingOffset;
+    hc4->lowLimit = (U32) startingOffset;
 }
 
 
@@ -304,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch (
     if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + dictIndexDelta;
+        matchIndex = dictMatchIndex + (int)dictIndexDelta;
         while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
@@ -314,9 +314,11 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
-                //     mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
-                // }
+                /*
+                if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
+                    mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
+                }
+                */
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
@@ -734,7 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t* const statePtr = &state;
 #endif
     int cSize;
-    statePtr->internal_donotuse.end = (void *)-1;
+    statePtr->internal_donotuse.end = (const BYTE*)-1;
     LZ4_resetStreamHC(statePtr, compressionLevel);
     cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
@@ -761,7 +763,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
     DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
-    LZ4_streamHCPtr->internal_donotuse.end = (void *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;

From 8db291bc1dc95ab58904b6961a5ac0967b148952 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 5 Apr 2018 17:41:15 -0400
Subject: [PATCH 164/257] Remove Match Upper Bounds Check

---
 lib/lz4hc.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 11d820f3f..fa2cbb74d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -235,7 +235,7 @@ LZ4HC_InsertAndGetWiderMatch (
     DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
                 matchIndex, lowLimit);
 
-    while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) {
+    while ((matchIndex>=lowLimit) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         if (matchIndex >= dictLimit) {
@@ -299,7 +299,7 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
     if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
@@ -720,6 +720,7 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
+    LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel);
     LZ4HC_init (ctx, (const BYTE*)src);
     if (dstCapacity < LZ4_compressBound(srcSize))
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
@@ -735,10 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int cSize;
-    statePtr->internal_donotuse.end = (const BYTE*)-1;
-    LZ4_resetStreamHC(statePtr, compressionLevel);
-    cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif
@@ -750,6 +748,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
 int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+    LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel);
     LZ4HC_init(ctx, (const BYTE*) source);
     return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
 }
@@ -782,7 +781,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
-    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);

From 3591fe8ab8b4391d1f9104b38b9ac19deac2a3e8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 15:09:59 -0400
Subject: [PATCH 165/257] Add Fast Reset Paths

---
 lib/lz4hc.c | 21 +++++++++++++++++++--
 lib/lz4hc.h |  2 ++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index fa2cbb74d..9deb90d5f 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -716,11 +716,11 @@ static int LZ4HC_compress_generic (
 
 int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
 
-int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
-    LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel);
+    LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel);
     LZ4HC_init (ctx, (const BYTE*)src);
     if (dstCapacity < LZ4_compressBound(srcSize))
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
@@ -728,6 +728,13 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
         return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
 }
 
+int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
+{
+    if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
+    LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel);
+    return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel);
+}
+
 int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
@@ -787,6 +794,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 
+void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+    LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+    DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
+    LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
+}
+
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 1b08425e3..3b0d0d3fb 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -270,7 +270,9 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
 #endif   /* LZ4_HC_SLO_098092834 */
 #endif   /* LZ4_HC_STATIC_LINKING_ONLY */

From 61c7ceffede9607721ca496ac13d788625f9f668 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 15:25:48 -0400
Subject: [PATCH 166/257] Use Fast Reset API in LZ4F

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index ea6a66881..d087a9461 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -743,7 +743,7 @@ static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSi
     if (cdict) {
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }
-    return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+    return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);
 }
 
 static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)

From 0a2abacd909d0e1e1be9a82de6c2a693264f8afa Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Mon, 16 Apr 2018 20:23:19 -0400
Subject: [PATCH 167/257] Use Fast Reset in LZ4F Again

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d087a9461..a43f59514 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -550,7 +550,7 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
         LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
-        LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level);
+        LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
         internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
     }
 }

From 22e16d5b509c56ad350ae42664ddcd3da7b97f1f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 13:55:58 -0400
Subject: [PATCH 168/257] Split DictCtx-using Code Into Separate Inlining Chain

---
 lib/lz4hc.c | 94 +++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 74 insertions(+), 20 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9deb90d5f..5c2b00194 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -79,6 +79,8 @@
 
 static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
 
+/*===   Enums   ===*/
+typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
 
 
 /**************************************
@@ -210,7 +212,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE** matchpos,
     const BYTE** startpos,
     const int maxNbAttempts,
-    const int patternAnalysis)
+    const int patternAnalysis,
+    const dictCtx_directive dict)
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
@@ -301,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
-    if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
+    if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
@@ -344,13 +347,14 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
                                  const BYTE* const ip, const BYTE* const iLimit,
                                  const BYTE** matchpos,
                                  const int maxNbAttempts,
-                                 const int patternAnalysis)
+                                 const int patternAnalysis,
+                                 const dictCtx_directive dict)
 {
     const BYTE* uselessPtr = ip;
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict);
 }
 
 
@@ -440,7 +444,8 @@ static int LZ4HC_compress_hashChain (
     int* srcSizePtr,
     int const maxOutputSize,
     unsigned maxNbAttempts,
-    limitedOutput_directive limit
+    const limitedOutput_directive limit,
+    const dictCtx_directive dict
     )
 {
     const int inputSize = *srcSizePtr;
@@ -472,7 +477,7 @@ static int LZ4HC_compress_hashChain (
 
     /* Main Loop */
     while (ip <= mflimit) {
-        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
+        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict);
         if (ml<MINMATCH) { ip++; continue; }
 
         /* saved, in case we would skip too much */
@@ -484,7 +489,7 @@ static int LZ4HC_compress_hashChain (
         if (ip+ml <= mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, patternAnalysis);
+                            maxNbAttempts, patternAnalysis, dict);
         else
             ml2 = ml;
 
@@ -531,7 +536,7 @@ static int LZ4HC_compress_hashChain (
         if (start2 + ml2 <= mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, patternAnalysis);
+                            maxNbAttempts, patternAnalysis, dict);
         else
             ml3 = ml2;
 
@@ -655,17 +660,19 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
     const char* const source, char* dst,
     int* srcSizePtr, int dstCapacity,
     int const nbSearches, size_t sufficient_len,
-    limitedOutput_directive limit, int const fullUpdate);
+    const limitedOutput_directive limit, int const fullUpdate,
+    const dictCtx_directive dict);
 
 
-static int LZ4HC_compress_generic (
+LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     LZ4HC_CCtx_internal* const ctx,
     const char* const src,
     char* const dst,
     int* const srcSizePtr,
     int const dstCapacity,
     int cLevel,
-    limitedOutput_directive limit
+    const limitedOutput_directive limit,
+    const dictCtx_directive dict
     )
 {
     typedef enum { lz4hc, lz4opt } lz4hc_strat_e;
@@ -704,12 +711,57 @@ static int LZ4HC_compress_generic (
         if (cParam.strat == lz4hc)
             return LZ4HC_compress_hashChain(ctx,
                                 src, dst, srcSizePtr, dstCapacity,
-                                cParam.nbSearches, limit);
+                                cParam.nbSearches, limit, dict);
         assert(cParam.strat == lz4opt);
         return LZ4HC_compress_optimal(ctx,
                             src, dst, srcSizePtr, dstCapacity,
                             cParam.nbSearches, cParam.targetLength, limit,
-                            cLevel == LZ4HC_CLEVEL_MAX);  /* ultra mode */
+                            cLevel == LZ4HC_CLEVEL_MAX, dict);  /* ultra mode */
+    }
+}
+
+static int LZ4HC_compress_generic_noDictCtx (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    assert(ctx->dictCtx == NULL);
+    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx);
+}
+
+static int LZ4HC_compress_generic_dictCtx (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    assert(ctx->dictCtx != NULL);
+    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+}
+
+static int LZ4HC_compress_generic (
+    LZ4HC_CCtx_internal* const ctx,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
+    limitedOutput_directive limit
+    )
+{
+    if (ctx->dictCtx == NULL) {
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else {
+        return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     }
 }
 
@@ -1019,7 +1071,8 @@ typedef struct {
 LZ4_FORCE_INLINE LZ4HC_match_t
 LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
                       const BYTE* ip, const BYTE* const iHighLimit,
-                      int minLen, int nbSearches)
+                      int minLen, int nbSearches,
+                      const dictCtx_directive dict)
 {
     LZ4HC_match_t match = { 0 , 0 };
     const BYTE* matchPtr = NULL;
@@ -1028,7 +1081,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
     int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
                                 ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */);
+                                nbSearches, 1 /* patternAnalysis */, dict);
     if (matchLength <= minLen) return match;
     match.len = matchLength;
     match.off = (int)(ip-matchPtr);
@@ -1044,8 +1097,9 @@ static int LZ4HC_compress_optimal (
     int dstCapacity,
     int const nbSearches,
     size_t sufficient_len,
-    limitedOutput_directive limit,
-    int const fullUpdate
+    const limitedOutput_directive limit,
+    int const fullUpdate,
+    const dictCtx_directive dict
     )
 {
 #define TRAILING_LITERALS 3
@@ -1073,7 +1127,7 @@ static int LZ4HC_compress_optimal (
          int best_mlen, best_off;
          int cur, last_match_pos = 0;
 
-         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
+         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict);
          if (firstMatch.len==0) { ip++; continue; }
 
          if ((size_t)firstMatch.len > sufficient_len) {
@@ -1143,10 +1197,10 @@ static int LZ4HC_compress_optimal (
 
              DEBUGLOG(7, "search at rPos:%u", cur);
              if (fullUpdate)
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict);
              else
                  /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict);
              if (!newMatch.len) continue;
 
              if ( ((size_t)newMatch.len > sufficient_len)

From b67de2a327644607c034250d105ba9374b0897e8 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 18 Apr 2018 15:52:04 -0400
Subject: [PATCH 169/257] Force Inline on HashChain

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5c2b00194..c050c59b2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -437,7 +437,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     return 0;
 }
 
-static int LZ4HC_compress_hashChain (
+LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     LZ4HC_CCtx_internal* const ctx,
     const char* const source,
     char* const dest,

From 0abc23f72e44dd9af86e7fb61a2497cee81ed320 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 13:02:55 -0400
Subject: [PATCH 170/257] Copy DictCtx into Working Context on Inputs Larger
 than 4 KB

---
 lib/lz4hc.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c050c59b2..c201559a2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -720,6 +720,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     }
 }
 
+static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock);
+
 static int LZ4HC_compress_generic_noDictCtx (
     LZ4HC_CCtx_internal* const ctx,
     const char* const src,
@@ -745,7 +747,16 @@ static int LZ4HC_compress_generic_dictCtx (
     )
 {
     assert(ctx->dictCtx != NULL);
-    return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+    if (*srcSizePtr > 4 KB) {
+        memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
+        LZ4HC_setExternalDict(ctx, (const BYTE *)src);
+        ctx->compressionLevel = cLevel;
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else {
+        int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
+        ctx->dictCtx = NULL;
+        return result;
+    }
 }
 
 static int LZ4HC_compress_generic (

From f4b13e17ea110498c41ba3ac8dc6df0438c37d29 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 17:51:10 -0400
Subject: [PATCH 171/257] Don't Clear the Dictionary Context Until No Longer
 Useful

---
 lib/lz4hc.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c201559a2..9a02787ea 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -746,15 +746,18 @@ static int LZ4HC_compress_generic_dictCtx (
     limitedOutput_directive limit
     )
 {
+    size_t position = ctx->end - ctx->base - ctx->lowLimit;
     assert(ctx->dictCtx != NULL);
-    if (*srcSizePtr > 4 KB) {
+    if (position >= 64 KB) {
+        ctx->dictCtx = NULL;
+        return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
+    } else if (position == 0 && *srcSizePtr > 4 KB) {
         memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
         LZ4HC_setExternalDict(ctx, (const BYTE *)src);
         ctx->compressionLevel = cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
         int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
-        ctx->dictCtx = NULL;
         return result;
     }
 }

From 14c577d4c9332197dc0003833e043608b4b4b239 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 19:36:34 -0400
Subject: [PATCH 172/257] Fix Signedness of Comparison

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 9a02787ea..20df8fa30 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -308,7 +308,7 @@ LZ4HC_InsertAndGetWiderMatch (
         ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {

From 0064e8ebc7a475d27ba658ca92e40de75c9cac72 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 13:14:37 -0400
Subject: [PATCH 173/257] Remove Commented Out Support for Match Continuation
 over Segment Boundary

---
 lib/lz4hc.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 20df8fa30..e09d8e0a6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -317,11 +317,6 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                /*
-                if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) {
-                    mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit);
-                }
-                */
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {

From 8f118cf6e9f3030f52414177d93ab447e8e24128 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:08:06 -0400
Subject: [PATCH 174/257] Remove inputBuffer from Context, Work Around its
 Absence

---
 lib/lz4hc.c | 13 +++++++------
 lib/lz4hc.h | 11 ++++++++---
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e09d8e0a6..dff42f00a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -999,8 +999,8 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
 {
     LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1;   /* Error : pointer is not aligned for pointer (32 or 64 bits) */
+    LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel);
     LZ4HC_init(ctx, (const BYTE*)inputBuffer);
-    ctx->inputBuffer = inputBuffer;
     return 0;
 }
 
@@ -1008,9 +1008,8 @@ void* LZ4_createHC (const char* inputBuffer)
 {
     LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
     if (hc4 == NULL) return NULL;   /* not enough memory */
+    LZ4_resetStreamHC(hc4, 0 /* compressionLevel */);
     LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer);
-    assert(sizeof(size_t) == sizeof(void*));
-    hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer;   /* ugly hack, circumvent -Wcast-qual */
     return hc4;
 }
 
@@ -1032,9 +1031,11 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c
 
 char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
 {
-    LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
-    int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
-    return (char*)(hc4->inputBuffer) + dictSize;
+    LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data;
+    const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit;
+    LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel);
+    /* avoid const char * -> char * conversion warning :( */
+    return (char *)(uptrval)bufferStart;
 }
 
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 3b0d0d3fb..cfc5d9e34 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -149,7 +149,6 @@ struct LZ4HC_CCtx_internal
     const uint8_t* end;         /* next block here to continue on current prefix */
     const uint8_t* base;        /* All index relative to this position */
     const uint8_t* dictBase;    /* alternate base for extDict */
-    void*      inputBuffer;     /* deprecated */
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
@@ -167,7 +166,6 @@ struct LZ4HC_CCtx_internal
     const unsigned char* end;        /* next block here to continue on current prefix */
     const unsigned char* base;       /* All index relative to this position */
     const unsigned char* dictBase;   /* alternate base for extDict */
-    void*          inputBuffer;      /* deprecated */
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
@@ -210,7 +208,14 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
-/* Deprecated Streaming functions; should no longer be used */
+/* Obsolete streaming functions; degraded functionality; do not use!
+ *
+ * In order to perform streaming compression, these functions depended on data
+ * that is no longer tracked in the state. They have been preserved as well as
+ * possible: using them will still produce a correct output. However, use of
+ * LZ4_slideInputBufferHC() will truncate the history of the stream, rather
+ * than preserve a window-sized chunk of history.
+ */
 LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
 LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
 LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API   int   LZ4_freeHC (void* LZ4HC_Data);

From ca833f928f64d0b88fbfd2a8a8bc11b8333e487d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:16:27 -0400
Subject: [PATCH 175/257] Also Reset the Chain Table

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index dff42f00a..124da86ac 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
     uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
-        MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+        LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
     startingOffset += MAX_DISTANCE;

From d7347f9eeaffe5351886b6174f9974bd541b9b7d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 14:52:13 -0400
Subject: [PATCH 176/257] Add API for Attaching Dictionaries

---
 lib/lz4frame.c |  3 +--
 lib/lz4hc.c    |  3 +++
 lib/lz4hc.h    | 27 +++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a43f59514..d4d9397f8 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -549,9 +549,8 @@ static void LZ4F_applyCDict(void* ctx,
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
-        LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse;
         LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
-        internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL;
+        LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);
     }
 }
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 124da86ac..690015ff8 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -887,6 +887,9 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
     return dictSize;
 }
 
+void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) {
+    working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL;
+}
 
 /* compression */
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index cfc5d9e34..fb8397ecb 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -279,5 +279,32 @@ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLeve
 
 int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
+/*! LZ4_attach_HC_dictionary() :
+ *  This is an experimental API that allows for the efficient use of a
+ *  static dictionary many times.
+ *
+ *  Rather than re-loading the dictionary buffer into a working context before
+ *  each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a
+ *  working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,
+ *  in which the working stream references the dictionary stream in-place.
+ *
+ *  Several assumptions are made about the state of the dictionary stream.
+ *  Currently, only streams which have been prepared by LZ4_loadDictHC() should
+ *  be expected to work.
+ *
+ *  Alternatively, the provided dictionary stream pointer may be NULL, in which
+ *  case any existing dictionary stream is unset.
+ *
+ *  A dictionary should only be attached to a stream without any history (i.e.,
+ *  a stream that has just been reset).
+ *
+ *  The dictionary will remain attached to the working stream only for the
+ *  current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the
+ *  dictionary context association from the working stream. The dictionary
+ *  stream (and source buffer) must remain in-place / accessible / unchanged
+ *  through the lifetime of the stream session.
+ */
+LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream);
+
 #endif   /* LZ4_HC_SLO_098092834 */
 #endif   /* LZ4_HC_STATIC_LINKING_ONLY */

From 3f087cf1cbcb0e0082ead149a621d259cb57e7ba Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:00:53 -0400
Subject: [PATCH 177/257] Add Comments on New Public APIs

---
 lib/lz4hc.h | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index fb8397ecb..a762d3fa2 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -275,8 +275,37 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_resetStreamHC_fast() :
+ *  When an LZ4_streamHC_t is known to be in a internally coherent state,
+ *  it can often be prepared for a new compression with almost no work, only
+ *  sometimes falling back to the full, expensive reset that is always required
+ *  when the stream is in an indeterminate state (i.e., the reset performed by
+ *  LZ4_resetStreamHC()).
+ *
+ *  LZ4_streamHCs are guaranteed to be in a valid state when:
+ *  - returned from LZ4_createStreamHC()
+ *  - reset by LZ4_resetStreamHC()
+ *  - memset(stream, 0, sizeof(LZ4_streamHC_t))
+ *  - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast()
+ *  - the stream was in a valid state and was then used in any compression call
+ *    that returned success
+ *  - the stream was in an indeterminate state and was used in a compression
+ *    call that fully reset the state (LZ4_compress_HC_extStateHC()) and that
+ *    returned success
+ */
 void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_compress_HC_extStateHC_fastReset() :
+ *  A variant of LZ4_compress_HC_extStateHC().
+ *
+ *  Using this variant avoids an expensive initialization step. It is only safe
+ *  to call if the state buffer is known to be correctly initialized already
+ *  (see above comment on LZ4_resetStreamHC_fast() for a definition of
+ *  "correctly initialized"). From a high level, the difference is that this
+ *  function initializes the provided state with a call to
+ *  LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a
+ *  call to LZ4_resetStreamHC().
+ */
 int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
 
 /*! LZ4_attach_HC_dictionary() :

From 209c9c29d1baf5b2f4fb1fc38b2612dd95d1b9a1 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:16:41 -0400
Subject: [PATCH 178/257] Add Some Simple Fuzzer Tests

---
 tests/fuzzer.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4fef..8b21f8e64 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -446,7 +446,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test compression HC using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed")
+
+        /* Test compression HC using fast reset external state */
+        FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()");
+        ret = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
 
         /* Test compression using external state */
         FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
@@ -810,6 +815,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
         }
 
+        /* Compress HC using external dictionary stream */
+        FUZ_DISPLAYTEST();
+        {
+            LZ4_streamHC_t LZ4_streamHC;
+
+            LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
+            LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST();
+            LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel);
+            LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC);
+            ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize);
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
+
+            FUZ_DISPLAYTEST();
+            decodedBuffer[blockSize] = 0;
+            ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
+            FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+            FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+            {   U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+                if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
+                FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
+            }
+        }
+
         /* Compress HC continue destSize */
         FUZ_DISPLAYTEST();
         {   int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;

From 7874cf06b3307a7ba5efdd141d068887480aaf11 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 15:30:08 -0400
Subject: [PATCH 179/257] Consts and Asserts and Other Minor Nits

---
 lib/lz4hc.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 690015ff8..e5eb11dd4 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -305,10 +305,13 @@ LZ4HC_InsertAndGetWiderMatch (
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
     if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
-        ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
+        /* bounds check, since we need to downcast */
+        assert(dictIndexDelta <= 1 GB);
+        assert(dictIndexDelta >= -1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
         matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) {
+        while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {
@@ -741,7 +744,7 @@ static int LZ4HC_compress_generic_dictCtx (
     limitedOutput_directive limit
     )
 {
-    size_t position = ctx->end - ctx->base - ctx->lowLimit;
+    const size_t position = ctx->end - ctx->base - ctx->lowLimit;
     assert(ctx->dictCtx != NULL);
     if (position >= 64 KB) {
         ctx->dictCtx = NULL;
@@ -752,8 +755,7 @@ static int LZ4HC_compress_generic_dictCtx (
         ctx->compressionLevel = cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
-        int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
-        return result;
+        return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
     }
 }
 
@@ -804,7 +806,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
+    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif

From 1d2500d44e97a46eb447745d02652e02435baadb Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:10:47 -0400
Subject: [PATCH 180/257] Handle Index Underflows Safely

---
 lib/lz4hc.c | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e5eb11dd4..843b539dc 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -221,7 +221,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const base = hc4->base;
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE;
+    const U32 ipIndex = (U32)(ip - base);
+    const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
     int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
@@ -304,14 +305,12 @@ LZ4HC_InsertAndGetWiderMatch (
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
-    if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) {
-        const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit;
-        /* bounds check, since we need to downcast */
-        assert(dictIndexDelta <= 1 GB);
-        assert(dictIndexDelta >= -1 GB);
+    if (dict == usingDictCtx && nbAttempts && ipIndex - lowLimit < MAX_DISTANCE) {
+        size_t const dictEndOffset = dictCtx->end - dictCtx->base;
+        assert(dictEndOffset <= 1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + (int)dictIndexDelta;
-        while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) {
+        matchIndex = dictMatchIndex + lowLimit - (U32)dictEndOffset;
+        while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 
             if (LZ4_read32(matchPtr) == pattern) {

From 86b381e40b6ec3394582a2113c7bc80f1d619bab Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:13:40 -0400
Subject: [PATCH 181/257] Fix Constant Value

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 843b539dc..48c42c6fd 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -101,7 +101,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
-    startingOffset += MAX_DISTANCE;
+    startingOffset += 64 KB;
     hc4->nextToUpdate = (U32) startingOffset;
     hc4->base = start - startingOffset;
     hc4->end = start;

From 756ed402da8161c7bc3749bebaac069259312d8b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 17:56:26 -0400
Subject: [PATCH 182/257] Sign-Extend -1 to Pointer Width

---
 lib/lz4hc.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 48c42c6fd..a973086d0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -830,8 +830,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr);
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
@@ -850,7 +849,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel);
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1;
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);

From 85cac61dd8823886132b4a9c2ff4a43b237da917 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:35:51 -0400
Subject: [PATCH 183/257] Don't Segfault on Malloc Failure

---
 lib/lz4hc.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a973086d0..b0325a79a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -830,9 +830,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-    LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    if (LZ4_streamHCPtr != NULL) {
+        LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+        LZ4_streamHCPtr->internal_donotuse.base = NULL;
+        LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    }
     return LZ4_streamHCPtr;
 }
 

From a8cb2feffdead7b49a09aad0dc7b13dcff206e5b Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:37:07 -0400
Subject: [PATCH 184/257] Tolerate Base Pointer Underflow

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b0325a79a..b70d03b65 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -97,7 +97,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     uptrval startingOffset = hc4->end - hc4->base;
     DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
-    if (startingOffset > 1 GB || startingOffset > (uptrval)start) {
+    if (startingOffset > 1 GB) {
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
     }
@@ -898,7 +898,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;

From fcc99d1f31bf8a913fa91a5455524b4a14b9a03d Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 19:37:28 -0400
Subject: [PATCH 185/257] Simpler loadDict() Reset

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b70d03b65..23d84f45c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -882,8 +882,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
+    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
-    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;

From 1895fa19a4f04b395f9ebb1fe38049ab4c909fc3 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:14:12 -0400
Subject: [PATCH 186/257] Remove Redundant Static Assert

---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 23d84f45c..f91ef4a93 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -859,7 +859,6 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 
 void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
-    LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel);
     LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;

From ee67f25576f111972bd5003c527d1ebee598dc71 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:18:30 -0400
Subject: [PATCH 187/257] Change vLimit Calculation

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f91ef4a93..a12f29808 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -316,7 +316,7 @@ LZ4HC_InsertAndGetWiderMatch (
             if (LZ4_read32(matchPtr) == pattern) {
                 int mlt;
                 int back = 0;
-                const BYTE* vLimit = ip + (dictCtx->end - matchPtr);
+                const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;

From a8a5dfd426cec6e3ac6c689a314aff5817de9ce3 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 20 Apr 2018 18:09:51 -0700
Subject: [PATCH 188/257] fixed clang performance in lz4_fast

The simple change from
`matchIndex+MAX_DISTANCE < current`
towards
`current - matchIndex > MAX_DISTANCE`

is enough to generate a 10% performance drop under clang.
Quite massive.
(I missed as my eyes were concentrated on gcc performance at that time).

The second version is more robust, because it also survives a situation where
`matchIndex > current`
due to overflows.

The first version requires matchIndex to not overflow.
Hence were added `assert()` conditions.

The only case where this can happen is with dictCtx compression,
in the case where the dictionary context is not initialized before loading the dictionary.
So it's enough to always initialize the context while loading the dictionary.
---
 lib/lz4.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bb6b619ba..3ca1126de 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -777,7 +777,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
                 if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;     /* match outside of valid area */
-                if ((tableType != byU16) && (current - matchIndex > MAX_DISTANCE)) continue; /* too far - note: works even if matchIndex overflows */
+                assert(matchIndex < current);
+                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far - note: works even if matchIndex overflows */
                 if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);      /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
@@ -918,8 +919,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 match = base + matchIndex;
             }
             LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
+            assert(matchIndex < current);
             if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1)
-              && ((tableType==byU16) ? 1 : (current - matchIndex <= MAX_DISTANCE))
+              && ((tableType==byU16) ? 1 : (matchIndex+MAX_DISTANCE >= current))
               && (LZ4_read32(match) == LZ4_read32(ip)) ) {
                 token=op++;
                 *token=0;
@@ -1304,7 +1306,11 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
 
     DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict);
 
-    LZ4_prepareTable(dict, 0, tableType);
+    /* It's necessary to reset the context,
+     * and not just continue it with prepareTable()
+     * to avoid any risk of generating overflowing matchIndex
+     * when compressing using this dictionary */
+    LZ4_resetStream(LZ4_dict);
 
     /* We always increment the offset by 64 KB, since, if the dict is longer,
      * we truncate it to the last 64k, and if it's shorter, we still want to

From d1f21883d652ac0abae91bc04924747c82bdbabd Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sat, 21 Apr 2018 00:11:51 -0700
Subject: [PATCH 189/257] fixed incorrect comment

---
 lib/lz4.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3ca1126de..22051623c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -776,10 +776,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 forwardH = LZ4_hashPosition(forwardIp, tableType);
                 LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType);
 
-                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;     /* match outside of valid area */
+                if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue;    /* match outside of valid area */
                 assert(matchIndex < current);
-                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue; /* too far - note: works even if matchIndex overflows */
-                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);      /* too_far presumed impossible with byU16 */
+                if ((tableType != byU16) && (matchIndex+MAX_DISTANCE < current)) continue;  /* too far */
+                if (tableType == byU16) assert((current - matchIndex) <= MAX_DISTANCE);     /* too_far presumed impossible with byU16 */
 
                 if (LZ4_read32(match) == LZ4_read32(ip)) {
                     if (maybe_extMem) offset = current - matchIndex;

From ab06ef97bb5bb50f642add1e80854a09c3d38068 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Mon, 23 Apr 2018 01:43:30 +0300
Subject: [PATCH 190/257] lz4.h: clarify the risks of using
 LZ4_decompress_fast()

The notes about "security guarantee" and "malicious inputs" seemed
a bit non-technical to me, so I took the liberty to tone them down
and instead describe the actual risks in technical terms.  Namely,
the function never writes past the end of the output buffer, so
a direct hostile takeover (resulting in arbitrary code execution
soon after the return from the function) is not possible.  However,
the application can crash because of reads from unmapped pages.

I also took the liberty to describe what I believe is the only sensible
usage scenario for the function: "This function is only usable if the
originalSize of uncompressed data is known in advance," etc.
---
 lib/lz4.h | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 0dfa19e00..db6353c0c 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -206,15 +206,17 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt
 /*!
 LZ4_decompress_fast() : **unsafe!**
 This function is a bit faster than LZ4_decompress_safe(),
-but doesn't provide any security guarantee.
+but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
     originalSize : is the uncompressed size to regenerate
                    Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
     return : number of bytes read from source buffer (== compressed size).
              If the source stream is detected malformed, the function stops decoding and return a negative result.
-    note : This function respects memory boundaries for *properly formed* compressed data.
-           However, it does not provide any protection against malicious input.
-           It also doesn't know 'src' size, and implies it's >= compressed size.
-           Use this function in trusted environment **only**.
+    note : This function is only usable if the originalSize of uncompressed data is known in advance.
+           The caller should also check that all the compressed input has been consumed properly,
+           i.e. that the return value matches the size of the buffer with compressed input.
+           The function never writes past the output buffer.  However, since it doesn't know its 'src' size,
+           it may read past the intended input.  Also, because match offsets are not validated during decoding,
+           reads from 'src' may underflow.  Use this function in trusted environment **only**.
 */
 LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 

From bb83cad98fdb15a7ade4cde582b98e836fb8ef11 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 23 Apr 2018 13:14:19 -0700
Subject: [PATCH 191/257] Fix input size validation edge cases

The bug is a read up to 2 bytes past the end of the buffer.
There are three cases for this bug, one for each test case added.

* An empty input causes `token = *ip++` to read one byte too far.
* A one byte input with `(token >> ML_BITS) == RUN_MASK` causes
  one extra byte to be read without validation. This could be
  combined with the first bug to cause 2 extra bytes to be read.
* The case pointed out in issue #508, where `ip == iend` at the
  beginning of the loop after taking the shortcut.

Benchmarks show no regressions on clang or gcc-7 on both my mac
and devserver.

Fixes #508.
---
 lib/lz4.c      |  8 ++++++--
 tests/fuzzer.c | 26 ++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bb6b619ba..870ab5a1c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1520,6 +1520,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                      /* targetOutputSize too high => just decode everything */
     if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1;  /* Empty output buffer */
     if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
+    if ((endOnInput) && unlikely(srcSize==0)) return -1;
 
     /* Main Loop : decode sequences */
     while (1) {
@@ -1529,11 +1530,13 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
         unsigned const token = *ip++;
 
+        assert(ip <= iend); /* ip < iend before the increment */
         /* shortcut for common case :
          * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
-         * it has not yet been benchmarked on ARM, Power, mips, etc. */
-        if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend)
+         * it has not yet been benchmarked on ARM, Power, mips, etc.
+         * NOTE: The loop begins with a read, so we must have one byte left at the end. */
+        if (((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
           & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
           & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
             size_t const ll = token >> ML_BITS;
@@ -1553,6 +1556,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         /* decode literal length */
         if ((length=(token>>ML_BITS)) == RUN_MASK) {
             unsigned s;
+            if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error;   /* overflow detection */
             do {
                 s = *ip++;
                 length += s;
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4fef..def52300e 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -487,6 +487,32 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
         FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
 
+        /* Test decoding with empty input */
+        FUZ_DISPLAYTEST("LZ4_decompress_safe() with empty input");
+        LZ4_decompress_safe(NULL, decodedBuffer, 0, blockSize);
+
+        /* Test decoding with a one byte input */
+        FUZ_DISPLAYTEST("LZ4_decompress_safe() with one byte input");
+        {   char const tmp = 0xFF;
+            LZ4_decompress_safe(&tmp, decodedBuffer, 1, blockSize);
+        }
+
+        /* Test decoding shortcut edge case */
+        FUZ_DISPLAYTEST("LZ4_decompress_safe() with shortcut edge case");
+        {   char tmp[17];
+            unsigned long i;
+            /* 14 bytes of literals, followed by a 14 byte match.
+             * Should not read beyond the end of the buffer.
+             * See https://github.com/lz4/lz4/issues/508. */
+            *tmp = 0xEE;
+            memset(tmp + 1, 0, 14);
+            tmp[15] = 14;
+            tmp[16] = 0;
+            ret = LZ4_decompress_safe(tmp, decodedBuffer, sizeof(tmp), blockSize);
+            FUZ_CHECKTEST(ret >= 0, "LZ4_decompress_safe() should fail");
+        }
+
+
         /* Test decoding with output size exactly what's necessary => must work */
         FUZ_DISPLAYTEST();
         decodedBuffer[blockSize] = 0;

From 672799e8149c4c9b89f767801209721a59012c91 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 23 Apr 2018 14:21:02 -0700
Subject: [PATCH 192/257] Fix compilation error and assert.

---
 lib/lz4.c      | 2 +-
 tests/fuzzer.c | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 870ab5a1c..40b2229a2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1530,7 +1530,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
         unsigned const token = *ip++;
 
-        assert(ip <= iend); /* ip < iend before the increment */
+        assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
         /* shortcut for common case :
          * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index def52300e..6fd27fcea 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -500,7 +500,6 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test decoding shortcut edge case */
         FUZ_DISPLAYTEST("LZ4_decompress_safe() with shortcut edge case");
         {   char tmp[17];
-            unsigned long i;
             /* 14 bytes of literals, followed by a 14 byte match.
              * Should not read beyond the end of the buffer.
              * See https://github.com/lz4/lz4/issues/508. */

From bd06fde104c7c29b8d748dad8f27a08c328b74fe Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 15:42:27 -0700
Subject: [PATCH 193/257] fullbench compiled without assert()

to better reflect release speed
---
 lib/Makefile   | 1 +
 tests/Makefile | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index 7b3123929..673406d05 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -181,6 +181,7 @@ uninstall:
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 	@echo lz4 libraries successfully uninstalled
 
 endif
diff --git a/tests/Makefile b/tests/Makefile
index 2b93c9f62..d4847b1e6 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -80,7 +80,8 @@ lz4c32:   # create a 32-bits version for 32/64 interop tests
 %.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h
 	$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
 
-fullbench  : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
+fullbench : DEBUGLEVEL=0
+fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 $(LZ4DIR)/liblz4.a:

From cd0663456f82b6d771e6aed01c6c4ff7f1bf4358 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 15:47:08 -0700
Subject: [PATCH 194/257] disable shortcut for LZ4_decompress_fast()

improving speed
---
 lib/lz4.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 40b2229a2..d794c3519 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1536,9 +1536,10 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
          * it has not yet been benchmarked on ARM, Power, mips, etc.
          * NOTE: The loop begins with a read, so we must have one byte left at the end. */
-        if (((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
-          & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
-          & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
+        if (endOnInput
+          && ((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
+            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
+            & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
             size_t const ll = token >> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */

From 644b7bd2b62b10da2b464d71655ae2ed523fdc33 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 15:52:44 -0700
Subject: [PATCH 195/257] fixed minor declaration issue with clang on msys

---
 programs/util.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/programs/util.h b/programs/util.h
index ff2510625..ef6ca77ac 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -30,8 +30,9 @@ extern "C" {
 *  Dependencies
 ******************************************/
 #include "platform.h"     /* PLATFORM_POSIX_VERSION */
-#include <stdlib.h>       /* malloc */
 #include <stddef.h>       /* size_t, ptrdiff_t */
+#include <stdlib.h>       /* malloc */
+#include <string.h>       /* strlen, strncpy */
 #include <stdio.h>        /* fprintf */
 #include <sys/types.h>    /* stat, utime */
 #include <sys/stat.h>     /* stat */

From 44bff3fd3b4fc6280cad3f8b930d6a404e3bb236 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 23 Apr 2018 19:26:02 -0700
Subject: [PATCH 196/257] re-ordered parenthesis

to avoid mixing && and &
as suggested by @terrelln
---
 lib/lz4.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index d794c3519..b2e08e3f6 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1538,8 +1538,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          * NOTE: The loop begins with a read, so we must have one byte left at the end. */
         if (endOnInput
           && ((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
-            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend))
-            & ((token < (15<<ML_BITS)) & ((token & ML_MASK) != 15)) ) {
+            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)
+            & (token < (15<<ML_BITS))
+            & ((token & ML_MASK) != 15) ) ) {
             size_t const ll = token >> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */

From 13271a88d7c21bee4ee61f619ef359afc707f23c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 24 Apr 2018 11:55:53 -0400
Subject: [PATCH 197/257] Revert Stream Size Const to Correct Value

---
 lib/lz4hc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index a762d3fa2..28e252889 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -175,7 +175,7 @@ struct LZ4HC_CCtx_internal
 
 #endif
 
-#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */
+#define LZ4_STREAMHCSIZE       (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */
 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
 union LZ4_streamHC_u {
     size_t table[LZ4_STREAMHCSIZE_SIZET];

From db9deb7b747d630a5778ed588d88dae82328dc62 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 18:56:01 -0400
Subject: [PATCH 198/257] Remove the Framebench Tool

---
 tests/Makefile     |   6 +-
 tests/framebench.c | 412 ---------------------------------------------
 2 files changed, 1 insertion(+), 417 deletions(-)
 delete mode 100644 tests/framebench.c

diff --git a/tests/Makefile b/tests/Makefile
index f6a4ff3e3..2b93c9f62 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen framebench
+all: fullbench fuzzer frametest datagen
 
 all32: CFLAGS+=-m32
 all32: all
@@ -99,9 +99,6 @@ fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -113,7 +110,6 @@ clean:
         fullbench$(EXT) fullbench32$(EXT) \
         fuzzer$(EXT) fuzzer32$(EXT) \
         frametest$(EXT) frametest32$(EXT) \
-        framebench$(EXT) \
         fasttest$(EXT) datagen$(EXT) checkTag$(EXT)
 	@rm -fR $(TESTDIR)
 	@echo Cleaning completed
diff --git a/tests/framebench.c b/tests/framebench.c
deleted file mode 100644
index 9752f23f3..000000000
--- a/tests/framebench.c
+++ /dev/null
@@ -1,412 +0,0 @@
-#include <assert.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-
-#define LZ4_STATIC_LINKING_ONLY
-#include "lz4.h"
-#include "lz4hc.h"
-#include "lz4frame.h"
-#include "lz4frame_static.h"
-
-#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } }
-
-typedef struct {
-  const char *run_name;
-  size_t iter;
-  LZ4_stream_t *ctx;
-  LZ4_streamHC_t *hcctx;
-  LZ4F_cctx *cctx;
-  LZ4F_dctx *dctx;
-  const char *dictbuf;
-  size_t dictsize;
-  char *obuf;
-  size_t osize;
-  const char* ibuf;
-  const char* isample;
-  size_t isize;
-  size_t num_ibuf;
-  char *checkbuf;
-  size_t checksize;
-  int clevel;
-  const LZ4F_CDict* cdict;
-  LZ4F_preferences_t* prefs;
-  const LZ4F_compressOptions_t* options;
-} bench_params_t;
-
-size_t compress_frame(bench_params_t *p) {
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressFrame_usingCDict(
-    cctx,
-    obuf,
-    osize,
-    isample,
-    isize,
-    cdict,
-    prefs);
-  LZ4F_CHECK(oused);
-
-  return oused;
-}
-
-size_t compress_begin(bench_params_t *p) {
-  LZ4F_cctx *cctx = p->cctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  const LZ4F_CDict* cdict = p->cdict;
-  LZ4F_preferences_t* prefs = p->prefs;
-  const LZ4F_compressOptions_t* options = p->options;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  prefs->frameInfo.contentSize = isize;
-
-  oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressUpdate(
-    cctx,
-    obuf,
-    oend - obuf,
-    isample,
-    isize,
-    options);
-  LZ4F_CHECK(oused);
-  obuf += oused;
-  oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options);
-  LZ4F_CHECK(oused);
-
-  return obuf - p->obuf;
-}
-
-size_t compress_default(bench_params_t *p) {
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_default(isample, obuf, isize, oend - obuf);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_extState(bench_params_t *p) {
-  LZ4_stream_t *ctx = p->ctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_fast_extState_fastReset(
-      ctx, isample, obuf, isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_hc(bench_params_t *p) {
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_HC(
-      isample, obuf, isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t compress_hc_extState(bench_params_t *p) {
-  LZ4_streamHC_t *hcctx = p->hcctx;
-  char *obuf = p->obuf;
-  size_t osize = p->osize;
-  const char* isample = p->isample;
-  size_t isize = p->isize;
-  int clevel = p->clevel;
-
-  char *oend = obuf + osize;
-  size_t oused;
-
-  oused = LZ4_compress_HC_extStateHC(
-      hcctx,
-      isample, obuf,
-      isize, oend - obuf, clevel);
-  obuf += oused;
-
-  return obuf - p->obuf;
-}
-
-size_t check_lz4(bench_params_t *p, size_t csize) {
-  (void)csize;
-  memset(p->checkbuf, 0xFF, p->checksize);
-  return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize,
-                                       p->dictbuf, p->dictsize)
-      && !memcmp(p->isample, p->checkbuf, p->isize);
-}
-
-size_t check_lz4f(bench_params_t *p, size_t csize) {
-  size_t cp = 0;
-  size_t dp = 0;
-  size_t dsize = p->checksize;
-  size_t cleft = csize;
-  size_t dleft = dsize;
-  size_t ret;
-  memset(p->checkbuf, 0xFF, p->checksize);
-  LZ4F_resetDecompressionContext(p->dctx);
-  do {
-    ret = LZ4F_decompress_usingDict(
-        p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft,
-        p->dictbuf, p->dictsize, NULL);
-    cp += cleft;
-    dp += dleft;
-    cleft = csize - cp;
-    dleft = dsize - dp;
-    if (LZ4F_isError(ret)) return 0;
-  } while (cleft);
-  return !memcmp(p->isample, p->checkbuf, p->isize);
-}
-
-
-uint64_t bench(
-    const char *bench_name,
-    size_t (*fun)(bench_params_t *),
-    size_t (*checkfun)(bench_params_t *, size_t),
-    bench_params_t *params
-) {
-  struct timespec start, end;
-  size_t i, osize = 0, o = 0;
-  size_t time_taken = 0;
-  uint64_t total_repetitions = 0;
-  uint64_t repetitions = 2;
-
-  if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0;
-
-  while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms
-    if (total_repetitions) {
-      repetitions = total_repetitions; // double previous
-    }
-
-    for (i = 0; i < repetitions; i++) {
-      params->iter = i;
-      if (params->num_ibuf == 1) {
-        params->isample = params->ibuf;
-      } else {
-        params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize));
-      }
-      o = fun(params);
-      if (!o) {
-        fprintf(
-            stderr,
-            "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n",
-            params->run_name, bench_name, params->clevel,
-            params->isize);
-        return 0;
-      }
-      osize += o;
-    }
-
-    if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0;
-
-    time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) -
-                 (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec);
-    total_repetitions += repetitions;
-  }
-
-  o = checkfun(params, o);
-  if (!o) {
-    fprintf(
-        stderr,
-        "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n",
-        params->run_name, bench_name, params->clevel,
-        params->isize);
-    return 0;
-  }
-
-  fprintf(
-      stderr,
-      "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n",
-      params->run_name, bench_name, params->clevel,
-      params->isize, osize / total_repetitions,
-      total_repetitions, time_taken, time_taken / total_repetitions,
-      ((double) 1000 * params->isize * total_repetitions) / time_taken
-  );
-
-  return time_taken;
-}
-
-int main(int argc, char *argv[]) {
-  char *run_name;
-
-  struct stat st;
-  size_t bytes_read;
-
-  const char *dict_fn;
-  size_t dict_size;
-  char *dict_buf;
-  FILE *dict_file;
-
-  const char *in_fn;
-  size_t in_size;
-  size_t num_in_buf;
-  size_t cur_in_buf;
-  char *in_buf;
-  FILE *in_file;
-
-  size_t out_size;
-  char *out_buf;
-
-  size_t check_size;
-  char *check_buf;
-
-  LZ4_stream_t *ctx;
-  LZ4_streamHC_t *hcctx;
-  LZ4F_cctx *cctx;
-  LZ4F_dctx *dctx;
-  LZ4F_CDict *cdict;
-  LZ4F_preferences_t prefs;
-  LZ4F_compressOptions_t options;
-
-  int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
-  unsigned int clevelidx;
-
-  bench_params_t params;
-
-  if (argc != 4) return 1;
-  run_name = argv[1];
-  dict_fn = argv[2];
-  in_fn = argv[3];
-
-  if (stat(dict_fn, &st)) return 1;
-  dict_size = st.st_size;
-  dict_buf = (char *)malloc(dict_size);
-  if (!dict_buf) return 1;
-  dict_file = fopen(dict_fn, "r");
-  bytes_read = fread(dict_buf, 1, dict_size, dict_file);
-  if (bytes_read != dict_size) return 1;
-
-  if (stat(in_fn, &st)) return 1;
-  in_size = st.st_size;
-  num_in_buf = 256 * 1024 * 1024 / in_size;
-  if (num_in_buf == 0) {
-    num_in_buf = 1;
-  }
-
-  in_buf = (char *)malloc(in_size * num_in_buf);
-  if (!in_buf) return 1;
-  in_file = fopen(in_fn, "r");
-  bytes_read = fread(in_buf, 1, in_size, in_file);
-  if (bytes_read != in_size) return 1;
-
-  for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) {
-    memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size);
-  }
-
-  check_size = in_size;
-  check_buf = (char *)malloc(check_size);
-  if (!check_buf) return 1;
-
-  memset(&prefs, 0, sizeof(prefs));
-  prefs.autoFlush = 1;
-  if (in_size < 64 * 1024)
-    prefs.frameInfo.blockMode = LZ4F_blockIndependent;
-  prefs.frameInfo.contentSize = in_size;
-
-  memset(&options, 0, sizeof(options));
-  options.stableSrc = 1;
-
-  out_size = LZ4F_compressFrameBound(in_size, &prefs);
-  if ((size_t)LZ4_compressBound(in_size) > out_size) {
-    out_size = LZ4_compressBound(in_size);
-  }
-  out_buf = (char *)malloc(out_size);
-  if (!out_buf) return 1;
-
-  if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1;
-  if (cctx == NULL) return 1;
-
-  ctx = LZ4_createStream();
-  if (ctx == NULL) return 1;
-
-  hcctx = LZ4_createStreamHC();
-  if (hcctx == NULL) return 1;
-
-  cdict = LZ4F_createCDict(dict_buf, dict_size);
-  if (!cdict) return 1;
-
-  fprintf(stderr, "dict  size: %zd\n", dict_size);
-  fprintf(stderr, "input size: %zd\n", in_size);
-
-  params.run_name = run_name;
-  params.ctx = ctx;
-  params.hcctx = hcctx;
-  params.cctx = cctx;
-  params.dctx = dctx;
-  params.dictbuf = dict_buf;
-  params.dictsize = dict_size;
-  params.obuf = out_buf;
-  params.osize = out_size;
-  params.ibuf = in_buf;
-  params.isize = in_size;
-  params.num_ibuf = num_in_buf;
-  params.checkbuf = check_buf;
-  params.checksize = check_size;
-  params.clevel = 1;
-  params.cdict = NULL;
-  params.prefs = &prefs;
-  params.options = &options;
-
-  for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) {
-    params.clevel = clevels[clevelidx];
-    params.prefs->compressionLevel = clevels[clevelidx];
-    params.cdict = NULL;
-
-    bench("LZ4_compress_default"         , compress_default    , check_lz4 , &params);
-    bench("LZ4_compress_fast_extState"   , compress_extState   , check_lz4 , &params);
-    bench("LZ4_compress_HC"              , compress_hc         , check_lz4 , &params);
-    bench("LZ4_compress_HC_extStateHC"   , compress_hc_extState, check_lz4 , &params);
-    bench("LZ4F_compressFrame"           , compress_frame      , check_lz4f, &params);
-    bench("LZ4F_compressBegin"           , compress_begin      , check_lz4f, &params);
-
-    params.cdict = cdict;
-
-    bench("LZ4F_compressFrame_usingCDict", compress_frame      , check_lz4f, &params);
-    bench("LZ4F_compressBegin_usingCDict", compress_begin      , check_lz4f, &params);
-  }
-
-  return 0;
-}

From 5ed1463bf4668a0e668679ae1e458aa73aa9a6ec Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 19 Apr 2018 21:00:19 -0400
Subject: [PATCH 199/257] Remove Debug Log Statements

---
 lib/lz4frame.c | 21 ---------------------
 lib/lz4hc.c    | 14 +++++---------
 2 files changed, 5 insertions(+), 30 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index d4d9397f8..844c8d1e6 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -95,18 +95,6 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
-#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
-#  include <stdio.h>
-static int g_debuglog_enable = 1;
-#  define DEBUGLOG(l, ...) {                                  \
-                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
-                    fprintf(stderr, __FILE__ ": ");           \
-                    fprintf(stderr, __VA_ARGS__);             \
-                    fprintf(stderr, " \n");                   \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
 
 /*-************************************
 *  Basic Types
@@ -469,7 +457,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
-    DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict);
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -492,7 +479,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 
 void LZ4F_freeCDict(LZ4F_CDict* cdict)
 {
-    DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict);
     if (cdict==NULL) return;  /* support free on NULL */
     FREEMEM(cdict->dictContent);
     LZ4_freeStream(cdict->fastCtx);
@@ -544,7 +530,6 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 static void LZ4F_applyCDict(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
-    DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict);
     if (level < LZ4HC_CLEVEL_MIN) {
         LZ4_resetStream_fast((LZ4_stream_t *)ctx);
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
@@ -571,8 +556,6 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     BYTE* headerStart;
 
-    DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict);
-
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     MEM_INIT(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
@@ -791,7 +774,6 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
-    DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
@@ -930,9 +912,6 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
     BYTE* dstPtr = dstStart;
 
     size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
-
-    DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr);
-
     if (LZ4F_isError(flushSize)) return flushSize;
     dstPtr += flushSize;
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a12f29808..4126ef81a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -88,7 +88,6 @@ typedef enum { noDictCtx, usingDictCtx } dictCtx_directive;
 **************************************/
 static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 {
-    DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4);
     MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
     MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
 }
@@ -96,7 +95,6 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
 static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 {
     uptrval startingOffset = hc4->end - hc4->base;
-    DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start);
     if (startingOffset > 1 GB) {
         LZ4HC_clearTables(hc4);
         startingOffset = 0;
@@ -830,11 +828,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
     LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    if (LZ4_streamHCPtr != NULL) {
-        LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-        LZ4_streamHCPtr->internal_donotuse.base = NULL;
-        LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
-    }
+    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
+    LZ4_streamHCPtr->internal_donotuse.base = NULL;
+    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
     return LZ4_streamHCPtr;
 }
 
@@ -881,8 +877,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
-    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
@@ -897,7 +893,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;

From 2be3905fa4974fd20fea4d9eda5a0030c9d19e93 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 18:41:41 -0400
Subject: [PATCH 200/257] Integrate lz4frame_static.h Declarations into
 lz4frame.h

---
 lib/lz4frame.h        | 120 ++++++++++++++++++++++++++++++++++++++++++
 lib/lz4frame_static.h | 117 ++--------------------------------------
 2 files changed, 124 insertions(+), 113 deletions(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index e80b1a1da..bd715bd47 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -409,3 +409,123 @@ LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always su
 #endif
 
 #endif  /* LZ4F_H_09782039843 */
+
+#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843)
+#define LZ4F_H_STATIC_09782039843
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* These declarations are not stable and may change in the future. They are
+ * therefore only safe to depend on when the caller is statically linked
+ * against the library. To access their declarations, define
+ * LZ4F_STATIC_LINKING_ONLY.
+ *
+ * There is a further protection mechanism where these symbols aren't published
+ * into shared/dynamic libraries. You can override this behavior and force
+ * them to be published by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. Use at
+ * your own risk.
+ */
+#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
+#define LZ4FLIB_STATIC_API LZ4FLIB_API
+#else
+#define LZ4FLIB_STATIC_API
+#endif
+
+
+/* ---   Error List   --- */
+#define LZ4F_LIST_ERRORS(ITEM) \
+        ITEM(OK_NoError) \
+        ITEM(ERROR_GENERIC) \
+        ITEM(ERROR_maxBlockSize_invalid) \
+        ITEM(ERROR_blockMode_invalid) \
+        ITEM(ERROR_contentChecksumFlag_invalid) \
+        ITEM(ERROR_compressionLevel_invalid) \
+        ITEM(ERROR_headerVersion_wrong) \
+        ITEM(ERROR_blockChecksum_invalid) \
+        ITEM(ERROR_reservedFlag_set) \
+        ITEM(ERROR_allocation_failed) \
+        ITEM(ERROR_srcSize_tooLarge) \
+        ITEM(ERROR_dstMaxSize_tooSmall) \
+        ITEM(ERROR_frameHeader_incomplete) \
+        ITEM(ERROR_frameType_unknown) \
+        ITEM(ERROR_frameSize_wrong) \
+        ITEM(ERROR_srcPtr_wrong) \
+        ITEM(ERROR_decompressionFailed) \
+        ITEM(ERROR_headerChecksum_invalid) \
+        ITEM(ERROR_contentChecksum_invalid) \
+        ITEM(ERROR_frameDecoding_alreadyStarted) \
+        ITEM(ERROR_maxCode)
+
+#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
+
+/* enum list is exposed, to handle specific errors */
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
+
+LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
+
+
+
+/**********************************
+ *  Bulk processing dictionary API
+ *********************************/
+typedef struct LZ4F_CDict_s LZ4F_CDict;
+
+/*! LZ4_createCDict() :
+ *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
+LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
+
+
+/*! LZ4_compressFrame_usingCDict() :
+ *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ *  cctx must point to a context created by LZ4F_createCompressionContext().
+ *  If cdict==NULL, compress without a dictionary.
+ *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ *  If this condition is not respected, function will fail (@return an errorCode).
+ *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ *  but it's not recommended, as it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer.
+ *           or an error code if it fails (can be tested using LZ4F_isError()) */
+LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dst, size_t dstCapacity,
+    const void* src, size_t srcSize,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* preferencesPtr);
+
+
+/*! LZ4F_compressBegin_usingCDict() :
+ *  Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+ *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * `prefsPtr` is optional : you may provide NULL as argument,
+ *  however, it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer for the header,
+ *           or an error code (which can be tested using LZ4F_isError()) */
+LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dstBuffer, size_t dstCapacity,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* prefsPtr);
+
+
+/*! LZ4F_decompress_usingDict() :
+ *  Same as LZ4F_decompress(), using a predefined dictionary.
+ *  Dictionary is used "in place", without any preprocessing.
+ *  It must remain accessible throughout the entire frame decoding. */
+LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
+    LZ4F_dctx* dctxPtr,
+    void* dstBuffer, size_t* dstSizePtr,
+    const void* srcBuffer, size_t* srcSizePtr,
+    const void* dict, size_t dictSize,
+    const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif  /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index be587e6e2..925a2c5c3 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -36,121 +36,12 @@
 #ifndef LZ4FRAME_STATIC_H_0398209384
 #define LZ4FRAME_STATIC_H_0398209384
 
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/* lz4frame_static.h should be used solely in the context of static linking.
- * It contains definitions which are not stable and may change in the future.
- * Never use it in the context of DLL linking.
- *
- * Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at
- * your own risk.
+/* The declarations that formerly were made here have been merged into
+ * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward,
+ * it is recommended to simply include that header directly.
  */
-#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
-#define LZ4FLIB_STATIC_API LZ4FLIB_API
-#else
-#define LZ4FLIB_STATIC_API
-#endif
-
 
-/* ---   Dependency   --- */
+#define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
 
-
-/* ---   Error List   --- */
-#define LZ4F_LIST_ERRORS(ITEM) \
-        ITEM(OK_NoError) \
-        ITEM(ERROR_GENERIC) \
-        ITEM(ERROR_maxBlockSize_invalid) \
-        ITEM(ERROR_blockMode_invalid) \
-        ITEM(ERROR_contentChecksumFlag_invalid) \
-        ITEM(ERROR_compressionLevel_invalid) \
-        ITEM(ERROR_headerVersion_wrong) \
-        ITEM(ERROR_blockChecksum_invalid) \
-        ITEM(ERROR_reservedFlag_set) \
-        ITEM(ERROR_allocation_failed) \
-        ITEM(ERROR_srcSize_tooLarge) \
-        ITEM(ERROR_dstMaxSize_tooSmall) \
-        ITEM(ERROR_frameHeader_incomplete) \
-        ITEM(ERROR_frameType_unknown) \
-        ITEM(ERROR_frameSize_wrong) \
-        ITEM(ERROR_srcPtr_wrong) \
-        ITEM(ERROR_decompressionFailed) \
-        ITEM(ERROR_headerChecksum_invalid) \
-        ITEM(ERROR_contentChecksum_invalid) \
-        ITEM(ERROR_frameDecoding_alreadyStarted) \
-        ITEM(ERROR_maxCode)
-
-#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
-
-/* enum list is exposed, to handle specific errors */
-typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
-
-LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
-
-
-
-/**********************************
- *  Bulk processing dictionary API
- *********************************/
-typedef struct LZ4F_CDict_s LZ4F_CDict;
-
-/*! LZ4_createCDict() :
- *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
- *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
-LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
-LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
-
-
-/*! LZ4_compressFrame_usingCDict() :
- *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
- *  cctx must point to a context created by LZ4F_createCompressionContext().
- *  If cdict==NULL, compress without a dictionary.
- *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
- *  If this condition is not respected, function will fail (@return an errorCode).
- *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
- *  but it's not recommended, as it's the only way to provide dictID in the frame header.
- * @return : number of bytes written into dstBuffer.
- *           or an error code if it fails (can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
-    LZ4F_cctx* cctx,
-    void* dst, size_t dstCapacity,
-    const void* src, size_t srcSize,
-    const LZ4F_CDict* cdict,
-    const LZ4F_preferences_t* preferencesPtr);
-
-
-/*! LZ4F_compressBegin_usingCDict() :
- *  Inits streaming dictionary compression, and writes the frame header into dstBuffer.
- *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
- * `prefsPtr` is optional : you may provide NULL as argument,
- *  however, it's the only way to provide dictID in the frame header.
- * @return : number of bytes written into dstBuffer for the header,
- *           or an error code (which can be tested using LZ4F_isError()) */
-LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
-    LZ4F_cctx* cctx,
-    void* dstBuffer, size_t dstCapacity,
-    const LZ4F_CDict* cdict,
-    const LZ4F_preferences_t* prefsPtr);
-
-
-/*! LZ4F_decompress_usingDict() :
- *  Same as LZ4F_decompress(), using a predefined dictionary.
- *  Dictionary is used "in place", without any preprocessing.
- *  It must remain accessible throughout the entire frame decoding. */
-LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
-    LZ4F_dctx* dctxPtr,
-    void* dstBuffer, size_t* dstSizePtr,
-    const void* srcBuffer, size_t* srcSizePtr,
-    const void* dict, size_t dictSize,
-    const LZ4F_decompressOptions_t* decompressOptionsPtr);
-
-
-#if defined (__cplusplus)
-}
-#endif
-
 #endif /* LZ4FRAME_STATIC_H_0398209384 */

From 2dfc7cbe826342905faa479175f6f050e8445aff Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 20 Apr 2018 20:55:38 -0400
Subject: [PATCH 201/257] Change Over Includes in the Project

---
 lib/lz4frame.c    | 3 ++-
 programs/lz4io.c  | 2 +-
 tests/frametest.c | 3 ++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 844c8d1e6..b616463de 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -73,7 +73,8 @@ You can contact the author at :
 /*-************************************
 *  Includes
 **************************************/
-#include "lz4frame_static.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
 #define LZ4_STATIC_LINKING_ONLY
 #include "lz4.h"
 #define LZ4_HC_STATIC_LINKING_ONLY
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 6d0d0d018..ccf4fa123 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -56,8 +56,8 @@
 #include "lz4io.h"
 #include "lz4.h"       /* still required for legacy format */
 #include "lz4hc.h"     /* still required for legacy format */
+#define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
-#include "lz4frame_static.h"
 
 
 /*****************************
diff --git a/tests/frametest.c b/tests/frametest.c
index 74d9c88fe..01f13007a 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -41,7 +41,8 @@
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock(), CLOCKS_PER_SEC */
 #include <assert.h>
-#include "lz4frame_static.h"
+#define LZ4F_STATIC_LINKING_ONLY
+#include "lz4frame.h"
 #include "lz4.h"        /* LZ4_VERSION_STRING */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */

From 27c6eec18d38d78f058e87e415ac979129342861 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Tue, 24 Apr 2018 18:50:03 -0400
Subject: [PATCH 202/257] Multiply-Include Header to Check Guard Macro
 Correctness

---
 tests/frametest.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/frametest.c b/tests/frametest.c
index 01f13007a..7d69ff79a 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -41,8 +41,11 @@
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock(), CLOCKS_PER_SEC */
 #include <assert.h>
+#include "lz4frame.h"   /* include multiple times to test correctness/safety */
+#include "lz4frame.h"
 #define LZ4F_STATIC_LINKING_ONLY
 #include "lz4frame.h"
+#include "lz4frame.h"
 #include "lz4.h"        /* LZ4_VERSION_STRING */
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */

From ff9b4cf82678f9643d256129d06098b692072584 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Wed, 25 Apr 2018 01:40:12 +0300
Subject: [PATCH 203/257] lz4_Block_format.md: clarify on short inputs and
 restrictions

It occurred to me that the formula "The last 5 bytes are always
literals", on the list of "assumptions made by the decoder", is
remarkably ambiguous.  Suppose the decoder is presented with 5 bytes.
Are they literals?  It may seem that the decoder degenerates
to memcpy on short inputs.  But of course the answer is no,
so the formula needs some clarification.

Parsing restrictions should be explained as well, otherwise they look
like arbitrary numbers.  The 5-byte restriction has been mentioned
recently in connection with the shortcut in LZ4_decompress_generic,
so I add that.  The second restriction is left to be explained
by the author.

I also took the liberty to explain that empty inputs "are either
unrepresentable or can be represented with a null byte".  This wording
may actually have some merit: it leaves for the implementation,
as opposed to the spec, to decide whether the encoder can compress
empty inputs, and whether the decoder can produce an empty output
(which the implementation should further clarify).
---
 doc/lz4_Block_format.md | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md
index 4e39b41f5..dd4c91bec 100644
--- a/doc/lz4_Block_format.md
+++ b/doc/lz4_Block_format.md
@@ -109,15 +109,24 @@ Parsing restrictions
 There are specific parsing rules to respect in order to remain compatible
 with assumptions made by the decoder :
 
-1. The last 5 bytes are always literals
+1. The last 5 bytes are always literals.  In other words, the last five bytes
+   from the uncompressed input (or all bytes, if the input has less than five
+   bytes) must be encoded as literals on behalf of the last sequence.
+   The last sequence is incomplete, and stops right after the literals.
 2. The last match must start at least 12 bytes before end of block.
    Consequently, a block with less than 13 bytes cannot be compressed.
 
 These rules are in place to ensure that the decoder
 will never read beyond the input buffer, nor write beyond the output buffer.
 
-Note that the last sequence is also incomplete,
-and stops right after literals.
+1. To copy literals from a non-last sequence, an 8-byte copy instruction
+   can always be safely issued (without reading past the input), because
+   the literals are followed by a 2-byte offset, and the last sequence
+   is at least 1+5 bytes long.
+2. TODO: explain the benefits of the second restriction.
+
+Empty inputs are either unrepresentable or can be represented with a null byte,
+which can be interpreted as a token without literals and without a match.
 
 
 Additional notes

From b4eda8d08f307dfc3cbd0d06baea4e6c581b70de Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Mon, 23 Apr 2018 08:37:44 +0300
Subject: [PATCH 204/257] lz4.c: refactor the decoding routines

I noticed that LZ4_decompress_generic is sometimes instantiated with
identical set of parameters, or (what's worse) with a subtly different
sets of parameters.  For example, LZ4_decompress_fast_withPrefix64k is
instantiated as follows:

    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize,
		full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);

while the equivalent withPrefix64k call in LZ4_decompress_usingDict_generic
passes 0 for the last argument instead of 64 KB.  It turns out that there
is no difference in this case: if you change 64 KB to 0 KB in
LZ4_decompress_fast_withPrefix64k, you get the same binary code.

Moreover, because it's been clarified that LZ4_decompress_fast doesn't
check match offsets, it is now obvious that both of these fast/withPrefix64k
instantiations are simply redundant.  Exactly because LZ4_decompress_fast
doesn't check offsets, it serves well with any prefixed dictionary.

There's a difference, though, with LZ4_decompress_safe_withPrefix64k.
It also passes 64 KB as the last argument, and if you change that to 0,
as in LZ4_decompress_usingDict_generic, you get a completely different
binary code.  It seems that passing 0 enables offset checking:

    const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));

However, the resulting code seems to run a bit faster.  How come
enabling extra checks can make the code run faster?  Curiouser and
curiouser!  This needs extra study.  Currently I take the view that
the dictSize should be set to non-zero when nothing else will do,
i.e. when passing the external dictionary via dictStart.  Otherwise,
lowPrefix betrays just enough information about the dictionary.

    * * *

Anyway, with this change, I instantiate all the necessary cases as
functions with distinctive names, which also take fewer arguments and
are therefore less error-prone.  I also make the functions non-inline.
(The compiler won't inline the functions because they are used more than
once.  Hence I attach LZ4_FORCE_O2_GCC_PPC64LE to the instances while
removing from the callers.)  The number of instances is now is reduced
from 18 (safe+fast+partial+4*continue+4*prefix+4*dict+2*prefix64+forceExtDict)
down to 7 (safe+fast+partial+2*prefix+2*dict).  The size of the code is
not the only issue here.  Separate helper function are much more
amenable to profile-guided optimization: it is enough to profile only
a few basic functions, while the other less-often used functions, such
as LZ4_decompress_*_continue, will benefit automatically.

This is the list of LZ4_decompress* functions in liblz4.so, sorted by size.
Exported functions are marked with a capital T.

$ nm -S lib/liblz4.so |grep -wi T |grep LZ4_decompress |sort -k2
0000000000016260 0000000000000005 T LZ4_decompress_fast_withPrefix64k
0000000000016dc0 0000000000000025 T LZ4_decompress_fast_usingDict
0000000000016d80 0000000000000040 T LZ4_decompress_safe_usingDict
0000000000016d10 000000000000006b T LZ4_decompress_fast_continue
0000000000016c70 000000000000009f T LZ4_decompress_safe_continue
00000000000156c0 000000000000059c T LZ4_decompress_fast
0000000000014a90 00000000000005fa T LZ4_decompress_safe
0000000000015c60 00000000000005fa T LZ4_decompress_safe_withPrefix64k
0000000000002280 00000000000005fa t LZ4_decompress_safe_withSmallPrefix
0000000000015090 000000000000062f T LZ4_decompress_safe_partial
0000000000002880 00000000000008ea t LZ4_decompress_fast_extDict
0000000000016270 0000000000000993 t LZ4_decompress_safe_forceExtDict
---
 lib/lz4.c | 132 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 79 insertions(+), 53 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index bb6b619ba..2fad34f4f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -92,6 +92,7 @@
 *  Dependency
 **************************************/
 #define LZ4_STATIC_LINKING_ONLY
+#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
 #include "lz4.h"
 /* see also "memory routines" below */
 
@@ -1666,6 +1667,8 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 }
 
 
+/*===== Instantiate the API decoding functions. =====*/
+
 LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
 {
@@ -1687,9 +1690,63 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 {
     return LZ4_decompress_generic(source, dest, 0, originalSize,
                                   endOnOutputSize, full, 0, withPrefix64k,
-                                  (BYTE*)(dest - 64 KB), NULL, 64 KB);
+                                  (BYTE*)dest - 64 KB, NULL, 0);
+}
+
+/*===== Instantiate a few more decoding cases, used more than once. =====*/
+
+LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */
+int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, withPrefix64k,
+                                  (BYTE*)dest - 64 KB, NULL, 0);
+}
+
+/* Another obsolete API function, paired with the previous one. */
+int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
+{
+    /* LZ4_decompress_fast doesn't validate match offsets,
+     * and thus serves well with any prefixed dictionary. */
+    return LZ4_decompress_fast(source, dest, originalSize);
+}
+
+LZ4_FORCE_O2_GCC_PPC64LE
+static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                               size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, noDict,
+                                  (BYTE*)dest-dictSize, NULL, 0);
 }
 
+LZ4_FORCE_INLINE
+int LZ4_decompress_safe_withPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                   size_t dictSize)
+{
+    if (dictSize >= 64 KB - 1)
+        return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
+    return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+}
+
+LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */
+#define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict
+int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                const char* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+}
+
+LZ4_FORCE_O2_GCC_PPC64LE
+static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,
+                                       const char* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, 0, originalSize,
+                                  endOnOutputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+}
 
 /*===== streaming decompression functions =====*/
 
@@ -1730,25 +1787,26 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
     If it's not possible, save the relevant part of decoded data into a safe buffer,
     and indicate where it stands using LZ4_setStreamDecode()
 */
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
-    if (lz4sd->prefixEnd == (BYTE*)dest) {
-        result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
-                                        endOnInputSize, full, 0,
-                                        usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+    if (lz4sd->prefixSize == 0) {
+        result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
+        if (result <= 0) return result;
+        lz4sd->prefixSize = result;
+        lz4sd->prefixEnd = (BYTE*)dest + result;
+    } else if (lz4sd->prefixEnd == (BYTE*)dest) {
+        result = LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize);
         if (result <= 0) return result;
         lz4sd->prefixSize += result;
         lz4sd->prefixEnd  += result;
     } else {
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
-        result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
-                                        endOnInputSize, full, 0,
-                                        usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
+        result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize,
+                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = result;
         lz4sd->prefixEnd  = (BYTE*)dest + result;
@@ -1757,25 +1815,21 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
     return result;
 }
 
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
-    if (lz4sd->prefixEnd == (BYTE*)dest) {
-        result = LZ4_decompress_generic(source, dest, 0, originalSize,
-                                        endOnOutputSize, full, 0,
-                                        usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+    if (lz4sd->prefixSize == 0 || lz4sd->prefixEnd == (BYTE*)dest) {
+        result = LZ4_decompress_fast(source, dest, originalSize);
         if (result <= 0) return result;
         lz4sd->prefixSize += originalSize;
         lz4sd->prefixEnd  += originalSize;
     } else {
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
-        result = LZ4_decompress_generic(source, dest, 0, originalSize,
-                                        endOnOutputSize, full, 0,
-                                        usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
+        result = LZ4_decompress_fast_extDict(source, dest, originalSize,
+                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = originalSize;
         lz4sd->prefixEnd  = (BYTE*)dest + originalSize;
@@ -1792,36 +1846,20 @@ Advanced decoding functions :
     the dictionary must be explicitly provided within parameters
 */
 
-LZ4_FORCE_O2_GCC_PPC64LE
-LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
-{
-    if (dictSize==0)
-        return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
-    if (dictStart+dictSize == dest) {
-        if (dictSize >= (int)(64 KB - 1))
-            return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0);
-        return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
-    }
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
-}
-
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
 {
-    return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
+    if (dictSize==0)
+        return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
+    if (dictStart+dictSize == dest)
+        return LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+    return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
 }
 
-LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
 {
-    return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
-}
-
-/* debug function */
-LZ4_FORCE_O2_GCC_PPC64LE
-int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
-{
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
+    if (dictSize==0 || dictStart+dictSize == dest)
+        return LZ4_decompress_fast(source, dest, originalSize);
+    return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize);
 }
 
 
@@ -1892,16 +1930,4 @@ char* LZ4_slideInputBuffer (void* state)
     return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary;
 }
 
-/* Obsolete streaming decompression functions */
-
-int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
-{
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
-}
-
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
-{
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
-}
-
 #endif   /* LZ4_COMMONDEFS_ONLY */

From bd92689798292f8ab8d2b48f31cd4b49bfa6d87b Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Wed, 25 Apr 2018 06:42:57 -0700
Subject: [PATCH 205/257] minor edit of block format

clarifying parsing restrictions near end of block.
---
 doc/lz4_Block_format.md | 60 ++++++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md
index dd4c91bec..543873056 100644
--- a/doc/lz4_Block_format.md
+++ b/doc/lz4_Block_format.md
@@ -1,6 +1,6 @@
 LZ4 Block Format Description
 ============================
-Last revised: 2015-05-07.
+Last revised: 2018-04-25.
 Author : Yann Collet
 
 
@@ -29,8 +29,8 @@ An LZ4 compressed block is composed of sequences.
 A sequence is a suite of literals (not-compressed bytes),
 followed by a match copy.
 
-Each sequence starts with a token.
-The token is a one byte value, separated into two 4-bits fields.
+Each sequence starts with a `token`.
+The `token` is a one byte value, separated into two 4-bits fields.
 Therefore each field ranges from 0 to 15.
 
 
@@ -42,46 +42,46 @@ If it is 15, then we need to add some more bytes to indicate the full length.
 Each additional byte then represent a value from 0 to 255,
 which is added to the previous value to produce a total length.
 When the byte value is 255, another byte is output.
-There can be any number of bytes following the token. There is no "size limit".
+There can be any number of bytes following `token`. There is no "size limit".
 (Side note : this is why a not-compressible input block is expanded by 0.4%).
 
-Example 1 : A length of 48 will be represented as :
+Example 1 : A literal length of 48 will be represented as :
 
   - 15 : value for the 4-bits High field
   - 33 : (=48-15) remaining length to reach 48
 
-Example 2 : A length of 280 will be represented as :
+Example 2 : A literal length of 280 will be represented as :
 
   - 15  : value for the 4-bits High field
   - 255 : following byte is maxed, since 280-15 >= 255
   - 10  : (=280 - 15 - 255) ) remaining length to reach 280
 
-Example 3 : A length of 15 will be represented as :
+Example 3 : A literal length of 15 will be represented as :
 
   - 15 : value for the 4-bits High field
   - 0  : (=15-15) yes, the zero must be output
 
-Following the token and optional length bytes, are the literals themselves.
+Following `token` and optional length bytes, are the literals themselves.
 They are exactly as numerous as previously decoded (length of literals).
 It's possible that there are zero literal.
 
 
 Following the literals is the match copy operation.
 
-It starts by the offset.
+It starts by the `offset`.
 This is a 2 bytes value, in little endian format
 (the 1st byte is the "low" byte, the 2nd one is the "high" byte).
 
-The offset represents the position of the match to be copied from.
+The `offset` represents the position of the match to be copied from.
 1 means "current position - 1 byte".
-The maximum offset value is 65535, 65536 cannot be coded.
+The maximum `offset` value is 65535, 65536 cannot be coded.
 Note that 0 is an invalid value, not used.
 
-Then we need to extract the match length.
+Then we need to extract the `matchlength`.
 For this, we use the second token field, the low 4-bits.
 Value, obviously, ranges from 0 to 15.
 However here, 0 means that the copy operation will be minimal.
-The minimum length of a match, called minmatch, is 4.
+The minimum length of a match, called `minmatch`, is 4.
 As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes.
 Similar to literal length, on reaching the highest possible value (15),
 we output additional bytes, one at a time, with values ranging from 0 to 255.
@@ -90,18 +90,18 @@ A 255 value means there is another byte to read and add.
 There is no limit to the number of optional bytes that can be output this way.
 (This points towards a maximum achievable compression ratio of about 250).
 
-Decoding the matchlength reaches the end of current sequence.
+Decoding the `matchlength` reaches the end of current sequence.
 Next byte will be the start of another sequence.
 But before moving to next sequence,
 it's time to use the decoded match position and length.
-The decoder copies matchlength bytes from match position to current position.
+The decoder copies `matchlength` bytes from match position to current position.
 
-In some cases, matchlength is larger than offset.
-Therefore, match pos + match length > current pos,
+In some cases, `matchlength` is larger than `offset`.
+Therefore, `match_pos + matchlength > current_pos`,
 which means that later bytes to copy are not yet decoded.
 This is called an "overlap match", and must be handled with special care.
-The most common case is an offset of 1,
-meaning the last byte is repeated matchlength times.
+A common case is an offset of 1,
+meaning the last byte is repeated `matchlength` times.
 
 
 Parsing restrictions
@@ -114,19 +114,23 @@ with assumptions made by the decoder :
    bytes) must be encoded as literals on behalf of the last sequence.
    The last sequence is incomplete, and stops right after the literals.
 2. The last match must start at least 12 bytes before end of block.
-   Consequently, a block with less than 13 bytes cannot be compressed.
+   The last match is part of the penultimate sequence,
+   since the last sequence stops right after literals.
+   Note that, as a consequence, blocks < 13 bytes cannot be compressed.
 
 These rules are in place to ensure that the decoder
-will never read beyond the input buffer, nor write beyond the output buffer.
+can speculatively execute copy instructions
+without ever reading nor writing beyond provided I/O buffers.
 
 1. To copy literals from a non-last sequence, an 8-byte copy instruction
-   can always be safely issued (without reading past the input), because
-   the literals are followed by a 2-byte offset, and the last sequence
-   is at least 1+5 bytes long.
-2. TODO: explain the benefits of the second restriction.
-
-Empty inputs are either unrepresentable or can be represented with a null byte,
-which can be interpreted as a token without literals and without a match.
+   can always be safely issued (without reading past the input),
+   because literals are followed by a 2-byte offset,
+   and last sequence is at least 1+5 bytes long.
+2. Similarly, a match operation can speculatively copy up to 12 bytes
+   while remaining within output buffer boundaries.
+
+Empty inputs can be represented with a zero byte,
+interpreted as a token without literals and without a match.
 
 
 Additional notes

From 5603d30f81c99fbe55f48d953e3df728720ff982 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Thu, 26 Apr 2018 07:46:26 +0300
Subject: [PATCH 206/257] lz4.c: fixed the LZ4_decompress_safe_continue case

The previous change broke decoding with a ring buffer.  That's because
I didn't realize that the "double dictionary mode" was possible, i.e.
that the decoding routine can look both at the first part of the
dictionary passed as prefix and the second part passed via dictStart+dictSize.

So this change introduces the LZ4_decompress_safe_doubleDict helper,
which handles this "double dictionary" situation.  (This is a bit of
a misnomer, there is only one dictionary, but I can't think of a better
name, and perhaps the designation is not all too bad.)  The helper is
used only once, in LZ4_decompress_safe_continue, it should be inlined
with LZ4_FORCE_O2_GCC_PPC64LE attached to LZ4_decompress_safe_continue.

(Also, in the helper functions, I change the dictStart parameter type
to "const void*", to avoid a cast when calling helpers.  In the helpers,
the upcast to "BYTE*" is still required, for compatibility with C++.)

So this fixes the case of LZ4_decompress_safe_continue, and I'm
surprised by the fact that the fuzzer is now happy and does not detect
a similar problem with LZ4_decompress_fast_continue.  So before fixing
LZ4_decompress_fast_continue, the next logical step is to enhance
the fuzzer.
---
 lib/lz4.c         | 53 ++++++++++++++++++++++++++++++++---------------
 tests/fullbench.c |  2 +-
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 2fad34f4f..eb3da213c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1713,26 +1713,17 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int origin
 
 LZ4_FORCE_O2_GCC_PPC64LE
 static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
-                                               size_t dictSize)
+                                               size_t prefixSize)
 {
     return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
                                   endOnInputSize, full, 0, noDict,
-                                  (BYTE*)dest-dictSize, NULL, 0);
-}
-
-LZ4_FORCE_INLINE
-int LZ4_decompress_safe_withPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize,
-                                   size_t dictSize)
-{
-    if (dictSize >= 64 KB - 1)
-        return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
-    return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+                                  (BYTE*)dest-prefixSize, NULL, 0);
 }
 
 LZ4_FORCE_O2_GCC_PPC64LE /* Exported under another name, for tests/fullbench.c */
 #define LZ4_decompress_safe_extDict LZ4_decompress_safe_forceExtDict
 int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
-                                const char* dictStart, size_t dictSize)
+                                const void* dictStart, size_t dictSize)
 {
     return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
                                   endOnInputSize, full, 0, usingExtDict,
@@ -1741,13 +1732,26 @@ int LZ4_decompress_safe_extDict(const char* source, char* dest, int compressedSi
 
 LZ4_FORCE_O2_GCC_PPC64LE
 static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize,
-                                       const char* dictStart, size_t dictSize)
+                                       const void* dictStart, size_t dictSize)
 {
     return LZ4_decompress_generic(source, dest, 0, originalSize,
                                   endOnOutputSize, full, 0, usingExtDict,
                                   (BYTE*)dest, (const BYTE*)dictStart, dictSize);
 }
 
+/* The "double dictionary" mode, for use with e.g. ring buffers: the first part
+ * of the dictionary is passed as prefix, and the second via dictStart + dictSize.
+ * These routines are used only once, in LZ4_decompress_*_continue().
+ */
+LZ4_FORCE_INLINE
+int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize,
+                                   size_t prefixSize, const void* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
+                                  endOnInputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
+}
+
 /*===== streaming decompression functions =====*/
 
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
@@ -1787,26 +1791,38 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
     If it's not possible, save the relevant part of decoded data into a safe buffer,
     and indicate where it stands using LZ4_setStreamDecode()
 */
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
     if (lz4sd->prefixSize == 0) {
+        /* The first call, no dictionary yet. */
+        assert(lz4sd->extDictSize == 0);
         result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = result;
         lz4sd->prefixEnd = (BYTE*)dest + result;
     } else if (lz4sd->prefixEnd == (BYTE*)dest) {
-        result = LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, lz4sd->prefixSize);
+        /* They're rolling the current segment. */
+        if (lz4sd->prefixSize >= 64 KB - 1)
+            result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
+        else if (lz4sd->extDictSize == 0)
+            result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize,
+                                                         lz4sd->prefixSize);
+        else
+            result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize,
+                                                    lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize += result;
         lz4sd->prefixEnd  += result;
     } else {
+        /* The buffer wraps around, or they're switching to another buffer. */
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
         result = LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize,
-                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
+                                             lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = result;
         lz4sd->prefixEnd  = (BYTE*)dest + result;
@@ -1850,8 +1866,11 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed
 {
     if (dictSize==0)
         return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize);
-    if (dictStart+dictSize == dest)
-        return LZ4_decompress_safe_withPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+    if (dictStart+dictSize == dest) {
+        if (dictSize >= 64 KB - 1)
+            return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize);
+        return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize);
+    }
     return LZ4_decompress_safe_extDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize);
 }
 
diff --git a/tests/fullbench.c b/tests/fullbench.c
index f489392d1..1939aeb85 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -277,7 +277,7 @@ static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int in
 }
 
 #ifndef LZ4_DLL_IMPORT
-extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const char* dict, int dictSize);
+extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
 
 static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
 {

From 1148173c5dd1ad9b672c63fd0da110e3c2d66274 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 13:01:59 -0700
Subject: [PATCH 207/257] introduced ability to parse for decompression speed

triggered through an enum.

Now, it's still necessary to properly expose this capability
all the way up to the cli.
---
 lib/lz4hc.c | 53 ++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4126ef81a..3593da774 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -199,6 +199,7 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
 }
 
 typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
+typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e;
 
 LZ4_FORCE_INLINE int
 LZ4HC_InsertAndGetWiderMatch (
@@ -211,7 +212,8 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE** startpos,
     const int maxNbAttempts,
     const int patternAnalysis,
-    const dictCtx_directive dict)
+    const dictCtx_directive dict,
+    const HCfavor_e favorDecSpeed)
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
@@ -240,7 +242,10 @@ LZ4HC_InsertAndGetWiderMatch (
     while ((matchIndex>=lowLimit) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
-        if (matchIndex >= dictLimit) {
+        assert(matchIndex < ipIndex);
+        if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
+            /* do nothing */
+        } else if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
             assert(longest >= 1);
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
@@ -326,14 +331,12 @@ LZ4HC_InsertAndGetWiderMatch (
                 }
             }
 
-            {
-                U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
+            {   U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex);
                 dictMatchIndex -= nextOffset;
                 matchIndex -= nextOffset;
             }
         }
     }
-
     return longest;
 }
 
@@ -349,7 +352,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
 }
 
 
@@ -484,7 +487,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (ip+ml <= mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, patternAnalysis, dict);
+                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
         else
             ml2 = ml;
 
@@ -531,7 +534,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (start2 + ml2 <= mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, patternAnalysis, dict);
+                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
         else
             ml3 = ml2;
 
@@ -651,12 +654,14 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     return 0;
 }
 
+
 static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
     const char* const source, char* dst,
     int* srcSizePtr, int dstCapacity,
     int const nbSearches, size_t sufficient_len,
     const limitedOutput_directive limit, int const fullUpdate,
-    const dictCtx_directive dict);
+    const dictCtx_directive dict,
+    HCfavor_e favorDecSpeed);
 
 
 LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
@@ -711,7 +716,9 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
         return LZ4HC_compress_optimal(ctx,
                             src, dst, srcSizePtr, dstCapacity,
                             cParam.nbSearches, cParam.targetLength, limit,
-                            cLevel == LZ4HC_CLEVEL_MAX, dict);  /* ultra mode */
+                            cLevel == LZ4HC_CLEVEL_MAX,   /* ultra mode */
+                            dict,
+                            favorDecompressionSpeed);
     }
 }
 
@@ -1082,23 +1089,26 @@ LZ4_FORCE_INLINE LZ4HC_match_t
 LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
                       const BYTE* ip, const BYTE* const iHighLimit,
                       int minLen, int nbSearches,
-                      const dictCtx_directive dict)
+                      const dictCtx_directive dict,
+                      const HCfavor_e favorDecSpeed)
 {
     LZ4HC_match_t match = { 0 , 0 };
     const BYTE* matchPtr = NULL;
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
                                 ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */, dict);
+                                nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
     if (matchLength <= minLen) return match;
+    if (favorDecSpeed) {
+        if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
+    }
     match.len = matchLength;
     match.off = (int)(ip-matchPtr);
     return match;
 }
 
-
 static int LZ4HC_compress_optimal (
     LZ4HC_CCtx_internal* ctx,
     const char* const source,
@@ -1109,7 +1119,8 @@ static int LZ4HC_compress_optimal (
     size_t sufficient_len,
     const limitedOutput_directive limit,
     int const fullUpdate,
-    const dictCtx_directive dict
+    const dictCtx_directive dict,
+    HCfavor_e favorDecSpeed
     )
 {
 #define TRAILING_LITERALS 3
@@ -1125,6 +1136,7 @@ static int LZ4HC_compress_optimal (
     BYTE* oend = op + dstCapacity;
 
     /* init */
+    favorDecSpeed = favorCompressionRatio;
     DEBUGLOG(5, "LZ4HC_compress_optimal");
     *srcSizePtr = 0;
     if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
@@ -1137,7 +1149,7 @@ static int LZ4HC_compress_optimal (
          int best_mlen, best_off;
          int cur, last_match_pos = 0;
 
-         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict);
+         LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed);
          if (firstMatch.len==0) { ip++; continue; }
 
          if ((size_t)firstMatch.len > sufficient_len) {
@@ -1207,10 +1219,10 @@ static int LZ4HC_compress_optimal (
 
              DEBUGLOG(7, "search at rPos:%u", cur);
              if (fullUpdate)
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed);
              else
                  /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict);
+                 newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed);
              if (!newMatch.len) continue;
 
              if ( ((size_t)newMatch.len > sufficient_len)
@@ -1258,7 +1270,10 @@ static int LZ4HC_compress_optimal (
                          price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                      }
 
-                     if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
+                    assert(opt[pos].price > 1);
+                    assert((U32)favorDecSpeed <= 1);
+                     if (pos > last_match_pos+TRAILING_LITERALS
+                      || price <= opt[pos].price - (int)favorDecSpeed) {
                          DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
                                      pos, price, ml);
                          assert(pos < LZ4_OPT_NUM);

From a2edeac201a7c1c7869d3754cd4dd5d49997357e Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 26 Apr 2018 16:53:40 -0400
Subject: [PATCH 208/257] Limit Dictionary Size During LZ4F Decompression

Fixes lz4/lz4#517.
---
 lib/lz4frame.c | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index b616463de..4d6d39cc7 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1502,11 +1502,19 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }   }
 
             if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
+                const char *dict = (const char *)dctx->dict;
+                size_t dictSize = dctx->dictSize;
+                int decodedSize;
+                if (dict && dictSize > 1 GB) {
+                    /* the dictSize param is an int, avoid truncation / sign issues */
+                    dict += dictSize - 1 GB;
+                    dictSize = 1 GB;
+                }
                 /* enough capacity in `dst` to decompress directly there */
-                int const decodedSize = LZ4_decompress_safe_usingDict(
+                decodedSize = LZ4_decompress_safe_usingDict(
                         (const char*)selectedIn, (char*)dstPtr,
                         (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
-                        (const char*)dctx->dict, (int)dctx->dictSize);
+                        dict, (int)dictSize);
                 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC);   /* decompression failed */
                 if (dctx->frameInfo.contentChecksumFlag)
                     XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
@@ -1538,10 +1546,19 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }
 
             /* Decode block */
-            {   int const decodedSize = LZ4_decompress_safe_usingDict(
+            {
+                const char *dict = (const char *)dctx->dict;
+                size_t dictSize = dctx->dictSize;
+                int decodedSize;
+                if (dict && dictSize > 1 GB) {
+                    /* the dictSize param is an int, avoid truncation / sign issues */
+                    dict += dictSize - 1 GB;
+                    dictSize = 1 GB;
+                }
+                decodedSize = LZ4_decompress_safe_usingDict(
                         (const char*)selectedIn, (char*)dctx->tmpOut,
                         (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
-                        (const char*)dctx->dict, (int)dctx->dictSize);
+                        dict, (int)dictSize);
                 if (decodedSize < 0)  /* decompression failed */
                     return err0r(LZ4F_ERROR_decompressionFailed);
                 if (dctx->frameInfo.contentChecksumFlag)

From 2becd69bb1b29be8b1414c8ef6cc6269bb184f58 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 26 Apr 2018 17:25:12 -0400
Subject: [PATCH 209/257] Add _destSize() to Fullbench

---
 tests/fullbench.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index f489392d1..22199b7d9 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -184,6 +184,11 @@ static int local_LZ4_compress_default_small(const char* in, char* out, int inSiz
     return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
 }
 
+static int local_LZ4_compress_destSize(const char* in, char* out, int inSize)
+{
+    return LZ4_compress_destSize(in, out, &inSize, LZ4_compressBound(inSize)-1);
+}
+
 static int local_LZ4_compress_fast0(const char* in, char* out, int inSize)
 {
     return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0);
@@ -422,12 +427,13 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
             case 0 : DISPLAY("Compression functions : \n"); continue;
             case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break;
             case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break;
-            case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break;
-            case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break;
-            case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break;
-            case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break;
-            case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break;
-            case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
+            case 3 : compressionFunction = local_LZ4_compress_destSize; compressorName = "LZ4_compress_destSize"; break;
+            case 4 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break;
+            case 5 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break;
+            case 6 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break;
+            case 7 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break;
+            case 8 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break;
+            case 9 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
 
             case 10: compressionFunction = local_LZ4_compress_HC; compressorName = "LZ4_compress_HC"; break;
             case 12: compressionFunction = local_LZ4_compress_HC_extStateHC; compressorName = "LZ4_compress_HC_extStateHC"; break;

From 0858362f28195bb1d88bfa60075d3ec19ff73a9c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Thu, 26 Apr 2018 15:42:16 -0400
Subject: [PATCH 210/257] Merge _destSize Compress Variant into
 LZ4_compress_generic()

---
 lib/lz4.c | 256 ++++++++++++++----------------------------------------
 1 file changed, 66 insertions(+), 190 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index eaabe5b5e..b4d3140de 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -453,7 +453,7 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 /*-************************************
 *  Local Structures and types
 **************************************/
-typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
+typedef enum { notLimited = 0, limitedOutput = 1, fillOutput = 2 } limitedOutput_directive;
 typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
 
 /**
@@ -637,6 +637,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                  const char* const source,
                  char* const dest,
                  const int inputSize,
+                 int *inputConsumed, /* only written when outputLimited == fillOutput */
                  const int maxOutputSize,
                  const limitedOutput_directive outputLimited,
                  const tableType_t tableType,
@@ -679,6 +680,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
     DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType);
     /* Init conditions */
+    if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
     assert(acceleration >= 1);
@@ -795,9 +797,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
         /* Encode Literals */
         {   unsigned const litLength = (unsigned)(ip - anchor);
             token = op++;
-            if ((outputLimited) &&  /* Check output buffer overflow */
+            if ((outputLimited == limitedOutput) &&  /* Check output buffer overflow */
                 (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
                 return 0;
+            if ((outputLimited == fillOutput) &&
+                (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) {
+                op--;
+                goto _last_literals;
+            }
             if (litLength >= RUN_MASK) {
                 int len = (int)litLength-RUN_MASK;
                 *token = (RUN_MASK<<ML_BITS);
@@ -822,6 +829,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
          * - token and *token : position to write 4-bits for match length; higher 4-bits for literal length supposed already written
          */
 
+        if ((outputLimited == fillOutput) &&
+            (op + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit)) {
+            /* the match was too close to the end, rewind and go to last literals */
+            op = token;
+            goto _last_literals;
+        }
+
         /* Encode Offset */
         if (maybe_extMem) {   /* static test */
             DEBUGLOG(6, "             with offset=%u  (ext if > %i)", offset, (int)(ip - (const BYTE*)source));
@@ -855,9 +869,17 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 DEBUGLOG(6, "             with matchLength=%u", matchCode+MINMATCH);
             }
 
-            if ( outputLimited &&    /* Check output buffer overflow */
-                (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) )
-                return 0;
+            if ((outputLimited) &&    /* Check output buffer overflow */
+                (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) {
+                if (outputLimited == limitedOutput)
+                  return 0;
+                if (outputLimited == fillOutput) {
+                    /* Match description too long : reduce it */
+                    U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255;
+                    ip -= matchCode - newMatchCode;
+                    matchCode = newMatchCode;
+                }
+            }
             if (matchCode >= ML_MASK) {
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
@@ -939,10 +961,17 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
 
 _last_literals:
     /* Encode Last Literals */
-    {   size_t const lastRun = (size_t)(iend - anchor);
+    {   size_t lastRun = (size_t)(iend - anchor);
         if ( (outputLimited) &&  /* Check output buffer overflow */
-            ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) )
-            return 0;
+            (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) {
+            if (outputLimited == fillOutput) {
+                /* adapt lastRun to fill 'dst' */
+                lastRun  = (olimit-op) - 1;
+                lastRun -= (lastRun+240)/255;
+            }
+            if (outputLimited == limitedOutput)
+                return 0;
+        }
         if (lastRun >= RUN_MASK) {
             size_t accumulator = lastRun - RUN_MASK;
             *op++ = RUN_MASK << ML_BITS;
@@ -952,9 +981,13 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
             *op++ = (BYTE)(lastRun<<ML_BITS);
         }
         memcpy(op, anchor, lastRun);
+        ip = anchor + lastRun;
         op += lastRun;
     }
 
+    if (outputLimited == fillOutput) {
+        *inputConsumed = (int) (((const char*)ip)-source);
+    }
     return (int)(((char*)op) - dest);
 }
 
@@ -966,17 +999,17 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
     LZ4_resetStream((LZ4_stream_t*)state);
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (inputSize < LZ4_64Klimit) {;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > MAX_DISTANCE)) ? byPtr : byU32;
-            return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1000,28 +1033,28 @@ int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
-            return LZ4_compress_generic(ctx, src, dst, srcSize, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
         }
     } else {
         if (srcSize < LZ4_64Klimit) {
             const tableType_t tableType = byU16;
             LZ4_prepareTable(ctx, srcSize, tableType);
             if (ctx->currentOffset) {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration);
             } else {
-                return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+                return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
             }
         } else {
             const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
             LZ4_prepareTable(ctx, srcSize, tableType);
-            return LZ4_compress_generic(ctx, src, dst, srcSize, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
+            return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
         }
     }
 }
@@ -1060,172 +1093,15 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m
     LZ4_resetStream(&ctx);
 
     if (inputSize < LZ4_64Klimit)
-        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16,                        noDict, noDictIssue, acceleration);
+        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16,                        noDict, noDictIssue, acceleration);
     else
-        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration);
-}
-
-
-/*-******************************
-*  *_destSize() variant
-********************************/
-
-static int LZ4_compress_destSize_generic(
-                       LZ4_stream_t_internal* const ctx,
-                 const char* const src,
-                       char* const dst,
-                       int*  const srcSizePtr,
-                 const int targetDstSize,
-                 const tableType_t tableType)
-{
-    const BYTE* ip = (const BYTE*) src;
-    const BYTE* base = (const BYTE*) src;
-    const BYTE* lowLimit = (const BYTE*) src;
-    const BYTE* anchor = ip;
-    const BYTE* const iend = ip + *srcSizePtr;
-    const BYTE* const mflimit = iend - MFLIMIT;
-    const BYTE* const matchlimit = iend - LASTLITERALS;
-
-    BYTE* op = (BYTE*) dst;
-    BYTE* const oend = op + targetDstSize;
-    BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */;
-    BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */);
-    BYTE* const oMaxSeq = oMaxLit - 1 /* token */;
-
-    U32 forwardH;
-
-
-    /* Init conditions */
-    if (targetDstSize < 1) return 0;                                     /* Impossible to store anything */
-    if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0;            /* Unsupported input size, too large (or negative) */
-    if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0;   /* Size too large (not within 64K limit) */
-    if (*srcSizePtr<LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
-
-    /* First Byte */
-    *srcSizePtr = 0;
-    LZ4_putPosition(ip, ctx->hashTable, tableType, base);
-    ip++; forwardH = LZ4_hashPosition(ip, tableType);
-
-    /* Main Loop */
-    for ( ; ; ) {
-        const BYTE* match;
-        BYTE* token;
-
-        /* Find a match */
-        {   const BYTE* forwardIp = ip;
-            unsigned step = 1;
-            unsigned searchMatchNb = 1 << LZ4_skipTrigger;
-
-            do {
-                U32 h = forwardH;
-                ip = forwardIp;
-                forwardIp += step;
-                step = (searchMatchNb++ >> LZ4_skipTrigger);
-
-                if (unlikely(forwardIp > mflimit)) goto _last_literals;
-
-                match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base);
-                forwardH = LZ4_hashPosition(forwardIp, tableType);
-                LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base);
-
-            } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
-                || (LZ4_read32(match) != LZ4_read32(ip)) );
-        }
-
-        /* Catch up */
-        while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
-
-        /* Encode Literal length */
-        {   unsigned litLength = (unsigned)(ip - anchor);
-            token = op++;
-            if (op + ((litLength+240)/255) + litLength > oMaxLit) {
-                /* Not enough space for a last match */
-                op--;
-                goto _last_literals;
-            }
-            if (litLength>=RUN_MASK) {
-                unsigned len = litLength - RUN_MASK;
-                *token=(RUN_MASK<<ML_BITS);
-                for(; len >= 255 ; len-=255) *op++ = 255;
-                *op++ = (BYTE)len;
-            }
-            else *token = (BYTE)(litLength<<ML_BITS);
-
-            /* Copy Literals */
-            LZ4_wildCopy(op, anchor, op+litLength);
-            op += litLength;
-        }
-
-_next_match:
-        /* Encode Offset */
-        LZ4_writeLE16(op, (U16)(ip-match)); op+=2;
-
-        /* Encode MatchLength */
-        {   size_t matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
-
-            if (op + ((matchLength+240)/255) > oMaxMatch) {
-                /* Match description too long : reduce it */
-                matchLength = (15-1) + (oMaxMatch-op) * 255;
-            }
-            ip += MINMATCH + matchLength;
-
-            if (matchLength>=ML_MASK) {
-                *token += ML_MASK;
-                matchLength -= ML_MASK;
-                while (matchLength >= 255) { matchLength-=255; *op++ = 255; }
-                *op++ = (BYTE)matchLength;
-            }
-            else *token += (BYTE)(matchLength);
-        }
-
-        anchor = ip;
-
-        /* Test end of block */
-        if (ip > mflimit) break;
-        if (op > oMaxSeq) break;
-
-        /* Fill table */
-        LZ4_putPosition(ip-2, ctx->hashTable, tableType, base);
-
-        /* Test next position */
-        match = LZ4_getPosition(ip, ctx->hashTable, tableType, base);
-        LZ4_putPosition(ip, ctx->hashTable, tableType, base);
-        if ( (match+MAX_DISTANCE>=ip)
-            && (LZ4_read32(match)==LZ4_read32(ip)) )
-        { token=op++; *token=0; goto _next_match; }
-
-        /* Prepare next loop */
-        forwardH = LZ4_hashPosition(++ip, tableType);
-    }
-
-_last_literals:
-    /* Encode Last Literals */
-    {   size_t lastRunSize = (size_t)(iend - anchor);
-        if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) {
-            /* adapt lastRunSize to fill 'dst' */
-            lastRunSize  = (oend-op) - 1;
-            lastRunSize -= (lastRunSize+240)/255;
-        }
-        ip = anchor + lastRunSize;
-
-        if (lastRunSize >= RUN_MASK) {
-            size_t accumulator = lastRunSize - RUN_MASK;
-            *op++ = RUN_MASK << ML_BITS;
-            for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
-            *op++ = (BYTE) accumulator;
-        } else {
-            *op++ = (BYTE)(lastRunSize<<ML_BITS);
-        }
-        memcpy(op, anchor, lastRunSize);
-        op += lastRunSize;
-    }
-
-    /* End */
-    *srcSizePtr = (int) (((const char*)ip)-src);
-    return (int) (((char*)op)-dst);
+        return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration);
 }
 
 
+/* Note!: This function leaves the stream in an unclean/broken state!
+ * It is not safe to subsequently use the same state with a _fastReset() or
+ * _continue() call without resetting it. */
 static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
 {
     LZ4_resetStream(state);
@@ -1234,10 +1110,10 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
         return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
     } else {
         if (*srcSizePtr < LZ4_64Klimit) {
-            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16);
+            return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1);
         } else {
             tableType_t const tableType = ((sizeof(void*)==4) && ((uptrval)src > MAX_DISTANCE)) ? byPtr : byU32;
-            return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, tableType);
+            return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, tableType, noDict, noDictIssue, 1);
     }   }
 }
 
@@ -1397,9 +1273,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source) {
         if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration);
         else
-            return LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);
+            return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration);
     }
 
     /* external dictionary mode */
@@ -1417,15 +1293,15 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
                  * so that the compression loop is only looking into one table.
                  */
                 memcpy(streamPtr, streamPtr->dictCtx, sizeof(LZ4_stream_t));
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration);
             }
         } else {
             if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration);
             } else {
-                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
+                result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration);
             }
         }
         streamPtr->dictionary = (const BYTE*)source;
@@ -1444,9 +1320,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     LZ4_renormDictT(streamPtr, srcSize);
 
     if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) {
-        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1);
     } else {
-        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
+        result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
     }
 
     streamPtr->dictionary = (const BYTE*)source;

From 3792d00168edd060c58ceaecffb97d43dab27094 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 15:18:44 -0700
Subject: [PATCH 211/257] favorDecSpeed feature can be triggered from lz4frame

and lz4hc.
---
 lib/lz4frame.c |  3 +++
 lib/lz4frame.h |  9 +++++----
 lib/lz4hc.c    |  8 ++++++--
 lib/lz4hc.h    | 18 +++++++++++++-----
 4 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index b616463de..06a0f7bd8 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -612,6 +612,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         /* frame init only for blockLinked : blockIndependent will be init at each block */
         LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
+    if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
+          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, preferencesPtr->favorDecSpeed);
+    }
 
     /* Magic Number */
     LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index bd715bd47..fb434ff76 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -173,13 +173,14 @@ typedef struct {
 
 /*! LZ4F_preferences_t :
  *  makes it possible to supply detailed compression parameters to the stream interface.
- *  It's not required to set all fields, as long as the structure was initially memset() to zero.
+ *  Structure is presumed initially memset() to zero, representing default settings.
  *  All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
-  unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
-  unsigned reserved[4];            /* must be zero for forward compatibility */
+  int      compressionLevel;    /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
+  unsigned autoFlush;           /* 1: always flush, to reduce usage of internal buffers */
+  unsigned favorDecSpeed;       /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */  /* >= v1.8.2 */
+  unsigned reserved[3];         /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
 LZ4FLIB_API int LZ4F_compressionLevel_max(void);
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 3593da774..b90d60b15 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -876,6 +876,11 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
     LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
 }
 
+void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor)
+{
+    LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0);
+}
+
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
@@ -1120,7 +1125,7 @@ static int LZ4HC_compress_optimal (
     const limitedOutput_directive limit,
     int const fullUpdate,
     const dictCtx_directive dict,
-    HCfavor_e favorDecSpeed
+    const HCfavor_e favorDecSpeed
     )
 {
 #define TRAILING_LITERALS 3
@@ -1136,7 +1141,6 @@ static int LZ4HC_compress_optimal (
     BYTE* oend = op + dstCapacity;
 
     /* init */
-    favorDecSpeed = favorCompressionRatio;
     DEBUGLOG(5, "LZ4HC_compress_optimal");
     *srcSizePtr = 0;
     if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 28e252889..bb5e07373 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -152,7 +152,8 @@ struct LZ4HC_CCtx_internal
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
-    int        compressionLevel;
+    short      compressionLevel;
+    short      favorDecSpeed;
     const LZ4HC_CCtx_internal* dictCtx;
 };
 
@@ -169,7 +170,8 @@ struct LZ4HC_CCtx_internal
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
-    int            compressionLevel;
+    short          compressionLevel;
+    short          favorDecSpeed;
     const LZ4HC_CCtx_internal* dictCtx;
 };
 
@@ -253,9 +255,9 @@ LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") LZ4LIB_API  int   LZ4_resetStr
  * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
  */
 int LZ4_compress_HC_destSize(void* LZ4HC_Data,
-                            const char* src, char* dst,
-                            int* srcSizePtr, int targetDstSize,
-                            int compressionLevel);
+                             const char* src, char* dst,
+                             int* srcSizePtr, int targetDstSize,
+                             int compressionLevel);
 
 /*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental)
  *  Similar as LZ4_compress_HC_continue(),
@@ -275,6 +277,12 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
+/*! LZ4_favorDecompressionSpeed() : v1.8.2 (experimental)
+ *  Parser will select decisions favoring decompression over compression ratio.
+ *  Only work at highest compression settings (level >= LZ4HC_CLEVEL_OPT_MIN)
+ */
+void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor);
+
 /*! LZ4_resetStreamHC_fast() :
  *  When an LZ4_streamHC_t is known to be in a internally coherent state,
  *  it can often be prepared for a new compression with almost no work, only

From 5c7d3812d90aeaf072d14f6b5d935711da6f14c7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 15:49:32 -0700
Subject: [PATCH 212/257] fasterDecSpeed can be triggered from cli with
 --favor-decSpeed

---
 lib/lz4frame.c    | 2 +-
 lib/lz4hc.c       | 2 +-
 programs/lz4cli.c | 2 ++
 programs/lz4io.c  | 8 ++++++++
 programs/lz4io.h  | 7 ++++++-
 5 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 06a0f7bd8..9d88644cf 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -613,7 +613,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
     if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
-          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, preferencesPtr->favorDecSpeed);
+          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
     }
 
     /* Magic Number */
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b90d60b15..39ab5fb5d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -718,7 +718,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
                             cParam.nbSearches, cParam.targetLength, limit,
                             cLevel == LZ4HC_CLEVEL_MAX,   /* ultra mode */
                             dict,
-                            favorDecompressionSpeed);
+                            ctx->favorDecSpeed);
     }
 }
 
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 42392ebcf..ba519b462 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -140,6 +140,7 @@ static int usage_advanced(const char* exeName)
     DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
     DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
     DISPLAY( "--[no-]sparse  : sparse mode (default:enabled on file, disabled on stdout)\n");
+    DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n");
     DISPLAY( "Benchmark arguments : \n");
     DISPLAY( " -b#    : benchmark file(s), using # compression level (default : 1) \n");
     DISPLAY( " -e#    : test all compression levels from -bX to # (default : 1)\n");
@@ -355,6 +356,7 @@ int main(int argc, const char** argv)
                 if (!strcmp(argument,  "--no-content-size")) { LZ4IO_setContentSize(0); continue; }
                 if (!strcmp(argument,  "--sparse")) { LZ4IO_setSparseFile(2); continue; }
                 if (!strcmp(argument,  "--no-sparse")) { LZ4IO_setSparseFile(0); continue; }
+                if (!strcmp(argument,  "--favor-decSpeed")) { LZ4IO_favorDecSpeed(1); continue; }
                 if (!strcmp(argument,  "--verbose")) { displayLevel++; continue; }
                 if (!strcmp(argument,  "--quiet")) { if (displayLevel) displayLevel--; continue; }
                 if (!strcmp(argument,  "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
diff --git a/programs/lz4io.c b/programs/lz4io.c
index ccf4fa123..b52c1f32f 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -116,6 +116,7 @@ static int g_blockIndependence = 1;
 static int g_sparseFileSupport = 1;
 static int g_contentSizeFlag = 0;
 static int g_useDictionary = 0;
+static unsigned g_favorDecSpeed = 0;
 static const char* g_dictionaryFilename = NULL;
 
 
@@ -221,6 +222,12 @@ int LZ4IO_setContentSize(int enable)
     return g_contentSizeFlag;
 }
 
+/* Default setting : 0 (disabled) */
+void LZ4IO_favorDecSpeed(int favor)
+{
+    g_favorDecSpeed = (favor!=0);
+}
+
 static U32 g_removeSrcFile = 0;
 void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
 
@@ -548,6 +555,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
     prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum;
     prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
+    prefs.favorDecSpeed = g_favorDecSpeed;
     if (g_contentSizeFlag) {
       U64 const fileSize = UTIL_getFileSize(srcFileName);
       prefs.frameInfo.contentSize = fileSize;   /* == 0 if input == stdin */
diff --git a/programs/lz4io.h b/programs/lz4io.h
index b21b8b6fd..22c5e3e65 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -94,10 +94,15 @@ int LZ4IO_setNotificationLevel(int level);
 /* Default setting : 0 (disabled) */
 int LZ4IO_setSparseFile(int enable);
 
-/* Default setting : 0 (disabled) */
+/* Default setting : 0 == no content size present in frame header */
 int LZ4IO_setContentSize(int enable);
 
+/* Default setting : 0 == src file preserved */
 void LZ4IO_setRemoveSrcFile(unsigned flag);
 
+/* Default setting : 0 == favor compression ratio
+ * Note : 1 only works for high compression levels (10+) */
+void LZ4IO_favorDecSpeed(int favor);
+
 
 #endif  /* LZ4IO_H_237902873 */

From 0fb3a3b199ed0a92a402c5979c514a329b85462a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 26 Apr 2018 17:02:20 -0700
Subject: [PATCH 213/257] fixed a number of minor cast warnings

---
 examples/frameCompress.c | 3 ++-
 lib/lz4frame.c           | 2 +-
 lib/lz4hc.c              | 9 ++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 972f716f8..9bfea483f 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -21,7 +21,8 @@ static const LZ4F_preferences_t kPrefs = {
       0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
     0,   /* compression level; 0 == default */
     0,   /* autoflush */
-    { 0, 0, 0, 0 },  /* reserved, must be set to 0 */
+    0,   /* favor decompression speed */
+    { 0, 0, 0 },  /* reserved, must be set to 0 */
 };
 
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9d88644cf..03382669c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -613,7 +613,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
         LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
     if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
-          LZ4_favorDecompressionSpeed(cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
+          LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
     }
 
     /* Magic Number */
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 39ab5fb5d..35eac1ae3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -708,6 +708,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     assert(cLevel >= 0);
     assert(cLevel <= LZ4HC_CLEVEL_MAX);
     {   cParams_t const cParam = clTable[cLevel];
+        HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio;
         if (cParam.strat == lz4hc)
             return LZ4HC_compress_hashChain(ctx,
                                 src, dst, srcSizePtr, dstCapacity,
@@ -717,8 +718,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
                             src, dst, srcSizePtr, dstCapacity,
                             cParam.nbSearches, cParam.targetLength, limit,
                             cLevel == LZ4HC_CLEVEL_MAX,   /* ultra mode */
-                            dict,
-                            ctx->favorDecSpeed);
+                            dict, favor);
     }
 }
 
@@ -756,7 +756,7 @@ static int LZ4HC_compress_generic_dictCtx (
     } else if (position == 0 && *srcSizePtr > 4 KB) {
         memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal));
         LZ4HC_setExternalDict(ctx, (const BYTE *)src);
-        ctx->compressionLevel = cLevel;
+        ctx->compressionLevel = (short)cLevel;
         return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit);
     } else {
         return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx);
@@ -873,7 +873,7 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev
 {
     if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
     if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;
-    LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
+    LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel;
 }
 
 void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor)
@@ -1274,7 +1274,6 @@ static int LZ4HC_compress_optimal (
                          price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                      }
 
-                    assert(opt[pos].price > 1);
                     assert((U32)favorDecSpeed <= 1);
                      if (pos > last_match_pos+TRAILING_LITERALS
                       || price <= opt[pos].price - (int)favorDecSpeed) {

From ce4e1389cc38c3c26f40ecef9c4fa6438411fa90 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Fri, 27 Apr 2018 07:06:37 +0300
Subject: [PATCH 214/257] fuzzer.c: enabled ring buffer tests for
 decompress_fast

Ring buffer tests were performed only with LZ4_decompress_safe_continue,
leaving my buggy changes to LZ4_decompress_safe_continue undetected.
The tests are now replicated and performed in a similar manner for both
LZ4_decompress_safe_continue and LZ4_decompress_safe_continue (except
for the small buffer case where only one function can be tested,
because part of the dictionary is overwritten with the output).

I also updated function names in the messages (changed them to the
actual ones).  The error was reported for LZ4_decompress_safe(),
which I found misleading.
---
 tests/fuzzer.c | 121 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 81 insertions(+), 40 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 244cc4fef..cadda2164 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -914,8 +914,8 @@ static void FUZ_unitTests(int compressionLevel)
 
         /* ring buffer test */
         {   XXH64_state_t xxhOrig;
-            XXH64_state_t xxhNew;
-            LZ4_streamDecode_t decodeState;
+            XXH64_state_t xxhNewSafe, xxhNewFast;
+            LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
             const U32 maxMessageSizeLog = 10;
             const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
             U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
@@ -923,26 +923,36 @@ static void FUZ_unitTests(int compressionLevel)
             U32 rNext = 0;
             U32 dNext = 0;
             const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
+            int compressedSize;
 
             XXH64_reset(&xxhOrig, 0);
-            XXH64_reset(&xxhNew, 0);
+            XXH64_reset(&xxhNewSafe, 0);
+            XXH64_reset(&xxhNewFast, 0);
             LZ4_resetStream(&streamingState);
-            LZ4_setStreamDecode(&decodeState, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
             while (iNext + messageSize < testCompressedSize) {
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
                 crcOrig = XXH64_digest(&xxhOrig);
 
                 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
-                result = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
-                FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed");
+                compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
+                FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed");
 
-                result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-                XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                { U64 const crcNew = XXH64_digest(&xxhNew);
-                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
+                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewSafe);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
+
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
+
+                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewFast);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
                 /* prepare next message */
                 iNext += messageSize;
@@ -1064,8 +1074,8 @@ static void FUZ_unitTests(int compressionLevel)
 
         /* ring buffer test */
         {   XXH64_state_t xxhOrig;
-            XXH64_state_t xxhNew;
-            LZ4_streamDecode_t decodeState;
+            XXH64_state_t xxhNewSafe, xxhNewFast;
+            LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
             const U32 maxMessageSizeLog = 10;
             const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
             U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
@@ -1073,26 +1083,36 @@ static void FUZ_unitTests(int compressionLevel)
             U32 rNext = 0;
             U32 dNext = 0;
             const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
+            int compressedSize;
 
             XXH64_reset(&xxhOrig, 0);
-            XXH64_reset(&xxhNew, 0);
+            XXH64_reset(&xxhNewSafe, 0);
+            XXH64_reset(&xxhNewFast, 0);
             LZ4_resetStreamHC(&sHC, compressionLevel);
-            LZ4_setStreamDecode(&decodeState, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
             while (iNext + messageSize < testCompressedSize) {
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
                 crcOrig = XXH64_digest(&xxhOrig);
 
                 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
-                result = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
-                FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+                compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+                FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
+
+                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-                result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewSafe);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
-                XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                { U64 const crcNew = XXH64_digest(&xxhNew);
-                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
+
+                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                { U64 const crcNew = XXH64_digest(&xxhNewFast);
+                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
                 /* prepare next message */
                 iNext += messageSize;
@@ -1106,8 +1126,8 @@ static void FUZ_unitTests(int compressionLevel)
 
         /* small decoder-side ring buffer test */
         {   XXH64_state_t xxhOrig;
-            XXH64_state_t xxhNew;
-            LZ4_streamDecode_t decodeState;
+            XXH64_state_t xxhNewSafe, xxhNewFast;
+            LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
             const U32 maxMessageSizeLog = 12;
             const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
             U32 messageSize;
@@ -1115,11 +1135,14 @@ static void FUZ_unitTests(int compressionLevel)
             U32 iNext = 0;
             U32 dNext = 0;
             const U32 dBufferSize = 64 KB;
+            int compressedSize;
 
             XXH64_reset(&xxhOrig, 0);
-            XXH64_reset(&xxhNew, 0);
+            XXH64_reset(&xxhNewSafe, 0);
+            XXH64_reset(&xxhNewFast, 0);
             LZ4_resetStreamHC(&sHC, compressionLevel);
-            LZ4_setStreamDecode(&decodeState, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
+            LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
 #define BSIZE1 65537
 #define BSIZE2 16435
@@ -1129,15 +1152,22 @@ static void FUZ_unitTests(int compressionLevel)
             XXH64_update(&xxhOrig, testInput + iNext, messageSize);
             crcOrig = XXH64_digest(&xxhOrig);
 
-            result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
-            FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+            compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+            FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
+
+            result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+            FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-            result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-            FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
+            XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+            { U64 const crcNew = XXH64_digest(&xxhNewSafe);
+              FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
-            XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-            { U64 const crcNew = XXH64_digest(&xxhNew);
-              FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
+            result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+            FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
+
+            XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+            { U64 const crcNew = XXH64_digest(&xxhNewFast);
+              FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
             /* prepare next message */
             dNext += messageSize;
@@ -1151,17 +1181,28 @@ static void FUZ_unitTests(int compressionLevel)
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
                 crcOrig = XXH64_digest(&xxhOrig);
 
-                result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
-                FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+                compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+                FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
 
-                result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
+#if 1           /* Because the ring buffer is small, decompression overwrites part of the output which
+                 * is first used as dictionary.  Hence only one decompression function can be tested. */
+                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+                FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
+                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                {   U64 const crcNew = XXH64_digest(&xxhNewSafe);
+                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during small decoder-side ring buffer test");
+                }
+#else
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
 
-                XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                {   U64 const crcNew = XXH64_digest(&xxhNew);
+                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                {   U64 const crcNew = XXH64_digest(&xxhNewFast);
                     if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
-                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during small decoder-side ring buffer test");
                 }
+#endif
                 /* prepare next message */
                 dNext += messageSize;
                 totalMessageSize += messageSize;

From d81a434c3dc032779dba6f22a58b127922ef332f Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Fri, 27 Apr 2018 15:00:11 +0300
Subject: [PATCH 215/257] lz4.c: fixed the LZ4_decompress_fast_continue case

The change is very similar to that of the LZ4_decompress_safe_continue
case.  The only reason a make this a separate change is to ensure that
the fuzzer, after it's been enhanced, can detect the flaw in
LZ4_decompress_fast_continue, and that the change indeed fixes the flaw.
---
 lib/lz4.c | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index eb3da213c..916acf098 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1752,6 +1752,15 @@ int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compresse
                                   (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
 }
 
+LZ4_FORCE_INLINE
+int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize,
+                                   size_t prefixSize, const void* dictStart, size_t dictSize)
+{
+    return LZ4_decompress_generic(source, dest, 0, originalSize,
+                                  endOnOutputSize, full, 0, usingExtDict,
+                                  (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize);
+}
+
 /*===== streaming decompression functions =====*/
 
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
@@ -1831,21 +1840,32 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
     return result;
 }
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
     int result;
 
-    if (lz4sd->prefixSize == 0 || lz4sd->prefixEnd == (BYTE*)dest) {
+    if (lz4sd->prefixSize == 0) {
+        assert(lz4sd->extDictSize == 0);
         result = LZ4_decompress_fast(source, dest, originalSize);
         if (result <= 0) return result;
+        lz4sd->prefixSize = originalSize;
+        lz4sd->prefixEnd = (BYTE*)dest + originalSize;
+    } else if (lz4sd->prefixEnd == (BYTE*)dest) {
+        if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0)
+            result = LZ4_decompress_fast(source, dest, originalSize);
+        else
+            result = LZ4_decompress_fast_doubleDict(source, dest, originalSize,
+                                                    lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
+        if (result <= 0) return result;
         lz4sd->prefixSize += originalSize;
         lz4sd->prefixEnd  += originalSize;
     } else {
         lz4sd->extDictSize = lz4sd->prefixSize;
         lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
         result = LZ4_decompress_fast_extDict(source, dest, originalSize,
-                                             (const char*)lz4sd->externalDict, lz4sd->extDictSize);
+                                             lz4sd->externalDict, lz4sd->extDictSize);
         if (result <= 0) return result;
         lz4sd->prefixSize = originalSize;
         lz4sd->prefixEnd  = (BYTE*)dest + originalSize;

From 938e4849ae3bb33665aeee2647aa3e2077af0b5a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 08:43:40 -0700
Subject: [PATCH 216/257] updated NEWS, in preparation for v1.8.2

---
 NEWS | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/NEWS b/NEWS
index 9ac41f298..c3b47f112 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,13 @@
 v1.8.2
+perf: *much* faster dictionary compression on small files, by @felixhandte
 perf: slightly faster HC compression and decompression speed
 perf: very small compression ratio improvement
+fix : compression compatible with low memory addresses (< 0xFFFF)
+fix : decompression segfault when provided with NULL input, by @terrelln
+cli : new command --favor-decSpeed
 cli : benchmark mode more accurate for small inputs
+fullbench : can measure _destSize() variants, by @felixhandte
+doc : clarified block format parsing restrictions, by Alexey Tourbin (@svpv)
 
 v1.8.1
 perf : faster and stronger ultra modes (levels 10+)

From d294dd7fc684b1f46130358ff5c218e835082b16 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 09:04:09 -0700
Subject: [PATCH 217/257] ensure favorDecSpeed is properly initialized

also :
- fix a potential malloc error
- proper use of ALLOC macro inside lz4hc
- update html API doc
---
 doc/lz4_manual.html      | 12 ++++----
 doc/lz4frame_manual.html | 63 +++++++++++++++++++++++++++++++++++++---
 lib/lz4.c                |  6 ++--
 lib/lz4hc.c              | 10 +++----
 4 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index f8639fe72..ddd272431 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -126,15 +126,17 @@ <h1>1.8.2 Manual</h1>
 
 <pre><b>int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 </b><p>This function is a bit faster than LZ4_decompress_safe(),
-but doesn't provide any security guarantee.
+but it may misbehave on malformed input because it doesn't perform full validation of compressed data.
     originalSize : is the uncompressed size to regenerate
                    Destination buffer must be already allocated, and its size must be >= 'originalSize' bytes.
     return : number of bytes read from source buffer (== compressed size).
              If the source stream is detected malformed, the function stops decoding and return a negative result.
-    note : This function respects memory boundaries for *properly formed* compressed data.
-           However, it does not provide any protection against malicious input.
-           It also doesn't know 'src' size, and implies it's >= compressed size.
-           Use this function in trusted environment **only**.
+    note : This function is only usable if the originalSize of uncompressed data is known in advance.
+           The caller should also check that all the compressed input has been consumed properly,
+           i.e. that the return value matches the size of the buffer with compressed input.
+           The function never writes past the output buffer.  However, since it doesn't know its 'src' size,
+           it may read past the intended input.  Also, because match offsets are not validated during decoding,
+           reads from 'src' may underflow.  Use this function in trusted environment **only**.
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 459bac89f..53ea7eb19 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -18,6 +18,7 @@ <h1>1.8.2 Manual</h1>
 <li><a href="#Chapter8">Compression</a></li>
 <li><a href="#Chapter9">Decompression functions</a></li>
 <li><a href="#Chapter10">Streaming decompression functions</a></li>
+<li><a href="#Chapter11">Bulk processing dictionary API</a></li>
 </ol>
 <hr>
 <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -89,12 +90,13 @@ <h1>1.8.2 Manual</h1>
 
 <pre><b>typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       </b>/* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */<b>
-  unsigned autoFlush;              </b>/* 1 == always flush, to reduce usage of internal buffers */<b>
-  unsigned reserved[4];            </b>/* must be zero for forward compatibility */<b>
+  int      compressionLevel;    </b>/* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */<b>
+  unsigned autoFlush;           </b>/* 1: always flush, to reduce usage of internal buffers */<b>
+  unsigned favorDecSpeed;       </b>/* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */  /* >= v1.8.2 */<b>
+  unsigned reserved[3];         </b>/* must be zero for forward compatibility */<b>
 } LZ4F_preferences_t;
 </b><p>  makes it possible to supply detailed compression parameters to the stream interface.
-  It's not required to set all fields, as long as the structure was initially memset() to zero.
+  Structure is presumed initially memset() to zero, representing default settings.
   All reserved fields must be set to zero. 
 </p></pre><BR>
 
@@ -293,5 +295,58 @@ <h1>1.8.2 Manual</h1>
   and start a new one using same context resources. 
 </p></pre><BR>
 
+<pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
+</b></pre><BR>
+<a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre>
+
+<pre><b>LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
+</b><p>  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict 
+</p></pre><BR>
+
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dst, size_t dstCapacity,
+    const void* src, size_t srcSize,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* preferencesPtr);
+</b><p>  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+  cctx must point to a context created by LZ4F_createCompressionContext().
+  If cdict==NULL, compress without a dictionary.
+  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+  If this condition is not respected, function will fail (@return an errorCode).
+  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+  but it's not recommended, as it's the only way to provide dictID in the frame header.
+ @return : number of bytes written into dstBuffer.
+           or an error code if it fails (can be tested using LZ4F_isError()) 
+</p></pre><BR>
+
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dstBuffer, size_t dstCapacity,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* prefsPtr);
+</b><p>  Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ `prefsPtr` is optional : you may provide NULL as argument,
+  however, it's the only way to provide dictID in the frame header.
+ @return : number of bytes written into dstBuffer for the header,
+           or an error code (which can be tested using LZ4F_isError()) 
+</p></pre><BR>
+
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
+    LZ4F_dctx* dctxPtr,
+    void* dstBuffer, size_t* dstSizePtr,
+    const void* srcBuffer, size_t* srcSizePtr,
+    const void* dict, size_t dictSize,
+    const LZ4F_decompressOptions_t* decompressOptionsPtr);
+</b><p>  Same as LZ4F_decompress(), using a predefined dictionary.
+  Dictionary is used "in place", without any preprocessing.
+  It must remain accessible throughout the entire frame decoding. 
+</p></pre><BR>
+
 </html>
 </body>
diff --git a/lib/lz4.c b/lib/lz4.c
index eaabe5b5e..deaec7b71 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -161,11 +161,11 @@
 *  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
-#define ALLOC(s) malloc(s)
+#define ALLOC(s)          malloc(s)
 #define ALLOC_AND_ZERO(s) calloc(1,s)
-#define FREEMEM        free
+#define FREEMEM(p)        free(p)
 #include <string.h>   /* memset, memcpy */
-#define MEM_INIT       memset
+#define MEM_INIT(p,v,s)   memset((p),(v),(s))
 
 
 /*-************************************
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 35eac1ae3..1c7096b55 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -805,7 +805,7 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src
 int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
-    LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
+    LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
 #else
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
@@ -834,10 +834,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 **************************************/
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) {
-    LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
-    LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
-    LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
+    if (LZ4_streamHCPtr==NULL) return NULL;
+    LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT);
     return LZ4_streamHCPtr;
 }
 
@@ -857,6 +856,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
+    LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0;
     LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 

From 7d11e344130a16a529e3ee15a16a10f2e67faa3c Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 27 Apr 2018 13:57:10 -0400
Subject: [PATCH 218/257] Rename LZ4F_applyCDict() -> LZ4F_initStream()

---
 lib/lz4frame.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4d6d39cc7..e2be99021 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -528,7 +528,15 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 }
 
 
-static void LZ4F_applyCDict(void* ctx,
+/**
+ * This function prepares the internal LZ4(HC) stream for a new compression,
+ * resetting the context and attaching the dictionary, if there is one.
+ *
+ * It needs to be called at the beginning of each independent compression
+ * stream (i.e., at the beginning of a frame in blockLinked mode, or at the
+ * beginning of each block in blockIndependent mode).
+ */
+static void LZ4F_initStream(void* ctx,
                             const LZ4F_CDict* cdict,
                             int level) {
     if (level < LZ4HC_CLEVEL_MIN) {
@@ -610,7 +618,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->cdict = cdict;
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
         /* frame init only for blockLinked : blockIndependent will be init at each block */
-        LZ4F_applyCDict(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
+        LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
     }
 
     /* Magic Number */
@@ -705,7 +713,7 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
-    LZ4F_applyCDict(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level);
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
@@ -722,7 +730,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
-    LZ4F_applyCDict(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level);
     if (cdict) {
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }

From 5076aa3e352ecbd67cd71314cb1fafcb319aed56 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 27 Apr 2018 13:59:02 -0400
Subject: [PATCH 219/257] Remove Redundant LZ4_resetStream() Call

---
 lib/lz4frame.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e2be99021..c4e0ae047 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -457,7 +457,7 @@ struct LZ4F_CDict_s {
 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
-    LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+    LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
@@ -471,7 +471,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
         return NULL;
     }
     memcpy(cdict->dictContent, dictStart, dictSize);
-    LZ4_resetStream(cdict->fastCtx);
     LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
     LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
     LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);

From fefc40fc0afe8c1f5a02e65b94bb65515fd949c4 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Fri, 27 Apr 2018 14:10:27 -0400
Subject: [PATCH 220/257] Avoid Possibly Redundant Table Clears When Loading HC
 Dict

---
 lib/lz4frame.c | 2 +-
 lib/lz4hc.c    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index c4e0ae047..8cf0c2fb2 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -472,7 +472,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
     }
     memcpy(cdict->dictContent, dictStart, dictSize);
     LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
-    LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
+    LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
     LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
     return cdict;
 }
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4126ef81a..90f52f082 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -877,8 +877,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
     }
+    LZ4_resetStreamHC(LZ4_streamHCPtr, ctxPtr->compressionLevel);
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
-    LZ4HC_clearTables (ctxPtr);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
     if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;

From 72e99c8939d7f005487403ab54fa65ac31ee4317 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 11:44:47 -0700
Subject: [PATCH 221/257] lz4hc : minor editions for clarity

---
 lib/lz4hc.c | 75 ++++++++++++++++++++++++++---------------------------
 1 file changed, 37 insertions(+), 38 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4126ef81a..077855281 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -138,6 +138,7 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
 {
     int back = 0;
     int const min = (int)MAX(iMin - ip, mMin - match);
+    assert(min <= 0);
     assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31));
     assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31));
     while ( (back > min)
@@ -222,7 +223,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const U32 ipIndex = (U32)(ip - base);
     const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
-    int const delta = (int)(ip-iLowLimit);
+    int const lookBackLength = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
@@ -242,34 +243,35 @@ LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
+            assert(matchPtr >= lowPrefixPtr);
+            assert(matchPtr < ip);
             assert(longest >= 1);
-            if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - delta + longest - 1)) {
+            if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-                    int const back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
+                    int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
                     mlt -= back;
-
                     if (mlt > longest) {
                         longest = mlt;
                         *matchpos = matchPtr+back;
                         *startpos = ip+back;
-                }   }
-            }
+            }   }   }
         } else {   /* matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == pattern) {
+                const BYTE* const dictLowLimit = dictBase + hc4->lowLimit;
                 int mlt;
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
-                    mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
-                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictBase+lowLimit) : 0;
+                    mlt += LZ4_count(ip+mlt, lowPrefixPtr, iHighLimit);
+                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictLowLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;
-                    *matchpos = base + matchIndex + back;
+                    *matchpos = base + matchIndex + back;   /* virtual pos, relative to ip, to retrieve offset */
                     *startpos = ip + back;
         }   }   }
 
@@ -317,7 +319,7 @@ LZ4HC_InsertAndGetWiderMatch (
                 const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
+                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;
@@ -456,14 +458,14 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     BYTE* op = (BYTE*) dest;
     BYTE* oend = op + maxOutputSize;
 
-    int   ml, ml2, ml3, ml0;
+    int   ml0, ml, ml2, ml3;
+    const BYTE* start0;
+    const BYTE* ref0;
     const BYTE* ref = NULL;
     const BYTE* start2 = NULL;
     const BYTE* ref2 = NULL;
     const BYTE* start3 = NULL;
     const BYTE* ref3 = NULL;
-    const BYTE* start0;
-    const BYTE* ref0;
 
     /* init */
     *srcSizePtr = 0;
@@ -476,31 +478,27 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (ml<MINMATCH) { ip++; continue; }
 
         /* saved, in case we would skip too much */
-        start0 = ip;
-        ref0 = ref;
-        ml0 = ml;
+        start0 = ip; ref0 = ref; ml0 = ml;
 
 _Search2:
-        if (ip+ml <= mflimit)
+        if (ip+ml <= mflimit) {
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
                             maxNbAttempts, patternAnalysis, dict);
-        else
+        } else {
             ml2 = ml;
+        }
 
-        if (ml2 == ml) { /* No better match */
+        if (ml2 == ml) { /* No better match => encode ML1 */
             optr = op;
             if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
             continue;
         }
 
-        if (start0 < ip) {
-            if (start2 < ip + ml0) {  /* empirical */
-                ip = start0;
-                ref = ref0;
-                ml = ml0;
-            }
-        }
+        if (start0 < ip) {   /* first match was skipped at least once */
+            if (start2 < ip + ml0) {  /* squeezing ML1 between ML0(original ML1) and ML2 */
+                ip = start0; ref = ref0; ml = ml0;  /* restore initial ML1 */
+        }   }
 
         /* Here, start0==ip */
         if ((start2 - ip) < 3) {  /* First Match too small : removed */
@@ -528,14 +526,15 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         }
         /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
 
-        if (start2 + ml2 <= mflimit)
+        if (start2 + ml2 <= mflimit) {
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
                             maxNbAttempts, patternAnalysis, dict);
-        else
+        } else {
             ml3 = ml2;
+        }
 
-        if (ml3 == ml2) {  /* No better match : 2 sequences to encode */
+        if (ml3 == ml2) {  /* No better match => encode ML1 and ML2 */
             /* ip & ref are known; Now for ml */
             if (start2 < ip+ml)  ml = (int)(start2 - ip);
             /* Now, encode 2 sequences */
@@ -580,11 +579,12 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         }
 
         /*
-        * OK, now we have 3 ascending matches; let's write at least the first one
-        * ip & ref are known; Now for ml
+        * OK, now we have 3 ascending matches;
+        * let's write the first one ML1.
+        * ip & ref are known; Now decide ml.
         */
         if (start2 < ip+ml) {
-            if ((start2 - ip) < (int)ML_MASK) {
+            if ((start2 - ip) < OPTIMAL_ML) {
                 int correction;
                 if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
                 if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
@@ -601,14 +601,13 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         optr = op;
         if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
 
-        ip = start2;
-        ref = ref2;
-        ml = ml2;
+        /* ML2 becomes ML1 */
+        ip = start2; ref = ref2; ml = ml2;
 
-        start2 = start3;
-        ref2 = ref3;
-        ml2 = ml3;
+        /* ML3 becomes ML2 */
+        start2 = start3; ref2 = ref3; ml2 = ml3;
 
+        /* let's find a new ML3 */
         goto _Search3;
     }
 

From 19b1267d44a6bd3fbb1b682d515a3ab5dc70ec50 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 27 Apr 2018 12:46:49 -0700
Subject: [PATCH 222/257] fix lz4hc -BD non-determinism

related to chain table update
---
 lib/lz4hc.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 077855281..184ca8cab 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -699,8 +699,6 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
     ctx->end += *srcSizePtr;
     if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe something to review */
     cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);
-    assert(cLevel >= 0);
-    assert(cLevel <= LZ4HC_CLEVEL_MAX);
     {   cParams_t const cParam = clTable[cLevel];
         if (cParam.strat == lz4hc)
             return LZ4HC_compress_hashChain(ctx,
@@ -892,7 +890,8 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
     DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock);
-    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end >= ctxPtr->base + ctxPtr->dictLimit + 4)
+        LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;

From 69242a8a0821fb3157f1d28e0c07f3210a7fca52 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Sat, 28 Apr 2018 07:16:46 +0300
Subject: [PATCH 223/257] lib/Makefile: show commands with V=1

`make V=1` will now show the commands executed to build the library.
A similar technique is used in e.g. linux/Makefile.

The bulk of this change is produced with the following vim command:

    :g!/^\t@echo\>/s/^\t@/\t\$(Q)/
---
 lib/Makefile | 62 ++++++++++++++++++++++++++++------------------------
 1 file changed, 34 insertions(+), 28 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index bb4558273..d63de18be 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -85,32 +85,38 @@ all: lib
 all32: CFLAGS+=-m32
 all32: all
 
+ifeq ($(V), 1)
+Q =
+else
+Q = @
+endif
+
 liblz4.a: $(SRCFILES)
 ifeq ($(BUILD_STATIC),yes)  # can be disabled on command line
 	@echo compiling static library
-	@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
-	@$(AR) rcs $@ *.o
+	$(Q)$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
+	$(Q)$(AR) rcs $@ *.o
 endif
 
 $(LIBLZ4): $(SRCFILES)
 ifeq ($(BUILD_SHARED),yes)  # can be disabled on command line
 	@echo compiling dynamic library $(LIBVER)
 ifneq (,$(filter Windows%,$(OS)))
-	@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
+	$(Q)$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
 	dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib
 else
-	@$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
+	$(Q)$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
-	@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
-	@ln -sf $@ liblz4.$(SHARED_EXT)
+	$(Q)ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf $@ liblz4.$(SHARED_EXT)
 endif
 endif
 
 liblz4: $(LIBLZ4)
 
 clean:
-	@$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib
-	@$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
+	$(Q)$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib
+	$(Q)$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER)
 	@echo Cleaning library completed
 
 
@@ -152,41 +158,41 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 
 liblz4.pc: liblz4.pc.in Makefile
 	@echo creating pkgconfig
-	@sed -e 's|@PREFIX@|$(PREFIX)|' \
+	$(Q)sed -e 's|@PREFIX@|$(PREFIX)|' \
          -e 's|@LIBDIR@|$(LIBDIR)|' \
          -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
          -e 's|@VERSION@|$(LIBVER)|' \
           $< >$@
 
 install: lib liblz4.pc
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
-	@$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
+	$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
 	@echo Installing libraries
 ifeq ($(BUILD_STATIC),yes)
-	@$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
-	@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+	$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 endif
 ifeq ($(BUILD_SHARED),yes)
-	@$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+	$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
 endif
 	@echo Installing headers in $(INCLUDEDIR)
-	@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	@$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
+	$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	$(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
 	@echo lz4 libraries installed
 
 uninstall:
-	@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
-	@$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 	@echo lz4 libraries successfully uninstalled
 
 endif

From 45f8603aae389d34c689d3ff7427b314071ccd2c Mon Sep 17 00:00:00 2001
From: Alexey Tourbin <alexey.tourbin@gmail.com>
Date: Sat, 28 Apr 2018 11:14:40 +0300
Subject: [PATCH 224/257] lz4.c: two-stage shortcut for LZ4_decompress_generic

---
 lib/lz4.c | 107 +++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 82 insertions(+), 25 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 06ff611d5..13303742c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1397,6 +1397,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     const int safeDecode = (endOnInput==endOnInputSize);
     const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
 
+    /* Set up the "end" pointers for the shortcut. */
+    const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
+    const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
 
     /* Special cases */
     if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                      /* targetOutputSize too high => just decode everything */
@@ -1406,39 +1409,90 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
     /* Main Loop : decode sequences */
     while (1) {
-        size_t length;
         const BYTE* match;
         size_t offset;
 
         unsigned const token = *ip++;
+        size_t length = token >> ML_BITS; /* literal length */
 
         assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
-        /* shortcut for common case :
-         * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
-         * this shortcut was tested on x86 and x64, where it improves decoding speed.
-         * it has not yet been benchmarked on ARM, Power, mips, etc.
-         * NOTE: The loop begins with a read, so we must have one byte left at the end. */
-        if (endOnInput
-          && ((ip + 14 /*maxLL*/ + 2 /*offset*/ < iend)
-            & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)
-            & (token < (15<<ML_BITS))
-            & ((token & ML_MASK) != 15) ) ) {
-            size_t const ll = token >> ML_BITS;
-            size_t const off = LZ4_readLE16(ip+ll);
-            const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-            if ((off >= 8) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
-                size_t const ml = (token & ML_MASK) + MINMATCH;
-                memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
-                memcpy(op + 0, matchPtr + 0, 8);
-                memcpy(op + 8, matchPtr + 8, 8);
-                memcpy(op +16, matchPtr +16, 2);
-                op += ml;
-                continue;
+
+        /* A two-stage shortcut for the most common case:
+         * 1) If the literal length is 0..14, and there is enough space,
+         * enter the shortcut and copy 16 bytes on behalf of the literals
+         * (in the fast mode, only 8 bytes can be safely copied this way).
+         * 2) Further if the match length is 4..18, copy 18 bytes in a similar
+         * manner; but we ensure that there's enough space in the output for
+         * those 18 bytes earlier, upon entering the shortcut (in other words,
+         * there is a combined check for both stages).
+         */
+        if ((endOnInput ? length != RUN_MASK : length <= 8) &&
+            /* strictly "less than" on input, to re-enter the loop with at least one byte */
+            likely((endOnInput ? ip < shortiend : 1) && (op <= shortoend)))
+        {
+            /* Can we copy the literals with a single memcpy invocation?  Sometimes we can't
+             * copy 16 bytes, because they can clobber the dictionary in the ring buffer. */
+            if (!endOnInput /* only 8 bytes */ || /* nothing to clobber */ dict != usingExtDict) {
+                /* Copy the literals. */
+                memcpy(op, ip, endOnInput ? 16 : 8);
+                op += length; ip += length;
+
+                /* The second stage: prepare for match copying, decode full info.
+                 * It if doesn't work out, the info won't be wasted. */
+                length = token & ML_MASK; /* match length */
+                offset = LZ4_readLE16(ip); ip += 2;
+                match = op - offset;
+
+                /* Do not deal with overlapping matches. */
+                if ((length != 15) && (offset >= 8) &&
+                    (dict==withPrefix64k || match >= lowPrefix))
+                {
+                    /* Copy the match. */
+                    memcpy(op + 0, match + 0, 8);
+                    memcpy(op + 8, match + 8, 8);
+                    memcpy(op +16, match +16, 2);
+                    op += length + MINMATCH;
+                    /* Both stages worked, load the next token. */
+                    continue;
+                }
+            } else {
+                /* Save the literal length, can't copy 16 bytes just yet. */
+                size_t ll = length;
+
+                /* Prepare for the second satge. */
+                length = token & ML_MASK;
+                offset = LZ4_readLE16(ip+ll);
+                match = op + ll - offset;
+
+                if ((length != 15) && (offset >= 8) &&
+                    (dict==withPrefix64k || match >= lowPrefix))
+                {
+                    /* Copy the literals. */
+                    memcpy(op, ip, 16);
+                    op += ll; ip += ll + 2;
+                    /* Copy the match. */
+                    memcpy(op + 0, match + 0, 8);
+                    memcpy(op + 8, match + 8, 8);
+                    memcpy(op +16, match +16, 2);
+                    op += length + MINMATCH;
+                    /* Both stages worked, load the next token. */
+                    continue;
+                }
+
+                /* So we took the literlas, but the second stage didn't work. */
+                memcpy(op, ip, 8);
+                if (ll > 8)
+                    memcpy(op + 8, ip + 8, 8);
+                op += ll; ip += ll + 2;
             }
+
+            /* The second stage didn't work out, but the info is ready.
+             * Propel it right to the point of match copying. */
+            goto _copy_match;
         }
 
         /* decode literal length */
-        if ((length=(token>>ML_BITS)) == RUN_MASK) {
+        if (length == RUN_MASK) {
             unsigned s;
             if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error;   /* overflow detection */
             do {
@@ -1472,11 +1526,14 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         /* get offset */
         offset = LZ4_readLE16(ip); ip+=2;
         match = op - offset;
-        if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error;   /* Error : offset outside buffers */
-        LZ4_write32(op, (U32)offset);   /* costs ~1%; silence an msan warning when offset==0 */
 
         /* get matchlength */
         length = token & ML_MASK;
+
+_copy_match:
+        if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error;   /* Error : offset outside buffers */
+        LZ4_write32(op, (U32)offset);   /* costs ~1%; silence an msan warning when offset==0 */
+
         if (length == ML_MASK) {
             unsigned s;
             do {

From aaeeb2572bfa197bf29fd804953c93cba9db57be Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sat, 28 Apr 2018 10:42:52 -0700
Subject: [PATCH 225/257] ignore windows+msys artefacts

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 117b02d14..829270b93 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,7 @@ bin/
 # Mac
 .DS_Store
 *.dSYM
+
+# Windows / Msys
+nul
+ld.exe*

From 5a2501a90d4a0fa581bce334fb23c2c4df69842c Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sun, 29 Apr 2018 07:42:24 -0700
Subject: [PATCH 226/257] added a test case for LZ4_decompress_fast_usingDict

with a separated dictionary
since a joined dictionary is now detected as prefix64K.

Also : fixed a minor warning under msys
---
 programs/util.h   |  2 +-
 tests/fullbench.c | 34 ++++++++++++++++++++++------------
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/programs/util.h b/programs/util.h
index ef6ca77ac..d74db0d74 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -194,7 +194,7 @@ extern "C" {
         return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
     }
 
-#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2))
+#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || (defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2) ) )
 
     #include <time.h>
     typedef struct timespec UTIL_time_t;
diff --git a/tests/fullbench.c b/tests/fullbench.c
index ee2966f88..c06e2301d 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -267,13 +267,20 @@ static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int
     return outSize;
 }
 
-static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
+static int local_LZ4_decompress_fast_usingDict_prefix(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
     LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536);
     return outSize;
 }
 
+static int local_LZ4_decompress_fast_usingExtDict(const char* in, char* out, int inSize, int outSize)
+{
+    (void)inSize;
+    LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65535);
+    return outSize;
+}
+
 static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
@@ -460,7 +467,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 double averageTime;
                 clock_t clockTime;
 
-                PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize);
+                PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, compressorName, (int)benchedSize);
                 { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; }     /* warming up memory */
 
                 nb_loops = 0;
@@ -471,7 +478,8 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                     if (initFunction!=NULL) initFunction();
                     for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
                         chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
-                        if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
+                        if (chunkP[chunkNb].compressedSize==0)
+                            DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
                     }
                     nb_loops++;
                 }
@@ -482,13 +490,13 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 if (averageTime < bestTime) bestTime = averageTime;
                 cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
                 ratio = (double)cSize/(double)benchedSize*100.;
-                PROGRESS("%1i- %-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
+                PROGRESS("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
             }
 
             if (ratio<100.)
-                DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
+                DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
             else
-                DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000);
+                DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000);
         }
 
         /* Prepare layout for decompression */
@@ -509,11 +517,12 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
         }
         for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
             chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize);
-            if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
+            if (chunkP[chunkNb].compressedSize==0)
+                DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
         }
 
         /* Decompression Algorithms */
-        for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (g_decompressionTest); dAlgNb++) {
+        for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) {
             const char* dName;
             int (*decompressionFunction)(const char*, char*, int, int);
             double bestTime = 100000000.;
@@ -524,7 +533,8 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
             {
             case 0: DISPLAY("Decompression functions : \n"); continue;
             case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break;
-            case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break;
+            case 2: decompressionFunction = local_LZ4_decompress_fast_usingDict_prefix; dName = "LZ4_decompress_fast_usingDict(prefix)"; break;
+            case 3: decompressionFunction = local_LZ4_decompress_fast_usingExtDict; dName = "LZ4_decompress_fast_using(Ext)Dict"; break;
             case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break;
             case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
             case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break;
@@ -555,7 +565,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 clock_t clockTime;
                 U32 crcDecoded;
 
-                PROGRESS("%1i- %-29.29s :%10i ->\r", loopNb, dName, (int)benchedSize);
+                PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, dName, (int)benchedSize);
 
                 nb_loops = 0;
                 clockTime = clock();
@@ -574,14 +584,14 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
                 averageTime = (double)clockTime / nb_loops / CLOCKS_PER_SEC;
                 if (averageTime < bestTime) bestTime = averageTime;
 
-                PROGRESS("%1i- %-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
+                PROGRESS("%2i-%-34.34s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
 
                 /* CRC Checking */
                 crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
                 if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
             }
 
-            DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
+            DISPLAY("%2i-%-34.34s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
         }
       }
       free(orig_buff);

From e28ae0af078a632f0c66647396c69d4661364b79 Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Sun, 29 Apr 2018 08:46:39 -0700
Subject: [PATCH 227/257] updated NEWS for v1.8.2

mentioning work from @svpv
---
 NEWS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS b/NEWS
index c3b47f112..90cafc6e6 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,7 @@ v1.8.2
 perf: *much* faster dictionary compression on small files, by @felixhandte
 perf: slightly faster HC compression and decompression speed
 perf: very small compression ratio improvement
+perf: improved decompression binary size and speed, by Alexey Tourbin (@svpv)
 fix : compression compatible with low memory addresses (< 0xFFFF)
 fix : decompression segfault when provided with NULL input, by @terrelln
 cli : new command --favor-decSpeed

From 4c696613a071bc31d111bed9f9bf85e392a80901 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 30 Apr 2018 15:55:33 -0700
Subject: [PATCH 228/257] clarified streaming decompression function

restrictions for ring buffer
---
 lib/lz4.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index db6353c0c..274526021 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -317,15 +317,19 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const
  *  If less than 64KB of data has been decoded all the data must be present.
  *
  *  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
- *  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
- *    In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- *  - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- *    maxBlockSize is implementation dependent. It's the maximum size of any single block.
+ *  - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ *    maxBlockSize is the maximum size of any single block. It is implementation dependent, and can have any value (presumed > 16 bytes).
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including small ones ( < 64 KB).
- *  - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ *  - Decompression buffer size is _at least_ 64 KB + 8 bytes + maxBlockSize.
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including larger than decoding buffer.
+ *  - Decompression buffer size is exactly the same as compression buffer size,
+ *    and follows exactly same update rule (block boundaries at same positions).
+ *    If the decoding function is provided with the exact decompressed size of each block,
+ *    then decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ *    If the decoding function only knows the compressed size,
+ *    then buffer size must be a minimum of 64 KB + 8 bytes + maxBlockSize.
  *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
  *  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
 */

From 8c574990a908233196080e70c12605f5f3d8eadd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 30 Apr 2018 16:08:16 -0700
Subject: [PATCH 229/257] lz4hc changed variable

to reduce confusion
dictLowLimit => dictStart
---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f2dc21b5f..d42bb5a9d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -264,7 +264,7 @@ LZ4HC_InsertAndGetWiderMatch (
         } else {   /* matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == pattern) {
-                const BYTE* const dictLowLimit = dictBase + hc4->lowLimit;
+                const BYTE* const dictStart = dictBase + hc4->lowLimit;
                 int mlt;
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
@@ -272,7 +272,7 @@ LZ4HC_InsertAndGetWiderMatch (
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
                     mlt += LZ4_count(ip+mlt, lowPrefixPtr, iHighLimit);
-                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictLowLimit) : 0;
+                back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;
                 mlt -= back;
                 if (mlt > longest) {
                     longest = mlt;

From 1949bf11e3903ab6c615ac625587426dc11b4beb Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 30 Apr 2018 18:50:56 -0700
Subject: [PATCH 230/257] added visual test dir to .gitignore

---
 visual/.gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/visual/.gitignore b/visual/.gitignore
index dea92fc6c..276f8f553 100644
--- a/visual/.gitignore
+++ b/visual/.gitignore
@@ -6,5 +6,5 @@
 *.sdf
 *.suo
 *.user
-
+ver*/
 VS2010/bin/

From 6a7d501fed8444cb8a78cd55133916921bfbb07f Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Mon, 30 Apr 2018 18:56:16 -0700
Subject: [PATCH 231/257] renamed variable for clarity

lowLimit -> lowestMatchIndex
---
 lib/lz4hc.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d42bb5a9d..26295fcea 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -223,7 +223,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
     const U32 ipIndex = (U32)(ip - base);
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
+    const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
     int const lookBackLength = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
@@ -237,10 +237,10 @@ LZ4HC_InsertAndGetWiderMatch (
     /* First Match */
     LZ4HC_Insert(hc4, ip);
     matchIndex = HashTable[LZ4HC_hashPtr(ip)];
-    DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
-                matchIndex, lowLimit);
+    DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",
+                matchIndex, lowestMatchIndex);
 
-    while ((matchIndex>=lowLimit) && (nbAttempts)) {
+    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
         DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         assert(matchIndex < ipIndex);
@@ -308,13 +308,13 @@ LZ4HC_InsertAndGetWiderMatch (
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
         }   }   }   }
-    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
+    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
-    if (dict == usingDictCtx && nbAttempts && ipIndex - lowLimit < MAX_DISTANCE) {
+    if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) {
         size_t const dictEndOffset = dictCtx->end - dictCtx->base;
         assert(dictEndOffset <= 1 GB);
         dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
-        matchIndex = dictMatchIndex + lowLimit - (U32)dictEndOffset;
+        matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset;
         while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) {
             const BYTE* const matchPtr = dictCtx->base + dictMatchIndex;
 

From 1a191b3f8d26b50a7c1d41590b529ec308d768cd Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 10:33:12 -0700
Subject: [PATCH 232/257] simplify shortcut

---
 lib/lz4.c | 77 ++++++++++++++++---------------------------------------
 1 file changed, 22 insertions(+), 55 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c6f042642..b46910f76 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1429,62 +1429,29 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          */
         if ((endOnInput ? length != RUN_MASK : length <= 8) &&
             /* strictly "less than" on input, to re-enter the loop with at least one byte */
-            likely((endOnInput ? ip < shortiend : 1) && (op <= shortoend)))
+            likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)))
         {
-            /* Can we copy the literals with a single memcpy invocation?  Sometimes we can't
-             * copy 16 bytes, because they can clobber the dictionary in the ring buffer. */
-            if (!endOnInput /* only 8 bytes */ || /* nothing to clobber */ dict != usingExtDict) {
-                /* Copy the literals. */
-                memcpy(op, ip, endOnInput ? 16 : 8);
-                op += length; ip += length;
-
-                /* The second stage: prepare for match copying, decode full info.
-                 * It if doesn't work out, the info won't be wasted. */
-                length = token & ML_MASK; /* match length */
-                offset = LZ4_readLE16(ip); ip += 2;
-                match = op - offset;
-
-                /* Do not deal with overlapping matches. */
-                if ((length != 15) && (offset >= 8) &&
-                    (dict==withPrefix64k || match >= lowPrefix))
-                {
-                    /* Copy the match. */
-                    memcpy(op + 0, match + 0, 8);
-                    memcpy(op + 8, match + 8, 8);
-                    memcpy(op +16, match +16, 2);
-                    op += length + MINMATCH;
-                    /* Both stages worked, load the next token. */
-                    continue;
-                }
-            } else {
-                /* Save the literal length, can't copy 16 bytes just yet. */
-                size_t ll = length;
-
-                /* Prepare for the second satge. */
-                length = token & ML_MASK;
-                offset = LZ4_readLE16(ip+ll);
-                match = op + ll - offset;
-
-                if ((length != 15) && (offset >= 8) &&
-                    (dict==withPrefix64k || match >= lowPrefix))
-                {
-                    /* Copy the literals. */
-                    memcpy(op, ip, 16);
-                    op += ll; ip += ll + 2;
-                    /* Copy the match. */
-                    memcpy(op + 0, match + 0, 8);
-                    memcpy(op + 8, match + 8, 8);
-                    memcpy(op +16, match +16, 2);
-                    op += length + MINMATCH;
-                    /* Both stages worked, load the next token. */
-                    continue;
-                }
-
-                /* So we took the literlas, but the second stage didn't work. */
-                memcpy(op, ip, 8);
-                if (ll > 8)
-                    memcpy(op + 8, ip + 8, 8);
-                op += ll; ip += ll + 2;
+            /* Copy the literals. */
+            memcpy(op, ip, endOnInput ? 16 : 8);
+            op += length; ip += length;
+
+            /* The second stage: prepare for match copying, decode full info.
+             * If it doesn't work out, the info won't be wasted. */
+            length = token & ML_MASK; /* match length */
+            offset = LZ4_readLE16(ip); ip += 2;
+            match = op - offset;
+
+            /* Do not deal with overlapping matches. */
+            if ((length != 15) && (offset >= 8) &&
+                (dict==withPrefix64k || match >= lowPrefix))
+            {
+                /* Copy the match. */
+                memcpy(op + 0, match + 0, 8);
+                memcpy(op + 8, match + 8, 8);
+                memcpy(op +16, match +16, 2);
+                op += length + MINMATCH;
+                /* Both stages worked, load the next token. */
+                continue;
             }
 
             /* The second stage didn't work out, but the info is ready.

From 93cf628a08e82256abc77d1fd144bda78b7ee1df Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 12:56:37 -0700
Subject: [PATCH 233/257] introduce LZ4_decoderRingBufferSize()

fuzzer : fix and robustify ring buffer tests
---
 lib/lz4.c      | 33 +++++++++++++++----
 lib/lz4.h      | 56 ++++++++++++++++++++-----------
 tests/fuzzer.c | 89 ++++++++++++++++++++++++++++++--------------------
 3 files changed, 116 insertions(+), 62 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index b46910f76..71fe8f33e 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1431,7 +1431,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             /* strictly "less than" on input, to re-enter the loop with at least one byte */
             likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)))
         {
-            /* Copy the literals. */
+            /* Copy the literals */
             memcpy(op, ip, endOnInput ? 16 : 8);
             op += length; ip += length;
 
@@ -1688,12 +1688,11 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
     return 0;
 }
 
-/*!
- * LZ4_setStreamDecode() :
- * Use this function to instruct where to find the dictionary.
- * This function is not necessary if previous data is still available where it was decoded.
- * Loading a size of 0 is allowed (same effect as no dictionary).
- * Return : 1 if OK, 0 if error
+/*! LZ4_setStreamDecode() :
+ *  Use this function to instruct where to find the dictionary.
+ *  This function is not necessary if previous data is still available where it was decoded.
+ *  Loading a size of 0 is allowed (same effect as no dictionary).
+ * @return : 1 if OK, 0 if error
  */
 int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
 {
@@ -1705,6 +1704,26 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
     return 1;
 }
 
+/*! LZ4_decoderRingBufferSize() :
+ *  when setting a ring buffer for streaming decompression (optional scenario),
+ *  provides the minimum size of this ring buffer
+ *  to be compatible with any source respecting maxBlockSize condition.
+ *  Note : in a ring buffer scenario,
+ *  blocks are presumed decompressed next to each other
+ *  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
+ *  at which stage it resumes from beginning of ring buffer.
+ * @return : minimum ring buffer size,
+ *           or 0 if there is an error (invalid maxBlockSize).
+ */
+
+int LZ4_decoderRingBufferSize(int maxBlockSize)
+{
+    if (maxBlockSize < 0) return 0;
+    if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0;
+    if (maxBlockSize < 16) maxBlockSize = 16;
+    return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize);
+}
+
 /*
 *_continue() :
     These decoding functions allow decompression of multiple blocks in "streaming" mode.
diff --git a/lib/lz4.h b/lib/lz4.h
index 274526021..410f4809e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -293,45 +293,62 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxD
 *  Streaming Decompression Functions
 *  Bufferless synchronous API
 ************************************************/
-typedef union LZ4_streamDecode_u LZ4_streamDecode_t;   /* incomplete type (defined later) */
+typedef union LZ4_streamDecode_u LZ4_streamDecode_t;   /* tracking context */
 
 /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
- *  creation / destruction of streaming decompression tracking structure.
- *  A tracking structure can be re-used multiple times sequentially. */
+ *  creation / destruction of streaming decompression tracking context.
+ *  A tracking context can be re-used multiple times.
+ */
 LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 
 /*! LZ4_setStreamDecode() :
- *  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
+ *  An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
  *  Use this function to start decompression of a new stream of blocks.
  *  A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+ *  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  * @return : 1 if OK, 0 if error
  */
 LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
 
+/*! LZ4_decoderRingBufferSize() : v1.8.2
+ *  Note : in a ring buffer scenario (optional),
+ *  blocks are presumed decompressed next to each other
+ *  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
+ *  at which stage it resumes from beginning of ring buffer.
+ *  When setting such a ring buffer for streaming decompression,
+ *  provides the minimum size of this ring buffer
+ *  to be compatible with any source respecting maxBlockSize condition.
+ * @return : minimum ring buffer size,
+ *           or 0 if there is an error (invalid maxBlockSize).
+ */
+LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
+#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs))  /* for static allocation; mbs presumed valid */
+
 /*! LZ4_decompress_*_continue() :
  *  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
  *  A block is an unsplittable entity, it must be presented entirely to a decompression function.
- *  Decompression functions only accept one block at a time.
+ *  Decompression functions only accepts one block at a time.
  *  The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
- *  If less than 64KB of data has been decoded all the data must be present.
+ *  If less than 64KB of data has been decoded, all the data must be present.
  *
- *  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
+ *  Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
+ *  - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
+ *    maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
+ *    In which case, encoding and decoding buffers do not need to be synchronized.
+ *    Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
+ *  - Synchronized mode :
+ *    Decompression buffer size is _exactly_ the same as compression buffer size,
+ *    and follows exactly same update rule (block boundaries at same positions),
+ *    and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
+ *    _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
  *  - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- *    maxBlockSize is the maximum size of any single block. It is implementation dependent, and can have any value (presumed > 16 bytes).
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including small ones ( < 64 KB).
- *  - Decompression buffer size is _at least_ 64 KB + 8 bytes + maxBlockSize.
- *    In which case, encoding and decoding buffers do not need to be synchronized,
- *    and encoding ring buffer can have any size, including larger than decoding buffer.
- *  - Decompression buffer size is exactly the same as compression buffer size,
- *    and follows exactly same update rule (block boundaries at same positions).
- *    If the decoding function is provided with the exact decompressed size of each block,
- *    then decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- *    If the decoding function only knows the compressed size,
- *    then buffer size must be a minimum of 64 KB + 8 bytes + maxBlockSize.
- *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- *  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
+ *
+ *  Whenever these conditions are not possible,
+ *  save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
+ *  then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
 */
 LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
 LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
@@ -341,6 +358,7 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod
  *  These decoding functions work the same as
  *  a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
  *  They are stand-alone, and don't need an LZ4_streamDecode_t structure.
+ *  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  */
 LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
 LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 1fbda8aa0..28f14bf87 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -47,6 +47,7 @@
 #include <stdio.h>      /* fgets, sscanf */
 #include <string.h>     /* strcmp */
 #include <time.h>       /* clock_t, clock, CLOCKS_PER_SEC */
+#include <assert.h>
 #define LZ4_STATIC_LINKING_ONLY
 #define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
@@ -953,6 +954,7 @@ static void FUZ_unitTests(int compressionLevel)
     const unsigned cycleNb= 0;
     char testInput[testInputSize];
     char testCompressed[testCompressedSize];
+    size_t const testVerifySize = testInputSize;
     char testVerify[testInputSize];
     char ringBuffer[ringBufferSize];
     U32 randState = 1;
@@ -1197,19 +1199,28 @@ static void FUZ_unitTests(int compressionLevel)
             }
         }
 
-        /* small decoder-side ring buffer test */
+        /* Ring buffer test : Non synchronized decoder */
+        /* This test uses minimum amount of memory required to setup a decoding ring buffer
+         * while being unsynchronized with encoder
+         * (no assumption done on how the data is encoded, it just follows LZ4 format specification).
+         * This size is documented in lz4.h, and is LZ4_decoderRingBufferSize(maxBlockSize).
+         */
         {   XXH64_state_t xxhOrig;
             XXH64_state_t xxhNewSafe, xxhNewFast;
             LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
-            const U32 maxMessageSizeLog = 12;
-            const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
-            U32 messageSize;
+            const int maxMessageSizeLog = 12;
+            const int maxMessageSize = 1 << maxMessageSizeLog;
+            const int maxMessageSizeMask = maxMessageSize - 1;
+            int messageSize;
             U32 totalMessageSize = 0;
-            U32 iNext = 0;
-            U32 dNext = 0;
-            const U32 dBufferSize = 64 KB;
+            const int dBufferSize = LZ4_decoderRingBufferSize(maxMessageSize);
+            char* const ringBufferSafe = testVerify;
+            char* const ringBufferFast = testVerify + dBufferSize + 1;   /* used by LZ4_decompress_fast_continue */
+            int iNext = 0;
+            int dNext = 0;
             int compressedSize;
 
+            assert((size_t)(dBufferSize + 1 + dBufferSize) < testVerifySize);   /* space used by ringBufferSafe and ringBufferFast */
             XXH64_reset(&xxhOrig, 0);
             XXH64_reset(&xxhNewSafe, 0);
             XXH64_reset(&xxhNewFast, 0);
@@ -1217,38 +1228,40 @@ static void FUZ_unitTests(int compressionLevel)
             LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
             LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
 
-#define BSIZE1 65537
-#define BSIZE2 16435
+#define BSIZE1 (dBufferSize - (maxMessageSize-1))
 
             /* first block */
-            messageSize = BSIZE1;
+            messageSize = BSIZE1;   /* note : we cheat a bit here, in theory no message should be > maxMessageSize. We just want to fill the decoding ring buffer once. */
+            assert(messageSize < dBufferSize);
             XXH64_update(&xxhOrig, testInput + iNext, messageSize);
             crcOrig = XXH64_digest(&xxhOrig);
 
             compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
             FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
 
-            result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
-            FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
+            result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize);
+            FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
 
-            XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+            XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
             { U64 const crcNew = XXH64_digest(&xxhNewSafe);
               FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
 
-            result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+            result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
             FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
 
-            XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+            XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
             { U64 const crcNew = XXH64_digest(&xxhNewFast);
               FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
 
-            /* prepare next message */
+            /* prepare second message */
             dNext += messageSize;
             totalMessageSize += messageSize;
-            messageSize = BSIZE2;
-            iNext = 132000;
-            memcpy(testInput + iNext, testInput + 8, messageSize);
-            if (dNext > dBufferSize) dNext = 0;
+            messageSize = maxMessageSize;
+            iNext = BSIZE1+1;
+            assert(BSIZE1 >= 65535);
+            memcpy(testInput + iNext, testInput + (BSIZE1-65535), messageSize);  /* will generate a match at max distance == 65535 */
+            FUZ_CHECKTEST(dNext+messageSize <= dBufferSize, "Ring buffer test : second message should require restarting from beginning");
+            dNext = 0;
 
             while (totalMessageSize < 9 MB) {
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
@@ -1256,32 +1269,36 @@ static void FUZ_unitTests(int compressionLevel)
 
                 compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
                 FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
-
-#if 1           /* Because the ring buffer is small, decompression overwrites part of the output which
-                 * is first used as dictionary.  Hence only one decompression function can be tested. */
-                result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
-                XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
+                DISPLAYLEVEL(5, "compressed %i bytes to %i bytes \n", messageSize, compressedSize);
+
+                /* test LZ4_decompress_safe_continue */
+                assert(dNext < dBufferSize);
+                assert(dBufferSize - dNext >= maxMessageSize);
+                result = LZ4_decompress_safe_continue(&decodeStateSafe,
+                                                      testCompressed, ringBufferSafe + dNext,
+                                                      compressedSize, dBufferSize - dNext);   /* works without knowing messageSize, but under assumption that messageSize < maxMessageSize */
+                FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed");
+                XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewSafe);
-                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
-                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during small decoder-side ring buffer test");
+                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext);
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test");
                 }
-#else
-                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
-                FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
 
-                XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
+                /* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */
+                result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
+                FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed");
+                XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewFast);
-                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
-                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during small decoder-side ring buffer test");
+                    if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext);
+                    FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test");
                 }
-#endif
+
                 /* prepare next message */
                 dNext += messageSize;
                 totalMessageSize += messageSize;
                 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
                 iNext = (FUZ_rand(&randState) & 65535);
-                if (dNext > dBufferSize) dNext = 0;
+                if (dNext + maxMessageSize > dBufferSize) dNext = 0;
             }
         }
     }

From 999a8488f60cb076944d69d2aea9828f890a4e43 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 13:57:33 -0700
Subject: [PATCH 234/257] removed test that might be optimized away

under UB rule "no overflow on int"
---
 tests/fuzzer.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 28f14bf87..5dd75b389 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1232,7 +1232,6 @@ static void FUZ_unitTests(int compressionLevel)
 
             /* first block */
             messageSize = BSIZE1;   /* note : we cheat a bit here, in theory no message should be > maxMessageSize. We just want to fill the decoding ring buffer once. */
-            assert(messageSize < dBufferSize);
             XXH64_update(&xxhOrig, testInput + iNext, messageSize);
             crcOrig = XXH64_digest(&xxhOrig);
 
@@ -1276,7 +1275,7 @@ static void FUZ_unitTests(int compressionLevel)
                 assert(dBufferSize - dNext >= maxMessageSize);
                 result = LZ4_decompress_safe_continue(&decodeStateSafe,
                                                       testCompressed, ringBufferSafe + dNext,
-                                                      compressedSize, dBufferSize - dNext);   /* works without knowing messageSize, but under assumption that messageSize < maxMessageSize */
+                                                      compressedSize, dBufferSize - dNext);   /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */
                 FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed");
                 XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
                 {   U64 const crcNew = XXH64_digest(&xxhNewSafe);

From 85be6b8f6d5a91a8834e913b5f547ded49f7f714 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 14:22:35 -0700
Subject: [PATCH 235/257] increased nbAttempts for lz4 -12

shaves one more kilobyte from silesia.tar
---
 lib/lz4hc.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index bb1b1a636..0859ea610 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -681,19 +681,19 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
         U32 targetLength;
     } cParams_t;
     static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = {
-        { lz4hc,    2, 16 },  /* 0, unused */
-        { lz4hc,    2, 16 },  /* 1, unused */
-        { lz4hc,    2, 16 },  /* 2, unused */
-        { lz4hc,    4, 16 },  /* 3 */
-        { lz4hc,    8, 16 },  /* 4 */
-        { lz4hc,   16, 16 },  /* 5 */
-        { lz4hc,   32, 16 },  /* 6 */
-        { lz4hc,   64, 16 },  /* 7 */
-        { lz4hc,  128, 16 },  /* 8 */
-        { lz4hc,  256, 16 },  /* 9 */
-        { lz4opt,  96, 64 },  /*10==LZ4HC_CLEVEL_OPT_MIN*/
-        { lz4opt, 512,128 },  /*11 */
-        { lz4opt,8192, LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
+        { lz4hc,     2, 16 },  /* 0, unused */
+        { lz4hc,     2, 16 },  /* 1, unused */
+        { lz4hc,     2, 16 },  /* 2, unused */
+        { lz4hc,     4, 16 },  /* 3 */
+        { lz4hc,     8, 16 },  /* 4 */
+        { lz4hc,    16, 16 },  /* 5 */
+        { lz4hc,    32, 16 },  /* 6 */
+        { lz4hc,    64, 16 },  /* 7 */
+        { lz4hc,   128, 16 },  /* 8 */
+        { lz4hc,   256, 16 },  /* 9 */
+        { lz4opt,   96, 64 },  /*10==LZ4HC_CLEVEL_OPT_MIN*/
+        { lz4opt,  512,128 },  /*11 */
+        { lz4opt,16384,LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
     };
 
     DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr);

From c25eb1666654c378a6f9e74c9975181cb2767a2a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:05:42 -0700
Subject: [PATCH 236/257] random lz4f clarifications

the initial intention was to update lz4f ring buffer strategy,
but lz4f doesn't use ring buffer.
Instead, it uses the destination buffer as much as possible,
and merely copies just what's required to preserve history
into its own buffer, at the end.
Pretty efficient.

This patch just clarifies a few comments and add some assert().
It's built on top of #528.

It also updates doc.
---
 doc/lz4_manual.html | 54 ++++++++++++++++++++++----------
 lib/lz4frame.c      | 76 ++++++++++++++++++++++++++++-----------------
 2 files changed, 85 insertions(+), 45 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index ddd272431..e079db216 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -206,38 +206,59 @@ <h1>1.8.2 Manual</h1>
 
 <pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
-</b><p>  creation / destruction of streaming decompression tracking structure.
-  A tracking structure can be re-used multiple times sequentially. 
+</b><p>  creation / destruction of streaming decompression tracking context.
+  A tracking context can be re-used multiple times.
+ 
 </p></pre><BR>
 
 <pre><b>int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-</b><p>  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
+</b><p>  An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
   Use this function to start decompression of a new stream of blocks.
   A dictionary can optionnally be set. Use NULL or size 0 for a reset order.
+  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  @return : 1 if OK, 0 if error
  
 </p></pre><BR>
 
+<pre><b>int LZ4_decoderRingBufferSize(int maxBlockSize);
+#define LZ4_DECODER_RING_BUFFER_SIZE(mbs) (65536 + 14 + (mbs))  </b>/* for static allocation; mbs presumed valid */<b>
+</b><p>  Note : in a ring buffer scenario (optional),
+  blocks are presumed decompressed next to each other
+  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
+  at which stage it resumes from beginning of ring buffer.
+  When setting such a ring buffer for streaming decompression,
+  provides the minimum size of this ring buffer
+  to be compatible with any source respecting maxBlockSize condition.
+ @return : minimum ring buffer size,
+           or 0 if there is an error (invalid maxBlockSize).
+ 
+</p></pre><BR>
+
 <pre><b>int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
 </b><p>  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
   A block is an unsplittable entity, it must be presented entirely to a decompression function.
-  Decompression functions only accept one block at a time.
+  Decompression functions only accepts one block at a time.
   The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
-  If less than 64KB of data has been decoded all the data must be present.
-
-  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
-  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
-    In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
-  - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
-    maxBlockSize is implementation dependent. It's the maximum size of any single block.
+  If less than 64KB of data has been decoded, all the data must be present.
+
+  Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
+  - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
+    maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
+    In which case, encoding and decoding buffers do not need to be synchronized.
+    Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
+  - Synchronized mode :
+    Decompression buffer size is _exactly_ the same as compression buffer size,
+    and follows exactly same update rule (block boundaries at same positions),
+    and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
+    _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
+  - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
     In which case, encoding and decoding buffers do not need to be synchronized,
     and encoding ring buffer can have any size, including small ones ( < 64 KB).
-  - _At least_ 64 KB + 8 bytes + maxBlockSize.
-    In which case, encoding and decoding buffers do not need to be synchronized,
-    and encoding ring buffer can have any size, including larger than decoding buffer.
-  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
-  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
+
+  Whenever these conditions are not possible,
+  save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
+  then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
 </p></pre><BR>
 
 <pre><b>int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
@@ -245,6 +266,7 @@ <h1>1.8.2 Manual</h1>
 </b><p>  These decoding functions work the same as
   a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
   They are stand-alone, and don't need an LZ4_streamDecode_t structure.
+  Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
  
 </p></pre><BR>
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 6cf2a06e8..f57db249c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -96,6 +96,19 @@ You can contact the author at :
 
 #define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG)
+#  include <stdio.h>
+static int g_debuglog_enable = 1;
+#  define DEBUGLOG(l, ...) {                                  \
+                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
+                    fprintf(stderr, __FILE__ ": ");           \
+                    fprintf(stderr, __VA_ARGS__);             \
+                    fprintf(stderr, " \n");                   \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
+
 
 /*-************************************
 *  Basic Types
@@ -408,6 +421,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     LZ4_stream_t lz4ctx;
     LZ4F_cctx_t *cctxPtr = &cctx;
 
+    DEBUGLOG(4, "LZ4F_compressFrame");
     MEM_INIT(&cctx, 0, sizeof(cctx));
     cctx.version = LZ4F_VERSION;
     cctx.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
@@ -1198,24 +1212,31 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoP
 
 /* LZ4F_updateDict() :
  * only used for LZ4F_blockLinked mode */
-static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
+static void LZ4F_updateDict(LZ4F_dctx* dctx,
+                      const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
+                      unsigned withinTmp)
 {
     if (dctx->dictSize==0)
         dctx->dict = (const BYTE*)dstPtr;   /* priority to dictionary continuity */
 
-    if (dctx->dict + dctx->dictSize == dstPtr) {  /* dictionary continuity */
+    if (dctx->dict + dctx->dictSize == dstPtr) {  /* dictionary continuity, directly within dstBuffer */
         dctx->dictSize += dstSize;
         return;
     }
 
-    if (dstPtr - dstPtr0 + dstSize >= 64 KB) {  /* dstBuffer large enough to become dictionary */
-        dctx->dict = (const BYTE*)dstPtr0;
-        dctx->dictSize = dstPtr - dstPtr0 + dstSize;
+    if (dstPtr - dstBufferStart + dstSize >= 64 KB) {  /* history in dstBuffer becomes large enough to become dictionary */
+        dctx->dict = (const BYTE*)dstBufferStart;
+        dctx->dictSize = dstPtr - dstBufferStart + dstSize;
         return;
     }
 
-    if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {
-        /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */
+    assert(dstSize < 64 KB);   /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */
+
+    /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */
+
+    if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {   /* continue history within tmpOutBuffer */
+        /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */
+        assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart);
         dctx->dictSize += dstSize;
         return;
     }
@@ -1236,7 +1257,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize,
 
     if (dctx->dict == dctx->tmpOutBuffer) {    /* copy dst into tmp to complete dict */
         if (dctx->dictSize + dstSize > dctx->maxBufferSize) {  /* tmp buffer not large enough */
-            size_t const preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
+            size_t const preserveSize = 64 KB - dstSize;
             memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
             dctx->dictSize = preserveSize;
         }
@@ -1246,7 +1267,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize,
     }
 
     /* join dict & dest into tmp */
-    {   size_t preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
+    {   size_t preserveSize = 64 KB - dstSize;
         if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
         memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
         memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
@@ -1313,7 +1334,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }
             dctx->tmpInSize = 0;
             if (srcEnd-srcPtr == 0) return minFHSize;   /* 0-size input */
-            dctx->tmpInTarget = minFHSize;   /* minimum to attempt decode */
+            dctx->tmpInTarget = minFHSize;   /* minimum size to decode header */
             dctx->dStage = dstage_storeFrameHeader;
             /* fall-through */
 
@@ -1470,8 +1491,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                     U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
                     if (readCRC != calcCRC)
                         return err0r(LZ4F_ERROR_blockChecksum_invalid);
-                }
-            }
+            }   }
             dctx->dStage = dstage_getBlockHeader;  /* new block */
             break;
 
@@ -1512,13 +1532,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }   }
 
             if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
-                const char *dict = (const char *)dctx->dict;
+                const char* dict = (const char*)dctx->dict;
                 size_t dictSize = dctx->dictSize;
                 int decodedSize;
                 if (dict && dictSize > 1 GB) {
                     /* the dictSize param is an int, avoid truncation / sign issues */
-                    dict += dictSize - 1 GB;
-                    dictSize = 1 GB;
+                    dict += dictSize - 64 KB;
+                    dictSize = 64 KB;
                 }
                 /* enough capacity in `dst` to decompress directly there */
                 decodedSize = LZ4_decompress_safe_usingDict(
@@ -1552,18 +1572,16 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 } else {  /* dict not within tmp */
                     size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
                     dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
-                }
-            }
+            }   }
 
             /* Decode block */
-            {
-                const char *dict = (const char *)dctx->dict;
+            {   const char* dict = (const char*)dctx->dict;
                 size_t dictSize = dctx->dictSize;
                 int decodedSize;
                 if (dict && dictSize > 1 GB) {
                     /* the dictSize param is an int, avoid truncation / sign issues */
-                    dict += dictSize - 1 GB;
-                    dictSize = 1 GB;
+                    dict += dictSize - 64 KB;
+                    dictSize = 64 KB;
                 }
                 decodedSize = LZ4_decompress_safe_usingDict(
                         (const char*)selectedIn, (char*)dctx->tmpOut,
@@ -1586,8 +1604,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
 
                 /* dictionary management */
-                if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
-                    LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1);
+                if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
+                    LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/);
 
                 dctx->tmpOutStart += sizeToCopy;
                 dstPtr += sizeToCopy;
@@ -1596,8 +1614,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                     dctx->dStage = dstage_getBlockHeader;  /* get next block */
                     break;
                 }
+                /* could not flush everything : stop there, just request a block header */
+                doAnotherStage = 0;
                 nextSrcSizeHint = BHSize;
-                doAnotherStage = 0;   /* still some data to flush */
                 break;
             }
 
@@ -1634,7 +1653,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 selectedIn = dctx->tmpIn;
             }   /* if (dctx->dStage == dstage_storeSuffix) */
 
-        /* case dstage_checkSuffix: */   /* no direct call, avoid scan-build warning */
+        /* case dstage_checkSuffix: */   /* no direct entry, avoid initialization risks */
             {   U32 const readCRC = LZ4F_readLE32(selectedIn);
                 U32 const resultCRC = XXH32_digest(&(dctx->xxh));
                 if (readCRC != resultCRC)
@@ -1658,8 +1677,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 
             if (dctx->dStage == dstage_storeSFrameSize)
         case dstage_storeSFrameSize:
-            {
-                size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
+            {   size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
                                              (size_t)(srcEnd - srcPtr) );
                 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
@@ -1673,7 +1691,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 selectedIn = dctx->header + 4;
             }   /* if (dctx->dStage == dstage_storeSFrameSize) */
 
-        /* case dstage_decodeSFrameSize: */   /* no direct access */
+        /* case dstage_decodeSFrameSize: */   /* no direct entry */
             {   size_t const SFrameSize = LZ4F_readLE32(selectedIn);
                 dctx->frameInfo.contentSize = SFrameSize;
                 dctx->tmpInTarget = SFrameSize;
@@ -1692,7 +1710,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 LZ4F_resetDecompressionContext(dctx);
                 break;
             }
-        }
+        }   /* switch (dctx->dStage) */
     }   /* while (doAnotherStage) */
 
     /* preserve history within tmp whenever necessary */

From 543223851fe4b334ff6380d71d4ca94c822fcc8a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:41:15 -0700
Subject: [PATCH 237/257] updated benchmark for v1.8.2

---
 README.md | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 596fdac7d..5d0aa48f7 100644
--- a/README.md
+++ b/README.md
@@ -43,8 +43,8 @@ Benchmarks
 -------------------------
 
 The benchmark uses [lzbench], from @inikep
-compiled with GCC v6.2.0 on Linux 64-bits.
-The reference system uses a Core i7-3930K CPU @ 4.5GHz.
+compiled with GCC v7.3.0 on Linux 64-bits (Debian 4.15.17-1).
+The reference system uses a Core i7-6700K CPU @ 4.0GHz.
 Benchmark evaluates the compression of reference [Silesia Corpus]
 in single-thread mode.
 
@@ -53,23 +53,22 @@ in single-thread mode.
 
 |  Compressor            | Ratio   | Compression | Decompression |
 |  ----------            | -----   | ----------- | ------------- |
-|  memcpy                |  1.000  | 7300 MB/s   |   7300 MB/s   |
-|**LZ4 fast 8  (v1.7.3)**|  1.799  |**911 MB/s** | **3360 MB/s** |
-|**LZ4 default (v1.7.3)**|**2.101**|**625 MB/s** | **3220 MB/s** |
-|  LZO 2.09              |  2.108  |  620 MB/s   |    845 MB/s   |
-|  QuickLZ 1.5.0         |  2.238  |  510 MB/s   |    600 MB/s   |
-|  Snappy 1.1.3          |  2.091  |  450 MB/s   |   1550 MB/s   |
-|  LZF v3.6              |  2.073  |  365 MB/s   |    820 MB/s   |
-|  [Zstandard] 1.1.1 -1  |  2.876  |  330 MB/s   |    930 MB/s   |
-|  [Zstandard] 1.1.1 -3  |  3.164  |  200 MB/s   |    810 MB/s   |
-| [zlib] deflate 1.2.8 -1|  2.730  |  100 MB/s   |    370 MB/s   |
-|**LZ4 HC -9 (v1.7.3)**  |**2.720**|   34 MB/s   | **3240 MB/s** |
-| [zlib] deflate 1.2.8 -6|  3.099  |   33 MB/s   |    390 MB/s   |
+|  memcpy                |  1.000  |13100 MB/s   |  13100 MB/s   |
+|**LZ4 default (v1.8.2)**|**2.101**|**730 MB/s** | **3900 MB/s** |
+|  LZO 2.09              |  2.108  |  630 MB/s   |    800 MB/s   |
+|  QuickLZ 1.5.0         |  2.238  |  530 MB/s   |    720 MB/s   |
+|  Snappy 1.1.4          |  2.091  |  525 MB/s   |   1750 MB/s   |
+|  [Zstandard] 1.3.4 -1  |  2.877  |  470 MB/s   |   1380 MB/s   |
+|  LZF v3.6              |  2.073  |  380 MB/s   |    840 MB/s   |
+| [zlib] deflate 1.2.8 -1|  2.730  |  100 MB/s   |    380 MB/s   |
+|**LZ4 HC -9 (v1.8.2)**  |**2.721**|   40 MB/s   | **3920 MB/s** |
+| [zlib] deflate 1.2.8 -6|  3.099  |   34 MB/s   |    410 MB/s   |
 
 [zlib]: http://www.zlib.net/
 [Zstandard]: http://www.zstd.net/
 
-LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance.
+LZ4 is also compatible and well optimized for x32 mode,
+for which it provides some additional speed performance.
 
 
 Installation

From ba168ee5c7f9e641c54a9a9a2668a0acec153121 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:41:15 -0700
Subject: [PATCH 238/257] updated benchmark for v1.8.2

---
 README.md | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 596fdac7d..406792a11 100644
--- a/README.md
+++ b/README.md
@@ -43,33 +43,32 @@ Benchmarks
 -------------------------
 
 The benchmark uses [lzbench], from @inikep
-compiled with GCC v6.2.0 on Linux 64-bits.
-The reference system uses a Core i7-3930K CPU @ 4.5GHz.
+compiled with GCC v7.3.0 on Linux 64-bits (Debian 4.15.17-1).
+The reference system uses a Core i7-6700K CPU @ 4.0GHz.
 Benchmark evaluates the compression of reference [Silesia Corpus]
 in single-thread mode.
 
 [lzbench]: https://github.com/inikep/lzbench
 [Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 
-|  Compressor            | Ratio   | Compression | Decompression |
-|  ----------            | -----   | ----------- | ------------- |
-|  memcpy                |  1.000  | 7300 MB/s   |   7300 MB/s   |
-|**LZ4 fast 8  (v1.7.3)**|  1.799  |**911 MB/s** | **3360 MB/s** |
-|**LZ4 default (v1.7.3)**|**2.101**|**625 MB/s** | **3220 MB/s** |
-|  LZO 2.09              |  2.108  |  620 MB/s   |    845 MB/s   |
-|  QuickLZ 1.5.0         |  2.238  |  510 MB/s   |    600 MB/s   |
-|  Snappy 1.1.3          |  2.091  |  450 MB/s   |   1550 MB/s   |
-|  LZF v3.6              |  2.073  |  365 MB/s   |    820 MB/s   |
-|  [Zstandard] 1.1.1 -1  |  2.876  |  330 MB/s   |    930 MB/s   |
-|  [Zstandard] 1.1.1 -3  |  3.164  |  200 MB/s   |    810 MB/s   |
-| [zlib] deflate 1.2.8 -1|  2.730  |  100 MB/s   |    370 MB/s   |
-|**LZ4 HC -9 (v1.7.3)**  |**2.720**|   34 MB/s   | **3240 MB/s** |
-| [zlib] deflate 1.2.8 -6|  3.099  |   33 MB/s   |    390 MB/s   |
+|  Compressor             | Ratio   | Compression | Decompression |
+|  ----------             | -----   | ----------- | ------------- |
+|  memcpy                 |  1.000  |13100 MB/s   |  13100 MB/s   |
+|**LZ4 default (v1.8.2)** |**2.101**|**730 MB/s** | **3900 MB/s** |
+|  LZO 2.09               |  2.108  |  630 MB/s   |    800 MB/s   |
+|  QuickLZ 1.5.0          |  2.238  |  530 MB/s   |    720 MB/s   |
+|  Snappy 1.1.4           |  2.091  |  525 MB/s   |   1750 MB/s   |
+|  [Zstandard] 1.3.4 -1   |  2.877  |  470 MB/s   |   1380 MB/s   |
+|  LZF v3.6               |  2.073  |  380 MB/s   |    840 MB/s   |
+| [zlib] deflate 1.2.11 -1|  2.730  |  100 MB/s   |    380 MB/s   |
+|**LZ4 HC -9 (v1.8.2)**   |**2.721**|   40 MB/s   | **3920 MB/s** |
+| [zlib] deflate 1.2.11 -6|  3.099  |   34 MB/s   |    410 MB/s   |
 
 [zlib]: http://www.zlib.net/
 [Zstandard]: http://www.zstd.net/
 
-LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance.
+LZ4 is also compatible and well optimized for x32 mode,
+for which it provides some additional speed performance.
 
 
 Installation

From 1e130d23e3b5f86e30e5dd7d3c1fc6d93d63e53f Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 2 May 2018 16:52:33 -0700
Subject: [PATCH 239/257] updated NEWS in preparation for v1.8.2

---
 NEWS | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 90cafc6e6..0139e6123 100644
--- a/NEWS
+++ b/NEWS
@@ -1,13 +1,13 @@
 v1.8.2
 perf: *much* faster dictionary compression on small files, by @felixhandte
+perf: improved decompression speed and binary size, by Alexey Tourbin (@svpv)
 perf: slightly faster HC compression and decompression speed
 perf: very small compression ratio improvement
-perf: improved decompression binary size and speed, by Alexey Tourbin (@svpv)
 fix : compression compatible with low memory addresses (< 0xFFFF)
 fix : decompression segfault when provided with NULL input, by @terrelln
 cli : new command --favor-decSpeed
 cli : benchmark mode more accurate for small inputs
-fullbench : can measure _destSize() variants, by @felixhandte
+fullbench : can bench _destSize() variants, by @felixhandte
 doc : clarified block format parsing restrictions, by Alexey Tourbin (@svpv)
 
 v1.8.1

From 5406c2e479f5bb2b594a43e74e28719f4640450f Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" <w@felixhandte.com>
Date: Wed, 2 May 2018 23:29:07 -0400
Subject: [PATCH 240/257] Only Reset the LZ4 Stream when Init'ing a Streaming
 Block

---
 lib/lz4.h      | 24 +++++++++++++++++++-----
 lib/lz4frame.c | 19 ++++++++++++++-----
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index db6353c0c..d8238f567 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -358,6 +358,15 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
 #ifdef LZ4_STATIC_LINKING_ONLY
 
 /*! LZ4_resetStream_fast() :
+ *  Use this, like LZ4_resetStream(), to prepare a context for a new chain of
+ *  calls to a streaming API (e.g., LZ4_compress_fast_continue()).
+ *
+ *  Note:
+ *  Using this in advance of a non- streaming-compression function is redundant,
+ *  and potentially bad for performance, since they all perform their own custom
+ *  reset internally.
+ *
+ *  Differences from LZ4_resetStream():
  *  When an LZ4_stream_t is known to be in a internally coherent state,
  *  it can often be prepared for a new compression with almost no work, only
  *  sometimes falling back to the full, expensive reset that is always required
@@ -367,13 +376,17 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or
  *  LZ4_streams are guaranteed to be in a valid state when:
  *  - returned from LZ4_createStream()
  *  - reset by LZ4_resetStream()
- *  - memset(stream, 0, sizeof(LZ4_stream_t))
+ *  - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged
  *  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
  *  - the stream was in a valid state and was then used in any compression call
  *    that returned success
  *  - the stream was in an indeterminate state and was used in a compression
- *    call that fully reset the state (LZ4_compress_fast_extState()) and that
- *    returned success
+ *    call that fully reset the state (e.g., LZ4_compress_fast_extState()) and
+ *    that returned success
+ *
+ *  When a stream isn't known to be in a valid state, it is not safe to pass to
+ *  any fastReset or streaming function. It must first be cleansed by the full
+ *  LZ4_resetStream().
  */
 LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
 
@@ -384,8 +397,9 @@ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
  *  to call if the state buffer is known to be correctly initialized already
  *  (see above comment on LZ4_resetStream_fast() for a definition of "correctly
  *  initialized"). From a high level, the difference is that this function
- *  initializes the provided state with a call to LZ4_resetStream_fast() while
- *  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+ *  initializes the provided state with a call to something like
+ *  LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a
+ *  call to LZ4_resetStream().
  */
 LZ4LIB_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 6cf2a06e8..91e5a4352 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -537,9 +537,18 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
  */
 static void LZ4F_initStream(void* ctx,
                             const LZ4F_CDict* cdict,
-                            int level) {
+                            int level,
+                            LZ4F_blockMode_t blockMode) {
     if (level < LZ4HC_CLEVEL_MIN) {
-        LZ4_resetStream_fast((LZ4_stream_t *)ctx);
+        if (cdict != NULL || blockMode == LZ4F_blockLinked) {
+            /* In these cases, we will call LZ4_compress_fast_continue(),
+             * which needs an already reset context. Otherwise, we'll call a
+             * one-shot API. The non-continued APIs internally perform their own
+             * resets at the beginning of their calls, where they know what
+             * tableType they need the context to be in. So in that case this
+             * would be misguided / wasted work. */
+            LZ4_resetStream_fast((LZ4_stream_t*)ctx);
+        }
         LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
     } else {
         LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
@@ -617,7 +626,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->cdict = cdict;
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
         /* frame init only for blockLinked : blockIndependent will be init at each block */
-        LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel);
+        LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked);
     }
     if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
           LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
@@ -715,7 +724,7 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
-    LZ4F_initStream(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
     if (cdict) {
         return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
     } else {
@@ -732,7 +741,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in
 
 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
-    LZ4F_initStream(ctx, cdict, level);
+    LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
     if (cdict) {
         return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
     }

From 2e2c9f6ff353e9f1a4d23274eb4e5b7a5f7d654d Mon Sep 17 00:00:00 2001
From: Cyan4973 <cyan@fb.com>
Date: Thu, 3 May 2018 07:56:33 -0700
Subject: [PATCH 241/257] fix comments / indentation

as requested by @terrelln
---
 lib/lz4.c | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 71fe8f33e..3860c5173 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1427,10 +1427,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          * those 18 bytes earlier, upon entering the shortcut (in other words,
          * there is a combined check for both stages).
          */
-        if ((endOnInput ? length != RUN_MASK : length <= 8) &&
+        if ( (endOnInput ? length != RUN_MASK : length <= 8)
             /* strictly "less than" on input, to re-enter the loop with at least one byte */
-            likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)))
-        {
+          && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {
             /* Copy the literals */
             memcpy(op, ip, endOnInput ? 16 : 8);
             op += length; ip += length;
@@ -1442,9 +1441,9 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             match = op - offset;
 
             /* Do not deal with overlapping matches. */
-            if ((length != 15) && (offset >= 8) &&
-                (dict==withPrefix64k || match >= lowPrefix))
-            {
+            if ( (length != ML_MASK)
+              && (offset >= 8)
+              && (dict==withPrefix64k || match >= lowPrefix) ) {
                 /* Copy the match. */
                 memcpy(op + 0, match + 0, 8);
                 memcpy(op + 8, match + 8, 8);
@@ -1709,13 +1708,12 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
  *  provides the minimum size of this ring buffer
  *  to be compatible with any source respecting maxBlockSize condition.
  *  Note : in a ring buffer scenario,
- *  blocks are presumed decompressed next to each other
- *  up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
- *  at which stage it resumes from beginning of ring buffer.
+ *  blocks are presumed decompressed next to each other.
+ *  When not enough space remains for next block (remainingSize < maxBlockSize),
+ *  decoding resumes from beginning of ring buffer.
  * @return : minimum ring buffer size,
  *           or 0 if there is an error (invalid maxBlockSize).
  */
-
 int LZ4_decoderRingBufferSize(int maxBlockSize)
 {
     if (maxBlockSize < 0) return 0;

From dc4270710739296602235caa5cfd065feafd87b7 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 15:38:32 -0700
Subject: [PATCH 242/257] created LZ4HC_FindLongestMatch()

simplified match finder
only searching forward and within current buffer,
for easier testing of optimizations.
---
 lib/lz4hc.c | 104 ++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 88 insertions(+), 16 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0859ea610..132673ec1 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1083,6 +1083,80 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 }
 
 
+int
+LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
+                        const BYTE* const ip,
+                        const BYTE* const iHighLimit,
+                        int longest,
+                        const BYTE** matchpos,
+                        const int maxNbAttempts)
+{
+    U16* const chainTable = hc4->chainTable;
+    U32* const HashTable = hc4->hashTable;
+    const BYTE* const base = hc4->base;
+    const U32 ipIndex = (U32)(ip - base);
+    const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
+    int nbAttempts = maxNbAttempts;
+    U32 const pattern = LZ4_read32(ip);
+    U32 matchIndex;
+
+    DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
+    /* First Match */
+    LZ4HC_Insert(hc4, ip);
+    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+    DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",
+                matchIndex, lowestMatchIndex);
+
+    /* find first match */
+    while ( (longest <= MINMATCH)
+      && (matchIndex>=lowestMatchIndex)
+      && (nbAttempts) ) {
+        const BYTE* const matchPtr = base + matchIndex;
+        assert(matchIndex < ipIndex);
+        assert(matchIndex >= dictLimit);
+        assert(matchPtr >= lowPrefixPtr);
+        assert(matchPtr < ip);
+        assert(longest >= 1);
+        nbAttempts--;
+        if (LZ4_read32(matchPtr) == pattern) {
+            int const mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+            if (mlt > longest) {
+                longest = mlt;
+                *matchpos = matchPtr;
+                break;
+        }   }
+
+        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
+            matchIndex -= nextOffset;
+        }
+    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
+
+    assert(longest > MINMATCH);
+    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
+        const BYTE* const matchPtr = base + matchIndex;
+        assert(matchIndex < ipIndex);
+        assert(matchIndex >= dictLimit);
+        assert(matchPtr >= lowPrefixPtr);
+        assert(matchPtr < ip);
+        assert(longest >= 1);
+        nbAttempts--;
+        if (LZ4_read16(ip + longest - 1) == LZ4_read16(matchPtr + longest - 1)) {
+            if (LZ4_read32(matchPtr) == pattern) {
+                int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+                if (mlt > longest) {
+                    longest = mlt;
+                    *matchpos = matchPtr;
+        }   }   }
+
+        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
+            matchIndex -= nextOffset;
+        }
+    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
+
+    return longest;
+}
+
+
 typedef struct {
     int off;
     int len;
@@ -1100,9 +1174,8 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx,
-                                ip, ip, iHighLimit, minLen, &matchPtr, &ip,
-                                nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    //int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */
@@ -1112,19 +1185,18 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     return match;
 }
 
-static int LZ4HC_compress_optimal (
-    LZ4HC_CCtx_internal* ctx,
-    const char* const source,
-    char* dst,
-    int* srcSizePtr,
-    int dstCapacity,
-    int const nbSearches,
-    size_t sufficient_len,
-    const limitedOutput_directive limit,
-    int const fullUpdate,
-    const dictCtx_directive dict,
-    const HCfavor_e favorDecSpeed
-    )
+
+static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
+                                    const char* const source,
+                                    char* dst,
+                                    int* srcSizePtr,
+                                    int dstCapacity,
+                                    int const nbSearches,
+                                    size_t sufficient_len,
+                                    const limitedOutput_directive limit,
+                                    int const fullUpdate,
+                                    const dictCtx_directive dict,
+                                    const HCfavor_e favorDecSpeed)
 {
 #define TRAILING_LITERALS 3
     LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS];   /* ~64 KB, which is a bit large for stack... */

From e00ba49cdeb6a6d7b219565286e45d23801429d6 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 15:40:01 -0700
Subject: [PATCH 243/257] updated API documentation

---
 doc/lz4_manual.html | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index e079db216..e5044fe70 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -278,7 +278,16 @@ <h1>1.8.2 Manual</h1>
 <BR></pre>
 
 <pre><b>void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
-</b><p>  When an LZ4_stream_t is known to be in a internally coherent state,
+</b><p>  Use this, like LZ4_resetStream(), to prepare a context for a new chain of
+  calls to a streaming API (e.g., LZ4_compress_fast_continue()).
+
+  Note:
+  Using this in advance of a non- streaming-compression function is redundant,
+  and potentially bad for performance, since they all perform their own custom
+  reset internally.
+
+  Differences from LZ4_resetStream():
+  When an LZ4_stream_t is known to be in a internally coherent state,
   it can often be prepared for a new compression with almost no work, only
   sometimes falling back to the full, expensive reset that is always required
   when the stream is in an indeterminate state (i.e., the reset performed by
@@ -287,13 +296,17 @@ <h1>1.8.2 Manual</h1>
   LZ4_streams are guaranteed to be in a valid state when:
   - returned from LZ4_createStream()
   - reset by LZ4_resetStream()
-  - memset(stream, 0, sizeof(LZ4_stream_t))
+  - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged
   - the stream was in a valid state and was reset by LZ4_resetStream_fast()
   - the stream was in a valid state and was then used in any compression call
     that returned success
   - the stream was in an indeterminate state and was used in a compression
-    call that fully reset the state (LZ4_compress_fast_extState()) and that
-    returned success
+    call that fully reset the state (e.g., LZ4_compress_fast_extState()) and
+    that returned success
+
+  When a stream isn't known to be in a valid state, it is not safe to pass to
+  any fastReset or streaming function. It must first be cleansed by the full
+  LZ4_resetStream().
  
 </p></pre><BR>
 
@@ -304,8 +317,9 @@ <h1>1.8.2 Manual</h1>
   to call if the state buffer is known to be correctly initialized already
   (see above comment on LZ4_resetStream_fast() for a definition of "correctly
   initialized"). From a high level, the difference is that this function
-  initializes the provided state with a call to LZ4_resetStream_fast() while
-  LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
+  initializes the provided state with a call to something like
+  LZ4_resetStream_fast() while LZ4_compress_fast_extState() starts with a
+  call to LZ4_resetStream().
  
 </p></pre><BR>
 

From d358e33faa87e0293f27f272a8579de73f4ba938 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 16:01:24 -0700
Subject: [PATCH 244/257] Added CDict speed graph to be used for release
 statement

---
 doc/images/usingCDict_1_8_2.png | Bin 0 -> 81858 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 doc/images/usingCDict_1_8_2.png

diff --git a/doc/images/usingCDict_1_8_2.png b/doc/images/usingCDict_1_8_2.png
new file mode 100644
index 0000000000000000000000000000000000000000..94341988e66589d3a390f47b9697bb5681f4c038
GIT binary patch
literal 81858
zcmeFZcTiJn_dklgpdun5(lm&Kt|HO|j+%g=zzIliN|6rIOMuvr4k7`R76p;s0-*&G
zia-PbsUfsTF9AXgC4}6~Iq!MD@4WZ^{<<@F=FWV#BW7>*&RWl^pS7N~4gXVH{p{(>
zr&(B7&T2e_=(4aJ=>z^2oj(j*Q9xak1O7Q`cTejc3rlG%>)x|t!1)#Hhq_uUECJV9
zSYEzmVc7*Py_{oV@xH;rvhb9JMe!pG3-^oE8rWUn7sst0sY9559jA=8Mu00Pyfn0+
zCnk;^;o_4ybEsDZc$h^4a_@=X&|=yNzl+mb6_!_Cg2Oa9S;H>ggJzvR`d&=yo-_Ec
z=!WCHWB1-ZEEf9=KHcbg1^FwooW)A$RNdQ*>sP;2zCU{S)HA%Y%udiL=KHE2%h$2-
z^kAvk#!(WfgZ}fA3^o2o0U99x&vEHw^Z9>111!W{**^Y1rw{I}oc*7(cdvi*{?FN&
zV+{vF`PXsj^lP_&KC`ggxOkNOKW9(DCvk`VeOmwL`M@LpK70A)e+m9u82>BMe~W_U
z|4XB3QEVnw89n}PU!_FBbIhmTy8K)5yW#m#yTdPKI-eEQf~;=WAA(<eaKPQ~(~D`M
zw-$7KGq7_;cI-mb#d#16$@>K25Vx9Y?ydC|(W$Qia%?Dpy;dWlL25@>_@GHBC|_N}
zv(bpFEH5y<PSaP=Tgz#g&|R+6L6U9u9Ub`a#hthDMG=FZ^+DpwB2FAp9b?8Ip(T;Y
z*+}Qyu(A%wCpSr^MBI`-m+?a+Q4Qw|WpCvn-hEd&3~CE(*BQ7#TH9%l;;5cEiE;(?
zsLsw4p`A*0ovCU`kSA}~7v{RJA25>I(3Ak>Yic;890@RTv0pP{QT4W<$`wT~nAeO(
zcXg7ia&d0KT%dpVCN5(m`^(uz2u<gtk|d{X`AqeCM!71rqb$QyHL`PzD=j}SxBB{0
zy!vkBokXtEu9tLen#tJ<^ReIc+0)khpWN4z)H0b~&@V9Dcy_ks{4Fvz5ye$a*ZHD{
z|3*lZZB1ggEq~tB3UyG@zK@){&ksutoujm5EH)z*6xMd(b*q&G*l<)j*vjK$M_EB)
za(ds!>$$ebjEy3Ot~s*bx8Q~OSGO;|>abmJ6P!>FF_#t#&MY){xZ+`3Y^@o-p>s*^
z0TRZE<Q4W96LFB!hM3DIwq6duq|ZY5QP$=j8*%9@VaDS7c^3Hy#ZXAKH^`oek=b=p
ztrqKHGR7{!)F_q(po#C?YcunPp~aA><A0dcTnpd0yxJv=p4M+BbJ>n*T|RSxH0J)H
zCfsaY(Uq}&d-1AAnkNe(Y?U8M;vMci3)yvQdKYoa5+B4CCrO=di)l?{&pJx(3iF`E
zmn(^9NTs?TO$mI6G(|nzVa*uvzEhC{&cu)9^qV5F+SQHV>0>df*F8eq2exXWXc|GW
z9Y4ByOMqo|>B(5wwEN_FahHm`F;DwTm)<F19{Zn1+@c>M$fg*G@#y_*IMhM4dY?+n
zgz(fYj5xDvom`tiQk&lZmT)D=`(n!VwlOF1H1^S~raN4=#glWz$sTG`ikY{A3JImd
zHf&z$mT7j8=&hlGn#^MSP6Rm9+o#}SHJU2gu;r97vU_GJm>b3)V-@(8z}X_>i&$On
zlZOp;{n}K5&31!`)eZdk^Bqc)slmm)TWUdFkLgvot(m>q`D)|7>N-L?{wIB!mmSV;
z<BzgA3Ao5;L`JF`d{+s#@`aEt$tM@yqq3%%0nRi`%fIH4)|;D?1m$5EDCB)1&<d4l
zSzv~b0A8Za;!O9pGHlmH<>muqFlz@fqNmuh!PKT5>pLjXBe-)5KPP26EY=q2MBN8z
z(m9TCa%7Eaj}}AbE#Y2?U?IzW3ozRDF5`OGts(7=`*D)P;o28%%lYwAAk&tL&U<m0
z3%*15VQM0g^Ol(M5O5}HV!Z)mI_CR3Aa0+~8XMt^&tA&-1X3S8Qn|sMu`#@}7!+q6
z%jx!*leDj}$tyI7#xdZzxS14XD37vI-j|06n>lyb?v@zd(1!jos_>d7KJA1xbXO72
z$!&Z4+^}ZkA~QcTs^ZVt8ZdlVx<(VZ<D{pCiv+J1Mq5augSy!K4W??9)C~p1+ZMRA
zb+|}t8oDw4-5EbCcL?ek8>VoZR06Flv7g;XfHw%*Ih-$p(9iaTRum{;5(u<1b~s@n
zor7p@(?uBc^dU+?ko`}E*Edxe#$HK`Jvl%C(hI_X;I`fp_UlGrvkxl*_h$pbJt)rj
zRcgja^ySU1LOuzPj)#4mYNGiC3t}F{)rdEL#)&URSJYy0t%`I{++GcM+Gwkg07IcH
zr(T*TG#)rvO>Csc+Ka`U;S`(O<_?xnWVch-S<dPqZ$U-8??r~YIkGqtEL<TVi4Jm7
zu8G)a#qjnIbUDIfK1piQ^Kv_&WQW=S4P<bNgW!CyQTc3Dj(Zp(vFm;JRJM~wcX~E7
zTGp9ll4+5FR4Uly%6dB*57`Qc_^Fbilu`Z=Hnc=tkWS_lt`Tyj0im+&F5V2;cmFoy
zX18)|r0(7*xUDhxg3wU4*^|mR0<AP?RXr|~HnyV=bs*RID8SS(x$8e|yJ;;MX=fpu
zd6CXcQ5;xQj9aKj&>x!vE=T3eyTfMav+=Bu-AQScD{W((e2Qcf2*T%Le}K_cRPV@T
zC1{149-g~oiPJZ!uDlj(D?NIxEl^4JT==a3pKtiNQWX#*&NnHNCc!T2BHT6xnP@oL
z&8O2%<c9e2v4uw58k%{u6|v~MKtBQToiCY&kcRgyUx7^d3fLvkwgrwCIYt9aZ#&_@
zOJ$j^5k0tym6VK|98ghws=dhC=6Mpil_C=6Y>e2u3>(rJc6&S5C7B`eEnGL|rwBx}
zfy5IhDKv@ILXN2S<$naj;gkGAwSORKBz7l=f~OB`To%@e>Q60_LeJhUSveWE?>znr
z1Xt-dr=_w%5mM~>W+FX;s9wFVOr@JBFx;-hSRR`K*2IU<w!-*0b4R)ncy8mG$iSg6
zbWKQ8FH{tT4*_Szce2|SJ1)+vR8T(+*F<`3-Bqmwm7LaxR^W?u!oYSa*;qtjfF}7A
zclFw=M6!fOn)?8mXqN(J4Ju3~c=v=O@jp|<+wA$tc95gwKUU7ZA8y#P&PY|@Bi4wa
zXX_LZ;5K{K-N~i>3%2DUk!K;*!@(^Hfc|;7jzw<Bc5QIlZs>8J=2Y-`UG9|y&XmPF
z3Ta}4zeIveRhqsQLqFl+#B>lmIK%f`gtKY_Y5V;{%k@^RjHm%V4}E0!9kPHBdYV6U
z4~%Y(D~kh}s$~Y<gQ<1-7=b(%PfgHnWt@q<!3s4#i&S6cu-(Vmiie>Uzj!=}o}yew
z+d6UNg)IuM53oWjV!!Th`tlSq4WbC&>b!~wi8meU?jd(_5^F`#wj+1UE@UBmW}84k
zX$iIv9v8c%o3NR5MfijrB2;T<(^Mb!PDl`hp3TNmKNGl;)TtL07A~zVNyaTvk_v>-
zbrJE&;fp*x>PxCy4B@sPS42g+934lvNc%VpCGwUfxldy%ryGb0o|Jds{Fqp{F0#K2
zTP6u{K)I-@+t5iSYLq7@&nx(-Rt7os3v!NT(Nnn;^5{0&dyyF<inD?OR~&~ezN+4`
zD+JG~c5yQ!oDt-6yby(Th0qj8(G5+~<c=Jqi0Kd;3>Les_EmMQHyE~joLpSQO~UQx
za6>xjcS)=`r!8*V^3Lf%&}%osO!|-WELEN5ck`FN%&5dEoBGjD0CKHhJrAg{iXXrs
zZavq!tD7^*Dj~DUPGVxGi-k-E)TajB?f-miqa#z1tzoDnjK;6pxiPFVAvCw8EL_H!
z%(~+l1;G_3>L3d9kJKoYuGc+~+@ikqg5sK=&d+e9QL=Jhe@o7oLqM>}5g#aFsocD_
z<-zg88L4Ddvf>qwCl7ajf#Nb_ik9}-NjQwBbR^$q*&x~%g9Ul?!IkCIY?uQkmh9E=
z6?6P&aHicbJQ8qiH;uR@s{X7jz&~y^kC1+Ir}EQ7@=&u8Y=%NK5=IA|K_KpbEk+1w
z1KuX;4jQgZ{H2Gy@Xm3>_O0M~+h-r{>P#u>`Ks4ZXZSM;aBco5v37W;H7JPjmXWc@
ze~NcR9Nk)M*b=s=>Nro4sMfKe&+;$}2qZ?Mf6!h`(B}&9I&pd55ZRqj(n>x*@02(v
ziJna+x!OXC5T4;k-e1udruSxlj>6D(&S~RMOg5DNv{Wa{$e4KQ@N^@h)-_<R(d%|H
zkmca~^9;Q@IenY<WDQQpSDp7EK~VsTOmpz>eB9!Uzffjg1waJXKCWEn@xkseHxaZg
zLwg#+bF4tx`#foXnT?OjR^{qyt_Yf8=Wq8BJe&Q%!7|~OF_N}=?g+#<%?z+elp9AD
zLVp?!qHlddP&pJF$$CET$Af!Jknyb%1<P06{34t!@uPphW~jKk_mJzoUwr5g<C8c_
z#4Qu%kNcq3W%DHvkZFhU830VEGlC(z%TNOsNoY)`YQ%{0wyVDhaAipd;vk6Fa);4k
zveHc<BudGhrB6$c&X_vg<-S0Jshq+PB4-xlcovj2tlho`noV;uf6P^Ff=o;tjL1O3
zmV`ahy1o?)nm$6-o#Q0y+VqmlpTTwuy(PftS>l2+j@y=M=OzF~+g(d3WVam*IvwV5
zy~S~*kdU~QXQF|mi3H^*%*7=5gsH&IkRFTC-2N)p(9`R+Yp=n+LbaXOVQrG>jnL5(
zVW;^nWPSHM3$5NfW&R9_*32~psRum|itX7dLb}Ct(-ec)9%H}ZlD>_)<*SII9a1eh
zt2;YX*F(c<<Sy<wYF0~*<%0}8YrCB?3etb{?dn5zj~eHkd&g4m{cZZw`hB4|<zeQ+
zYXXgY3g2yj>{c|ypPx2JVOnDerO)&XEM>QXU^BfHh7X_>h<C?Xp)?)H((Z#Yp$sMM
z`Jpgp($JnOD5#^D%u%=YHD~NJEDaCw&JVv#ys){WYeVl^YrGf7(DO|b2LSPKr(Gl@
zW@yWTVUDbmT$GJGr+)!z&Y@t<=3u0c4Ec>M1F!oEnmw3el;(_U!qTjhJ0lljIlOnZ
z`ksY6MD)jj<CLw(mgQhvuc_IP3TEjZYbb=a$hLG2aI*?)k+>z(e7FT*O!3hWV`Y_c
zvh*z)CwX^cGv@dFTLKRr-LdDE=R73C1_WXAKo>w4o7TKxx)bfs0Mz7UsbkZn(;fJR
zu+Q$(Cq0S^fKEw~i5J#O9wL^q*nk-I)_Bk}Ib0Vv|22g`L+uuGfYhUz!$3R^QV)?d
z%`t-_J&tW1$uMw>T#!27e~Po80qHCFZ2u4um$@HPIHnKl^g=PWuW>^v5NjW2Ax!rW
zKPMQi)>TdiiG^9`gJCnRsr!}y(Bh~8fd3xs=DN!Q+<b;I2Y2IY*|CEy$9Pg@Gk$Kh
zaA`q`<~Lb;p%ul<HEB?f2%#~(<?nDQ<T}GWZ%P{zM`vS>rUF(!Gp7$EtLl|naOKdR
zCn9nYu_ZGK+4U(>7HeyajE&r%Ogx5<boR~eB!lQWr<YFQOzVU+>D%I*&<f{5EulE;
z;_w0gd<=lXp><N|X}i4%Yna+FkFt?XJ6M)i%){vDH)v@^0X{E=05eP;#)bA3g4*t{
z)afCE86O8gaN7I1TOjPW*GdOYiaJk1@0+>+O3M<ZH{FB1>v|XFOvWyBa%TnY^<0Cs
zCDew4Qd&B$dC=j3hU~T*i{dg+1sR4b_cgSg(yYpPTiI<_@;w_pFK-lBcw0cYrhQ80
z@s#7CPzK5_KrW6Fs&rE)Y(vs)`I^Vr?9aw@Jx%0=t94g^c!RjAJ(a`wVrm+vj6YeL
zZ93}2PU`n>kM#IP-)j}w(M8skwQ<8E`LN7NQ*n<F)3hia0GW3Vw?nGof$gI(k41U%
zI+EbuY8oWnF*8=BhBTc_rFz5KWJiS|^F(8cut$e$N?oRAO7-6wX#y~>vx$Z7RFu!e
z;Bd;%^y>lHSvlkF8H)JuWJ@lse90c71oI;SfcXq83Za8EUFxk&C+z#iZa@0!HtVAo
zw^Tg3g`87;O+A|-0eP|mp;0+$sgL_m(g*t4fdKk}*}*y&Ia4Ac7V&YN)tni#DjcL>
zq0cp45qw;)NF}JiU=(;dZ!aw)b>zXq47cso?R}s~VwCkltKmbVKGi^-@M516MpN-L
z*wRgT?dBqu5|Y{0l~Epxsu9zy#&oBE;CHoh8Am|$GzJaYZBDEA$XIki7<vL=k=f=K
zXH9o|r{a2nRF+nDHKRby&jrs#LI*!qhH0)S8+j%Du#<)@r;p1=%qv3`)-}WgI7v(k
zL!AiSU{QI^saih}D_mO-Mte<7FAnFMzMr&^fy5W=8YBDbI!ozxiAruf2Tt?T-#rYR
zZ<@0M)IjU+)2yM!jJr!*5gz+ht9#l&ZdGzke`v`Zxd_t^HnEHP&?dZ0x{98Sr@&2S
z;F-AAn)We2*G%wqaXEK{v+v4|AVh&|hev4Gbg^q7^d~F+M_s2GUF-bAGz)o&woj^u
zIv2b%XHP_UIH3b<^9ZGSam|k)6?jd$>6lTg#cHADs5NurNBVDitRav~-0*+cv=8{4
z<C@;HDBXPxl;HiZfD{(mCJE#sQxWZ{l9NB*F|G+eAChR>QNET>9Il}2A$PEDfu_iH
z_1NccpcPCBq98j7Z&-{2Ymy!XL3ozg)S)~oPV3|oU-|Ukh207;z>ZgIk>h@KOSIYZ
zRqO+OjV+zNYN@s{T4Wa|>0+R&xF+52O#~?WC|L$Va~474!1)cnq5hy0+8z<W_`aXP
zi~&taF4DT*Z9YklG;F5I<`;rL^_Kg4@bu*+eh|FE2U`TT!<k<P6e9;HkzWj5H&$H4
zp|Bvbu#reeR~fXa1`}UXMIh&vRf;13C_fYlr5)9G5ajuF-=?c8iy0tai3`31w<GEh
zFHTQocND@rE`}U|cE0$Nd_;!)$Q+THA1v86HuiBHHDLYzA;L1&5d`Pk+m~ja=dgL!
z_3W=kfC)%l_Jb;dJu5JC!x5-E27^3-Jgzva9pSO%rHR<8MuaUo&}gXyt}2PP8$<Hq
zZTBJl5F2LrBLhSR3ITw`U^)?E9Br7q!A_!5t1p0tiNyk7z;7bupq5h{By`&4S?H%&
zm(g8NDD>%_MUfa~Ovl;=wiL|TP)XyM9&ovNt94cn##OW}CET`Wr(mdSgO%>5h2I(x
zbPeb8d9n%wC+Bq`kZHx870EcsRrhhJjK%&&4!5vH@{wLQ{{;mA-IbHC10=s8Ae}il
z!!d3Elz({gU_;Q!hlnghO%G84KQpA8AnOVxi>~(wo~i-0jpo7^Q$TitaaIvwx~^Fl
zgwTi-W!CUifxzL?+3)m7w4snEf`e41T5EFOrX5DvDTdCYM%Us1owzNf{%RPtiv;JR
z_EG><#?%-0bB*3S=N5sM``}g!Zrdpf1I0s0`5~1_Kq1~cGXPe|(@jO&x)eXc1_gDM
z&+?P)ue9yBr`K9e?Kb!h`I?Ps0f1JD#u2A-kBi0!KnB?DIq|l_cx$`zoKAfxPoZZK
zn!x#*T5ufQESXWhcNx@!QtmcH?)a#TiS-aw5t_(n5Ls;-x?R5K!>OF7!f@Zx$4qBz
zGk|dotX~mmn<=WG-NN^TebhvB0%$f`%^(duUFS^F(G9M7mhqFgm-&fM3J=>~e|~j)
zAQ(twOz-T62%jIrK(oU~RAZyB3)Cv0tj%?PI<T;A{|&f_F@7ShDMpMvW24JJQq1Ft
z`CKiBc$=f8Uv2?--B9+@HFQvWrBWC=;K3W8R!EG8F|&;nmA<qf?a`NR2Co9I%XY!T
zHa25bgxdlchto5VPlLXIGkuBAKXc2FP27Y%oca01L_LyZq=M+Z=fe0f9oKatQyD*~
zgPwWe`_^4M5F~mQ$vyqkee;zHpx#RIL5XjvEUvi|@?9N3ZUT0`W6nOPRUk2re@p_(
z+`Jhc*lzKqIv-<Xoz4U|pzto{wfo2r>Ykq#a?NST5Yj1MF-M`GI?_>{uuSB-jMA)&
zC7IGgjo!6buz*_|Y?Z3VLqsj#1*%Qe>Lm1c85Dj1XXF7uJQ!g(g+ny3R0lf$<yHs1
zR<_ntbm(kw&yy#5)>=cH1#x<i6vr@UhYRki5uHl|PYm>AwK`-SVj%}jT}~2WsJjXf
zfOjJ$!fduHUS{{&k9bH0*<8-{vZ@6#IIRW>G_Dd<F52QSK%cpS`By`kncGYL_PoUe
zM$^{3Y|c&c6uXoC9?Vco00B1(-o2_xU$mQr?3SfxLujccB(!$vvkc_J+b|ovUPaUg
zNxDYXGEm&GTZTKYj>0&eGiEVCK@1V}Xk35{kZC{FXiU}8C5HD;$(XE{?PnnCHp+pl
z>1@NUu$ZH(21pU(Sq#qi4cEdxFN^`14)DoZB16vc3fKIpesT8dZcQm6y|~25%M@8x
znmnpGm17+zb3JjMh{8XyX-`M>=_7+Xq^a!ll#TT3nt1J{HK6K_Yld_Rj4wuylO_S@
z)aX$O-jwUEhG<3KPPhO~hVnj&boSrMZVy}b@Y=oiwRqi42526Y@3ccJ78OdBA)RwW
z6{(}L`v8Hlwa19KCDbg92!eNZEK+Mxn+K)zAdYRLT^eo2{IMXiLl^3{j)VvnUsN_Q
z0Fb~i1olW8CM)bg%iayh2QTH<NN{<JN#~h8u+f=Xa?C`^Jl@tt%B*YBsX>-sYv<ln
zXQ*34Maf_Fbv87mVK~o>)SV{|mip7uu<VEo-+Z90O5eO1x31E^?FMlmGQG;|?<v4g
zycyb9h?~LrvknqH#K4?faHb?tn~Od0NQ2xTQt^mHFnp^sT@Hry_e#*eZqBLT2C6Hl
zB=G2M{hY7X1MvLdnuY~d9q|2Nf1|0|8;!6A-&fv154E)vB`K*>bisCixfOTNg^^8y
z#;g$&mmqZjWcs!6BN4c1Jj)IM2y%yL1*$6dkvlqlivTKe3fhW5Z%H1jSY+#xypUxs
z(^R+d0cc9z7GZv9qPY*^AvVE4je(6#6Y+R5{T$PzHYVbsCj5LK$L7>FR{9W6_cVyT
zst0tO0w8&R{(<b)>7KF%`qTb9?l3h|)6xgX9YTR32+ah$&`&B&5}xaHJYP3k{svSb
zU2~)pLy;Tamy0+_s#eoL9lu-NwZWCeaOSOuOf__mVF>Hdg0CiHQLUx34?FXd%8P@_
zf#h*@avcN@sFOsA<r^AV2Hu|<yr~2Cb}^{@T?=dbVx(z3s0^~hC*e*!ce{N*Cc;^8
z_Pq?qjv8ITmnO9*iMC6r2o!_wny0k^0d8;kHhVS4!A7Z3qY5WK3`DQ=+5SlAIJf*M
z{U`s})wUfSZ@;2BKXssjQ|MOnx@~0B<x`HmT&}d5Ly+8B*Gs++Rm&q!ra={$I1v?Y
z{t;=9`T@T3q?yKGj3#n_b-x!%d;Brh@;tx8)mtsNfNfr&(K5IdYCoC@qQ~x#r#NhP
z9#)xp*#LVIl6L+!`xmPlPx2;YUk~??t5;l*R=jxCqqR)wL{z<LM#;lC$|7&)vWl7e
zsm?1U$T0Vp8{ZST-KP#=6}f>;v#0fMu&kE7hgkUZ>r0>bX-ygVszq^)SBVoRt3;qt
z$A7W1ytF>|$2&MXaC5!rVU{V`J1$(do3YFI0%@xavmF9cQw2z=ztl&Pbb~>GhYaP1
zZk%KJeOLQfxctz|BLQ3oMEXvh5;@KClwLX!ih{^Vg>TqVsGRP1-PLtm+(&$)!0x!Y
zmOI^hr64|EOT;0McU;_%#b}n_i@|dC5n<gdSeo!VfO?L|>BkRQ>bcs=Iczt)A7na(
zT@v<fP%=WUWg*?ddWzSLIbmussskF|AJ&}X4!$_QJOc3e_+^<E%MDs-7DbFb^U%4$
z%u5Gj11u~;rJMTflTWV=Y+k&1?zf|}tArIFkVymC!B%|ru>TkP;FsR!2xnYx1~#Vt
zq67T+!{Xg<-YB$q_On?OsO9KCm#?_pUpedg^SwAoc~NCh7dZ&|s_NlZ6$bm8LEtum
z)A_(R_Xck~D!=~6cl&>yW0|r&iEFqL$oM#yzjm;B_^;!N^n;bN$DirG{6_t6;%6Ga
zx+GUbUi$aMz{}b*4RV*>YB0Y=bpP`_3!(jh+()ll`v1lInZy4vPcV0I(|=9|@V|Du
zaq>Tg3O*mO@#;V35xfrA{SS%0JO|J|`X56Hzpic{AND~$*1+#uAA|5Hr*!*hZ(gva
za%k_$>2Dv*bNqF@PrADOj`?&xuv?ieY~R{WvneJpW23=$y%PHA!~}_+8XuE&n)uJ%
zENi!pl2-(s;d;p8Byc;{u;z-=y8UqaZ#dd8@5R3*@t4$?uZR}imFAG{btuI((s`&3
zeaU8+BT#3!$RXwAe|@7R_TlBz`scHPruGiQW{NS>=3+P@bP(esPlo9dWLL<#%~4FU
zN3a^Be$m&o*Hw#K)*LF8`zX0Hd_pO*h>d%6g=?NUAV?%?7^B>qjNg_KU^N-@ufe$5
zx#U>zc-@ov*txC+r*E|p-I*rrouE{xmr-7VqpU^8SoXXv`>Cr9a#P(#BS3{?B@Qzv
z-i+C?@S@lOn_eQLvfiBYvDZ{ZBeu@72Vy%kn!LL)movVnXDG);Y#=DH6v~U9mt`mI
z;`Fpvt)Z*&kW=drSa)uZU^l9tXk(&{0f@;*&Xqh7?{}<E*1FiN9`1VkBEO0_=+w3C
z<FBa_N6EC2v1^jnSo;9Ox6*2$g}gBLdgFRyf~4!?N1U1WFo!#VKm1%d1GO49N7yHS
zTbvJULj36!18(9c#QYwH#Z|_T1#=r^h1hQzeV6|&@mpewYt>Y@+5XSMmt`-@{s=vm
zC+l_r*j(ltVSVkkU2G1qrtknA+UNU21voMIwyYLs5l<dw54hn4*9$I>NTe#Q@B4o~
z9jmCM{$0L``zZP8aLk88u-J%O2>l`TwdnJ?kE@G;Mmp+Ov2or?`=5ojiT`3A<FJ$U
z<mTlk;s^ZvdiKh(+4u$bfz57X9s`SVI+S`W^;oD@dv9Pr+kdX{?!savsHLF}aE3R(
ziDH1)6dpU^>SN-O{)m6AhV#XpzufZG@05YV@~rZ#kK1249!qUD{@ir9|Fk@-BEaYQ
zlQ;*-gXPzk^W_-S8>b#qlsJ&bzZ~g*AN*h8{IBKw|H?ZQ-WWFjzGuOiuvYxJKw?8T
z&%byHp1iei$HCC6Qp271gLaF{Q<HQ5M4Vi}=?Uh4XUSd@7P~`sX$@od4DT%SKT-7W
z59#@#`3e7_WH`_~SEpqezwu*mOMOw9z8|KJEg)^%b5kaXQ^vj4XtR|<8Wm!^D`${L
z7RRz}!EJw%gp^uR>vW|{=?MXUN1kJ;9qqrK9Z-C6=RadlOe5B01vS=>yoJwe9<fdk
zODkWvKmWDPvuF?F8zX62HAtDMkE}~f3o9yLgis5=$idz8+wc_?7{PL*J!Hk*T;*GO
ztgZRretO$7J;2h~M9OiY$KxNOvUh+i#`%IP_3Yod--Hziw$8Lc4qw~d^r{{^189Ou
zCelBZOAVig6hJ6ZgRCEiw^}{6b`{+5{J9i7>5HPuS%@-ShP)@@Z55cWxnhccsdQOn
zwP{(b+gJav;CO>yOZIbFD3mVJvNbu!7kbFfooj?!QQ{!Vb^-wG0K^=`XR}7qClHaD
zI-2o%-8WLVt=u3oAzxH~H&P#;xNqf#66dAS+|#=Jm!?hYit9$|{8k=}Mpmv2!Ae7t
zHhZ;u4zHpxFZNPt)K8jqa!!VsThDJwo``9BStifAe3(BG2=2%1{zgE51#YSFMQ~hx
zP@XVZ-CuggLe!I6awgaUn^%L^oqXLC*m4X@|A;7?fDqjbz|)q0(&q(q&#7RoO*-8#
z9psYKL;p`->0G%IGd}9EB{NcxQoH(m)Z)#3k<H6rk=1!zC${wS+^bzeE+<6}qr6mH
z@y~P?kqGvsnDKaI^2TEK&G_`c#N!Tpxf#db;?YJ4-aPvB%-?C~_i4zIlD~u_N~VGj
zKRfVf6<&j@rIZy(vppOsv;8AOcfYCpPrB!lk6<*$6oDk}n`@Y=iOGIjiu++bG$YF$
zxXk8Hm<^_ENA<L1#|05z%1F-2pE(F%oiEQl0uaLMG63$q`C~=vAhG&hqs@OUD%FW~
z37A5aYSA}I{N2T|y8BM$PKoXI<C|<IsiXJlTY*cf>X>Phtldpa5!u&ShT@dcY*6N!
zeR<M9Nl)4KbieD;h3vllg`4~|L%Rd*|FyhB|19t9e=P5^nW=s0wT<`87XNokJhP-?
z@-l+Hsn>?6<}yke04<VCQq3RGw;tshy0mkS9O+LlJfcqwGk=6tySA4wa&26?r7t)z
zGGt>_@;U!(vZnt&*X`c*N{Pqq(%;i6_kCgCAA0Z|KR^uX_g1w2d%1Qm0`4_(XLDcc
z$BxS6T2$V_(!8B|$#y!0`zq740)O0yKo(cbufr4%6F>fZ1{_bL%DXsp3-ILiy_{0!
zJ#o$VEE)6F$*<A!hcvt0>~T74^X*Yf!onksrOPYQ&e&s`BjHs-158@jr3VK#`LgUc
zFyZ^(-u{%eR5*StLQ>)Rc$}~`7p;iBCf_%jt82;cbF-(WWS}N%t}-USMB2g7Mu(V4
zwvYX8-*x9hRsm^Zo_`DDSK~YS>(`ifYyT-$enXMctmY4cYno}jDD&JVd8|rSXwhjg
z<tL)+Dp|E0_m?-d3HBY^s*@*LnLJy@BcH3fF3J6$Is43kU`lc#@LO^Xv$c3w+g_!Q
z3)Rr%ZIEtc3Eh!s#VmjhW+KHZ5n4+gbUJ(~x1XGEnrZ%J^s=ybmGX_;Cep;IaJxvE
z^p@u%Q6=XlpM$3n#T6)tT8ixN;L5|2!K>Mm<S5mSF!%qXjyjagc+$7$6UHaG-qk6)
zKI`2Xc9KggMv9uSVm6C!O5{L?V@3P3e=LgyB}-WF>U*pXrS3R)t3ozo_7BDKwRfE2
zCYqaZGp5iS?b>fGD0O*TV$9=1oZGUp*%(V#iC?a_t%(|6ZtXakZvN>Q*k3|e)#vg5
zmq_mZ-<8#3Jx++<;f`7CSMYjV+^P8~zWh(q$m?3snEKut3C|baOSevlTs$?b{DqKk
zD!i%qF~Qaj{thoN#zA!!$yImG98^=^7Z`rw+%!txm0cwM!Y9wT*K*ly5fk5(s~2UB
zMLpLE+3%+ts`|i(e0y=&9ZJ@6QjK&(PkNnt;=MxNcXDUC#%5Pz&Wir#cN)clkYe=i
z<Y8OYMxlQ(4Frz=wy<Tw@}T%xpH^A{_pVJ>faZ?sx8{^#Opa$Jje5kS-o<=3=G>=c
zCB==s>9AkxQcH_DE7FbX;|1D3-!h)jo9@>JT?o8TfpW!t+qukiNiH6WhRz^z6-Hs}
zemPvGs?zPhw#Q!89mzzQCzeQDEv=u{OcE#U#~9}GnfbrSa!9D?2uGjs?`$1zRZ7VY
zZ`&*Nd6b&2nMS#w$1vl*JN<V&ih2Dz<-fu;rKdHgM5vk3*w$~9W>)l0cP7_ln`C0z
z*$--qMJ|+iY>5XMNLg?ITE8sk<MR$I?HBN@p_AJecPVwTZWz9me4>3*l-L?*#EQmX
zv%8oEo?GscokgH_+5YxI(aP)J+eHRM?tGixKe44cx2SXJ&hx<NRoS1?PN*qgV>LGu
z6}z3(0@v2{uILnHUs24QWLBbU46fEoM*YU$(tfIXl>9{eM(?42l1%*ZdSk;(gZz#<
zM~rUU#y2jP6QY|0nyj?|NhP<byZE|y*v^fc@-q)oz11r$97~EDR2ch1hja)YtwNHD
z<#Q$CrKS%n-<ysR)A6+0{Ggfl*+F9){=MaTku$!8TmICJXL6w=T*$k1xpZfHgEppN
z)`c-m+GD@S$x8>hRi2gh$BN6KW16rP-<i2ZyK$jL^&97wfmaX?YJBb0Lw4q`Or+hG
z^(nSlkEsratd+AaI0|!X<%E>A)Ng$9*(`4!D~|hV{#u2$IQhvVsAea+fFTiG;mX9o
zO1#euiFRH+u&#RJ5}+b%GN9N1U7J7b=<k?(`^rFpdBfolkV`s5Ren3W!l-mfbxKe%
zQUlAo_L@txS^uF4yWt_b=zD0p#)`Kt-zH6_7@y$kV-D4_%gH;*B*bSf1i24)p$ytI
zc{X-<ZKy`85;CxfUS5h1!~ac{Aa#`du5jR-L0yadrP#SqyMQ$RWcL@+AB$cBU67fJ
zf-mmaJ(NgwX6T4AFXZ~VS(NDI%Io#4%5!vH&VAm5<0Zob-O6YQnZu`VON~`k!OSBy
z?rv{1WNV3I&IcCI5<c<<O6{0+?&)KgW4ig;Fv`Aa%=TY)nHo~?V8!JL$VAO6=3}Qk
zOV#1WPwIh2if06yJmv0%P;ybSyjgAto5r=O6;8=Dm8`z#w>In1M@Md3bS&Jg6!534
z`i{BZaPHflZB=#Ke=^*j(zZs`;lklo-b4))^H*d&PopDEZoK+8x%1rdVroqfM+V`&
zvj!q94H$)uvFJ}+?PMVdtYjj>mmrj_k~Ezd+SzO@A-NS?(_%IMr`yKE$mxEe%;UmI
z_Iu{|%87G7cczmRqtf6F7mpo)uI%lXWii6;+e=~&jms57=(oDznQn$!e9>>z3Q0(;
zpOf63Za0q)Nuwir-+b%Fyty~~TWlPkSNi7$xjvdy<V-MGIlC(Nx2aXIbAJ&v`))tw
z67y3h*|~JD4!X~X#aEp9mH6C6!O+z8gY9=qbghR)Vn@pvl3QtQfP<H8<TS>^(g$Ot
z*hxo+Hm^m@w+C=NqPz-L1r|Yx+3&Y<xasc2OYs<)QyiG~b5Zc|l3ZHlE~L~hCXuZi
zOufu%n{_2W`}PmIB?N4kG~UP(i^l8!kt-%C$zRwWD#4W{=JqR5)v4XzVc(@om)sfX
z!gIAfuh`80QEn>^C6fbr`l^}Lk!;y+{pv@)5j<AtI@yZ<ZcIuIH^s(0z@fc77AZ!{
zY;|8>pi2m}CN*Cid}i*m?$=afHV11H@__!>vt=D%RPR2(b6=(^FKtR~O>HUnuO>A%
z;MNs)F9tVlC26~jt?P8*+}Omph^tw#49#Svoij@Y6^7X-LY1FXJWA0MR3Bq5I^pq`
zI;)GYubv$=9Tb{Rq+(FtyW9*l_V$=>M+Xp;tMWxWnCH;}+-~ab6nEoY!~UW}f*(oS
zeY~}sp91hk>Z=+p?+;>TY6uhLsyQ-M<>2l)5ge7$ZZ98iuKInp(uk&`Rp3<0Cz00I
zYNc>{dLSTjnku`tSR#E##8g5D#b;Gv<haDc{<O5uKj`b`Vs_`Ww26_#CY8q*-@G<h
zmZpsGqX$Rou1j<A8vV|hFSOE9-oN3`3GbiGaWE=P7EtHr7uOvY3f}*D0(2Tj6e@Y}
z>2(U+%?VrZTd5A=v>w$uJr+4UwIx5K^tnS>pcNkL())8VuvpSXRdn#PT_G*}STTKU
z?pM_>E^;ovpjaS^Iv1mJQEEmhZ&D~E(um1;G2Vy^=6}Q~HK&wiz1q07oldRFUs+8@
z;oLLrr{z?6%S*qD6f+bz`frISM)T~#Nn}gu)%jxE6S$q3P{yMqX(jj7H*7KfRblgQ
ztX|n-rR)vLT}XD^ebT#C-lQD+X0$AqpGnfLP?y9dtDuU)oe-;6Y;tGj)_&)}TTbwc
zbNaYZ3lC*s!QNMB4XA{O-C5iTWP~5SL?d@aCWO^EiTgBnxs)`wRDmS*2kn-csfBo8
zv9~|EWK&{|oEAa{dj~!mpG{B<)VW8=q2din%DqWlLT08WT|--JkIVW50sB1CZ|grN
zY5Yz$e!9DEXf2gSpbNAXPzq?<V>Qz-=n%Ify^Etib_V5fx4MnK^GE=1p5?qh-CC(X
zulsbd6qn$}T@~o{=QK}V(BQ0&vw>r=@0V!YI<ru=R+e+x;)dkE!!j)lv6ra6H>tT3
zYJbxg%M|oM_+T|B?Bs11lRK1rSA8z-cMk>Ct9D52ODVf}NA_?RIAK8h-%Jhxh49t*
zvR^K%@e?}cDV{DH7@-!1=mU#LR*ccP{gjQoW{;`wOI?zQz3}PP<TBN{XQj9A3bU_A
zw&VQ<ta3eVY@{Qz#>z5hOeD1KJXSyif74~#NYR`8vbj?!?MK@;5m5b9!$p~M|87<@
z7Yn!^ISCQh>Q4Sv9QfP4c1y;k#U$yGsfhm~wXSMYT_A{#v{w*gUh2|25&_=q3uA9B
zY-bCVE|F3d95c;z6jn*e>gy%ji`*77qIaf0UE&jYLT{vwKEO<ajF*jt+k<e9Z*}Rs
z+?SlqakDpxHH-7^gEf4)$8tk#{7VwlYjr~{MCBp57&gwioW5^x)=q<kq}ID1kV7SU
zuZd@e@5UG<aojZkV{mqdDDeGF&2Ygrwr%=WUi&%}1vXQ-X-QbEZ<`5hXyw*V&z3|~
zsDRpoS~~`8dseAp%nyi}Rkpkl%irtP>iOlOFBtZI)9c4v`CLr>{a%SQ-4$21XFTB+
zhz`Ik<eJyz*9itYELLqF5;U7zOv2-tA?QrYMO9)=K|j-ug4Sjp<JL4yqDb9Usp)u`
z!Mz(ZrtrHjWIDL)gtjW8+szFBRfeIW^YPYn4^2VDOWIB1!}R8#F4elGMLxeg(kjfP
z+MNdK>U$`<@0&l=YeDM|-MIR1doft8zSPq(o1r^nB)RDN`N_nq;SLVA-`W;Aq}o7k
zH~P)kfeGrlPtD|k*=&0!bCG{pN(190PHsP^*(0NB4?Z~9p~2PO8>zZ%RodscGz;JP
zG7ubuO7)z?L9)0~MR)CAkq&K*#v7F^{_@^*g{RIbRGN3awhag(6`3_?)E&p*U{}x)
zcuS|ObVvG*lcql1r|+`<Vq86SjXMDQOT0aRb0MRz;g8Liy6_?Udu6nwBzh}h?FH&E
zLex9QzhQhp-IYti!8}-mxF?YF*?b(Uo;ciQ|7z`8c)Fa#=E+hiCqE;inarb|JI)n5
zl-;o-nZ;_c_|eMA5CO}A<zD8VjF2GJe0MRA{aCG<+Q825$xLO5#GJ!&DTFQ_e4|yh
zHzIY@PHTbNl2F5Ud8?%P!zI5}D8(~jP~`XhRhjrMlr_*p{|5SZpbvSA{;|W+CT5$Q
zIXYk!x%sxHzrVhTPj2nc4i9c>!3F%rkQ3435wof~j>}n^5xcE)4-0Km{tlqcgTDBl
z9O6tgyv^#=GIAxNd2RLtaUU&}ZK>o|%J(q<o6@}IJ|B&dNL+|k4IZ%aAEUIt6jrwg
z!ake!)*r50?a|?VjzMJXD5sd<RnSXB*5K;w*OzNbeBwHfxguoftA_=!7UI8xs#{Bm
zB3^~d?t4qaX~Y0NYCL`u6#J$wwt}{KX>>L=k$(;4SX*hCqubwlMmpyS#<QtNY}4zv
z$m&r4uP&$dv*<G#Jw;IhUqbqrGfIVhxErxEnoH7?>;-+DJ_4c0p%)#KB3T3d9n8>8
z{b51-y=S?8U7|3?am($XzE`;RF2n}39e2-;Ek=SeDw9!Uwk2sM=vq4BdYRl~rC}l{
z*H*KE_3N>!zSG5TUdtm;DN}j{UUn$93EtTHmU^&jasBE)67S$cbzeOjq$X}#kzSQF
zSbD4A`#Cd$l5EPzjjQ0RVOT%A*ec(N-R4bf#dyFhC9PN0X*sW%EOILr&!F~CP)}U)
zKj@gf7*O7(Fsj12+FuQ5ta3fF5%vGt^XgTrL}IemyJ8@ja`UH=t#T3DnZ*Vd16{s9
zT2E>vGBGU{y{Yht^`(<1CCz?hYQ>5!wfijw)fp4-3A9?c5jQ1c>w@~)PYI;$ju9tF
zd*12u>;4*9j(fM9Q@u&gjh<ztf4bABo@*y52-RpcQ+%9Aks^3(avB#gVTr@k_-V^=
zlQYWZiHJLi({6T{X_qC^3Cyrp?I-53ESd$)??G-*vob9)jDC|aIrxCwB}sAL`?7*M
zN7(G<&G%o_cWhFOH5k~+OL9{i9T$J3Y?|Gk-0{F7Wz@HEdMtRl?i{LMZTw~CsAOUe
z-0f!~tm=bv>9xB}`*8~Fs1S14B>;@C$nR-vzhBY0bYkoE{E2^xjox);F?IhQ>)j-<
z_h-YO?~c@6V%F-FJ@agh<x^c_lAGzTTG)QW48n&mRsP7MtcFa*22*2pJ{nl#_tf`B
zH6!tYLjQ^=Y0U$I(#K3Bup-y?ZA&HgC#RCQ9&PFbsPJyP)2Ww_yG6}k9Jwz4duwmH
zZ<ha?ua4OJ5tmJ;eC6Qf-pu~=!zpC>%>jAunxRtm6l5OPp7Fh`bljdhn`D5+m3H$i
zjCk6tW4mhkoPQYCZF~U(iwiKxRi<(ivioqi1cEhPKF+(BK=xuym1C5>=Cl8Z#Uo7=
zih%|k<xUQa%D`5@=v#`MX6U*fNBF6xUg+`PzKN~@%l){ZvAbKlv&A|RUI9Z<O+TVB
zQX{P&UIg{@lO+<J7*=%CF4syk<)@`1M&7{@%^uzy!B~5Y&pCupt;|7bg=fDBly4kt
z2b;R76Gu9~O~+8M2y_3h-;1OCE%HfA!YOwghQ6DbVI#NXCdoX^bIUfdRWiw)EoqVT
zSomevC4J`l?i()W?bvO)^3vs1%fD+W!iTalh#FJWcUjAb_wL=#+m%qcpmhrzG0(-k
zyQh*=F4Y<^8fgDZ{7v1&X(Xbo1<QKZ!f%~g8NcS{S1}cqpxHlnr~IpKmvC^7`DX3y
zKN5K{l+Ts+h8omb;da#khpU^sCeC%Y5Sgzn9?kgG`Y==0CzY>@W5zT0hywB@clTel
znX@@dZneA&7Bs5}BoQjD`UYcB#;b07N+~JAT5(;zOAe9K0~v$QxY|38%w9K=xG6L!
zGcMJysQ+F51>bSQm!f~!*>$&<WoBfpj^jH40~-Yn1F!p!A==gk7^~3}yyKC|-<$1g
zK)^c|l8goIe0qpOeiULcnn7e+u5CY+vxa~~;(0G1dY>J#Yf!TrmC%fo2DR5Cx@O3U
z)XX^x3(tKA;uJ|xSv_#Dte#Lw#51g3o#HgmKG$AZJrl>4@=$&7t|s<ND2<hJcBpz}
zAbMy?X-roPG1^R7!vy#>7FeF%+yC`ORcoUv+ixV6ZHls|_N$B6GLRY_+}<k7+oSSg
zKhZ>(dn+an>!<dCCi}LOYSML%o_@B>NJ`L(`HHX5U;?HnQFZ3=o%+AGKpQlUOv%~>
zsfFJDy}Mj%SoTA3WTj@@cH!%>oFOmL22LWAe0uN7%8}X^7eyN?uMb;t-p0c)YSVid
zRi+9DK5uUUYbNQs*^yCgn6{mA4rm!L=be*Qe4EJaOFE@%dm+RmIj6zBYMhXPw}V9~
zi5IGf`e~U%n%cKsQ&*{Tonpxn;J@kfl5be9zZbNaFqA;7!<>Z2>Lv)F2Ej}$+hRK0
z@|8}C>)e((u2`wS+#si$L6QDBe79zSl$b<={`{eo-eMT0B*45M_@hdBRR<mFdC&l~
ztG|8Z)|kSrEBTb+{XF<N;H6pzM)J4Q)HD4i&%-XANY&E0A8uO7p$WF#)n>iR?L_o-
zo7#2FBszO_)K9!>-h4ScVwRXcCNXW?wmyKNBo$d&?WMgMnV3SfEjU$52Iek5N?=(2
znxKY@OBI^<R>~Y=(!=rdB`?cXEFrb$VW7s`hOG}I`01k2sd>}cxEb?+J5Ta<tb?Sc
zDYV815AqzwF*MG|6i0i@(wv~OrWTziaTgv1UwW@v6WG^hwG&y6k|$MW*kRqc&J<Oz
zPxO^|oS>N*f#rTSV$Uco`ns1DFn@U5$!u`HG*WN;+D-25spBIqmA|%^rC~#@CmvtO
zVnZKcja=?7v89W&hwE0zXvB|h%W89;yPSvFj=Xf@h@umhEfu4}ao=L&(`V$4LY8K#
z8IR?}wMmCFb}}HpNM8l2$Be&;e9fQq$=S1;Is>g+smY0!+WNGL^Ak&^^&$heb!(M4
zaedatbEBQgd!|n3@PF)Sg#r~M)_rNSs{XC>zH)EB_MteZ{oOkS%`uukB;V2Y?&UsD
zp$DVfXzLmeE<1dw@UdLfVk4{(ahjZTFV{DLP_Yfv$FC8U(hQeON5e|*5u?>mft%Bk
ziQj$v#!JreCWxuMnY>Z;s>~A_^#<P1cX+wNbnr#A&yc#$6ZUDix($<7)|zCl8}ddC
zw5Ih}OUW|EnR5rdL=g#0*)wmf)SS1}w|C9zYcMS>WE-=(X5L9M0A{^7rh1<`y(xUY
z6mg3whmp<*_?i7#=boDUZ1o)>eVX%Z-{)!NL_F>(%4c&cXFAd#_(i46+Yn*QWlN2}
zR()(ZY^lPpb?7(M%Cw>|utq;zU^F3M95=qXwi&!QwmM|o`>hz0UxpaK&8$Ai5@195
zC^FwnZ)wke3wC4f_~M_)xun&t=98YQ^#mzNyx@D6dUb;JVASG9?#lw|?RhIF)L4d_
zB`1`5#l+X#rk!g8krZ>Dw4QtYO#epi$=ImxBo$z4;bo_ae?_Bl%JTEMCBfbNO1v&>
z5?KF^<LD2x8T;f=r%FJ@9%;=u*f{R{M0hiI1u~|@zNSJ#sNGYiVAsB1J<4&}gJPA(
zR}$)FGLFf3!=z47anp6U<@M#!PL9LJ+t*nIuk3$oI@~$q%@ut4R^YNe8EzSX_g6>J
zkpUA)4s~l?U{$+DfhFnB0ro^E-?(XNJW`Ch-cR2NLNV#<t^K<+I1X#(V$rv-#EJOS
zBtH{!j+HO8fdnC*KgdwMrwDD!8Ku=pV8*0kNE<Led4UMLuvd-v0=#Ogj_2MK6hkvK
zS2(627mD_zE#?gs^+h{oYQcA(^gh%YN~_qQjzgucEvXobF54q6eq+8fQ#Bu4Eq}<Y
z+^@oEx9#pa@jM{<4qS_HzBtcAnp>>1V@<8|ZDkDWELSlbon}?>D?<sD(y6#LpP|b-
zG58f%k6K;T+1?@sUv6KEg1Vhst@RL>Q_YCzz(t0BAGX^P8Wrs)YURA0w*nnt;>vJa
zzM_rCW)<ks*W*V9uc6&-$_UTx9;F5?Eb1w>Ezi|tQg+<A6)oocXjIlMiz+!E{l-^s
zkIctT!QsJ{TlAWBUi(9{>qF>u<yoo%;@0o)xg6QcMw|Y|W&z9vQofyzvWMcHm){gN
zE*qo+MTX@F!c+sbwz=|W-ZE`Cal@0jMN}yJ8))}B<V4F?O#}pCV7VSc{5C3wD-F0M
zlHu|FSJ;>A-Sm0iZ<cEu4kJ_=Ba{q3QF{$nOfscJDlSod2F95wcU8N*W~=Z0EQwTY
zJ`)`k6)1Y^*4-<C6`T0^@!kSvN8j5NFC$>&#q{e|BxwktY7}d$`nIdgC$4&Y9mC{$
zCtVUV_-zF<&f8?&DAZI8@e0r3?v|Vg{N5J!mf#B#4CCYTzv<EPgP!q7y)Z@~a2Z!d
zPJe`38=r<5Mw-K=iWWD&L&$H@m6}6$T21a)Bbuoe6k^$?T*qOoFKppQw6#?Jnx>>>
zHgq@?Qyh|-(U<unJ{H{X^&*y@;HVp@OR$bTT50ArGplX<2|yQa3~IDfgGR@N<KNS&
zO67~-KNq%hliryd`Bv|IYRDc=Gv@E7P7UR-2{j3wfTqXp$)g>oz7o`hwABY9PKI6L
zF1<Ms66V3~-NY8kv}L$eZ*F5p6t!8)8SOUwvhm)FKL?mI#!Nbl@^g1^Ve)Y(Cc8<u
zj4u?Xt(cF}cdUF7^;25WT&;XIFU<KpuyOsc^;yQ^XvVdr{1`m&pOPpAJp%HSl-X*B
zTYE!Y)mmGYfT4v3(~l~+)j?ro>PA{2#rSLPSkC|1(pFcs>$Eq$eTI6pjyj7Ap>tV0
zfkswy*i^e@&kCi--f%67S}#&pQgSd4@owgpe6Ktn%OOd_D0)?vg@w#w@Q1*JezKQQ
zy|J(G_DTYoi?L#is#LElfBrI5S0_({n&*~avJ*dW3elah)S8etr&^kFx&Io1`btyF
zOn!RJ2`Z5|6%!@M&n18EjZ-}*dHJN?$!#_@9bo$0aJjsAR8<ez4-XqYytREtY(!9!
zP4=4YSn+6=f1d87;OVj`MA*%n$A?v&Z+-i539-al_EW@`=T^Papwh=Ucka}n6FoVT
zkKoclqjybd(fn@~5WiN2Mq}7mF?i-mm(E||)*HGXt;C#f1L>i72J=>n#(I%?+XF|J
zaw=-8eZ)Vw53v`aH{Hpr7Mij<Uk%H3;{{VxmNz*Dh{r7lpS8oUjnxVSQWRQ}4<gUi
z%HINH*H1Dxj=pFA0fkh2|J5vT-nJ?}i_}=G2kAVCV_eQYO^rHA4s&td9jh!>Bh!&<
zOc*QF?Yu*^>NC3m9ZOZ^&vjh71_@ajbj8BC6NyVk6>nDE)hDE~+elQ*(~w2UVYj*}
z^cV7I=lMMhd}tTnS^63POetm8>V(?0sYNQi6LERbIHSxy>u<SDeQcVHfd7lI?+&E;
z|NgI3Dn$sD)sn14*{d{UXLF4kk?fUm&7zWoaP3vNbj`vwy4;&&WM*&IUKiKC-0Ql(
z*QN39{qFnu{qK4{U*~zA=Xu8CJdQzdF^zS%eTzoc=o6<D-p+L2d<`X4Pw&16H3j}?
zKiXvQ1G44I9{N_PQHOE>c2;XzCt^AEiV`+iQ4Ks#P`Fi0{hbhR=l~ENh^{EK$qo%E
zLpxjMy07VoKYHJlI!GPjbcLZEOhQ(#>O~2!b=$iOslyN3gt;Ythz@shXST)+f6xyu
zcNcZw=#gIRdbx3VX^Sb8sNT>5<Fy=W)izj9byc*GZ-~RW6%RYe*k2=@d8Cqdu=FP2
z1RzrO3#cTo&P_DP8F|SG=S`0HDdl>ALvOx&XPu>q6n!LSemc9Oij`YTJIge@lAI;j
zkMPa^sOj>`HQ2ti)ogPmP5F7^!kpnqejqvo{8)cj{Ak%^jvKY(4lwTpa-LcAgLYN4
z-t91E0IL28dMRzK5FbYZWgMZl5*U`;50#XVeXBjAl<j-jSdquYL11qC4m{H#|K#eb
zT}|y^kX5N51YF=JG|$nfXAYU!d#Bp5eK}TpVw@O{v?l7TkiG03$Bi4lz6{H)jr4$B
z6p;q`onvkVy!)%;pVm9Wuzo6p75pNdp|Oe7StWtP3C2#(G^xWpy|X)(nx2nBkNCY?
zJPa$oz$A_Ds2w|U!lSvwwg2p<>eW7|lI|?NS%;jp8C$)qIFlRW+}r!@oHUGbf<QHF
z2X^b53t%TtW+5J7avQ}<X_O<<P(~43j=i07az?s5k?S-<?ar{%U57m1nuZKw^KRix
zYA>A%7=DDC8ES6|DUzEvX@BIIHuE~jG;=~vD{2L|K!(me*ju&juvLLl+8UVen%W}r
zARb$Ga4T~UpI~QR&M35rH{xUN7zyyt<Ybk(10N2yILL=U`aQj@#Lfy%Pt>%uXAah$
z-O31@^w=U`y<l5X-b>#KxzO<*scw-(b%9rc1kI6=5{|a8D}t^Z{Y_>DsB6wogC&)R
z<pU2;ikxq5|Aa!FnL1qN1o5mp)^g*<O&Mw)i!K!7=dF*L7P7|1S&f-qB}NjZ?#g%a
zL)iUhbmu&3zOxIlcl*9`(ux+cp_O(Wh8MM%tOoJ0X~k-offn?)WQtg4UK7wu?pg@q
zpxY*NIpvEn;EmtEWyu(7z|xtaDnY`{f#yM+Lv5SUnuhuH;lnL@7Ldn~ck)U}euW$D
zbj&~*iD{0!wZ#2GnmAL4<yhe;eczx(C%ue==PEJb{6?N=KJPXns=}3k%NNR0bF@HT
z)Q1&u!15_A4UkF8`Dhxk2S7&y*r4Wz4*h&jz4}qtApNZ|7x^sO>}^a7)Xpk%nCKLa
zk35Ou4%nyLheEt;rRO?#=W5o1d{6k{sEgXmWNqR^&8tRxWu~PE=5ckxuTQo+Ti)L`
zM)rc1$Vh5gPv2TTozWcPq|T@V5XbeqRtBC1^l}8_fLVHQ^B@lbq8;u4?fYby<=wr<
zfJ4x5+81WVQQ>D5X~*0EHQ|)8u#&}szNbi4OgbR+WlybDU~*lRWE=?{GD?HQnpT+7
z1p35MsAd=e%-2Jz799G>Q5CAO1KtJTuK8b9_v~V=5}&$1A5I!#f(o;D?Q59<^~fm0
zKc8wfdFtk4s=V=e5j8?}%Cy_EGqLTqGK)xHFusz0=w?OWc3|Q8FSReiECynurRY{%
zds$<Wl*ygFe6tM?wq+>M6J($leok-Wf)ERk&`8oXMr)E|()W(3sgTmcH516ZTW?)A
zKX&P$x>N0N{+paRKpoH*)_pSALiO`&IsWL*v@fEzM3>Hs*toh5gdPl)VobkTs)=tD
zFTxNazyVWrd~~{y)iMwoTq;sM7Wk@T2NeuNEUQ<9_Us~;6f}q_O))m)G${NXvx}aV
zb-8;_&e9OAn_K+LRyS39E~79chvjMb{9Y3SVxMWE1t|IIMt78q%VzxQ5`H<fFJ{_T
z8F-<Z3Y$HjyC25azZ(lksdgp}j4#o1qqnCgXpwk#{&yX-=J3i8DMWTLI$}Z&%wnxS
z=MxNjusEI~5dGn_sf$3gi_ciF!`zC^vEXXrHWLP;|5iJSmfMq0=L5k+9n#dyUT{>v
z;N1ibV<@>DyFr~|z#lXLH?WT0(MEpr0~z~w@yEkYqTj0A0Cki#Y(Aw2e;<$EFE+oe
zg-=nTF_-J{EUL^oSO@^dAfN>~{q@eM{PZw16%9{k=|*vQ2=(w8cF}x#A@0=2&#Dqt
z-`jm0*g^(&KkY&BankiM-$N83Ci|3|a{bhz(K_Gr`b3#ZLGDS_%S+2bd;-xG+ya)X
zdYoO&4L*{B$3BVb!FlP}ARzcY8uCDnYn-3OW_QSu7CF65eU!<ugDnV@7Ts3b!<i4)
zDcCA1Er2rN^<E0HZ_fH){Z^XW)ksd~+0%-Nt|BUZ>^f^+CAHok$XF71jK8#@XuuCS
zS7I}y6oO7$pH(R4>ax7ZBJOGJ%;TL7TA@8`nCF}H=6QbDkh^eRLuE}x3;QbD6&<+F
z1S~@Lb$eD;CVxq_m7ZP6rCE{*SFpu;YiGAWbT)#FpY1<yAwd{2UW-u2jCGz{xbAM?
zrBKcs!V*0pSOMkjAbOCV;{E3QR`P*VHvOVT;jkzGk_S)?B`uOC_gy)-A5+9!_mK5o
z1{G-dR77FP0RYl*rmr9s|7z%=K--lA=JU}P#ewa2>X&K+_jqgJ$CHc&q6?{_K!n$t
z4ZdN%4|<sffKRwcO*NXq&e4{=v7Jvaxn~V2(pDVyHpoj}L)+xj$aoPDERDF(68(pb
zN4z`D%EULD?T*Tor$tmRf7$XCoJ&Zp9?7&QD}KrBC{}Pe`4F)X?-h_a(Bt{Gvo-X!
zgb2-@*&VTeavH5X{zT}^Mh4=^Jbyp4v3;*7Oob0>9eZLGRCOV-$tK5Ie_8XJ^5mt%
z{PHI22GfPdEh|s9FW<IAoOJN8-WnWn8(nxO$cM1q>rTo5h19JLersp!gS7M*w@ux`
zDk6s*@R^!>`;r#)uC=Dx%`1b&VhS#&l1XduFIDGrWricqV_=wwGWk_?<_4)2KiA<W
zx(YW8i@L$>#hofd;d+(?SK9}l76)GicFQ)`X%)Wd-i6dJ9Aex<$!iude>jlhde+XT
zi^n247urHPOSd@?RtPz!_0~;`Re)A;!85Hd#9ExDCLvmo<C51#{!{o8zA$DK<vV2u
zizcaTsUkj6O^FTWJl<X&q5s&zHZ=I)`gCq`6UV0c&7fKvUc}c6#I-h^zHVf3tNmT)
z6#kNYjjWpQ!3~lE3kJ1jaioR8V$Uo0qR$8DDa_+_c){UK55}XiUs&rsX^EVOwh8k#
zcDlV$`mGz?2O-MO71>NI`N#ig3|OGa+}^#lZzMuM2kdM*lz3c}v9~srR^LtN0VddV
zxno~wH0%|W*T3)$OU8sUYn?1v;hyzYZpTMP{6+mR|8GED$7aFF)uruocI$)4FGHvB
z+T>9$27<#q-r>Z#%;1zR`FUs1``iNq*)iXiTP||3qSqHXO*v775mHwgWQpt<m^^}M
z+A48;8mEKq+k&}`<I->UgRY}eU-8qe&J}ump8BYTZ<4ukQsMoNZU-A}@fqqJ`}y=J
z-D-W#bGe5OFosa>5*6`KK=$pT^S(KDtbt;*wFY%I?aQB23&??P+PvgSGCLC<-;Bxj
z&IDt!L}dk~lo68A4J-_DXieVlQl92-!TGfB<nXg=i?bq94FIf8QKXleB_v`Zq6_#0
zlH|i&x0pE$v+Vut)kkgH4rwS-r`hIw8QC)>%=2<7b43LqaK7<In04!hU>|ddb*ozh
zKg)~yWyk()?7>0VHO<c<T%5zSA~cWn-vU+Xm5V=t(~dDW%Y|*aL*p_(4Ix6r6RyT8
zJOUe)RSJQPf(lg{2JxBtzWT`eHAUDFS|Wzt8I!doh<Yr#s?v{}{_uuAhVG)LG3rFm
zMMB}{%8K|Ggf;86MZN4XP7`=Xs-JnvGCsoMx`V;xez>vmeA=L;Z({{tpC1#)h>X+s
zyo7u;ucc)-!=ZU3m(+Q|ip}~jdjot0Gdlvtbh)hGxf#q$E}jNC<{2YM#5cM)1P=@0
znY%!W<Tnf$9sQj#^XH+^%>0DoK9!<fsGoaxAeme`o3b*NQz+#A-FK!{c<P)3@O~{2
za`G-QIc>ewynW|RXKPsWJ5D}C-c6(Ix6<xwC4Aa8)A@S5&n|YUANkZXoTW)a!=tnC
zQKYt>Byq-Defmi$DrJGAwZtJs%uNn6HJ?oPtZT-=)u{W!(%3^T9dyOjnW}7LBr3#b
zSgGmjXFjTyzyW@D*{*ebMY-_D^@+}>BB>QrhWIQN$zr|sK!yCFtwl$y<#*rPX3d$#
z5~OF$BAl)E$5U^APZ;9gGU#`y9l{BFUvCJuMDw2}CC}p|Ts}|Y^VCV{Ro#_nW}U1a
zx$G*$JwqfXDZ&A<;r)WY8x3BVd%!NQun{_=!96**y+MU2l*|&6Gr^;__tjpuW}e<L
zegLA0QfmUy@FySFaa`Juee9E<Y4IZWvhl-{&6BE)RfwL4u4)QH6nkx~FQ<`pJ}X+Y
z`VMZjSl=v=m&uldd#S#Bo_zUTm1F`(!us5t>vec_A0{<tAV(XaPOECS)xW4W8sAuo
zzM1{fLMDq3k^9ZZ3vk!+rl*wbdi9DzT{k^i7nE`#w}aoY+UV)qfSXEhAuAL<3(OO+
zB7K5BhToh~>l-c9MHc7h>F2Gj+PUmViha2^iW2;7^(U7U5<XXH4T<8traB1pf<X2~
zpP`|Do3Yw&z+r2{gv@3IE+&o&@I`&X-ymf<%=hbbjxO|ts1>Of-}OQdA_+y)g<PH{
z!v&wK#2k#QU9DaQBvr5rT^RKVHU8-+9oTINEQ&5~tk+XR7;nDPoRm}&@fLo6w~E_I
z)WK|9Ik;SKN^LxZa!Z0$LB}C8d@v44;^eJhi%AH~KWqXfSaCE3Xs$-8Y%F=KD6&T#
z@i3&xv21XzHHZ23nSPv`8J}}UOXx)(7W}qM;FzbE>j#tYh+FNwa>SO+(MV$$uQnua
zy~an0$pp1!D$p#OVa%8}xL`hVcRqA5OjeT^j+T$XgF0+fS3i87eCp$1Qm(zC*=^)l
zY*VUN%2Io|`PHt=&cEC6n1L7vs_ngmJEU_;ak@N+EsxdYiCW-e7#^!vakI)=9cgH)
zW3`NnMxO7#nQm=gq?0vwRV3e#|7JZeQ=9sx9cj@MFK2di$P~>i;g-;lpPYv}cL#pE
zFDai29XT<{rQ~zM`eUQ&Czrt#<~fK@AoHpG?|EG&Sw4K#ZB2cA4mM%rY#+#jn|6Fl
zN=!Z;4NlnQ;pj841rlL3UOg;Kke)O}Bq_+8tU&5jzms*hCwGC}$*)T=;S-+>kdq3X
zY^7#jN95L}KYplD&n`k!=@Sx}&BMBUgEl^{<6H{{wtWiUK13g^^~A4poWANpW+f_o
zDk{{v&4v``JGlFN0!)58u}ehvB@!{~0nD#Ntr5sr;Xd&Z*Ym1+%fEFc9Ah|rAy(c+
zZn?%FV`8BJm#=i-xw~eTN{Je$K0jtomv=5<-(IC$@*TK|3tbYs<JTwbZg)2VN8AR@
zA7H>jm^s1C#WZY#bb&4$_8cLlov)ik`ie!!*?LcXl5#Nd;wpbM(H$RVETV-p^L)yW
zXPcw%<r5-BJKQl@ttb_exO8^*h2<(@BL8v<vv5D&2~*>ham>T&GankY>D<v^$&AV#
zl)kmnY#Tn|Z*I3);dYael%bBc5xWjiC%S9ThNv~-AH8#v5a^H{C|6kVt;~1Fb(0zl
z6Y}g(>m~hC3Fo_)sn~aH!6<*J?9Z0~GW|JM#m^=ZZ?mw)y(w-ZcNR~)#A(rWE;-hl
z2*TNf0L^^^9RAHpA|w}O_&HbpXtaVMe?lPOj;HXtE>c4t4uW7DzV<6r6M@{e<uY?M
z=SpxN!bju+11;Bdk`K6RTN~YgS5sT4=pNwHLAZICH_V{bKk73mksWdqFb4d3Gzdc}
zN9^f&<+Cg)Zzl3f;%y(TFGrao0~fZ>Tl&^u8&bV)p<N4!)sigO3)&J)-n=hexIh1R
znG@V8O^lSDjBkvAwiaF6mj;ZgNjUl7!u8`74>zApFsjIX8#-=mX7IUSPvQZfBs}`a
zHa{o;Z#&1k#<ZTbU2-C(E0ZHItxqQf-q9D)*IC0Cb;Q;sYs9%k`XZR$YaJJS;3~K_
zvu|MpC8Q*<-idm-tUOOUr~oC*msdO6O(bDymtv(X($+g88;bbg-Q3W!^}4$EYr#G&
zwf7V#?L1F+x<}-je>T^cX)}N>#@u}bhnB&aA%UkSg1a3>y{3iSmBw?cF_ccrq}n`5
zRbo*dqS#;CX!<+Z>*hN;K<bznU2DX;NL=tmo(YjF=6UXncoKwLTfEE0j1LA>F4T*>
z6ZlJX^{ngMf-SZa-1DTkP4%X`_Vp!|($ovz59SYJk6mp1x+m+!*cD2{C|tJ&sO2*K
zl%NntL-DN`!{XaK?MGmzMUQxXFg*`6T}_K!ZFR6s#f<N_d};+zTlPZV2~JQ2FQs=3
z4>>@2U26+(0MUdNHVSlAfu@qqAjQuE0k#U03C`>2zuF<<mwHNW&OoBjQ-Z8|j3Zdn
zy0>uon^>Yo7O9VwFYwxyoSaV@wge9Ze#Hc(dHX&B(=alB!?(Zis>l~gD-q@K0u=U_
z>vHFdm-(V1E?g;21{&|U_2}E-S|{e!{QM4dJp*Nf3kB=)K?+J}`cj=N=k6AzsF$i}
zwX#t^-Q0D5&As_He)(f*9i@vOGREA-{R46%E`F71P1AXl&w+)4pz6p|L1_Mo7XtJ0
zdCY8!T_IELsL?Jx=7w)wQR>CB#dnYyUO{eQE9`-%2m8}oa>Yi$FL70}pZ~GZUpK4f
zZEN`3?2u5P1gi_@icI@-Iqr4g{UeceJk!Z9)J451yDh`MG2$AjekQuN8*!!#x#Mf!
z4P3QZ_qk+NpD6=9GM)K+7|8YCv%-!aJdm^mud$U9wrJdHE72_xJYVun>ASzpeCW6h
zT5Fb6EhZaVe36u$M?^SmJ+-KEo417MBiFq=%Y4FOUiTr>M<W~-fqby|4{vag{>Nj-
zTq4KD*RB?HTE&m&dEwg!sGR5Kek6<(%^3d`0e6vz0EdOrvd8aq3xheMCMum&nac?`
z2_oI|YW#?mVoI>{k)-@c2S5^1xtht!EIygXoxFk_k9|THe-zV|rpRf7+YocL2`2Ka
zX>Op@!AsJzowFDevVxGNS}1GWeG3_h_XTbgRgu(V_vCwKW<CSyK=qB%jSU2v4}O~N
z(Hd$qAdPmiX)9L~Xg{9p?2{E5NZRr6XeCSd5Bo5fToq{DHz8RtQ1YQSTF*}%{ct(I
zz`L|Y@Fr)%TPXR`Jiy7qBT}~1nj6W7#j+WBt)T)|JOvL8w)cMuok=Wi9wJu{&{30@
z@ngB%M`e95uT?^su$VqZ)Hf~oKE0Tx&b&Q>+zm}Qw*eC9j4vfs`Xe`{@zWD^5?r4O
zhFJN@!WuJ+JX}I|b{gF$+geH5-RVq_5~$d(*OL^ZAN<f7tR{v2h-*MUEJC&D5i{59
zEoBZCec!jx)u%~=B*|%JO}+P-B2J>W3tg*;SAAOyu>#RD0r8=BV4Ee-e3YD3iL%nF
z(FjCpGswmzKcoie8r*FJ2bv{me!l*tG<;Ht)+Gf2nv-2hB3bN3A2sX}53olvYs4v(
z+U$5TXI)a|^tV@+0~=eELj-LvH{5hB_rn!Rr>&9KD_k$~q6G|Tv)jh%KgTf7`0*4d
z8=}rPyf0Qxjoxpuniae)l=*;ml!hH>%-i)ePc!}0tnSS17rE`T4)^Mf6yS%SSIJ*I
zQDd;~nDq8r^4{dV8PyBykTL#t=MPEmpDW`92k+;-dp;ae<mJzTMU<288k%@51x#S_
z)<BQ%Ama^@RzaIq1_#6NM^^eGo?C|mdt4oLr4$B!>Pi~GOR@9JkJdv)^ZN|@asUgf
zNvwKsah|T$K<)J7Vw53Tg(OVFeCq7ZGyg_LK6&}@YTOJN%52WrwQzV_4MRs%Z%e6Q
z_A2UI+C;nA6c<;MCRIU=xr1WT?_;JOwcJn)9E%uBEilOSsiJ;CXTC)NMNs<xnH;;`
z|C0Ii4c)jRIT^9)OQ9u+*9LBUzpE~%F6VN1>=wnpW}D+S*9h}QIZdh~o_Kl9vXe={
zRwb9qDr`3k%4oB@tVY_aGOtcj>Q}q%_BPsw%qSfc?I5Ft#Vi#Et8jP0a}H{!7sel9
zf@--r#9G?Sa^T7h0s?`QX8bl6?;1TS^z6)6N_R`*A3Ln2w0~qEO7|&8q}~x!1Uhem
zHID@z)UgsjK`mi*S9x`YiAh)cN^VS@6rtnQGG9H#3Q+F=`l4wYb{GqWfyOBcF-AMc
znI$D(K7PX+28DdoG#LxX^a`n!QajzoAmXh!9&vX?C8#`XJ)95aw8JexX}VDkgjHz*
z909{X$Jmem5nK~e9QA16E|3a<MdMy?yKJf%jisNJnS9k3)7rE+Fj7VZm`aJCObaj|
zlCA9hDe3Vqd$r@#Y{1f8H{q<eBQWI}`_bZ7cU%1BEChR*w<2)LsGr;g0Q!HouTlcw
zAleCqltVcrqaZ=d5W$IHJ_)P3*kMzXri%Ck<TsT6Pf`QQ5l}Y#Ch~aiuIR)QVp$lv
zYs%k3%L>27W;EOyO1!qkG8twLK8%&V>}YhRd0)FTCEot^^sC7`Qtc|{Y86blQ={I+
z_xt{q&5bXLHz)VITE`At>D3l+Fvd2`ocaArAnV@jkNsu-zEZ(JR}A5jdkw8S4}Ydt
zJ1xvzruJp@S@onb1B9CzGv{8&`JaV<`5dFro9Un{2JpUqQH9Q-7c7a_di;5ekW9_h
z<sP|WnE^lBop%y7;xu_bB@_&_16Wt`!>n;b-o@+oM#Jfq%&9Q~WXH|GNhgTd^5o+;
zzfT6Z1D2JpKivtZ9#f6DtV=riP0_h)eOI=DB}1*^ZVjCkB)nce^}KLy<R6Ptx@5{|
zejH0M746IV0T8?mm)m!bAaf@<@?d{!mA%YHje#0<96(aB)4fGu7Xe5-91p~%Qg~f5
z5IwG!v?uo;7HCpuFmi5t<GR|PFkK>9`PbjP^w_;l1(4&~+s}*YypDv6aUU~KATubF
zXats^?Kd(eD2du1wk7dE_tPNBU7d-h^Q+$~WQK%YC>T#ai@mTT`V>ga+|6&U49wj8
zHClgc@-4`T*@$|SV<&qT&Cj+0-8>+$ynbPMGW{$vHtU=0i4M4_bCvnP_<uy%d1jRY
zE?2erIP)g6k*O>O*y1|<q{Kp#lS91R`1-{}THzm8^QW71S*36HR&9^n&wM`Ux!CmH
zx$QpPWQm3Pks<702a-qDS+Z`Iu4C7@yeb@yKfdSbjs(p;VznkOg2s)5>k=Xl5)WC8
z6}DNzc%p48B-qV=xBc%Bjkv~s+7<i2#GcH?F+`mgmk<~CvCz-ny6c(mlaxy?555Zc
z?*@gR-Pp|ly0i5`A<?&>+ZHd&C9E9xPhRfJzZdHA7iC=7%jmHyJ!k-^UT0_<&v?(d
zZTOH`9vi>+hYu1j>Jli&rP8q5_(rjqz(=WF38AFy?0UL{mCf5nRN~}|vYplGDZdnN
zoUq96`-HAsJXqxvLiz3MgFkhOa$ExkM$b|fd|@YW^$YrNO8BPav3(_Xt{u{@FN&i`
zrj}m1AIqH&5F<M7?l&%43osEs-T_OeF4BErI^e?hf5@GvU7Q(_d$?*lKi#Bfqriu_
zf4TJnAf_5L{MJw<MsX_yOc9bLKw|S-+bjS)cIr9G7VdDJJNJeA+&R)1Ld-5St3nZe
zfN@7&*KTM@vv^$=?H&d>s8Ivv@%IPpwH##1Sfxlgq<(YFGCC2QvK}spEG`FG#%R)@
z&Oyi1D_aNp9#_n)Wo!p<Eu%cimC4jzqbD*0J{KkKG)eu;*B<&v4=6pZ{E20n9YUIw
z$0AE*yh$;@Wzi3O&Y|5dZ*B7N@z{TUyJJ^{soBn__Y}Ry8&)u{L6^}A@MlvIG#8|6
zV;$Ru^;2~Pl1d+^LJo!lcIRIU-gz=3L%C&X%hpq1f0Ox~j?^hA2!+)yn?RmqR6Cvc
zbS7tACpx%%-DAFz{lDM+gVL$y{KpUrCo@(U02{IP*(%e-j-8_Xx-vP6DwI~-df^*l
zD3KPh(mT)EPp^gg;wD9C$2>eVw=1Ao^dHPLLiXGVh&*WNI2!3NiVnidT?ogA`}-zr
ziH^|xXYVyncDNc!!Y@_WAuMO?ls35?4_xLx)@<6@p_!?iBuW+t$LA>;2fDGpI{)9*
z56g2=4!q|602GK#GA>P!#VIB!^x|eFE4|pc#wM;gQPQ^%RdM$H8501}(Ql%juDUDp
zJF5QXX2fz!J`~sgPR1M4#i_{amaD5Nbmv9nL9at`qr^Cz<#rX<oJo{^hG*RDYdOjn
z0K7evmjB%>NzE?XL`gsAYv{Ke;~?ZrGuE*MD$%*2a;hdU^2MBKo}gD0`Gf(7z}2&!
z^4HF{%$(Wr4R&5SN|o5w1Q3F)*(fBu?NILpfS6G(KS=vvItGNKN?o-{_N0(fPIn9I
zY)qJVa6cQsCcTCFg(K$$=wr^;+%|d$J?hbRMS1+S0JoT`c&y`hD8B;JO4UHySMg@~
zJk^E=fAB2<Z;kO8+9yX191YVI+H$*sR{b2BxIaA*t3NZ`#p#z_3QR`9Y6Bc^mbi8T
zz`rz9)@yn<?q<|!ysCbn6J`L0$f2V<1a7%>YBAmP1u&rIsO<RQAQD*?&Zg*JE}gRM
z{mM!7Md6!<_GK5i*)>WqvTHHp6_U6%+-@F1ouxU!!`mmT0*oqHpwYPVbnPQB7sPVS
zFR=F2LV=WZBZjAb)^f!rkvGvff$EKL1x$N}T2FwNbpJ)Jg+{60#<`mMC)~%j&neh7
zb?6{bUeEZ`{A1MgC7;p*>kc!tFE{*q-6XzUf~g$<A!S_2TE(ZA^(`l6!phr?pkxk!
zO1ovql31Xe<ygOFwPAFx<wXmR#iwm>r~-;<^WnJR>)53a1nVdn!%_m_X#%x<yp(k~
zo@PhF{@G4;cd||%`+X-SRnMVz5Q{>4%`Mzp^~rF6Vzpgv`+dy&EmG>}JAkNN5D8E!
z_U<AyGmYW`1gn!JHzG8*_bLs}T70bYi*qUtd+BtQV_h6MHdLd`Drj6PAIeHRvIhEY
zvzG$JtrRJ96#^Kjew(kAi>)HJ0PHI;C(fYPcy$;V0Ze!5-sY;G?iFRaWr=^O1aOhD
z0Nmk`iI=z~4S_)sZTXM0G~=5f8XoU6!sCmi>}D`k!EF5VCVC@ao6`^2u|o8u^9y0q
ze(z>`kFaD7&7jKiHQDR!ZTsKlY}d`lFp3WOO3acX15$_Rz-+Qty`9Dz!gJ4oW&4$j
z#;n49XWx=1jmT9WPl&DIEw+py`QMY;jM~|y`wffIKHVj|kF$n5+(udCcKB71LsnlV
zGDu7`Myxm_`(54i1g2nYujXo$+##`@RiGo=7f2Hp1ka+Mr4J&TN#0B2)sf*VYQ+-0
z7o~*<%sLYnA-YXU=e?bh3qeVmh4yoQ0Xjw!QvXS>1_cr&aks-aZv%Xx>qboB#~8c=
zS_5$Q0Nt&XloBwTpw=5;rjv3`1Q^6Z_)vqUWP``rGy;0&6VWHOiU&S)-*_z5PqKA4
z?2_jEAPTW_egc%DZ>yHU4(pX(SF2ik^pordp#RF-a8r0mcZLrc%;Ak=g83C=O*AI)
zGtFDQTmTa=vRw?n;bw7>U2eTP9q5|GB)D3Z$;E2~*`Dl`EO4E~=0H+yTMKTJ`s;+w
zn!{SKviVo-?T7hAY{W~ccz!FuQ!j7i`TdwTQG>T1OlSgdZz7bXmq8zky{69tC;`~*
zWdWgg3xKR;8acEG;k2yHIy7)W8U{WB0D!G0@<l^9<9Y6fKz~EpdQ?<co$%&XQ(Z#%
z>saLVrqf}i|MN}>Srr4I+_(osLNC$QS5@n|R$fs{Y?syE$>}-CYz%cs_%)80gs;8~
zM8!UZP?XTemanpteFe2S@t|2Vp6&5=8Fgy7!I&z5AbZvNGX$W~w&8HqMN*0H(8}Gs
zT*$3yPIlV@WxUdGJy4!?kZwII3(C`lk`itQE$;(*O7j&&>xAcGmI7-y%~Sw;&k`S9
zoBq%L2LuL4N?=f#uak8$WMJZ9(s+m%_L4<jc1Pt$y#@|88$Fv=lwq8P2nRh?=wqoi
zyAg=M@^)f=&b99|XGCnutu{OVOB{Ea;u0gjo8FGI;n5Lxi?aNt528uu-Zb*w-q;FW
zR_v)*fipLN_#aUA^wT|V18C=f2c&j4^zLW6B=cD3-po6tI75krSz}P}(xTj3A65HR
zni6nn(<4X8LqDB<Se5V&bDcQVZnB_E`Tp3n?$eMcVNUJYnxgSmv5Z#G8JWpt28f_h
z!;g1w*ClT=odKx+hWW)=QmH9#W|=oI?7&KCQ<`Ntw#B80`L6o+m}CsZcZ%|a{&?kf
zjrUF<$9ViYC6H@I1iTa`3z%li_`B>=1qCA+2gMwHhCN-L&2fE{p<9#eQ1;uh-n7Zi
z&wc3q#}}X4v5|=YdV}H|C0Y)2eyAMF<DI}@;<^{xKm+V6OhM&0FK%?L=O(Q$y<ci&
zO%=N$yYuAl>cq^4U4z%W>oT}sW|M_N{554u<h{M^cfR^I10(@CL|VT{AExj)WV-&6
z)Bi0w2}M$wVQp7RHWVJbg6meXaZ+d;1VS2(hwiJiWLfVU8~bWrwcq%~+r=Keqneec
zc4O$~Qw$He<zlWF@)Txj9+m@<N#}D9v@r;OE=hM#beZ$>3$obYr$1lw<E;l8`~IB=
znEHP9X56_b-kFjt6-7BfXSgOYRNS<QR|1vC*<~&{x(rW}9ixW<FZ~~T*rilz-;EaM
zZ9k>Z`Q{TLutvXZHW~Zxpn(G`cZ13UP%B5*k5a<EQ`A2-ZkMa-`M~pCcME{F)w97m
zHUz7DImleV&dmrQz&9-?qbd$JJO7s2zheI>jpfO#i(C%876hG%FZ2OjqxJg-?aiZR
za5=Yu6wU9C1-!!@U1Qx<)8^sNNC2Z(o(+{yfn%fWb3d5`b?uWhov5h^4H1a}vL<C3
zUqYX;1E~%m&-2@903s5}U3t6J@0E{p<Lh*^y@z=5z)Ii^vkRauVbSr6EzkTY(*0LJ
z-T6aA`sEA(-CkV076tE(WR<>yjE=Rd29@hofUV8oL4R2;{>O5Q?{35eUP8>7Kc2AQ
zwi0;zpqV?oq|DWAVLGJnA3FJ?N}=}SBAZelu#n8EPMC)(vvEEW$h&^mBMbn4nyWol
z0aAwl`Q1-$EIYY|eW}X)H23Pu4a>fVKkbS@-nuQ|<<VO;=@C?(rgoXHB!>-%s{b9g
z{>YlMJBr`SfGKUR9)r)c#{hDj#4<nvZ>^qedvIvySC;?q!vK+9-u=d6>iqahPDrPA
zy|zNxfDo{%VXT`~NjcAOU4Ozb|9?%u)gQmZUVqyd0_~F%n;av#SP2~8Qo_#<mLx<K
z0@>OLN@(*t*YKMI4rj};56W|WRJC1bB2|G3q}=?#S8+WkJB02chssyLNxc8hU=Dzw
zKikn%cimH*?8Ohfi!`@yLzOwH9|=eVS-o4gd2_Y7qKnm1;<ruxlZ{QPp*p{jlXrsX
zSa%&0XeDq4eee{g0Q%rJIQZEg-zLewOSv`$cuthy(`^>hGYcla(9F<_VrAP4tQW9(
z9xzM?H)IpF(1$3AkzZ-G$NIaevF3o?TP1KIHqwSGHcGf#XSQZ`+)7Nh9dsY-zUJPI
zv73dubbF5dSE|FdyD7bX?ZtX?QoeSjPLmQ-I^uIKXWnh2;45hUaQEdp)-i|Iwf~jT
zk2JrOi#P9c=n6a?N<KdEP)FUyiDPn<0g`V78ig?kjsV&1D*~B6GHW}iRLX2&Aj&6k
z!t%BP`6gt}Y&hWd7BL^%u0^8V{m3oOgmnOb_Mh84esup1ed*QIjyF0L%Eti0u2Sa)
z&rF0kRoOC_XOnMFiV}q(=$|=-BjPqca+aA>*HdD_w5uBR-bQe&9ELz7v=lMxPv}`I
zz}Y8>E-Rq_r9DV!iLkMDec|<c+%5Zn#|rh>rKL$`(x*?)pk_eYcSW3%bOJ(CN^)yA
z-~d+E-W6?THGcC=m`{pc{Z^7Q@iegCBU=7IKy0Ywtv!0463J1l<mj&|fXEN3MBF9Y
z39vrgNvi5oSihpp(_s^n+vTrfTPC*WRUM=MVV8Fq7VA#^SPohwTybG3V>4a&u4_AJ
zt`;3N$7o7blXS}cASLFbSSpH`e!z?N?FyAn_m!f!K7dKLvv~grNXrF4VgW738)5&K
zSR7y~&!d*@naVN!_s(Cp`Np9qC%fAIE@g{o1-97S4rvyX9NRg5AQk;P+#CYvnvcqx
z(4r~~58_tr;nV;??z<3|;yC-Yh83|E426mCU#zzIl-qGukQ$>DXTuA_%s%VdclM^Y
z%|_Q1KkTa#u|ICb^*~Cp`K%>H$=-+sIKZ}O?v`6wryezID2mP*1?v-e8y3dgLi>Y$
zlHmRhrXh@!2LmFokzBTfBU;*P*OL!eEb;>K2%X`Y&*oSEL_iFQD>wZsSG;RT->~@O
zumTQt6+_N<Gf5(Wk2S4F7+{~02M{?}MG(!`0ZILp`>HN>_?P8Q(eXQnzMDPzP2*U3
zx6*w&tHJB0-lp$y;Z{5q@U&WsT|N9gda!7M9yZiRu*8-Ww}GzLYTsUeJ$ws?sqovq
zm;k+JvoP(?Q>94QU+N%c@4Wi<iSMWu_#kdg0tVeYCD6f<A<%7AlG6i&;)rJ%eK2&!
zo)Xs{^8jx!qi~b^Gu)weY4=ZnX@dh6@ej_>T?S7hi47A;Y3t3lkr1qd*IOqam`XcW
zs?l)(x`(cB^LwT_#4W!JNqK3UbdASN#J&w=Te}rFce7P+Kl2g(XFnzWFWTWsSAQHX
zn{;)CXssl$*qJ(SiGUfvg%9W&U)H+c`00E@ZBCUSD3}!6Z%x#VB^qNx3OjxkIR1F+
z^BBGPEc$XgEL~`e$+onp@1VpR+HJxZ3)CdtfCx}Ovi?=n04@L}YTo4t)jNb@g&efA
zABRJYf`ZQ%F;AI--ugsMoXBsfko_g$Vg8?0tGjedY$u-$978e2VDe&0jd;jS<lyoa
zZS_OaTI}FQ0-;>vkdz_W3e~1TB5;FX4}Tf$UG?0ph13lW8}h6$!-iN3vM0T6ez{nU
z;uM9eBy=8VD5_)u1*erFaD-`WS`+cYa;Sm(TjJA&T?6+AtMjRy%TeNw8%mO|@^sha
zJ+tAbH_RM#lns1A^O#AS&aoQG|GFX;Ro5@waES5@i2PuN{#9Ms;VhN6KY&pZn9J&j
z|InV2dl;BDBj3sH^@`STD0}+DIm><A%o0~Uwuc}AQ)9Hb)f`_!)jfQ_tZWOO<o-Ve
z`z(<T)!=IE;#ZcZB+$2v7T1J70Ym%z_#RGf8)o!}18vlN7C`y+vX#KP>9!GYHLHdk
zJkU@7t0?zrZaLpS?kv24+SFPON|5830B`ucTh9cuJtEAobSZEkuTJXLZ^1O3#3C3*
z?T6`exw389waB#ux%eTWzW;G;BP&ZZGxjR2Enx;fg^|e|2KoRByCsES-?`$R$AdRt
z_rCBxc<Q{zEt`)?<9&y>4{twI4=RsLQQi<5QTS=T_cf35kL^|znQa3p2CjKOM#k{D
z?o6)|{JG#Z?#O_NrF{3QB!J;>9L)P>9C2c8BH{w9vBjl|5u~j9rhohrVipz=9r5#o
zyL)bP?b-8WkK(Nx+Q2y_0<AcdaNFR;722JDuCz&~W34c<k_k{6k;A;<9U-dU+8u1`
zH*_W=lmz$~nGiWg)mlNq-07w2dsFsplR_X~KtA>Tzwh<#yCE*Mp<7?Bk$RV$POc88
z3k4a>R%$2-*lvOMS_BjumWVyd?(pw7!mHYB3T|;mkobZBdLzaE=Vt%0!QOzAv`76{
z{BfihGgP0SBF~B(86V6?O3|AIbs8Gy>mcAgP%BXYMbGuXp8zRdv^9;tzXen)tSC;{
zzf8_o$NR?<S%5n|`?lkKPZgCy^)`-vr9=tBaVUNsa)z(p;6ACv#&OXjzjEWYjbcOY
zx9%<zLmS!mK2a7OO$f1W%Cq{r6iRVF_<MHhWS09eD7xpXYd&{*RyN7TPg!-Pi(U1d
zu;&{JeN`8pIh#^nglg4a3&R3*>F%Q#-E$6FE9K6)Kz8>(rOD<Odw?&#>hAz+<c^YD
z1U1%E<&$vqWGf{R@egax0aU$14<WGig)(Pb9EVXF;#Mw5(tbvK<#jm|#0B8U|G~sg
z@f&uktUE55xF~QS=-6A~%GTS0p+^r%mvNoB$Ft%@+y|K197zx%Op}}!Z!`-A{$Xr$
zoiDTPI-wn_{#zgSHU9|n0cuzv>!4AIFSyyN&M%b?-L(A#U@UI(q=?C95z9?m*7287
z1ztEa?0U9iRb=@f$lt+(8bTe1`4-&m`PU7;9Qu4loEO;OH3fK>%sSmx&rM*WHGxVw
zFc0u@!pHV0VUYk!*FeFCl3IZ(WO*;B(Ti)D7As*-E+zd%XF@4)p+Bjzq^0M$ddf$o
zI9eF83z?xg#{FF2q}(@+lrDGMIz7~~iCx)(6b=!s6RAWpo^@ysE%*9APUQPBwhCn!
z(2A#H2TWHbtpe=&I=WylQivhi#P2503Qj3$-Y2cGcZe`%CpG6?ZbywB6O`b7TV%g^
z)*nZ<6)3z_vtbYcAb;@yl*8Cu4lQ$eCNn+7F<OSrvhrBQgwP*p-kq@K)6+4xiy^%u
zsdE)wdBaGntb*-@$*J)q7CRO9Tz3mt^x$jXhrOy(uVdlQHP;;MjPH?|iRwUJ`9<AH
zx!{?^Im=1QWx&CxaRGREhX1b&Huy6=GQ5UBce~fij^Jm>gsy=XkaQSLj!FrQ+=gw*
zm9F?SpbOnxH{GGaAubQEpX%Wv41T_>6o=zJ*pv?NEDVq`Uz!wn_bi)E3f&sI11u)0
z4iE}w{xewxB){D9UINUMFYg1(X~cj2v>+NTK<`$Tm%83`d!{5(2Q`;EW2~VoTeexO
z88AC9Bl&(PDU2A%7p470vzCz3uF%5vzM~D7#%WBDA!>S_!<;B1s+;=Cug`zKf$H}u
z8H0xdy4plzln@TEC;Tp8K_XuY{>UVU_Jp+E-&VXP2Fxlc_?QlipM-6xyN1hD;-9^X
zm!$`W45rCSCEEkULC-C>ar3ItTbEMIK3b|7$Mri`IRt?0f$jpnyaOm!OXAA`{0i>W
zkc)1M0S(b5x1^P_w$#9<*nh1(eR^lc!ppr1b9dqncFm+TJ@V-fzyWRjmJPwH3WTUG
zZ6#=%?RwHNS)}0Tt*=jBGRA}%c`Pr}la~CH5~UZ_55xA!YR-BOUWGMxcXFaj$5rO4
zxw>jgcwC7eX&eT9Rs6Q(dkfhwLqj~yNOur09~RYf$UZPdBr<Y47j;SrJY}Pr@{I1|
zHr{gM^`cx$iNwnMrYK>(Tb-^S?fYrXha8n6WA%;fTho3%O)Qz?l<dXwz%}&9NQwcu
zec{A9CoVQJK1r>&KU7wUpq@$6h}~v&COtN-Z9hlC5XK0SJlOtG|EtihUel5DIei?y
zd|O>{EX_?4V%!tAL{s=#xav(R+$ZNE3)N@CzzfHvF1s0h{aUvbzFCE#BX?xMfiC;{
z+?49xv~d$)UPx0z@+`sRgpmFT8TH5aRkBtURxQ^=a2^8Z75_2dV|Jmrxx<|KTe*xd
zc}6%@M(?)y6UwOWq>(}dh8Y+mqRuemel5WWlrHCM8V)4AE7$o@QJ{_+@Y>5cDJwnC
zr8vPo8t|GZHx(Nr9<iExc`~@TqV9Z41}~_PuOZ6bxi`u}kB~A3a>xq+-~|q)93j6|
zYNr+4>8QAwSb@aG!Zr2up)PIX)$bA-6O}+UWR!S2^Ky_a`6w8b#?$WQO{6_CzaVuN
z{(J(rRfHu=8`_all;WhuO|H-QTuYC^YhW#hUvQJD-OJ_r=5L6tV|s9y>%;>sldLU6
z9?YB}K)`D|^i1jv#gzXjE|=DpNXdUa2<Y=Q+UEdugzpDnE-`Ctk4u;XnneV$YBSj_
zJT_0H)5ubv7z|s5dryr`W+_2(k8ZcxSlEZ9mmxHlj2(1d;p%;1W1&F8e3sv9x!zCN
zh}Y>bb;vkn!CT~he&atXGVgcUNC4v>+8-zUIlB9D-?;+5jl606vo&;(FDFqI7{eC?
z;$w+^4@3rWQY9EK-|48KmE5qD$0699+>~(v*y-FulD&_fBR95Ja$b7*CQp2&bZ9pj
z-#PF3_`dvo|1#kcF`;<iGa;TJ-^+higx(kSv41`V^nE`%c=XFOO`3WL>Pp7<7)%Hc
zFr7C+z`mrsR~`sAVG&XW^&qmc;huUj2h5xQtgP(iB`y9qrcR%4{MgLg1=^1@X@lKc
z%a7c#CpmPi8~GG6UPGRDnSsuQyVsFqH@%lNq7ttS^Pe;aaR7v}|1c#Kj(i~9`>Yic
zx_y7OgwgLb8UHn9p{F{^DoQii)nC%9OOfc&^64FuR~JH-b;^^zP?O?2^|KiJ_R8uJ
zZM}lQo2Cv<SZ@BadvWqu%Ou84+#88-rA<G$_Lr$|Q4F%~j+C5z?;8H|n^6WgY^B)R
zqYFT!dv?M4kD?rag{9el?f&sC4z|GtTxGAG<wlE|gw@HO?L`)<em;w2w~{k$1dju-
zeRQ|!j!BPO%PEsi8KQIKZo&^R>%F??V?qm`TPe`YfGDkb3a^q()H8gFI~Uml4Ud(`
zvSO!O<}T+}Zx!7dY6aKls=ZfGJM9?l^e<;>cjm)uPT`018)p-`4{jGj?CP6yqs!lQ
zjQcD2B^X_qj}O`MrxjV&s$b2w0ms>pp2?NfC)KJVxjIB{^C6}PQTceI7V{>)zW6bA
z{5NAXB1c+hR4Irr!@aBvKv;YRK=?oLH__HG3Wnx@+372XFz7|sC>;1@|EVr+d&^!f
zFZ(FtRPE~lNAuQXfj0u57R~R2fdy;DEojtdJ1u6Kgt%-E`n-A$UF69CAfQ1mO(BI0
zGoWribskLK*$3Xp(@4Yxxu`{pFENuo!gmM&Dnsq`9-tZGcj9jsWJF1b*ygY-j@Nir
z`KI_-_2883Y+Xv;0{mV&bWv>YppaxKqd2KNSN%b!+$iA#<8zsa&=%{%&LnOgk(u7K
z2i!6VodRqdXD@Dirt6x2tZO}!{=QS0`xF8(tp~yCd20nEUfc7ZyeKUtr4rpld^lVH
zQL=+*!RbJx{;2?6s}xl>Ph$AjcYy1zH+$#lc(?rt#{>qQNnWLS5#z(XHqJvxHgCLj
zcmNJa7(Wu@m6&|_elSy^ZbeomCq?cHD!1&RO)G997t@_EJC?rpFIne;y#e1vea%~o
zmz3<ReewMX8(xn}<{nMi3t7Mybt09Y$85WFwzm06J|J>g=NRdyzHl`M@pZ~vE;Yo9
zQ*)X@%R)t$1q!8>^Ljxl%Hx?wcorgCxH&s(6{&^+%;)dw7*Ng1rudbnPw^$e%pUoQ
z!ZL*ydFICysq6zU9=lh+^~uX54(Z+!^}q|SFXsoT_1R`cL{vC*6xbI=GAov7@{UxE
zb2G?39({TT8C_n4&pHOH7_2}%JAvmJ*1ArIf&-AmKhAUqlf~%IdgaiANHm+2C<H_b
zQqJf^KVZ?z&{Pt6|4HQS4IQz|+7|Jd9lP#(p{10pjy%U2JQce6OcUmrv&_?Z-qP=g
zix$lL;frfaaWkmg+vtiQ-)zR&&_C4|fH4E`F-i{j%d}|PWT^d$YBhWX4Sz3Jusx%k
z*(H|?<*6s-H?Ey#lOD49ph;tIiMRY1TBSNqSa66s;w|(EG8`;H*5U^<SNLoSV=WJ9
z&R#cajH41A)U9>}HDqOQ32+Hy=o(?0{wCR*-c$!o8~c61K+M)VwPGnql(v}CRnFp@
zU}7o%&ewcqB1@&PfT2vyLwK(48F$n&XV&eZTBGxMYvrfSm&>h5(p7du>5;-)WyH7#
zs9sLA+|ZW)>j!OXr5mLlynp*vJqnEEOOXcSlq#$1)`kOpZJ1x@Yz9s*7YZ`^qOUi}
zi#VeOKZv%K=^Tz}?B+LqOzEVL5+>(D@!y9J;ojxp3u7mhUfag6gE`9NZG<Yae4=2>
zvU5JMpuef=8U=lQU(^RvX)CGaY)QV#wL<z>;f0bT(Du3)<Bp#vy?)A+CPT$HD<+^w
zo=YC*!^|lb%Sx%xKvNzEe#a0b$h!3|&fhP!e2XiJjw|a16}x4D-}{~|Q<p9g8`-@a
z|HKLbCYbJ)#Qo#GWao(<3_3@|bZ7y@2<`h`S1JV4oOFO5X<+H+=TF%NB48OAA~z2}
zT?%b^_ol>~wydGWxlnhxCIOKnrvZUQmxLsjl+PQ38ip1HM!a0M3%UFrSK(jemxbOM
z`gYTPwX`*W_s2K?V}ci;Q0|nx+!yG9^LpwAZ<|S7*OU4;h7g9Vn5(nueUoWmx{}TJ
z-y1Ecjx;s;VIr<>;<W0?Jh1i9vH~C6N%yUw@|pwc_|uutvgJWvWBRho%dK^;5@h(Q
z;l~_X{P##Y8ATtbwmm=%EAgMuGN)+3kEfWz)kExq?-kx_%Bcwx1b8lYqRZAbePGOp
z@cV8{SJJl1VT^i7JW21NWs6C2t<SS9#f<A(87tpxqqw}4V?~`P*@Hcl{PFLqKLxdB
z#(w3HTvOuGi2Bu-#$?7qb2pO>h@740aIbp8>BTqcjRF@O*Yjgo->u|ob&G*9wz6kZ
zhG#Panb|<sEhyXlSHd0}N~!alM9w#`KPU+S%TCU1OuT;4&lw9PCn&9PRN$J1(p;8(
zntXw3Yd-+x<32`-p#Oxx9UGwqo_)PLWC_A6aAm94))C>;3(W-ut@ANaI@TlhFwrhq
z!nc~fX5ZR3CaC8=G{ll~0Jd{4CGq?ZP@rSiq2o(WZY*Zc>3t)BEzsN~-m9l;-ntjc
zM0$#8)t3)$nii3XK(VV@?WC-!`1uyfUH(_G14-#1ssyIMoji7vJ-uVAJ8D_-R9`zb
z{c16e{MyYcvgwBn_}A~8&P?y!9rX4h++%FMsSx8~2)`}D#;y{`o^;PnUWBYk!7%UH
zDLYV@TYn<rj_tx$?@aW_ao4gDf#9ezG&GaM2~;^ZsPi{XR!&*|<s3uy@8Wm^II}vw
zpp@p5eJ1(`3+FNMWx2P8GFbn<N1?MjSXaBnn^B;-T<x$g@3=fq6kAHQ;;8NE2JB_8
z1GvK<5`q5%vUcaiG&MllQSx!d#|Jzz1W3fM7xlSsz~~$fP&PV(Y#o4q{h=2?v(vvj
zHtcNI1w(22(E2>>sJVj}J^`)-sI6JwX8<0!D-Oy_@k<<>`_53{29VJwu)!E9H=nlH
zp0#2$@58O_Dk5D1pYnK;x4Qh=v)R`G!1G5chLS`16I`cJc8VR}(liaFie47XgyP4C
zwT2??F#bGf-~ex+!Zu{Tc>LHh1!w!S0+hhATj{dX&g&}oAX%%DFcmUxIPCm8s6=yd
zf7G;T35WO9h`(%mlKoHs1=+zGwIbD0EREJ8<56<?Zv6mJK?V<*$sAj9t<}LH6{v)S
zkMFAOA5#Evy9o6D_lfZ=q$N1Sovj!Of1xQ9#h_QM-88L3h|!+~Kl*m^Z+q_kxo5S|
z&~j(zRrWvu3rTp#GqEt{lq(lVgtIa}%0$`wzCry7j=QJy$0xg=J-Lqp;nhBuef!xR
zU1j3kG0PGrn=QvUxvp|OHz&ecQ#RLf=mO)0&G5rJVU%Zg$5Eve(000qddm;*s$v?0
z(8W}ruQ67fho`-qA3O=uq={T8&*%q|VnYw<02u3EK0N=X^nOYi_5H0AN6-u!)0e3Q
z5vz^leG4`%>tyRMgy-brK+0`Lw<x>*JsNS}Sqf;@p{h7qU<CW6WME<#ChGmZ!@W}=
z8l2`92Cb7SpL#Z{MR&#Q_i04~#lGEE4;qTc2cS^<L0eTU&jeM-9ti4Gnqq<Vjzj|s
zT#64oSd(k=itewUQpSbtw%z#u7*7f;EHYyFC7?}C_U!~z=a~Ium{Ire(9mm!UuyZg
zX!3@67uRryD^>NY9({Ye9I6ux_wwXpGFeDA{Qvx9Hfb7{idH1nfsZK-fj5218t`MA
zP(tRzktRNl(oZ8XOs|>#vgVKZkp1tC>=~r^+~T)!J+y(mY@XK~-+4#Dw(GQ5)%}L-
zTa`2Y(3#cNHLrdh?C+upW%4SZ8c8fyFx=tp`T@x}+G{gmDx7Ycm1TBO!an)p@ju>!
zKk6Gi$C)TdcTrL7!I)D<8OY6`4x0*BGcSGq?)*!L``0ReJOO`<vs{#W`D*t9Mb49b
zw{W%0FO<f8kocw+`{we7rz&wKTGr}cqUgPv|L2K@)Nk=JKpfB=;>AucG6R(2)Ok7!
zZKm60Fcx3Mt#+DIDW%)7pWQ^vyERzWuxUnRLw>uDSH{UhLl<O*&Ig>YkkhL-@@Gx{
z$EEqLet`a*1fbsvpQQnG2uk;XO-OEB{wd@+E3>&HFqJA`e8%s^fJy^pZYW@+04)G&
z;&;w%PGFnz#AqJzRN|jcIdqomv}aVs3Fm6@KJJ{Tc$a<h<`0AaN8uM3Si1u^|2o?S
z!}0lmuW$Kb#>Yc}AL2CY@l|hvP@KXqerfD)i#oY85Ew8`hgVoU0g+D5#}}iO!phe+
z5`_L{iv)jM$&>q;>^B})Wj3k9Ot48C&vWxT9h_4!i+{IZzkU^4t2pjM2xm&k-B9rx
zmv3r73gVi-+)yRHYUqhlWPG3}ocy;<s_q^HQtGUQ4=+Lcn&1{1zZQB#Vr>gPRp@w6
zYmz|;5_VK}MkYdI;xBfDGA)^6M+Sw^g5mBTzS+F({#+EM!1l^pV%eg<q#e)!7a1h4
z1?tN&^?&HWuSUM?<3}kv>jqBs*=KpM`I<2i-R+rK6kYl<f}YC`)0d3ZLk+nY^R$}-
zQ;T-Jsy_~gcmGYw;S929an6HqK_;ADKHf211^fV^W;{P<9~ADtW6Zy5n?3u7X8c;-
z<_C;OA(i*?w)+x#(!?FzB?Uul9hc?yR*R;ytABhQept2Y+{J4aiU0E-Y;gjnk5c|h
z4m0lop%^V(8$$A^xkNviWB?On>K{M+pFbD0ukg#oBJw8t<BYL(>Nbb(ag+At^o#m@
zPWq=t|Jt|L@xwb}6V$EH$<lz|$!Kamg3_XOtWW>r2K@K$0i^x1+V=XLx{%{j2xeHL
zUDWB4i#ZLYBCjQW*|OiyI)r-HelE!03#WATwsk1<dI%KD<iZA^B4N=9se-2<e>;_(
zz#cf211sVHE=umm(Zffb+-zF&*ShkF56Ax|N2%incRt8RzJ61C9d4WauHZ4oj%Y~Z
zPWbEdb>rmD^R*`AcPZR`(UQ{GHf3(3QNQ|BbMI_Ac#neg-}bft>vf8@7{@(I`E2mC
zP^DX2G_SlEjeyRS6HvlQtj2UowvX8b|E8!r9jA*nalj;|&hO}YVUN^q1rtt-GDIUo
z@5%yW{EOQvkFuaMd&H!j;E@NBNr2VaP4es-q33MQ(Zl`6&K}%H?P&J&uEll2TA=%J
z7}({plg7Gi0Ju<l<R$jFj_*=XHFqP5x8J?Dn4Eq#!)$ap_LWi`2lochi{i*!AS8f>
z{*k(abO3WRDWr-A_W_ysQ~L+3gQo2w>oV4K*2gr^7vW#%0b-Sx9;a86r_z$c9u^|J
z1)aZ|ST6%ceISa*Yjm-iOYJlpkQwC{_W}9-iB+fc3i&&H2#%b%ed<7$pB^w>TbJyP
z-qULArVwKBF*V2c|MB(RVNGURzcXVSMPLvG0Ra&ektRauMMb59^bS%Z9f5$fL`6WP
zNtX@+p%Z$jiHL#_ia@9VLNB3&7D5O~ZgA#1-#Pc5=RSGz=li~E?^S-kwf5S3UqWK_
z!^A4TRD5yWtL)o@+$SS@e@0#v_WshYLqe7s4&2HM4H9kVHu<7Qr!8-5BW-g2Q~Jbh
z%s+QE3)B=;twxSLmfDQvq-C=@652Y?I#1rUTeNxu5K9WE>&<7#z;Fet4zNzp0s<9k
z$e}!Ocb4!SjJlSmRAuRHI{nMbA^)+|lr9~qb?M|t;@|(0iS7GrX7Gf>b7=y+o^V)l
z12YzPazXb;`YwvqC)z7dj`+Lr@8=ob&p!gJL7w_unaI1U4U`m#VW=zhpF%g5NSoYB
zOYC)4G=HTw6&*4^L`>DEUcS?mzYIz{gwJC?FpOSn%2mfM%W9`S8FPB?VRf1ND*q6z
zLb(!_Sc9ONQ)<_UT^KHS*O5&*f!oeAOWo3UJ({`&gmzw*v!Cu_V{7!`ZP{^q+q~m8
zwcj{UL0cn7BDh=q&@*;O`rY7q{c6fb&Tab0BdwedInK#<DJ+zUq{<%gL>Kzq;9m5T
zElZ3`N7VXp>QBm7%`Noitp|6+Z+umL>-i&p3Zc8NEuxn-asv~(Q9d)BF)N>ndEh%G
zSSI!D%=a_jZ4grXlRxXWj`dqT2+bqfd9<<N^)r_~+`b&cbya$()aqrs@mj15uHYdg
z*k;Ol9m2X0-leT|=OyLMyG#CC&J2pllZdTF8_ZbY2K-6_xY3GFR9y7ikxkV>)3~k-
z_v3%^Yz;d8e>Hm?@x)N5UHatrS~0JIy3xe^<e-%SF87lA*#CV|k&Tu3%kPiL@a1u3
zRs)wjq<-5L)}V5XB9q**eeU&1r5f`7<O0usSn#Z|z}BDb(&JBN<v)LT@|a`Gf#WoN
z^#+^v_yBSVF*S7MPqW$DufVW?mNp_mC(0gq^_#!r4?IGMo=bg@4_T24USgp4n}0F!
z)cfCMh(m8p|97kBHs%}Dll1D^XdmxTYLECeD?eN#s2`^nT(I$G*t`Bx>W~$-Y~9B_
zYK~x+b#cv$5~Hy+({nBTw|GlsXRl&<wSQJYpE_&K`hOQUR?l7g@;>E8^6$=LQ^dDw
z6Az4)e?<Vbe%4KH8Gf&D!T$JPC}?)$9PeS$8f&3ls(|);t62XgzoFUF^hB>+aCleq
zKdoIgD@*Y2{WJIe1dAAaf3c_lRid|Zu(m1f9{nKbawgqReb)2$@e;4P)~?SAA9vL$
z^s!U#{o5Vuefj<1e|F)}-*R${>Eca2?Nqdd{=gJ!XDm=2R=>Z&d1N_IsgVZr>iBPk
zj+j65m`^&#r@qn@={?Ro9*0gH($j_Y8%^M)b;Ysd``Wh;|62v&{a;7F8rK6VrIZb~
zpFJFw1zN3dZj65ZUkv}M{ZCN;_rl6Q2{}Hq%kWtXjHm^JwITev=#Q<v*8e*!f5`ZU
z`26qYpFjFJ&;Jc=vg&FkIVh!6!Yn4lLNkB;VPs5Y>fdm%lk_(?|F_MHU(EPVin(uQ
z<$;Yx0I>J5gO7V;D}4(-4M<bo!kGVoR!9D}K45$FT0Z2T^yz>1ZPGg(h2ICAknNR=
zP!zrK^RDqwQ9+!zd6fR`C)@w30Q_4bUwwV+WRL8o+gl-!=SW*J>21&4r*Tp{ys1Ke
zVe}n{1AaF3A2j{9jVze}`q70aIF}VeGoTk;Vt8zvgKzQIr+6ETDwunGKzwujZ}a$f
z$G_)_E!d3SV(ynL1V8;R(C&2ouTMNY(X=P<uj#v00ndH*c#@<1+xyc-zy6D~{D)MR
zPAmSB>dhK`0ftYE)O(QTexl#CQ-#0gE%w8I(NnYHV}+BaxKX16>v5&ZJ%$ELN1&gw
z!{$A{)*&`&{eQEpSn_{+R{rmK;C{0Eo{QfzT5>G$hkU?l46|qOCw6nxJE$!SYx-SB
zxiA0XrvEN42=e|_X8)eb;}vVR!jR)>@?#CbE#Zw5S;E%dZ6pdoM0ZRhyLNS4BTJGR
zZ$K0Dp%>8nf3v9ntg<n)#7Uf!X5>(`rf9S5z$t$PV{~7XWE-c^URJb;3s^+yL%TIL
zfBDDM_PhULbkgBfBeIs49WMimAi=$_=<iUZzatL9JaMW6d{x=u_#?e@B<a45UMX<w
z=SrZTBV_h(uKpi2x)44mP=A$6(u}AE`4#ca=l$P_%9O{au?O3<$rk2AD2_r~P;C<Z
z(w*?u;xfzBjh{6UL1L|cxSQuXhTf48Ek4s@7|(b2{^U)Oqz1Pu$$v1t)A8eQVG!YJ
z)|95K>@iX?<gz6wzq|KtAecvbOm+1bvDYoV(lPh0-5>M`^TU3iuAPPVO?^}S)!R?G
z#kK0nAM?w_vol^d^D@<ezjb(6^E~3r^o0bF(?$P{&yXrKBUPVPKh-0KS4(cA{gBtc
zh$stf_aC_#&3Nbh9dT*zsV-}G50SL83q`=nA<2ax-y33_4{BzsGCj_8zjbJ5Vraf{
zbG`TK*0|40)Ww=TP7SN$c>!AgN4x3LaQb^bU0SMUug7jf=d>GAkmSKe>tQ?NqRy`J
zWtOi|XFm$d2p3sAthsJh)$x}8LOK4kSU*YkQo{*&jh2Ku#pJgut-s&C8`bXbSg|#A
zXu06i>81+V82LVnsk0gmp81fywZ|66YAyNNq1{%NJ}kOS+-$x=YNH(DZWZ><ngym2
zIQwn)mi=w35I=(W=Er#do!6~W0q*1GJvFhd6$_p`BCRie>weqeO&<qC!WnttRpXE4
z5_aXMvYv0d%iDj7Zsmu)g#4iJHKgp8v&&XnZ@*Jz_%BlseRUoE8qWK%#=3GbtWv*R
zeUtv(sA4bJP1wx^HDGg{j+MJex>K;p@O=|@oq<IS?8$e%P5b{{?(T&0MB8nvVFq)Z
zc3u7Q=r-}ghOm+-h1(L9SVbq|)rJ(>2nrem=<~YSoJ~K@Ex5~uoGE%#KM)cmo!RK=
z+4~92C~m(`d9nE9)XA8BR$MpxC@fF^(dQ<Na__Jqe}GLbnfoo4RB&0yp7|m|tlu|A
z;g1;CEWEqnx{K?M(0lGxlU!$~T8c?;i|Ri2jG{2s6}rfJTsf5v?5=RGgK})eO*%E^
z{B2T9^jXuaDL?gV+JxP@W?PKwoApaK?gTH(>z8X0#1AXg&@3INJ5KMht#Ad5ZI0Hs
zk>e{+4X){HY(bS7P>vC^!%jPG@<QB3EaX`4K$j8t877%oeS9&sgzuFNC@2*gCaTMG
ztvU1GKlrx|KY4OBcr0xXsg{hC)OZTLyl!B25g}y99bnOgbrVSje?UjGN{WOyw^R2E
z0R+GTGnB#t`MH}+ERFXEV$QCcg@px}h4J`52@IGjbcX7<{OQqDKa*(WKpMVrVo86`
z-#<1XTz)lJ{;xQi8sU9{9pJ<y13E!2GJzGf+3TBsT0}uP;+!Yz5a$J;gbJ$3-u<T`
zYmRlneO}tNRDTd6ozUvJE;~!31#hG1+6zL9g=5s&ZZa^T9>NoZ)z~(wa_MeR8Qt#B
z{iDt1ZeYvhGChN<!KumMV3UJNx5O5oxK?gk*EMhTMUrT$Bv;ik&goPs!LW>@rEIT-
z=#ox|_6$0%3pdt7a+*7*mmoRc3Y58qVRN1HNZr&t(urwDEah69eACbL#4N_zXIr4`
zb(saW`d26=$uQ*QG868L2-JORn5gId33`p;K|>18bou3Clp{~N(=}rf^3kDks^pq=
z3*<bDzy0LGOKJLbmbDf{AfOMe67!YI*}SWXO8}IqNq1*2Q>NzRe+8Qn>9be>JgKKb
zrLe|2Ve+^<i9oG%FUYpxGa6i}2_I>n#oTp?o4e8>)XBBK_ul(m#gpN;_l0V+0YU1C
z>6cPIdRv&jT`JNaknVf{3aWsfX-%7D*}nb%dSRKsMGIFCL7}VGK*)8_&fq~Irf|zm
z=(~n7V<`S}{fc>(t7G6}9_O{xjn(S>a~7I?Ao!Y+cE)nfM~OZfB*ekpa;N}$@z05D
zsq5c%hb#40LWyWexct~+%QVM4xFG1g{QvvW^;WtWzOMIIw+gNc*-?%;h@Wr8<LJ>t
z;bvUUh>cWR`I@7f2%yZ7R?lUAfE0Y0Mb?g7qDfMpP3G*Rj7V>lL}{j@lMYvgjXRa`
z2g{RX(=%vEpZCY2Z_XSXTfG|mf_iu|pC|FjF~TvTkZvheyq)EMS&VXFsl*|bWFdGK
zJ2KZ3iZ8aLFa6=QgZ4>>FW*%-n$LpizyeJ(p%O0yu|MT%WdQUwY7MutJqeksS2Mwc
zIvuGGr%(bH;f~1N&%fV9*~&&N(WB=-MsgD+C&Nzv&T|9ilBc?AMvXt&xm|Lc?|h$a
zk+Tfgb~ld8ddEGSuH)J>YS&eyDI&9gXR1H;uSH@-?<)yxz6g$`8qfHcqEBUaNcOD)
zb5?FyOKyArbuA_I7j{h8PS~y)e-z$`5O-xfTni9fA3qXjF3HGFXkKbX8(=lfX!|81
z`tZl<)32;?TlWMjE{@i^So&<cPhOAX@Uc)-_P*x4&3*szFs{O4N2d*xVg%^`G>-(f
z?JoKAsGETWPIPrGrIz#GhQt7D|5U%6qVGH>HgbI52&+CFQi$#_(VU((EhUNR_azj1
zToVONT+tU|Q?E`iT5t))zY~KVX%i&Kt28l9`Ul&}v!RNkvqejB0I#fmoU{`;l+yVk
zeGm@ib#8r;3#0=P9YTcFQn=|q`ar*euLi3KSh{b!ADKO!87w*n7Js&)+u|m|RBiO4
z^Hh4n@fN>oDps5BH7C*Fe;`_;#w++DB9FCF;?O#z)EPV+oS#}|c{QTqJIj7h5LtyX
zQ#rARP*2JJ<<}<=>SyX_dSWDGI)a??JHB0Lgb`UNFdZm;8P@LqXyvzo-*gRHU9A|h
zE55&xisn?rXE?0=a@7ActY5C`crDnZ^tkJIr>JyOpNvnKWVu-{YJ+q8d|Aj0P<iiA
zO4=w@x~aRR!r!8S4UI_u|GaGaY`nf>#H@!!fhkAOFmV>+tHz_*=~D@+l(vWp_22^d
z&PaC;)y`I3`=urn6N*WbJ`1+W7ZIDtZ%Z4RsdkPoDc0w(WK}KQv}`-M*$@1l4QyS8
z1}h<n!iv8_=*gsQRBcq9yy2jO4niXPDR<>hL;Pbe3mJq2atB+e&W!FzTQFREu~&Q$
zfW4_XFo_vvgsjngJIcXfNCyHxI}@sX@817X8W*}#SIhH!tYK<by%=w&UHTLf>S<r8
za90b`xYab{CK6kI^h*GX+DA5;V7Yw&s&?6zFg2lPb^We^N9arv=7B!}OK*W9aI^Nh
z0^C=t#=qrtKrNfZoO+VifDHGC1`@9SySrUC3wge61>_0VE8UpDmaBtS>DEnoOB|R{
z&al`ke(w76P^5r!C8Xt1%}cJ~+uz(<J*YicSO(UOoPI9DT{@TzG_-9tI{tOFyn5e_
zz`+yy;l4t-GZcod3JQgStf~Za*WXUre2`zE>Qfhd>_4<Db?(z#gI>^(Pc1LOUw*rl
z?nI3{vLsx2i!gy#pnnEuPMuY*be4vellD6==RoMpYY(r8b%@67^_7(E^%hkw|D&{D
zm~xXLBHO&`bUpu!P3gv`mKj4qV<?n926wO~58rtpl%&1KZ`Q!3YMix)Af%VuY8qzS
zMiF4ACHv~04sXej+{ff~l!!;Lg`6M?9`fDR5O|AgtT--mP%=k+a5$v%o15D7mD)Gi
z72C63W^=q|C^;-U$!dixyzU1xZ`(?DdUf5UW$sP-k*Gl2l88)z+dv10*k-`^$G<8#
zE(N+xDZsm)YD;(;l!O$7FRY^>yn65=fC$X0nh(Z(f{p(}LvNYMFrCoQ1KAQ<^JiiX
zorfu|(#c%3G5s{6%=Ny}c|)#9LMz@_RX+vpUGE?uuo;z^>0tJToyzmtoEFq`i`<~y
zv^bx`<s(Wk?Ul``K9S!12;6csg-34+;L>K5(?^&3wHceInl|eBHB;J_B^BjON}&kz
z2Yc^Y{z$hNb@*tWxQ;%m8?NL2eNEf{G<wssnKsz_lM?5W9`N>P!W+G5r_IoIn>6>+
zRYv%nJe@+E06X3NqvWzps!jvetDg(5e<Ck4o-k3bgLZ#(PB=?A%jkMw;%$p4pEYSe
zatEc;C4I2vew;N>OA72Z8MkAfQEsJ}*KMvXp;!=JRh{lMemU87+|U4f%SUbQQ!k&j
zhh9hP)KsNu;<8`ZQcJHMkR!(syHgAW@HQ@YEuSh9QNNScaFf#iHhIlo?%4qN;~{wd
z+z9EymWfB*@j%U2-BPSHrsivBv1I7r4lf^fmq#!>{}}M}-BGuuq{wH39ut<V|7$pt
zUGFQ-ehSx2R;&#=kSU38*VpAq2=uB(XL+jC%kYf|o8gY+3(9i<LQg*s$~jlKgW0jk
z@`C;e>TR3mfs2*?{%xA+ZX!BOSuZ;8Xj>ozn@GFbff}Q7=mw_7iVlzUS&ZCi7$pa|
z+Yo})nHgPZ=(ASzs693u)jO=f`9Y7)>$;}5E8m@%%}F->l#*<EPH{bFHF?#O+!Ya$
z<Q*`snvz2^BJ3%{z^T#zk&VQpBikGqPt``qwTVfJJ<3y3i(;KZ(+&}wt1S_GY0Bhh
zN(Toz;X8>{^^zse&%0<)v;>Kp`pocz=7kF-%V+ep$!Y+S)`*>vp8i<KO_1$2rS>H5
z_RAki@g(h=Y`X9J@DBl9U)p7pKU))~ikv(18nTF+pSpFF8a($F?U%%??Ai(<`C$-C
z<H?Q2@`u`X>5MLRq%?*R-I>SJ%_Z&Y73oq1RvYpvlVNeX0gA!FrI2>+66(RBdTfja
zX=A)N3KBiGSghN2+XCFsi-Zkt?tmO#cZ~&ZGQsTi%36ivgoghhkZ-?8pQ|li2p{+#
zrLFgj|BdM}?b5zqInH6@QyZndJf9yYpfalJS6>(mYPHc7?brik5y`qW!6gR^B!%~<
z<Q!iC+*ZnU4!8Nd%ZOa%kMD&uV7r4c_eAdw-s|6$hn8Y@s+zq%Je)o%*4r@h=4v$^
zKI)y>1?EgxLmr0(QQ{>kUOeir!_hA8z>7rUkZmugaAeLdGel745r_3i-D~&r*1uyc
zD^yz*@^1$On2#!`RUF^YUT<_=cpKaDFX>(WkzOhP`T1Pte!!s)(ynV=-Wnd;8U^d#
z{D^sTdcu6a-CmDA7dNyevPm|OECYBqXF|T1#7)<9=Q7Si>r&?+bD%;?QGMA`vRwUa
zKe}YjN5A#!%Ebv@E^!B>Bj7A5{Zm9~bz`$J=s6uBB+$8jaA?yl*yd`A!eQhm=CjHy
z4fmGPtHMfH8G1`tQQ)PAzy&I&@96H1&j_%s#5v}u(tM=YQ>W~+w2%3qps}r)ATiw0
zY_terUU*O22O?Y3L`KnB7B@X6B!JHR1B;!rTyk@kp`PpXV;MD6O8JsE#?5^j>x(tz
z9t-P``0QSWb&Gy1?aR3vZ2Lj?@R7EJ$1~uyt{3Lx)7@PiTern576GDL8DrzMp6Ish
zkRYkaRw$5PTO!9us6@2vQo-4&&U*y*^CBaw$3ohBN`JLYk$)=#MqMfCZZRn^AN_e`
zaz-8J_W`=JDM-EV$sDmn&*A83ONjWJByA0!Px<X=RyY)IbdJ;AwXU!6<1dRb1Hyld
zWKB!<eKOx!P}o6zI}>R$Nfzklu#r>(wk`<-XvDDaq9tK<#r4+C4i3x$V37^)`LT*K
z5obL_o&!qrXV><$FJzBO)^fhh=oqoNd-{d>*5-GU#Tk0yJ#PE3<|aeP^S>#z^i$8D
z=eu}=fs*5!^%&G5->K-&`#zB?m(*h_o~&?1I~_gBB2tExvxr$&Gp~NS>W0K`A(Uzo
z$$_YWpL)Zb1dI9xJ>^SpBzl6&wl{m>3b-eTc|G0X=uy5`{@M&kkwSw*_rKMmS>t8>
z?`k;=^<0qUFnP?Ou&1#1BdX&26<-gYre>eL5B+4FUNtSX^10N*@Q=k8Ro+9>GK%MB
z$>}LA(8KB}fP-F!K3JV6ewShWx$@S-c;=RK^z<((Ff+(?@&haZX|Z!9VySqaN#5oI
zXtthJjSUp`_k_=!u+nRE7H>`@UI}YzJ#P`k;31LjjDS8l&Q7?t6Es0)mew=}HJTBS
zRF9nKGJmou0@a55{1-Q%&-D>nz?#*RB)2}17YVz~<jCz3VL~Z>XiX52?+vUU*hkk@
zz3F~pdrDR~LVNnC>GJv)SAmbhxC?(5BBli{3ID#|E>7okurxMBkx7Mwcsdze#|7Iv
zyjtM`^!au}{M!fFy0P5udKHXX63#B;?rZnrW`Csdw3j!i>~E+>^4<o2lS3=PEb_yH
zrc0eWhh6@pzJqS=s?$7D>DE<TWuYn%t98xF1gp5P_4PIfv#xoq=T-so$a4~01L;qQ
zyTqRw{8Y5n?~m?gdQ3hc&O!E9&t-Y_(ks7B(Y&9h+lqp81^64-QfJ)I1cV7e_5g3h
zE+;91+n=i|?g4Y-p@Z738Rm#|Bxiswy3S|%16XD(=89i#c9zh0Idp{%ugBEHnu=zT
zmO+-?TG}yhRNNBH^n1syp^>wGc|~P2;qm<Zw}pel9^m~wWqBWk<NBn$xsH&yZw>o>
zbqY+FGHDKv3ZBG{ML&{Dla=g)VbK;`1h8_t(0LBmhfH-}@crFd{*8mxsZh@o@mmIN
znEtUEn$L)cu}5?Gqh!Z2D$ab~tB7aG@*I&gP;*wz9H`#(;h)htAN2||q{~uTX6$VK
zY9+~fF3Zd)kWH!$-BQJV#s=0I>vI<=<VcAJsZj>>%ZsuzDLW3669gmjRF=;WaP#6p
zlW1UTTn1*z^}<wTwL<fjUR*W%qSQ@2kB-!yO@HOlAiKrbMFD}wU`$QH&AT*+qMdYz
za8CXFQ|%0;jV~YxQGUVfh=(<S-)A#UGNdh4l@oh-c@|3+-NViz+k*io6h8^$Ejg^D
zZDHYiKgfOXyYhV^WnSJJJ0`hc3wWAV>O8}%>c3C@1SkON0Z3LKAU$}yHK^r?!70g#
zVS6rV-@smI$L^k4WD2dXH$4H6nCyE-mOqMq&f~u2CBlc54>RuY=vB(cCGlD<MQ?Oj
z#_SuHd+r}Go&xBWjU7$m#Xv_4P}&j9`J(t6{r=0u{aijl)KlKW)d}s}+AYBk)VqHg
z_+aa(YnTd64Yq@^x)JV?rn!<<=zPG{n`NpO5x<gKrLYqyiIq5%0_dkoPe^P}yctMf
zuwdxH1qTjS_0t*Xq1t!;Xq_l29$cN;uvX9rX&T7k_p`RxnM9=T_(rM@zXj`CIjsR;
z2YUEd^|0-Qy}h%`D?nr#NRF&+tgD;7Pu-dpuo<jd!tL@y2XQxUN|!wJQEsvtc`)oj
zc3lr5YIB2CcROj4iZXO{*q6H7ke@&5{JfZJtE%*#hRBAF1bH|L5kPnotbOqj*cJU3
z#kOfWX!u);fGCqsB5mf{Aw-w^XI-(8wz-S>s)P1k<of`>*-(crdcyRG_iFGky`D7A
zb?VIOJN}D^#GnWzmZNvhEGvPv^|PP3;@F0Chi|~-F7pN?6SDLA=B=+yY7O^_j9m4A
zLYH7dI{h|X=7Y6W@GAqi#H@ER3-Rcn@9EZqk}BBZ&~QX~wDl6<E>#hKM%#j|s1i2)
zV0b1A68!uN-nH-6EsUSZJV5si-X7mSK^_{$5=6|VwGhUu{w~Fes)}&~#x%u89`*wl
zxn+GVA5jE{i-OIbCCsG^9?#nZ1Ryw@J*yGvfh}vPdlf{nI)%IN0+&hL?=H`kvc}96
zm=lFFdH=_?cu!o5KDr9J4)<-x;##~Po^4<=EcWjeqqp)5)_v3>#4Rc7@kjrry=Q(I
zfOTbAR~h(RdHUw<i%e>0$zd<RX`5o_;Y6kxDaFX-!E-qocF-dSXY*C|^if=-@^mw$
z=qvMd{Y>j7qT?37GIgNbGas-ygqRyBUiZU4i;EsWZ`ydb_&tp>5&^$=zJJeeHph8o
zK7YY=W3Yy{vEL+7c`?KQq3M9yU3pDYq8`ahx|9CW<{Fn=UVFp(BXz|#Axy$<Tl*xl
z_B^R8#L|;^3g0=WFo;8#Q&_^E77Wy`gZwjCsB?%D_0mX0+1cS*FPWJ!RsS7?^>A+s
z`lAmV8upF0qpqePv|#CX2+bPbiN4hpi96oH0yZ}i9h(WYvDP#ub7ENYpXxrz`<l35
z_w^<WymYTVgLL~STYZNCA|C8TEsz)lHlV5WtLZZ2&M6GZQK?621xNx_yF*hWLBIkG
zyGmd$CP<3-`{}t<N;mB(Xw`VJ{f{>VoOlzpO|$F(8MHXs4+DYw8e^j}$8BBH#J<a|
z4(yoF8>y&w*B=ag-BFya2cu3|r?VZ`az@Z~<YZI1Hn@R0RCFfK6RVVBOJ1TzB)A}@
z&M2lAliR5SV1V<>31!NhGw5Z4WYhE&@@R->p9l}VSiB{xN{#u<{K|9Jxy_QX2@+&E
zTp5(Vku!7OgLrh{gu;^(p#i+HGmFEqE4IzA9p(;ehBTu_PIAhsbK$;KQ5ls9Lv=L8
zKSz5Sy<vtUh<pffsFZ>ot1|>slT5yLy^G_caL$-7Yg98GC6d*a1)AcjnD&J;Q6E66
zHWt;D0T@6;I-DNG6+vLH&(MJKxMQ?^NJHLgr`xGALleJ{G^JN#dnuMP@s?lbX?WMF
zfReJ*cV*IP7hUmF5F_65{--viQUd&NcNec}fma?HIhwpLY?6hfhC(-ANr`x<3Y`RG
zFD)~HUvkf#?!JHM_vj)*&xaX$zrm7TsGTKy6`;eEv+_zOk6V(A-KnE`5f>L27qs_e
z$1oliQDz@NKXW?l6Ly*R*q_ivw|<4B!x@EM`73-H!#Hy1SW#P|{JosLY5Ng6wzIm7
z*W((@xt$;2{)5yp&_=dVyh_kQv20p1G+EZjew5rBN|PgLc6}-f89iDQg*<564YnS}
zb)`^-^H_BbwSu*WQ`$<Mwav?Q+~*%fl>GA6C&n($)%0p#=Jkp1`sJZMwua||lv?jz
z4aRuie&whgSX<|^=U<1&bM%cTD&PZddN4=WB-?TOe%c6)z$+eK|7A5<Ew@*qe8w*O
zqBI3polsr6<oPd*!-$`f_VJuyV<!3A?dej4pVR{b=IU&K<0Ojgnd#R8O&cjWoyS01
zOEx;>1O=oQ(vF+g9r`D@`mOWgM*ak=fV;L+%aLogar3M+N{@tHcr(+XnfCfd?HZyV
zwykh6A&mb=nuU4!+35>isrz}C*JmxS26tpLi*cRBj?z2=O-s!qAox;FF(Y~8bam$T
z+wHd>!!y_2Z3K=BOwV!O#?bj)E^zMf1C(96;aIHsg_WM~2;%Fk*^cYkfuXwD=Mxet
zXpZLgF&Fd4`-6!rgXLaJHa20GOQj306+81QsaxIy2TUmU9li3?`#Jk_eUTqAjc(1c
zpj>DHjdh>Xoq-i4PgMUTs$i>@s&`wugj+~uWO6cPm@$R+U9rr0E~h6)Y@I^1uzT7X
z=xgw<l6oU$#n#VMrjIEIT4&1Gd?i+FNl9?Dz1v#0_H@<Cn+Y2C1y8bC>BbYi1F>Yz
zgUX~$;9FkrM7ftyNpzUd)?EMU<fCy{{OQPJBb<iCC~=O$3(z%sFmI?CAb?0d9G&~Y
zkL4jr!Fow5QTX^AKchH5zRgGFM2A+dEs76bAjGG!Z%!&QlvUN##duyIMStu~%^JDc
z$!x4EJ@!b|b`;46M_7(~!;%Qd3ZjMb^4G4~hW=0GFm#HP{wVFXSzwCyd#wsWvf+?g
zNs?HhUyL5WW$cGlzCNO1Y7aD2cN2o5iK;84`Kk|17Qhomc}N3ui~Ka%6d&I&)tPkw
zAT_yR`Pjcna0MxZ4CS@-TdWO5ZAF=nP!M;tCHnP%HzQolJJ|Tfq};?k1EzikplB~;
z{<6|fAerDa(;!*<N??>aKIv??Nui|7ao6062yH#H4z-IMFNhPN1@V=Dh8QACBI)!d
zB&Tt77diXwI*RrOE4u}is6GD(MME?k$Jz*h(;C0*B6}*63AN=mw((14A~nGyw^C>v
ztf}Ty_LQr3By8okipZ7(5A{+Y2^*vyB1|LmI~x!2M#$g(&b{^ORma>xg!Ii8-n5S!
zV7Jae<1VL650KsRn(<nnPHKTPs)>ISlPRyEu}kl4#?rE?k+<81J8F!J&;k2hv^B!e
zWEZED`w-vayWnrIY)>270kVg>miA-+$(oNrrKW7W_x|J6$Y}vTcO~SKjRh4nAIYkl
z{c3mHN1ti3!V!|-DO~2J)hxM&C?LnSCQaQYZ4LVjr56J3kTZCkce~yzn{(JCf=uIY
z!Jlk`F{G8nU7>?7OIRd%6^ttNFe*EG4Q-pP<y#695Q&42^@Mn_qxiiRCyskrx|HRQ
z)^`SCPn+`W{jQ#;4pz^LbdEfqLJKn=6-DoUI_%j+o+*9B16518i|mL^&@j7E@9OPr
zhOefU5cp~9K4$98$)$Y`6|8)T=sC^8SU}<j)JMC~u34^PuWbrhqvXTcDV1~$qs>Qm
z>VrD{J-Vi1+pFm%L~KeZy<&Z2;;S*DE*KO{$%=DOsM}+K3yeN#Qp|Jaiz}`D3j0}?
z4Fp$U_j<P;PUQPDdvBT;pahB~k}$66*te%l;=pysUGKIGA#eUNYBISkV4XNb#C-?w
z?r-IOM;&$d9N*thQ8q_D8&vI&*5Uxbqa3q}idA#X_m)Bc?vBTidP2w*I#Fe4*{I6g
z(T0C5@#=&dAX1#pKOa!3rv?3Xpwfu`u8J|@3LaYqAf$=7gJ<xFUi`hoN=8fG7`phy
z7WI%bLsbb$-)&RIPA%Kw6s8C`zh)@o410_BHX;)hRkUbXS#)KlULfDWWa@!|R`(ZA
z^W}03X7>~H3?yIl+si1<fDGyyq4Ny_Y!Wwf^*A~BTY6CXO=-bmUpV5<(uKIW&p88D
z8MUaxH1pxEr30dLvoLO?@^jpP1wVXxl5)SL3!c(7SAk<`R2yzx=~S4funcruH<Q7i
z^d*e60}xp(_VVL<b4DKT+HR=z@PvBOL<=49SX7+``0L`Ey!gEM0yocAekpn>LJwY}
zuUS?J4u3uV4*Yv{I$M&Myw^`AP?_kOEoZs2hi7>kW|I*XpjXzmyrgyK=mzjDe>tWk
zV>czI*>0dYx+LgP@cd^J(&OeB8_akgYFDBRBEeQL<9**NF>r5S1+C_n1SK@PqT;QQ
zK@nZnfM*!+9eDtiH@w!?htB$)K;1HNYtF;)!HNdWsHr)&9QS7H3fuRA_CIY$O|rkU
zlQdsDGtmA_ADK$9@+TCp9}w-u_OllS9$D<30Lb1JeGyqN(g7K7$DPR#>Pa8AV@HNX
z3X;YlR?-6&qiq#BBC&Ul81V};UD`HYmHr|gA~?Ah<@8no2R&fKl7J@aCAier{^6r$
z@#mUfU%wWa9`p3W)9IfU!X>x$F=JBg$b*1t&Sz{dCTBB0;m-oanAH#tO%MW-oVIa%
z1~~N9Mu&q(X^Lp+CWNG~a|hF~9nVk$A9Fp3B|%S1G=Zkz(rgAu_L$Iwk8|%zxb#3+
za?+w$SuarmTUv1Q(3Wte&2<^{=d}5;`OJ($?OG7vG`^I&n->Fe-_tcZHj_=T!iyr^
ztj9rRw_^70CSKQ;h>o1J-v+&3nUmF5@UQOp)&AikGw~Z8YMSn~hAQ$}U%A+rXp=Z}
z_gOuD3ENV)LF7tf>(RXmz_>P#`R-N}Ry=8II=%jYzJi9Hunh=wmL1bY1Uu{^Q;>=>
z0#2w|J8cP@{HJpkXtkp!1gr@qWYiNrodH|(^X|+!Cz=*eLGURW7^iLRFLk7U7vYjp
zAAXc=$Xha#jVYg*3Y@6-=7Li12D`UHV*(cjh}fvK>V<DXHp0DGdWpy<$Gs2xg86%;
zQU2y31!gcO_cdcONwB3Omnn{GB+N;Rf9U{?PS&m9_TL%UZI*!M*dmrBqH%k|af#3~
zWr^udopEXwtTyMyUTJ5FJW1cEI%$pgz^QXU`KA|%uVv~FJ1N8>b)6>`)&<1Oc7{Hr
zYQ4!^w_3M|3iNaweRJZ+wH>>xg@r{}*CJJF;dy07>o#RCbdSYnaHOk2uCpuE(4?0~
z1ioI>y*U84+S`^_cC0+~-B`&<O?tdhIgyk1qBHVJ(Qp)|h#eT7oOBuKH6tDPb_ERc
z^!K=m0c~{uv`UX_7%UeJO}NBaEpK;f+C1Y%M$@^L0V#rE$41c7!7F4{CyS66)l7cU
zp<wf(%cx)9q0_KPPEn<hN1*xeMfZHLcwI^1%A9yg$FBIzA507a<F*nSCj@)$FgDox
z)k?gs>A97!e=^J2;@jbRR=Pg+D5h*~1vB@P)}V(qqQ0hx8$w_8aVaB*7#rIOAtu^A
z8E4C9c1Q2O5sPSP3QnEp;^UE1$GuiD9r;Uujt_IVAr5EU)UE+e!N(uOlES>`SJM25
zG_;B10mUcfdXA75xzWI^WK6O<OcvC~(mcZQSaB(!fA7=UB|xX~+P9>uDTj5nOFoRD
zpOx<&JCj<$xLV|3UCPQRk7wEvKoMsED>2gx>#AuO@v1o|+|6?xKxa$sOelIuB+Njt
zRSNjnR^`No55slFJ-4mus8gp_c<zP0tdu@k*1kTWze9MsajjZnvwGa1y2ux7in!Hc
z=2u&et+8-|qlfXySO~H6lsXxCwa#l1VU?f#iPY1`d*C{GMbA|<m7AUvG%@Dff}`R}
zZTHD8??No6Inkh&96cgT56<J=QuqXx*LJqkN_vA%TEA}`#Ozx&Q<N#G#2g=`>KFcl
z4Q~tf5BJ>zqI9i-aCe(W0-kV7_DX4qE9X)_&vOG#)4q_O=E%E|rdE+d+0h&$f?_ol
zWbJgPz(<PdoF&M^XSB&L`;E}17|Wm-%srTDe&;L2%=+5x$Hu{qkgv5OI--i0w*}+2
zbEoBnLp|I4gvJgl!|n6ERe89Qm&^`bQ&y~z(Yyyo0bpK2-3n0!+S$)_zutr>(iaGV
zx9;lWOD}yscxKQzsoqsKd>$QCb{b$nlzzs$R~%Rfow=@PYV}dNu?!kh&Wm+(pzw8$
zdxl@7$aOaCuFKrJAs6)gFsX!GQ*`=rog&2SVaC3}nUXbu{$9~lAE)KKpxsIZUmtb#
zWsghVQWtk~Sr52^VLC6G3)k3&ADqRuemh5xsNbQsRzbd-EK~Yzxrm2t->CFISbaLb
zGk-aCN$TLsPm>5lGk!oYtH@V$BWL1vEo0nYL8zQpPW@@Jf^iZ440x6n@Q)mogtsHf
z>3#f$PlXJvvvEjxDpUE`<EdAb0hCpT9Bhd9{jpn?Y1{bEOB2Jjx=R4%8S0@!^OY-+
z=D7L}*OqP@0@$@F2HakjIyTq?&Z<ibBSLMssw8a5=!|CGNrg&s0BVeE8S~wuSatn(
z0(Wz1^{WtR`mMrx{H;7alNO&6uBcwvsPeMDhyY<KSXl*b>iE`U<qG#??R!+5&s|t9
zxOLfG8B+Sff;PKpb0AG7M7nQ|1uYVGq+?rWc6&4_^6-zHxRJELf#lrB1l)!q9Z>JF
z6>l*|8<BGZ&#-5fdeH1A&IweyFH60Awchx$!-6?b5jveaR@6J`@6l8Kwqb?qa$Vns
znasUf;MU4_cOPYD-qc<5Lm7PF;8@_;9%F3xE+bFP^@DXjhNn{RJ*8^ifR5cs>S*Nh
zg(jV8s>NmlP8$~LRGc%XNx(9f<Sc@;Tf9G!I)!NSEL)aW7at02NFO%P^TNO%l}s&i
zRsAO#{F<CNZ8~0(Yw*Sy7Kmzzlj5H8t=h<8JL?ec5WYcn)kqbzvSYp&;<zd_x&v3Y
zn!KUb*2vVVOMwTnxHljcO~!tU?IupIwul#6(eFYAw}VPT^vR9O3oBC}l7jTNy0Nbg
z3Twr#?5is#2bRuG)M*ZjipV5_?@)e{j?Pptm<uRy+~a%xXoFXU%(w|O_DpuK;Ylj)
z$j?>BWv`FS4PFAV9pr>K5lO;#(L#gk1D}cBY|CC}*#<iN;uU1}i+<vx!8n%U%3dB)
zK3OL&sEn38Y<EEy+NI-O2kAsB2h~jtHFu>W#TxD0U_68tH1XJM*n^3*m7c+lP~3uN
zV0j+i*cZM=uYpbbt-t?f!kR@5I2<2nQi%;?Q-V+lUqw$nYQSIHCcQU)na5w*&>Cn}
zevl>M!hFF9=G9eV0gz$J%-=F0Cy}%P&k*WuqO}`l{9NC$3N<A*IpaZ;>{ZVhsu9Fz
zuwf7Qp`q~B<z8J)wKX4)kPj-zvpQD;ydNjX$9b)Mr7v9B#<lw!=JECX<v7*bH1kV6
zBZ-u?@AQ4NG_i%v8ve`h0Y(&KS-*+URysLo3y)DD^R7a!drWWVcf_S*pQ)6)fpT-h
zn;wo82ceZ0Bx$$d87l|l<M#H3k$mkRKkn>iM-UY<+5K?+2L9H^K}(`Ce)Hpz#}r^6
zu<wkbK23vCmTQ8mM%$kmC>qf$Ga0{cuh#i-Pavw{vik%z0HXPQV`p<6BkkAy5vr2j
zse3KZWwpeenaNGEYOsML)Pu)&ufN1zl*6aHIZT?lDL_rl0CZ=ss(G{6gVz9#bN?+B
zV5r>0Nj&$xxFZ*VJ7~TS$K~XmE!}fCRRIzIsWLrdv8-FMf>tg;^jqNWxxPK>5wnyj
zq7w38_^51eQWA<Gw$M`(`;>X8BKG=H^>)Tri>ocgqQ>kUk(L`fAG=<E{<Kjdn>xBd
z>}f2Oky_7A6XVv(s-Ins#=}~_?<&|$6*WHw6}&8j0u+&Qh_wlI_X8{G64^DGmjnH&
zcHH~#+TqsR1Wg@9-1;YFi+<H7o=dIDOR#rkMc1}5EaYaa4t#xVu?a9*H(P|R95ilQ
zZq}MM$7apD02+)T-Oa&C(3obIfvsw7#u$Gn6OTWb-XA#}BHS{_<{e>gmw7I$s%sYW
zNmO_9yx3XcKTW?}@G)gd2y?y7MM881@!-#d7+@gh(}8lJz7p5_PdCz9f!_OFN+BUf
zmGu4;ercZbU`y_#VivDiaS-`KvwOY=%*ZpX(PEl&WD^WR$~l~OGjv*Y<Gw>Qi2$5;
zQG<$(T-t4DHd<K`HZW0<EQ4%2TNiEV0gYf8{(G|-=em$e-uNGF)k5JL+%?h%Mo3fS
z_&EFV4ayCBGWizyR@cntSO-iMy_r3&(30#cIAht`ROQ}z98EeiP$tnBRLG;NX#G<H
zAP5kItWV4VBUT#Z<Y;$JaW@-9e5jvVZt9C}GvZZ*d0>+<hfngu<2ZS*tk?Go@!2nA
z3mqITX6Uv-j#APoTX{_a{f|%kw>YB`wrVB&(3XmjI(<0)g?Z{s{7Q>L>jr{9F=bju
zU*~6Sj(RmyAl+daWR6s}>Qz&r+M?S!To&8&Bq4?)t;nfVWyj4gii5y&RH*`kjZb`W
z(ZQ04z8_5KY~?x%e^PbY(hcO}^eC9Gxk4iqa;g~1JMcW*eXBAN7#lC~!ybh1)%Kgv
z-|gkDwNC6(Vu4C7q=uZ@U1PH1F+N*T*0k*sb=gWM%2(<1*{tG;cK@E;_k1iWTJEp(
z7pkt^WEj&P4a|m$qX&s^r-EY7ax?Z|1O3J)Y|ju3cr`<W@nwl@3Y_hGRr!Li?gOL=
zmivX_H&s3pav<l*AOp77%BVN)s!o@_dH1sBXgCpfpwkp%AznxOc}xj86{yp;7L3pQ
zG__v3m#C#K`<FaHYcItWE;TSBnRqlN-03-z7GgrrTNcFmLTMjJLLi&o-}QZRv$<1N
zQ)Qw-KUvRW1Fulfyx1rHg5ZjRf<w}4A}g5bs>C*NmGu;o%UgA$Or?@LAjI<EPm~0>
zs=3+M>ouo-nZNHn4@2^6f=y2e#EiXoMn`^CAJk%`s9C1DC1Su^Tbk!YN(6Yid{1vU
zaM$Kq&pE_=`x0N4fD74vj}?o|iQz&Vq`nn{D1Bl#q<2oH@ioodx<w);CuM>?NbyFK
zJHvd5Cgpdt6DOq=yC!O*COsPaXPqtzhe-J3Pp0OtAguQT`+0p9Zg*)foq!qcp@56u
zw)?f4y6#+EpldTPWnAfPooZ~ADdp;IVKL4M5X$-HEA=9@tziZoNc>=cfBRNzv43lG
z04S2|^6JO-gFP=oYr%|%qFPoHcd_XE%LSEXF4>>^vGT%jSD>=jRvrKtu3Bg2Q?`kf
z4g#yIZ*9C0`yR*`IItT+HN<MS3B4IS@>CY4NqowW<z#b|5)KyKyPHrNqON@jsq6Ap
zZuNYqCor`M;pGOMxF?GPc3eg6E!N^D^s|(Ju_kcc;=zL$7sC*p=~8jxeUxJUQ#_Su
zp?h(<s_1($``LP|fuyi4A&x@@eAQw<D^N#C7+y3j((*MFKrsMbqmhw1^yP+?R3Yl&
z%hfo%o4l{fJ{jNEQaz`=ET6+{z<=<J08P9Nuw&Mz_1r7Yo(w^^jGd-_-XiHuZyCaI
zHf(20kfVSS|C;km2KWZ=nt@}bsLu$Z#;%;6Bmn3Xdn!wlXXN>Vt=E?8T<TmlMsS;M
zceHN!?eI>8uY@-pva64^84*#~p1tlpaS(8v-VREi{7>a;Oawd1L%%ooBbFcz$TW)6
zX!{0YBE;;ZG78;p>K#pk-`daGT}qme$3!=}yz*FGkiOV7H!^F4o7iaNY<rB_YzG@)
zvw5CPz<7y<o#Y+CP;lg?_@Nb@zCDXMgI<b{J_o3nurKpTpH>baB0z}?R<`v67rS{u
zPHKZOD<)T_$00KZvXm9lL6qz)W?<kNxgci(ZmU(`sWKp)KN*N<lIL59pFUqSxmckv
z0)--iI%|PWK!v5~bw6M5R+j2Y7I!)gGr@<uZ_+gk7C98&9NP$eqFBScMd&-Ug~5P1
z#qx^ei#*&C_S>s(hCjzKXG)QNB64o?_B7S6VrAht0!uRJbKNT`9|!R=17wuhyo7LI
z@jzJ$@DbtI<yd{1jiNZqlUZxO@sZiD>q-I7b`4E8$y`741qn&i(9o&KAxxTduyVyV
zuD7N7UsecMLt!giiz@ZTtcG!l9HlcV-vd3G2+=zaK))wl*WP5og*ZOiGUgrDVJs2D
zkF<>0S1&|paqk+_TUe9>I3*q(9O1e38t<;xmlyR@0`^OjeAsE-yG5BKQm?L{Vn<83
zf25V{;_Sy$?YO!}d(Q|V{OEz)Ws!#x=z%I#O0v5E{4zZ0Q`}i)>BA;^<5z8m)5^I_
zvk)hcc7=)g<Ky8SUC*OxjLJ06{Zc-JUK>*98FXqZLpfl3vArQUD=A^2FAl4wD#Ol%
z%Vh-s4y6#qmA(X*L>+Cp>QZ{Eh{K#*w|T_$XuGs5b<CFZwmd9FAAYh%u#zBkYuT_j
zZ`J2i?b+KLZ6kTqE?m==4Q^L3jy$t4Cg7v7QL!VQOzA;26j>sJHB(KbDz<jZme!j}
zU~QJu+332;ENPPOW+QGnKd!<hjqFpcx|LFQ_JrLir(}^ws(YKwV<c2|RkL@5T)<$*
z!5B)aTMiadja{fF9x5vBHXm}ST9b)pY3(i-?esg+t`jQHy%{U>*ZR|#782DAiQrr?
zReg#tvj8pa&)N~NqY_Ia1fNqQ&xD_40Xz+X>SxPPFVg7iKQAcbeJZf@8tJ)o!62V4
z5nSqU=Zw(Dp@XU8BsL(nAM5M)IPhtE+Wznv<}Bpr-037)@({H5TBF?2w=aJox2mO&
z>UnGK9_;aT#`i67#kL;d-<WO-f-=g$s(^<3B@+Cht!VtX^>iuy81+Dl)-NfzhS26s
z%jM}iXgT_201TEkM^F3ta8`F8RqDfGWiZ~)2t>=#57dZz!qK4uA+a^j*~7nF(5t0G
zOyNKfxh+_Gkd<*?XvG7iF|#Qt?`CRTyabs`!FXlEu03>vqw}jb&jQ}}UwaX7`(C!_
zUU>;mce??=eYW(_Bv}62zz^;gVpIBIHizdGZ~%CTN@QZ5AlD5LReTPzI6xH4oNe3>
zB7;T6(L<{o%&P~4&_MZ}FKamDh{c@@v`<i#3!<L6C1mBmU~02tDtwv~fGTSnTj0nz
zXqdq?%@*y)u4Qq%*GG|;0sXGmKro=q03u^--)*Y4T6O`i>0t(L(Zl(fu3?ZFcs)-y
zgncpj88HN+wI*3;F!!++D3L>nXoJ~qRE^ulGfrU|;o=^8Ha-Q-ZR%}Nn52?8oshZ@
zuq<lg3k}IZ!TvB6`CkD~8~5SEleCrC9QpoI?-SZ4bknsa+jI9SKmGy}DlUbs>>>Q?
zj$?TbMVL4o1<`smRo7Jnx~F3_0M0^J9IfD7a~gMsxFGVJ4f7>cZARoRgG!B0;*Jl8
z2S%rU`mSS5#F(4QeeX5<<}_!Od5mCSB1ZHUk>aN-Z*cm*bzTPGsu52znpZX6a|VVN
zBXcii=R4<<ztbd}yQ+fh2AEqs9AJmx{1~$>6aKO1fJ*=@6oKy<lHRv+xy#*P{YnLk
zz`2%+t8I6rOU>su*7DzGG{3aBczq#>gM0zn+e5BB%xG2Cy}1lhj5p1CtRwaVxw8Lu
z`U%IhAA0~O?&wR|*!HZ<!nR{dj_qf2avWNBDG&T;J?u1E_NcQD2jt3g))gjgA_V@J
zacdIC^@V$e)RH6NO4U=!6~56t<^7*7>e)Lc9<{wKw|4|6nU{w9zfsgY*Eb4(%A{`+
zCtZqK`2;URi4~M!p5TnzlD*$AjlXdC8OxYI%IwZdo2oLQ9DFNeqKxHs*-JBlrE@2!
zvn87hDXFK~Jbr8ob7rh%9azuNk8W@lW7knbZf;DzRW`tC><my2R`UUbah(kPdwlLL
z<Ht#^U)e4m-;)cNjZjpcX@#wHrR3OH@nIdf&zkc(ui>QYsa$7t>or_QJktxDJCIQl
z^gHRk2w>uv4-0-$j%Obc?bm%y#a6&59(^9kSCXgiwwR5kY&a?S8!f)D_Q=U^0``%}
z1L+0>m-lrPvjTg(gYFU<1Isq@Q!x4L9=ezJwG^G2Hf@^}9Z7iB5`huLurne0KEpO5
z;~%MywzBPrQ;kNfaZ1%LT0~drLf=H9?5oE8wkukr$mQ-a$sE@OAfrcCqNTq6{v${5
zOe_B_PihiLtDOP98OghcvjD1~NpmP`W1F^|!04HBo_u+Y9F7>A_K(m$oOo$=#r#3^
zC+Ye8PdcGs&dUw>{Une<3R_$+LqA!8NVWIpZW#g%i4~^?0&!V_n_s;<yzOg3Izl@7
zn2VDQJ@0cA%-omz8OTyaKVB!<!u`)OdFg()%(Ua+V*g_Qm^Q!kk22yfZ&<Ba%6tjc
zF@$Wl?Vw^?<hN4%8-d$@Gllr_l8nH$V3C1i_QYiCe-?xn<Ldy<u8zt=q_m&RDqR&#
zc@tlqYPG?*CcUgKSH)EFm(5{BhEkamq~Cz`!o{Kv16{UkEhIn1cQ7^0f2Ce{aMvcT
zkz|^ajF=(YvomF`nKbUD*diI%TY`4(AoX}Rrb!2ix&W-wd{aj{TjkTVIZd=|ytM1&
zD)|7scYx-T_XAuV^!-xziNiYa!(>v@WnBf(lZPwAItO@}VVGF6V7~StMwR)o@ct$T
zq%CI$y1^(8H}?~+l>@$9I*q~y1!58YqfD53;L@sCIMdADcyZ9#l?||;>26&rD};t_
zGi_^ose?>N*fWr@xg#jpJdaMjb!!4;Rl(kW?MDt|`Ac?$g!R;3l?kz!*xJlY6YZBg
zmPcvu7@q138fnyuA)kuD9lYaAZ=*4<ERW^Xh|Nt3lD)SCIY0Wul~pX3pmy1Enk7Rt
z+voPRpJWW=2v4FRa|cZn<bf4$;(CjRDY0?1qnUt1E{$4KMAo4*FRz<cc&S+sgohqf
z_^{T0jVr~tYGR9KzgBP_pOq5!i96CP`4Z7{-10jJ;hEda>GOQ=J}oeJqk@3ts1t3=
ze45)l@0;5rc~66K69<E6(MIdo!U?+Y=)C|1C9kT1rtc}hXblQMrGM`8$^!66lD_Ar
zN{)A;fSMdZG9hI3r&y23Qx8}uPZc<p7Dg~-_D{zY;hQf}yxvYzX@eWafc1@!?h^36
zW1nG@L7OL-o%&%*L1Rpl*Qaw?#nf3$f%Rd*yQk5y&spbBvac)wS3}xgA%r6+OgbbB
zEgB1-`D`)g&>eW2N#mL!1(tYZy??NZwR2+3Rz751uyDC1^K`m&&&j^+9Pc(k1F5u3
z8kaWpS?yR;C&6>F>sqeoqqhElZRz0yLA)NZ77A14&Q2h|5_Lx1P`M;3SJ-tB#bLc3
z-<P6XAACGC=L9SIth62I;&1nqJRMmiI>4X+N^67cCY|d~)8A4LDaf_PothAc3Unyi
z2oYFlY0FnQ_jr1lR0^}vYV}0gRxDI<kJh9TQ^MP<ha}gHQbXZM)|p+B=9H(*NmNv5
zAr7l0Y*p+rf^7TqTCUP<GYWz-vT{@mW6gK=@EM(1KOk&7z7z_F9R%GtU_r9C!v0Tt
z?-|xq_Qeb1*b6YDAfgl%fuRTj0!r0UkSY>-7XeXPC<4+QQ5caTUAj^wgf2aRi1a4X
zTj)eWiPX?S?mh{{GI##(^Soa#pPbA&`|Q2;>U-_q@-M3^kzHBm*jq`kljLzf>!`gZ
zzrXLxdh*0cRHSy0Gis=+{e62|SRb-oU_U%@BIP8Fue^C+XV!=4<;5Po{REXh2J<)_
zR|}H?2~JxhhsHj<*+yMO<p#eeK0Nk>l`2uc@*VotIT85hrM(Mn%QXY1yPiml<egxk
z^n`!CX@6kwaLYag>ep5ZqrQ^)Ctq<dMqAia_AeLB_+g*ekJ>Z#ixvilUxW(mtaFS_
zWqWW_?9*|}EAKO^SCceK^I|cD6I1tf;1-c1wAzK|vy#g(lfHJ8@^bvbG$!F8#sOTb
z`H2@Ww~HiiQ4Zu9g!H{f@9+95+Vehb^dNCq3l>xYBg|@qZkRcF`E%%76%kUq8*G(`
z3n~{~2KX9`hVTzF-Lcv0;}uSow`U%TEiNgC&sqsTTCRN|3TM`audvURQ(fh=nD$Te
zW@K(%!$hMg*B?<USC*LEXiqOb)|AuIkzP#G%|FGe!L+`9WHzj?ynQJ5CZ)vDj&x^l
z-@KOt?q)W68%>pt{)JqfUxoYOc8fY_{9|g_2qp<GF=V0J?5BSE{VmC}O5#C%!(ZnU
z%wxm@8xI~W6=06|Vq)+mWcCJczySXuX9Gf0g9`y5R<2{GC*IJ>WvVh*Xr)WMEk}K>
z5qtbGzrSMiAk|=}%g~VF%MlIg^?RS}ZBlNvJ4Tf)n+Z3V9!#?f_VfU0IOn-)(+&#f
z>k$G8=^D2+Cifo*)1R++q%*l+C`>%vYRX*uW#`n0&N9<nJryy~&I{=y$=_)HjZ$=W
zaoq^J?x8Pkr{2mn?_zQxEF@aTItO44<uYX~G)9ODx!0frMgxwySK@f5me&2Skpz&F
zFhkgNfuvx&)t%h-mDe9%tI!e6uI@c5f8=F%z~h;Hw?4e47Ez7?M_Q#xHtjr>c3v1w
zSrVg!Thxbq3;K4v4L@(e?3B8mH`^5yGPA`g82OF~M+Nt*iwU2pJKtO67xr{@PGv;4
zzx#6qCV1SBw^YKhb_T1i7tnDkL;4ZcQfO~S)q%a~aMo3)2p1%rt4L#DGu}!#wJ#kj
zBt0OF&=HK+L!d4AzR@p*c4%RfH98y_)SOGd4M|`}_FA#<OOMqE;hUD);zgX+oc-&&
zQI4O{Z=cI*8JJ$I6L;STAMKmYRxh&jZx&f|Qnqf6i1wu^|EBa2b1PtxXK_`dKX$l>
zCF9s^x_hsU84`)kZ@zMU0MQ`*x-kikeCU=|lZ3r^C44i{swG^%pl{J@O$47dD6B>(
z(8j#tk~{dW!i><RKe?RR_o*<`>lWqUsrA-Z0bag-ZuPvaaE<8`?xF<u0JnbLP8{a7
z=Ae5SuO*0MjEGCDfvmitVn#)7*<%sl=wh&)xc>UlAlyx(W<UtpmBphIiJzZdlFnL-
z@N^6>nk{rTFFRZC<<V~)aJugF#xn++s?Jq&L*pI>#y8vU2~(Ed`%1fhFIiV`9Pu>B
zmrDREYNV0u&$A+CuhU%?qEy)j0%I5aQ2NFDnef5$Rr5ML95bQgodxns9q;uei5u5&
zc&{cO52M0x+;AagW7#G7CVZ~G+eO0s!Cwv0HW;`h-cma)-egj?2F|JLfY{&SLd$_#
zZS-i~)XYH+d3`j+<BmKKx4+ZB*HdzNAS-wDU7OMANJ+JT7Uv{lSXE+1CNr(&%BM?Z
z*6R25C+ZMxgNeM<lCCR;opD{at@lAyp`h9$#bo+HaYE`OXY<m1^F4Kc#nmNW$h0WS
zDO$C1iXB{x%fLNy^Y#T7;DrIEKsvI$FQHR9{RMWGevJN9Uw`KFIGyQW_cVff)y9xZ
z5xn$WiTJv6S=bx_J?!F5W8bpLwSK3<_r2xAa@s|OCK+*^JCb>(W(BAsb`_=AcfF1+
z7e(ggU+7DjH`_~MV&`EieZvU}nfS8Kb*!did8&KxXeOhL)y@2+jq$rn$#&<knjapB
z8#&~iKA*M_8LpkVW>G4UeL5xz$xOU6Y3(qRZne-qQDx$%$$w9F!;YoPSN8<6x}-u2
zJKCWZ?<t{TfGPd5V4EHud?|Io-qR6rF7vc>4I>ku;inQlCeJV`?LuDr`e=^0SdR-H
zH#gsW^h@cA(?^tpX!~1;1INq}9wp1D-u!n3<AL8as*_7c@%{nMEmJm`HKSq2s82IJ
z8m7F~F)sA9UB~)Bsm?_I!=+40A^6m)bc=BO?3`=Q!uUv;li$MWIk7uVDX`oOZVuTv
zcb*TnA-W+kn$eHW<1MCoiC1k!lTk6p)gHU8u}<=aiy1sBE*w~9By=rF{@z!e`M1H#
zLbx}ao+&TiuaDj8=u^5hca)U)Uy-RbUd5aEFY)~Fw`u6H%U`9I_;TBgCSG9+hv^RV
zdme+++-~fTE=u-_^Im78l41Ce;KO**|A`^<qj)n<c!CK2f~^y#LByc?bWYwA<F_fs
zDrJ7JPw>^8iq9zhIvF$DbVfk^4^oBnRQ-HOti<EByEeL;eXDR>Lj=ED{o?CtPb@Wp
zIZE`h#e?oJ>C#@j88ecH{f~ZC%FJNt75{I3lj0BR-mtFECpSj&r#;q=<!A{x<KntD
zH`%eKY{f<yTHBl(7fZN2o^}X-2oHq!mK(EPUuAlCdUxWc<@L{tUkU_|X(Obgv6jC#
z<#r@-`nY-E3F70;S)Z~#Z9FvXjMK^t5ae(kRZS7N%38JBaI|DrlIF7&_Fdtcr|23L
znh$QAt4ys$tg7T+qPtqr=>h)(sXlpkvZFqn=}rMQo>RUfv^+fHPQl>2z8guz-$p<7
z$!WI?ddQ~d7FQQ74R$<#CtO9T3|F9!&v?T0MP&eyyMCj+D)4DKUEM?NwdOs4bC!Id
ztKTrESmG6WiiH2QW)V?ty>`v2;4*GWAv49u<4mvXSHD)g(-{;cN2XW#iA{%AFT0p#
z`$qG}BhOFRZ}1kKFqLaf2Pd%G0vgpW$<7rWvT8rkg*>NoROaNGj1czC_@Uaqcg<6>
z;BE)|(_B;<sQZ90KUl6ryI<Ip5u3VO5}(nN-ILu{gU9uhoAwtEdwVkr1>C9ZbQM%(
zOttb(b!krBV{WjdK{w@{a!jcwR%~c=4|ANP-uu!AE`~3C&(YsF604LJuAceE)Xdh*
z_Ig~qQ7Ze9SOxJ|=bWL`_TW_0Be8k?$+TkdqNYC|^DR9xQ?H0heVB?lDxq*urlj<)
ze6Qd`^z*KyfvEiaZ(Z;ISscdn-A^=~Rr25%_e&t~^IYv*T?uAotzyEs;FJzM_P`c)
zw;VJOUupDbW4Z0G%lPGvU;3G}xI21!1DX>Ebm!5{efsh4boQ+9Hg*%9=>#5|6N7ih
z{8-LZqo?sG1Kdp967>x8V+rItwbqNPH(F3`_Y*I2MF``@=7KrEp@V3Ov!?oxwHd5J
zAeA+2a8_ZBdD6StK-pTp;^Mk(@;-&b|1{GVnhn)=_Ys(uU9K!UmHWA$F)~>0n<!|q
zT`w8z>}DrCB|J3}S`%8d{Jo<yx7?KIi^Eg(ie`Q5;?CsG?Ej-in(<6(FlJc0e2k{s
zQ2CDogP&j2Q<VvO*%xQD*;hLj)*dG9p{x*p@xrU-8w%Kq`9TKMgtLo#kEco;7=*5g
zQUy+{yvSrNs}^ZTK6OrYOh!k15#pBQmSh`rOu$L!o@R*ZkP>+6keY4lVejHutOi%3
zf2<cl<nb_fYpYA^m{BvysLMce^WBM|WK=cq%Q8ji*b|%+mTX0^^%E5@fGcoDe6u%D
zQCi5iv;(JAF3~56gJa9bTpzbKNm;G3H2aDPgzR%-I#4$<a%%NDJ14k_nrc3iBzD^U
z#tJ2p30gsOEOPRoec`*!I?K7yBH1EYV+g?nx*BrjW==3DQhtIld<8+ofYaRdAyLkb
ztnRFCk(K)c-P+f3te1@yvD&Ya>%`LfK<OHEoyl9qA(m!}f>awlbLHl<+uaf?LQNX*
zN6WEFba6b$Go+73At#Ll=V$9``^v~419%eWO3as~<%>+=wdfCK5Y|ce-h)1$59=Hl
ztmAz~b&T=B5yP7wJAVs$ZpUukpPb=U6pLBt{Xn{k1a3!!GcGS6SGV%4LC?jl3lGJ7
z!=+<wQhg+<$9PzRYI$bMi;49<*+3@_>(;Tkc)6cmOSK7fX2UN(z_j{zWF%GEk~a?A
zDVxr!ORCzM1#tSGc_^tVtxHZ3Eu@%@od34l+XJn>%ovZ4kg!_2qH@BCDxeqsjsJ0W
zbDpzi^Y)|y7==nnRn%a_yN93&jSP0c(3>|lyhDg>wfeJt0|h!QEBDh$#0)bQF6*D0
z_YaSnePxOZ5&whz{Ozod_SQS&L9k7s#<HD<`Sv}#y;87H-U2h+%-+l%C$`gY-v}PV
z+;IELO_K}qG#Mm&YH!4pf8BU)6C>u5QdO1dZ_v~WQhd;T0e}!f25$Q30C^9a;p@WR
z;Flh*C4Q8BZh{MNavieYn<?N{58)*MKDd=r{dlQ5)cY|S00)RsLp88u;G1_MR2*+l
zFyC@V;YQu|xTQ$n_TzRb=eVJFLa6Rr(y+Yxi}d{$j^4W6nTRg@^8R(8u=n^7RJ#iB
zP#`cXDSMl{hhtVAFV^Ttpy%sYb1x$Ci5Z)-(~hQ!X{X*~K^yh8?zB>BE^5@N`sZ)c
zGwA=tF0nb_5lTd4hH+oi);2C>f0T6W&iPRhr5Ul%B}3o_#fJsOu$wg8Vp_HA!8N!G
z+rPqK75z#UXTL;sfV0gX7Q+G2B6yIpn>ov4yj<AXaxn*;%l$&&Ahg%-mOvn<w_yZZ
zmP(XPj077r>BsAUN*rZHp8082P4~DD#k^jc#Jq@OuU=;?Us#Drk@vD<E*4ic`ql&(
zSSp=eRJir^1LRkwgp*F`%i5OFEG;d!40Oly>R-}+H!bMM0I>3<lh1CuE;!dBUa841
z6LW^b_XpzYWmrw?%XpXPLOzj36hW=Xw6KUb3^GrS123kuT4{WBRpv8)rim}=D)&LC
zx^@>)ijfagry<Kc%nQ;7!qDh4<#p-7f^l7tm<5+TrPnBX3hobTFEfn_fnC>ey1zqt
z@{QYf;tWs!VlOjWnDK#YSP1F!4+ZV1IoNcD55CVFl254t{(>;Wf=MH3G&l=^w8zf9
zdB45NUq5+2XN<IekYB_!N=y3Sm%7iWY1H(qnvU9ayV&5J*^)1HV-I*Ie11{Y;#Eqs
z{KdPx?98Usm9z@zGkSakw#r%iDEP)&b*2_22?%5tPoWs9KjF*k^SvrC(YVoN%IvoM
z2l>4jf#(giq9a8vOeoBCb<av$NLN7!WDx92ycI;76@wAU53pc1y{nUQ>Yjj!U-l}y
zKAhcKM14v!BoIF#DY9xmZ=?t=ppyv;O%)Vn4j+PU=Wb*!msh~L)(5|Ou#mp^yfkU!
z^qk8lTr#uCSZQi$O&gO^F57psfdHQTJ)w%v!5g6fLke2na{_~l<xFiJOVbN1H}3C&
zqa}3U63u8uicoBCEhhjZfCL!=VEY0zW&1^7pAHkzKxIKvV@O@&+S#lx+Nb+Q8hy~k
zy>lUVo;i3d_|b*QL*`Y6S04k50D-_kfGx@eq&csLoFr4HNn|2j8tr5@Mlfx;DvnWo
zJ#G5p=n1-<d*96$GH&)Dl6dsA0)Vps>=x@rJIfEfPmY>85qs?PTbzh2Jy&1YoxtQ+
z{l{B^Q(s3q#5M4jad%j7Gve#`X}C(xLXxKV&pTNKcCum!+={*MGBQZ){G%IR52zfa
zce>TQv{E%n7c`y~U1_^bji*+uT3h9pmsX%aD1OcV*+mz}o|5aH#$z!-WjRyz84!RM
zWEFRS@O&W}oN~<tv{Poh^;ejmt`L8%o;3QFyXJk}`xgM*PV>U0ZG5G(TA1%<gt&S<
z$D%z)`MO?vtxHeN9}UtFIv%M+e;bxA*XSE1fTbgezKPzIhCY#b^-(y4>_yf;WLUJ0
zgU}ux01#-v^3B#3o7jal9iA_|G|yCIaREX#+g;-RIxIMh|GLkUB(WBO^LJNhhc)(<
zzLiEdg!}0T`9rXJ+nfnu6D@h~ZWbM#{pTU0Z+!lNpe?&)1ODRREeID7V@Rqh-kJ;7
z?p#!jRT_8*C5yrX8Pez$uVu;^dGI_y{n;U38tG=@u`)<c{@GF}?=_I@ZmZKTj9WuZ
z$({rp0QTe+H@LMAW}>%ecjEQ@f*LHihqgbDHw8d^wztF3WSY0qYV=tzlT$GFwxT7$
z{RKTAy}LtKdy`x4piL5YZ=tT&Kwv>JwH>qcPqUn=-yK!2ZBwGHMuFL=SY8^foaw~$
zO$-7@gcg-UwGFifmGF{zs5r8=yO~8a;PO&fpkqDmkFfJw08H{*u#C@XcUFa*x-_0D
zoYBu!FsXz?nP4uG(c+#O%Wk2?{kbbRowToK)md;#Qxkq%vSF3B>O0yJ(g@8!y0#=t
zebDzg|FqENWFZS2@cWkCg7ymFO))@ri%k8s;CvQ$wGGa;O8#kmf?HdK;=y3RjR%1l
zVco)N1jj@>k2?n)r?dHU<duz|FG{%VOZN4e13R+~vnR8{bC8|d-i)Pl^PG6me%vK-
zP8M8(Qo6HMhu!uIJ3ayTfx&lsIe6tBj*Bh41$QQsJ#b&{J-5%xYXsG3GM%)@hiD7u
z+{t9cdDqvjJ@cszDM8!}BUxgQqqNK70L|wp80e*U%yJ97`F<(!W{9duc`~?vNgfN_
z|C0e!Ss(9dQEh1*EDSQ%Tg=)p2KNYeWeOX=eH${Zk&wuF+O=WTuOUT!F@+8@nDk!L
z6d*g^=Kx&4RKJ*i+Nq$cvNp=t-bFK38yw&4N>A~FoR>*ZnflNf`?;*NyYJoV%u)l6
zCvvU}vX^RUBeP-|+|_*S&uWQt=tsL;#=m<QD0U*UWy*9+F~>PnpkT}YBP)ZhknKx?
z2~-LqTqcYj1|?G1Fe#Xod=g?=Fr_rj3`N6*WG7t@4C?SUO4C@@>1jb`j1U$K2CQZs
z1nNG{(Z+CGR40S6K^|d;U{XJO6~KIh2(N()G+T>ruQN@I)xTh}E_Iu#NH(TWs@}Qf
zb8?csVeVDd&{GL!@tpQa7pK<0!KhTRB3g7`l!<>ft_hEIh9A|L4b9msSaP4L(Or2w
z1yKGjOdTpu@ED<ypHZ@vw-sF*ec{ZG4F$J5-52brCGdA^)91PIeR{YH<-xkkN=iak
z7C=c1m4WTbRHXsC^aW{cGpi9Xc>+OCCJf<6zL)oBY6vT=ge^R5iD@cMTXMc!B0O}=
z>GgRnA7c&x=@4pVzif<2<(FkFQ^jfuq!u3IxbJ+d?C&GuOR1_-d#y(GU}`1R!lteZ
z1><M6YTg&AcndN1?vrF`v8ag+unw?h*SbR2d@ErG8EU_&t0+JyeK&A0+V-**c7$1_
zC$mwoY{_RTArQGx{~&LnH74LRvCIe0BOk~e{kz6l2&-=k+_Y=OK^okrP1D!pP-}fq
zLO7d>pe!Slgq)m0vO^b;M0iIToP=JMRjga#N`TOEapnE6<8+D++2m`X2|P>=bZU8-
zlY@ZO4FU6!FY$<ktGS#e^nW!SMIj_p0=XH!lKB`ivSd}n6NxPT?F#8?X@dU@N(ZrK
zP*d*sFdG3(MNe^1nS;ilo|_`Rp|)h@OMh~aqU_1r9-6V_L&$At)w+*bkUm?%1J=2r
zv3-jq=DBjI8+_3o9qwcxo%6ZQ&`Fwf6L<T|7Xa?wj`7tgP;Y_5`V+e!M{((+pTz<_
zv^z)O=2z}Msd2n`-y-!_xR2dZps+Y7WFtZAkrW{0>D+j<WxFi&_=HtRh;%WIqCrWf
z!1n{HitLM-Fv@_W){Y9yUyjHO2`reDYhFtJ?m2)OTlX>B`VG~VflXIDj7f15oZ(tH
zC5>JL7UKtiu=7-1+p03J&NZ$WRo8}|D(wnp`(=(^yB~IV9XmA7_RClzRK43{Er(fK
z_ugn@;veUq|7CKmHea_VmnYmtPY=PIUXW=4p%@e{lhp!9`0V-!dCW#{i&3)N80WTJ
z$aq3{JLJf3nBmG3?=73udt=cK9f%H`==4QR0tmvZ-y!k><#~c?bZ2&VnbLXSw(h-M
zZDnkN&LlhY{h-{t3f??y?mZxiLG+X&nBdPCw+q8}nYJ7;9hG?aycK4kd69z!?hI@^
zlrY$~@iDaCi(8&d50=aQPsfL|FqLRul#26na6LjAw)ijb!8^|jWV{N*V^<Y@qG*Eu
zfM5b4IF4<k)#hE>nJ*GA>ZCLmd+*pNv^hTl5@Yvx<F`jdaJXM_begpdF7ZXRl@FM7
zo1A6*WyXv<_S>T4wR3%~OK58AfZr?|-Oo=h{k_W=IqK^|^nJ$2Nhu!qUoAgj5u>f&
zQE%~No?iWGXc?EAVjcfH&vKp1pSW|!-$H<sTf7b-AFC_v?t-OheNdp|BW8GRK?L_#
z;A(u20Nug7>ze7D8Z{ZS)U6&@BtP{feVd_PA!BX2x4&On`0jWm50gh<9R-Et@3J?q
zt5)(D9VYMF_rBRufTzc{U`l-%a5To%3fTiMaSfr@GV2_+lmgB9tNOM|0rq!y&Rq8*
z|75|*zyl77G2I;}161xJ$=dJVbvni)n1TWeQt88mNi_Iwyw1E!hn{S)cUWF|(_&xB
zpp}owm?A{m&)=;L!0=OS(e`3PyR0<&@E<CbDPFm~5FZ=_KG8NXb@Az4VsVWg<)7D|
z=^Be+!(}!9@+8~NlRdSoOv+5k)e*Wt$^8Vst5NfSf-C|V(H1#QRUc^gDhAX7#hcx4
zUo@p@2(ioeB-NweW~@TnoaABtd4IV}$1WYaYq?D!t%F-p!j2Sv5DXS9<7R;;u<ual
z755fC_Q|AEC$5nKl@A?3erCfxow?~YyxTjuzb<IY%bxS#<0ez`@2Lwqt~#?{`&*@h
z0sr5cjz-z>Psx%gdRA-u<MsEpUFj~_&d1lJ(EtF>0OQWc_3M*6ib_MKljdE<SGSOc
zsLxwCpkZ6c;xAtsT5s);8+!?nKu>9U`LZSOh<^U=rf~H^vY>u;%;^j)u-|CHw*E5M
z&L>OR%uBOgES+aZp4mna-FB<9_TSh?X5K4WAd2MK<m)WRd|wvI$|b$?Q4>aT5Mn4Q
zeVF`Ja6XSQ;O<%cLzEpCZE?8{>|}m_Yf*w0A9*Jsl1_afp`{<~gE9k9K=5}kYsn>Y
z{>IP6WQ?)nV{W>sPN&cJn66O%QuXBa=3fa<JQKvE<c!)NRPW)R7b<Hci|jT!Few|>
z!Kc2pXLeU}Bv<dwSLEPt#t0}}R=B*%2kZ(-_(%w~kit0GQDwb{%!<w9(_X%>0g|~#
z++ZwGVN)R4+hi<;q3~nY#}8}UTXXV418EsA0mSiEG>S1XyU2p8bz+)hnnNDgCg_eM
z53<->M3Npvk978==gzroDY|Wda>%Tk9rrx2Mfh+71Ll&)(<4JVr<~fhc5&ysku`<e
z^JQJ3rFUX}AfDIOe*0fO4B)`Ywrhx60bU2X=56ZX2A3q#u5WK7l*s~xi6H9>pwPD-
zU3t1(SL<STt;UA2Ci^s5ydhU2r!t3>BI%~y*^HbwPW<!eZUmue27X@<Qk^@d7|OQ+
z^!BZNHsW{Blt%B5$#>}0UfvwNo^oDEDo5gnu}L7s%OAE~AOGq{y3*+C2Spc&DY6o>
zb+UEH<KGi@JEOP#h0#j}Je5{$?>(5D3}lvbKf#ZKf|=bJ0o-Uoh{n@9R)^v5hp7O!
z4Aot`{#TCOaSU6tN!5Qnj}4o=E;zruGRT5ES_kd}4gBZBK9dfD^JSHtf!{;>k^xEq
z%-R;;Jnc1QbkI3C2yD$yE?~`$vzS-sEPG;VToE{uKYM>Wo|T#Rk{?$({GKdO*FQlh
zD{OD;U3l_C5cnGC5B%qpep~00PRm6FbTxtyZWkRH<OzP<P;rZmG?hCkXGgsv89+W^
zcW1V-1h-lKot9**+t?7xLC?;PjsM%2HwsGvLXy?oK23o8>6%KwqX*e5@80)>BzMvE
znaddkU8Fa!kglgg05?Cn)&6tWxD(6_D#~3RX2%xyl2C;55>`6<ZHh(}B<0wX6^=;K
zx%-1`d#wnFGof5C<QboQl=hOXVW{sg--@)J<i3VuC24{yvoLKWA|wA~F3r>(QD5oF
zijKF61Chmnz;gl!$tqi*Mr5;S-)c-JpG~@Aj`fYYj*URW=$QOFPPr-pq#aEi@v9pM
znyI^RKEDHIdj;E;mP=8G0noFx^i{hbhf$Pe2DnGiO?B*Sg@!|;Tc#=~jcP-y3F`ag
zbp)Sq2UtD?<F)DjEIUu9>_-O6KH^Qwmsx<DSkEb<^2376<%*Pi?5%k~phZVra^ygN
zK6M%O|IfgcHXX~LcH#u#qP<X^cpc$!KyU=RK!Zat06-~Ww(_uPefbiJA5#kJW9bXB
z3$i=D_jtIrna?<JFXzB@05lzhv@<bF&rA%GmQCUg>=i6}kHl*d+t&h#OGU^F19E}f
zQaLn$zvK!;TyhpvBNC|$IPH-Us=8h@0^$b-uRZUd4bGt|d`S$d=M0MMQLT3-PW}$?
z988*5Cg2UqT2lvw2U}geS4(?Kw&)ujAPrR7R;Q2Vt*}#M7C>1S=FP!`$7J$OvznBy
z_r}0(waE2sh89&9=-aqmcDW)V_7TRml*#RLV}7==Bp3|w?$E!)u){LcHD<b_edY2G
zvxA&Jir(w<eCV^Ny-106bQ+WbLa3{|1;n(?r#z9<_eg*YD_%Dat;!;_J#n*1RnAy5
zbLFn<MvP`Y50f56*f<UVM+r#Xm2tz>x-a)g3XANp>s{8e-*3H@j$Og?fUHIO{_z=R
zA;Q<igk*xf<UD}uIeFXR|4Xp?b`8k_pIV86R`=<@;#68bL}0<gyaW)#<m{?ya(4eC
z2|g1_^u9rTjY9P~7YMqIXo8$t(vEfAg7{sAa92r5XA@e<kTb8(Jn#t)bd<~dHak4<
zFC8GD<UJ`FUeE6Z%+L~@qE93ZBz-@*gHQn>DQ&AMrL@eqa?000bOdP0@8ICK68Zqj
znrwV!qUggJ!Ywvn(&#Ya?HBUrfc*UA<nFkxS^lVmlv;X7Gym+jZZD@QG+pHL1DyoR
zYAJtrrxU*}Ja2m`FU{*8Wc8UBgxB4WMoUd%sNzHw0F#hgMEvTu7P0{-RHs`cC{&1l
z$QpgeJKOdzN6Zu?Yis*XjVgo-p97n}8{m=-BSg0&ib8+Fg6ZStZ<m#<|Jxdkh%^cq
zk1@KH;?>X^+Z5mRgp~ny44L8@cg;9i;0q?rZW;e&-QwY`R_Jt;P6WIa$_M`6jY+Gp
z3d$FCLgXMwbSP~R2I{IpPMcO$ZF>=|T%(pC@E}M9$FoUo?+gTBOK#}Vy`ANKxp|Xd
za?0|Q<wk%zaI=2l-FS!Xgmi20h`B9ugIi-EaMUN49K`ZQIUv)p)%axB%iZQ7=X)D+
zkV{`I{0J_yGu54ASd*QpdaIQSNA^9)n4@+B86G;ypXFzF4mJ)CBSfuwZGuxheNivr
z^-Il5Y+Ia3_PU_HK=C_?{41iPo*?+?Jmx~_vMoMj^nWNn^KOURo4L%=H6k@4Z2~uW
z;h@tHnUB9a=-Pd<VKEh?6q0Z0(%R5M8Iqm;4P@^aDp-Kzw~9XEqw$z!qjkDnsZ;A#
zmY=NP$l$SCUd@Oue>F}k$Mi+d8+5(x9er;UI0)GNJbAO)X|_q)T%~y)_|7fj?%!4*
zG?v{y2vBFGmyz2iL{(Zh881CPEv6{1J2%gUdO#MSt;7$o=u$J=?)V6|f=7L==v%!H
zZhP>7+yU0#f1H%={m8T}9aqZKm<PTsP)eTZDqaH@>wg&*II*sOlY$i+<Ek0EO1nx$
zapN()vAEo`rhDq<<NyASUf=h-fK=lZ{?W%v>R~bcT_u_9D*Bhn`er*V2XJ3^l<1Y@
zcSc?mH=e9&^Ze^(F&*F|D>45%k@<Z8a@k-y%5BX6**Pf9`VMrjA-nJleoG(OI$uV=
zV&OeaR-EL_7fCThovI-RzE`NG`ZTl$>%Xk!hmF<#3y17zL;uP`b!l{(MDZ}@j62))
zUl$S&G%(rIx7l}i=C|3Sg)NI9=pO#Q3Dedg^zj6GvKF~@R~vz@pMO%(FeK&hE8V5^
z47d0RR&ZW&{`CZ3(<2Uy^8z3^bihN7o3_Se+IyNbW>_(7v9c%wQyl3A!ZyCgE&qtK
z$oK(DHHiEenMe(6Yf+d+;|b%jCTQoOW>h=j3rMDbQ!xdo306}lY#laxM3ljy`*KVm
z34CT(*Ft3^!<wf$$s}!;7`+cRT=Eo_9vetQ?!NLv+VRRZ-?b-y?L!Te4*1OhG&HCW
z*w6V@X(954=hZ=RP+cIGdb*~R^uNq9VTVql3heDVT1;cQa0v8gHv>${?Aq`ZlG!-B
zAmrg6l;=;szAZ36A+=ZLJ7Pkum>3vG&yY9BMZhf`U7u5%HINmf`?!_j=NRSIjwN(g
z*bA~}jiBz~Up{>G(x<Htdw*4OL*tzW#XPsEX!l)04xGQS`P#lAoxho>&yAoeoxGcv
zYinIRf**o@m$gr!{#R5D(0yS@oj?t@f}ZDY%RDSsp%jz^6u;>yZJK4>Po4+nyC!X+
z##*G4Cz2{lN=7bQ&N~Db_+<l+i`-c2`_!Ply01NXCJ0``!S|xc1BN1XwNd}_pw%(Q
ztp{7gl%>(iqQ~5hiTOa(0)E48oN`u+eVbZIA_@VZTB|(((xSgSCVuzR7S9O=>NBN_
zYe)Td%4q?#Ci*FVyH@e$AbBY{Uo@t%*=gRXFF{rcq#9>x@8$i=z-~K~(yFGhm%@1-
zINBNU>w-D}3;>z&@&%|~1s2SykrdL;ed@3$ZZMO%MCT^!Dbz9j<Y8ab&R4(78)E4b
z$Pb3S<eCjE5q8W>&GkhED{>t`p8C)W^;{w?>|amJ=}g7gfo1rciTbhF2gKI1-?s}}
z+o9?C<UwGXib0cwZ{#7re=OrE-|qkd44m_vhJr(RwHdnvO0*N&>AsRQ+FAZ57fU&f
zeAelC=b%KPk{lOi)EqicB)Ei}<Z66*fUAwkNp}Vx6?;ki@6fUFvoa^u#JDBNz@@uU
zrRZ)DCOQ+>@*(}JyUvQ^#f$2i8afRcuuuZ`D4P>^dhF*5g|e5--=#plxM$_Oz*eOk
zIsYHT9lhTZ-afncHodJXB3_cRAp|k4u}lQVVy5D<AcR@3$#gG}O<{{s@GL~VK!eZ6
zXBd0@I2Us{F@OsUjx=vrCweE$cgcdtX*NVg?M#xUthlTeH&oK4Q_kt3R(5qjWS%GM
z?h=DY4=W5{;(eSO7yk3d<55PQ5Hhshm1omRi5BJUvTGVJDVIbf9_1q4!ND;lUImvi
z=?bp*Q8po%8q<e(sud%G5wIlN2|3!Bu->IWRpn^aX2q+H<Ncv-rpt6a1|g>!P{ywE
zRyWk^M=`>7<=yVtCd?NxN9^Q5HG{t?trjQ8R94a#)$Dou&}sTqz2GDy(Xw=KjgL+O
zy;g)gR|m_}>P+)Rlzwquj##?0@i^{F4%&#&cC-Ysl&dLT9J+fL+Hor<Ma@KRT#Qxi
zcV2tz9GVkiAiNxB{wEYwEbIxVJMF6UinCWQ<W9&dyN!N5%q%BT!_{MsV8TqDj2oXj
zQHUc-_{_^4nWF7%hzwftOnRE55GI232|tZuNR$a10oo=>M;4Sc_%MdWLoLaUmF;l}
zKq-~6s>NhhCf#1-G%>7?50RoJJe)4O&UnE7Qp}JK`u1yXBE>7?;R`F?-Mb272*%$0
zkMbWNACSU*C=9WwG1_!-iL*}fp~IKyd3=h}K(<4T_9*rIpK|5yHI$-OchjS}wb1j=
z!8nhnF7B$xZxdn7|A4&0h+)dJ3E7VJu0<R;l+Vi0zuYKOf4iQ?qkJw=%qPA~SQlOx
zt7cnIM`-{MR4yE?aU5pWP5V9sC6`f{b>M;%FP~9e=32-9d9hPV$-h}#g1EAggFn?l
zTk6+WgnAKN{@^uWxmVoLLs=UZKb(SsFj$Nf1`9m}2dE&5y`f}lz$u~a+P*f(S0+*E
z>J>qCX?_Bds=UXV`V*8M5OD6cx~EfIK1zC940;<nehu(Rp;Hoe1O8*k>@fUUd4<o8
z`0tj9ygNh=T-=UYfnp2A;1Y#t{=kkQVEVRVrtMt6jSbm>S~4@t_eFW~g5xmA-fl+&
z<kpzn7FA~t1I?@BeR`~=Deeq^zsS5qoSQ@xEAV!H9?=j~UO?J(?L^LXY4k)p*D;M#
znv{@n+H$f%kqo&2G;8bJOkDQx5m6e4KaFFKmA#zTr2hWdAdswua+sXt8l-~Q*PZpX
z>)|cMgS`ZGbx*P~{~-~+6_0d!Kl>*&D%pzPaX6iOOG}eabIH!QDq{jGq|Mzf`GNkR
zjm&#*Q87F$WjAUr!Z(-J)PGOYpNm;UOTKoM2BU%%m^TOQN&}ijH`nI6w3xr2_RmW+
zHPvQg!9Cln5}+UsTi&e1Gf{~%)UW;LS2AmtXANf!lQW2}dbm#~WHm16RdCL2t9r24
zzs!rbjf&oh9cK_&a305;ANJwXt53hHzu2YG4^2Cv0H=>WUAy~hv2<6&N#a5gnl3*K
zbXs9^uhag|&xDVx72zk)=1jm+jVu=p7Y)-|T|Ano$@gCrz}%|`2k<tvRnQHi@7U(4
zaSy4qOIeW2h%r)Y;U!oc0v<^yzXd9YzvD(IaS!nV=rIfBOA2x?NpP{?{BRDgX`?#O
zlXt<P|F)ajV@URzT*yAa%nD5Z7@i6*bic<L^q|{n=DOW|judqv$WG<Vif<7JHrINF
z?pqsy62Kr*!sOo$18$up6tz5xn@r9P6$NcKp6?QRCphjPN8Jv_6Db`<=vJ>~uVtu0
zZ#Ot2Cv3XUkj1s;{hVeBvnb)g5d^7x`ac$L#LozhdF#1`D=WRB%0qIg+;=h0Rx(61
zvSZ*hoe<CE<}?uN{U<#~|11;gV(F^fC<8(H!h+p`9aNL|OUR{2-Y7=zhnc=8MRbSw
z|5*FJs;LDXOo0&Py9m(m<-aI@=xJGaAif$WnulwT1qGX1rF}c`_DfP(F6d!umDJA|
z#t>%FuN4e6ocK<RcFv!+&Tj1+G|#f&-uZc%Q+7f3nSju22ise%(~8N#c1gxfgPC+E
z^JL#h$^TwZ23N9zeFnZpe=q6)R4;oSw42-xc;pM93cy?B6P=P}sf!Cf3qAsb2L1mr
z&AgU&RcSPb@N-MbzIsr<yZ_g?j1(E_MV)sOFMj-pkF5s}Ra}23aiI4kDR-epuB0gu
zif|7PbhKUcapPKsZY_L2O;QB+fE<@~`K!sF9_asizip)V%73pszhI=#wr0$q&w~X>
zKJ~mr<N7PGO~0Pxd?KY!`p}Iy;Po@yarqysUsjn8#@|8TNIp*QL{}Omz+=h)6$bxu
z_~T2Kd%`~2+FDUiOjt62gNk&Y{v7wskt6eb?=d4o$L9kkr}g^6g$@5z>ulu@W-1=v
zjG1yvUALd@+9uY|EksFcD!I?feAxVCXX7Jty;#r|aI2p+x$r25)X#c5Z`H-!1}b{|
z%`jJM?S9{0&-VcmsnsgkSPHThC*UvOWRp+Wd;GV0(o<2u7Nc87-TK}7A@TgB$2X`q
zgEAs%yk=2v{oaCD8|1S8bDkP;lg{)`DJbUHQ?a!q-utN-Wr3VH&|?e?FA}zc-xnLx
z{2v?IVR#yx4Udf)rdMi1$_dJF{m?snn#_9|ieqZCjwOSPszd+$k0C?V7Ok3BqUT6F
zvOXprf+y4T=NNHI^l}<iKzQyxXw&lF=EhP3pibvGbw!pdW1I(LMTH!>Nn7>Pun@c+
zWG@}6jp>~-orzxmoH-BOy7`&<+x0d1GDKHQQ4B_bT6Aq(v0Hm9IR3!_IY`#|I;Y}5
z%~JK}+u@GI(IjaiMbRW|YVtiBEV(`y_fo$3m<#7Lhf#(=%ztmGLG&aLrCao+py~NN
zl7#$w?TyBT`sutT^QcMW!v7di<CANiFDgM2A2BNsvpvi2TT5$f5gzrm6RqSV3EB`N
z|HqDwM|1))Hmqnrr{Yc4qdyjgPl=G3Bs|Z?J|!$hJ5nrq{l9dcjp5e;9iNU%ltAst
z5x-3ix5W?~Wutkel0{rW>UP(TZ8wc|fHdXsV8VM{Cq^NOPD4BIrt3D&SGv|ecJk}^
z+$y9p0NG1W`bgEF1nxKd(4`lYK`YnXCbA`Q#3Tcx|EZM%wGQd}(5|+XG^FudH>%Dl
z*ghxEH!NP|(F&BkJK1cjS(fp0X{Ya2Y3D4Hs|o5fhY10Pa1$inN8Fb$ns7dSs{tF)
zJc-eTr(N)*A;e1pb7+0C=L87>d%M6CoW>5m+73@y_9lYP(6RHVc5o9)*N(E%Wy*5Q
zGSo@NhY33Op#V_@<N;nQh;I%e6oizPH~5IdR0NVV7qH-T-uri_DCh__*HkwB;n{Js
zv;w0k#qfbgJ;Sf^%L@oGHODESzb6ZbX4hlHydds93>5_}GCk<-gGUo#RB~Fl2m=E-
zJ6Z3E%V~p+QXY<^$0^+2Nos?h<seLU5>0%~z`x=pKwTCkqootskp=DqP;Q)AiCgRD
z?c@#XAirSkQ1Cv)x%P%n$2%%7U+Y7I66|U*6^?T$u#DV$S51YV!;es#rxMFKYd5t2
zWnq@*OmMgyn@k#ZuTGkie^3qgor85eoNm;DX*DJ38dfPtYo+<si0-#P@&ylH>w3W9
z{0vG&aA{fnkcR+u+hxE}=P7DR)SQU~(obbL-UaKRcg1{q=M(e=ug!Yw*cC?W-o=Qt
zjhVr0=A^HGkWvw27TiCi52S(#vgEVr;op-bS*s++vz|{5=Ex}<l>+cZBSG?-pUj`)
z|Lr!5S$5ayOu=g7O=t4Y6osd%O|7p^P;i)T)G0FY&)>dHId#1AIYUDvAvfe<2u<4C
zw2M9-`+vLg!a!7JzhB@z!9~pHLmZvgl_m^W?1z{*)IGayQ<x2+T9SMNWU@~?2dz8<
zJa<i$MV><A+lVGddh{43iGheQaJw-&=T~!PaOAl~(T%xT*m$ss$)oxBw?!d>ZMo%g
z*=)`t+y%Dl@;-t#`iCP`oxDqvG@0ENIYg1MkJfMd7%yS3GIQ{E1j`tt<_tLV%ld@c
z=riHv=ENG!+1p_}+<SB}hijMD8E<9ZvNE<xbb7rJUF9;*j!|N3!d{><ZMnQ4#Gzi<
zV{nLx{(@8iXl<Y`TJgtwZT6)xMi|mdV#OjTUMHq&)&LaZM+b6e#tc0uu9a+{cA5G|
zwPzhi&$cwy1w+DO>`%&5h&weaA~puI2Fh+A;?t|zRqBDivT$hiu%D5h+fn57OA7tV
zUpXAp6e*K5PnZV2J`dzB{w#;Nzp`awz57_07i<x!zwW4I3MQl#`>>%Nb<>Bjlit!z
z0df5qO=gTPDDa}h(5?iwR}p`+9npg!L%En2KDO$M1|mHts*Yb{R!<KG-JE4R*NM)`
z^{ws+V{nZ|9ln;!0f7Y%R>Hu94a|9}Rv)Y_^m-&B52Lj%R7HNBd<FVb#m(?moff&D
z+=rJ}Th4{KCTZTdb$fX;9cF@ydsMEv&<vZ-YL2q_VhJ0^5v>Ag>ag}0g8f?>^AX>}
z&96Dsdb(~SV-tF^^bt1x?V0qjP`3+NUjBpWv;!U^EF3}hh3%lD+2r8|EiABc;$m1b
ztTW)MU0i#)RjBP4`&kt?dk5p_x8ZN;>hw31qP*qOA48@8@+~s0bj|k~Y&e>gZJ)<7
z+NY@oi>2n^Q&6l`*sHe5Gns*iugAg-;=<Q#y=Y32DUA8!W~nT2c`aPg!xuDZnCZIP
zR@{R9N?@#1Di|_0_wkGahq;3z^3^@X=+@0SSpMR)L2gn#nt862#{6q-OQj_-h?9s&
zt@|@I9WO_TQUK)-X)Q9@&dp_&w{ho8B)CjUP1ECO(lks{v-f@Wo~4#AZIwZ6#0eR2
z=MCNGm{MuUO-p20aC|X4B;M*0&k-K@ce;*z5Ak-XypO$IhN~s>E4NYuGn^KOdJBh(
zgLixGFxbUh&2A~|@_j&>M}7sWEYuw4YgY?ov#r+p3_V=seC*q9<wR-73GxO$&FwDM
zs9)~7-IJE|TZ(+Pvp>f-Nto`1ijG+CuA8!2lF3?s({LrHFgiF0aENsM4Kq4CbVTV!
z^HQRbyKKx9<HQoclQ7m+etHZ2s`zovQ6;$utH67dj7|Rt!o1QFEj*5WW9k9eV<>nC
zjkkld+go@8X4&a4pV0O;bM~p44L%v4&_@iT7`cII4z=Y?`Po1)$^;(_`PK+Q%xW*I
zgQ>a9EIwx8Q2(M8>n-{^k#nuhLrSJ9Sw7p#*=<I(T?%?XV34%~YYsNDTwF))Wk}9o
z!f$HRY+)46wqw^*=XvM#ukehoND`}aV;mm!8fsUl3=@|+WjXr1WA%h(_%3u5sB6fI
z)3a8YU70vas9zqkD}y~slb{+tX)Hf?Y<i!!b&E_Wu5*JX^`)a@04<oDM65T_P{uiw
zhHEmiW%!+<w*W{o_fCGKPsYn-2l*%0Jmv^sPPABEIb#sa8Rgzn@X|ZhL9;VUec|3=
ziW)=p8|xu#hEtWDxu;S62P2Bv;tUdHIs-+m)ZT?$<ek;xbL;-Bh5e{j{xVVX1Zb;}
zP>v2iuXSrV#ap$b*#7e<n1P*mcr!jc$G!1oQ_`lm95w+Bu_H!*^P>$?fzi0)IEDX*
zE2^>kngpCU$9`|7b)pJ!UUf-rT!7`fb`JjyGe+R({;@t8J{_8Kze`ffN3hjEV|En=
zlp>=?Q_#rOo<Ns{;It$QB63XvR-nrUgjqCGG7%Z@Y+VAhlaD6Y!+Q#X?z=^|HhV*o
ze+9@F(X%N@+Kso0v*UVZ_P%Qaoca-im&;Qp9qlPId#*}iHX0*!`vc6ABK?EcX3nJ6
zm!a+gPH9Vpbw0Av;!)d-^Au;~%oDtYt;mNZrk2wYbjpLlr=%)1l8`;6_P4Qr-SXgF
zq7G+V2)#EDoyMgE59IXl&<ZY)$+q&3n@r&a)j(PrrRO{DJq5z|IwC*joSHKHz;sCO
z=8KTd8?>1*(futKlddRVV-}q*gC|~5?=Z*#`Y7arRE~W#l{`gz`kRaM7f!8)c^7yK
zrzuZ{udgT8mw$%O`?q)U!1ls}bI`;uHGt48rrMjEm5fPj)4^a1$~9y|39qM*addvx
zLOBG72sVj+OyXRVQ8D6BZ)|0-4Mw~(HB}08wtEFygP}rUO@yR+35|gIkbB~c-Zd7f
z>%74^<60u~P8=D!lb16iC4vyvGS2#D>%Vb)c?4vWOG#c_@MirQpFuyj+Okn6W%xbH
z7_O<8j^iIIVabLt<m-gVeqHR}nMs`W!YuMWdA8N)V>)am>$#WB1X;o;-f?kA-sJYx
ze6aC6<zuAlTX-HD4EynSl#CCpovS38MXiVKpqdNI+4iz@8=z=&lZnnzd1>H;!PLY3
zK;6gI<GMY6VP6=*=$c2$b-_Df3YcedYM8t>mC{5_rJBYrZ1`0et($Q;0fu67K!=o1
za#oprLmYOfd&)|I^_dfP;`A-+T35qqXi?^$P!(!J<K;^U&mnW?q=K#EO#*|ozDaF=
zD+^l`)jNinUvq;M<iENn7+oric?Rr<UCRcyl%wc+u9XzCY>xAQl5A(7->NP`@l88u
zt6HXsaJ1v#w-0L`C{RUw)Uyu*FxYZ}dhAj;4hqj$Da;HO3FrfG&q<ruUPM|=AB@Q^
zQIpc`${-A}QEkv8P~tTCMiSK&nTz@yr_Ehu_LUdhoE2>2F2O{0%4B=_-+vOI+GE^y
zQnqKa_hG<aDQ&oS00>D&tIdg9QLf}2#$RS&H3e$=Mw0!B7gspUr2{U9BK~YUs4Z9O
z6am(GR4h@m?zB6b@!==ErZD(l!Izw{1QndA;A2^D#JG7W@ly#*j5C_MjUiR3Sq9=_
zah8tw1RC?&Tt09_)#ib0uM)6qRRBB?Tr8Ju<?KA8y18kHNzA~9-CJU4ny5*NWRoQ<
zhtRa3n|V4ku&FGtat|&OdH=-8m5KMEYsk3v%uHFe#X!@&NyrGat*TE)?r3c7af&>S
zoRsz3_?YTE3+-!7DC2_7$;C}rMHg^_wXVwhbl8n4^xb}i_$u~KYPhT=b=b&w*{Dt*
z%*S`JHys_71nAhuC*_0DF{R_zM#S&yvI@@GoGgfw^IQGP=Tpo?Ufb=|`OL^j-@sny
z^7ao0V4dp*THL_c!bcX_>f#S{MuYtcp8Q9wA_>=%XV`T>eQy?_s-lXxDDa%__ca-Z
zaX*;SCwVQ&i39}>^<xxfn6n!SN~u1Z_vY6D<3|b1W^tG&Y17IrS%;ftL(I9{RcY=H
zq*5XG91x6L$uk~n-ZbX%<;DxQWV6}wiZ{9U+!0kEbmaZ9F7SFUCuXh_mVQz$w3Ik=
zFzJ+o=(_xetAO<>Dc+@FhCk22z-dp93uu63vc7Kl#ws}d>n8jNknMiFzkDFlj@5h}
zvpjjBN>0T86D3S60UMCl#j_#bISE6`#)#YH?0ZtG)q=|h=@m1I4yWwq3L$T63JRAf
zq7O|=OU82Vi!KfS%$h440oEKk5}PlN0msJG6ofGcKk$^Clc(s8TJk}6rn%&vrnsX{
z)JeQ@YA?k>k(;W`;#^f`g@1L@p)YNO0GEk7wI`OF>%fPkI03ESKDZxe1JL_eLNYyS
z2zcL5f*H>9bGHfNc@t0hVp~|J00S&50Ub{9M3E|srEPRRhh@zj&I{*N-wPi1N8(*m
zt3?3`#19wiVn0T&J911B2J=HXJ+6jCY0EJNgaeM#A|${%+tKx}fJR(AkUC#iHpEp`
zyvG_Oq4t#g<NnDd5ic{U-u#yQeFodjOnEk)O{F-$ppjBFhMu?1Fxqi`mz?$}PjLD;
zmL6g@%c~~TrW^r7vQrGqKH=VTm%92dCwiOr!}Y6N7_ffu$6&?q=5cc<hz!EHrzj}I
zXnzyrwvF^|t$fhmEKdnNK_y0YS4=L4;tdXS1E@W%<kk7}EaN5r^!$;lMS8&hzAb`g
zRGE-{UnEb{2ZU)tSKn0M%$V&*+1VL!@-puMgIxJ`_S#jqU=Wd%gQ5e9!f!|*CW2Lq
z&qcb{z5l8D<0W}0Dv?8X#So(AI-mlUVx!9)1eF~%MQCUmfaarjWe&DKA$nxs<NuLD
z9w7gE2S5z$?fAI!Py%T1sd^OS43QoD`GdoNJ5te8q?v+wupMx?Mhp5Sk^k?|Yw&Vt
z_0G#IMivKEPw&1AGKzN&@4hVI<#>(NdgtX8@UqabE}T5p2dzsV_gd^u1ggW>px%9%
z?ts$=;)k72*@5E()qm_t0d=AqdemUTyg~P%a8H+hRP4=P&}-LB2xhOoT-G7>b}t9)
zv&f$1O0Ad3kZ`#zm-Zn=PAUNX%{xic3A}Qv((u4H>MLDmD4uZ6zI_e@$p?*P!SEb{
z&A3w(Q0C~h6e1cJxy=Amx$Ft>g8z%kV?)fOg{b9%$O>Fv6$nn6o;x&_eT0HyNcZvN
zfepImr2`aiL>xTW)M5-r{F#COZLs;}L@q9<cm=>1#RtUo=+!T`^xEjwucbT;POA6l
zEZ~(@TgFW0$(nr>N?j9{?U~nd^OskQJ!eli1%PtO;^Qvl-9(_&*;4132PG8|$(@px
z`Kl(8QzOg<qMq<Rjqb3)v>Yxl!6(GIH9uC0`Vok@2qQcOgw|}cn>Xvj{xFon2lTOl
z#W1Bq-|RO{w4lEkR_cMn!}6*5ZiLB%5>yJxVB?bbY9LMPEJRkdWu-Vmfa+>X@zVQ{
zrOh&L;DG6UIUpZeUQRp#qa)75x+@?y2wHt$iGvGLsV_b9g7?7kHyUXN;%K?mdIGaM
z4b(!V8$pG_RL#K&nb2}#>Kq8UJgiVOqhNkd5W}oA)_WkXx#L!1DywGt&CR$Z<f7?1
z(_!G;jJ|~kVJ;%4wm4w9dRVES%cKGCZ2T%@#$a{X1QwtO2x^)h4XUp(73W|}aj$X+
zE}<Q>a;fah!JO-JeG;!YiF2TXT#xO6N2`*41YN|m_(A~i%7P6xN=}u?sySbnNk<%u
zivyR&yWh!-CV{oPD};qyoT|1vN&&cN?)Z_yClI5(D;vO(X7^PCEb@8sIha*C2x((N
zQ$PfOT<ik(ng$xovu_|OoIyUOCPlu7AE}^o&SV{j7Q2C{;s$#-HNll+g=GhGMSI8&
zEJvxX2H@<p5goeV1$K!b5euNZ1acEpd*U!r*HRHNts9@NrPe1KTvEff{GO^}12d7F
z=<kE2mVj@mFh;Qe-kN(l2P4|&1?s`_FKFs-P{ZgX``lVJ!GLvIuo`VyE!@LSS!@t>
z(qk_mEAKv~se}O*1#~y7Z<d8pF1VLLYs{XwlEm4W3u?<#8?7q!VYm`u#rm+gg=(F|
z{<5GPwB7U#4uJJ^xB`|>oNaOsHamV+ZMbFOpLV?h5Qjb#mqko+KwMdT2VvF~yLnLw
zu~B0|fMG|<HMH=iGClKwbi`DD!H{Cx0@derv%!d-+(cgYR@x}0;R*yI`!}o^pSaW3
z{H|=zPp(HfRgqV^!Qgty9jwQ(vURz<85S@@2K|eG1qOmIm^8FFwGm)`VZbqvr3XI_
zx!~nE`aJLxZK~p+y|QWEp`c>>EZ_!}M9B)4>VructjD%5BqSl?aBpvNOesm2CWc&$
zvu_ds7sve@e4=>m=UPPCmFkx>HwaN6%tQ9hpZeAWRDt<#6clfgzwsaC@q<_)t3kdI
zd#ub`h7eGJw}hp;2j-yL%mdm(CFS%sTrP@(O>cQfIJF=O>vY-dJ!{f7w0;$um}o{w
z@!+4}BWp@PmSX!-WChcC)d)8^OZYp9u}NmbEkK}Fx{H8#Too{=cbC+Wif5Nq!(?l>
zVmc*3WsQdW7)R>;aG5z2*vvbXw^%{yhaM80yW!!lX?0JJFiWs0Bc=(hgJ2?dXP(Ik
zVtLgz&Gk1#dpGUe-)n*>zFLw6kk7|irG7aMQzi&1hc0GFp6fWPfE|hJu*+6ml`O@D
zw0j2tO|Wu1P9~JNQu143{pJO4k5`CkrOiG^n40`d&oH3QwA%7$TraQ5`cR3PN0aq?
zPt~Pz$SU%n{O=teii9*OMM?Q*<QQ#I`pJRwCT*Wl{WZpVoNBFj5m7BfjwD8M<CzNx
zNe`DRgFAsHxFQTP1amFb$2}G$nrNZrhb1P1Nobq^^0nfg2I;HD65!|{R}KI^v~UlX
zRkK{w&09*H?&2V7(F=gbYM1xG#NxmlRM#FqV|`*y(tu`db$2;xh*!IcWIH+>;ST{=
zBhUf_j&PZdX}tCf2dH}TSWgL=VfJuHM*PmvCpq;xJmiMhy>RGpNr#s-Enp!O9pC~L
zk2kdYPmGEu<Wj3ajt(-5Z$wUQRE(3Xf%WI`iBOaTWQU#{G0@B1XU#}5U<^I7E#UG6
z^fUJE#zDGWE5`+_T==e4o4R}G&=3C!jwMHTEM&)}BhQ+5jW1NdT|go-7?{FZ@z}n%
zIz*x_ZpR&<p!i@9PROaGGG<)OF*<$^cdTjQ`Rb>i&#Alt?nBBUyJr!=>mdE<LU$nQ
zt4}hbr@tl79on(v883&=K4QJ31yFY~4uK|+Q9Cc3059AeK0(hea62Q;wa>A@t;TF?
zunb`SxBhue1K9ele?J13lK$V%z_R>5nvmJx|HcsG#64-VqE54PSQ5X;O5MJh@z>p_
F{|Els%iaJ0

literal 0
HcmV?d00001


From 434ace7244b063c418b8d9f5c29330411a1df237 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 3 May 2018 16:31:41 -0700
Subject: [PATCH 245/257] implemented search accelerator

greatly improves speed compared to non-accelerated,
especially for slower files.

On my laptop, -b12 :
```
calgary.tar :  4.3 MB/s =>  9.0 MB/s
enwik7      : 10.2 MB/s => 13.3 MB/s
silesia.tar :  4.0 MB/s =>  8.7 MB/s
```

Note : this is the simplified version,
without handling dictionaries, external buffer, nor pattern analyzer.
Current `dev` branch on these samples gives :
```
calgary.tar :  4.2 MB/s
enwik7      :  9.7 MB/s
silesia.tar :  3.5 MB/s
```

interestingly, it's slower,
presumably due to handling of dictionaries.
---
 lib/lz4hc.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 132673ec1..6c50db5d2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1099,6 +1099,7 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
+    int fromMStart = 0;
 
     DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
     /* First Match */
@@ -1132,8 +1133,8 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
     assert(longest > MINMATCH);
-    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        const BYTE* const matchPtr = base + matchIndex;
+    while ((matchIndex-fromMStart>=lowestMatchIndex) && (nbAttempts)) {
+        const BYTE* const matchPtr = base + matchIndex - fromMStart;
         assert(matchIndex < ipIndex);
         assert(matchIndex >= dictLimit);
         assert(matchPtr >= lowPrefixPtr);
@@ -1146,6 +1147,21 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
                 if (mlt > longest) {
                     longest = mlt;
                     *matchpos = matchPtr;
+                    matchIndex -= fromMStart;   /* beginning of match */
+                    if (1 && matchIndex + longest <= ipIndex) {
+                        U32 distanceToNextMatch = 1;
+                        int pos;
+                        for (pos = 0; pos <= longest - MINMATCH; pos++) {
+                            U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                            if (candidateDist > distanceToNextMatch) {
+                                distanceToNextMatch = candidateDist;
+                                fromMStart = pos;
+                            }
+                        }
+                        if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                        matchIndex -= distanceToNextMatch - fromMStart;
+                        continue;
+                    }
         }   }   }
 
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);

From a7cc0b590a634e3c2612976c1dc83e74f8448d43 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 4 May 2018 13:35:10 -0700
Subject: [PATCH 246/257] Fix make install

* Uninstall didn't remove the pkg-config correctly.
* Fix `mandir`
* Allow overriding either upper- or lower-case location variables, but
  always use the lower case variables.
* Add test case that ensures overriding both upper- and lower-case
  variables is the same, and that the directory is empty after uninstall.
---
 lib/Makefile          | 70 ++++++++++++++++++++++---------------------
 programs/Makefile     | 62 +++++++++++++++++++-------------------
 tests/Makefile        |  5 +++-
 tests/test_install.sh | 21 +++++++++++++
 4 files changed, 91 insertions(+), 67 deletions(-)
 create mode 100755 tests/test_install.sh

diff --git a/lib/Makefile b/lib/Makefile
index d63de18be..abb6c075a 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -62,7 +62,7 @@ ifeq ($(shell uname), Darwin)
 	SHARED_EXT = dylib
 	SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
 	SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
-	SONAME_FLAGS = -install_name $(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+	SONAME_FLAGS = -install_name $(libdir)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
 	SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR)
 	SHARED_EXT = so
@@ -132,20 +132,22 @@ listL120:  # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (n
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
-# support both lower and uppercase (BSD), use uppercase in script
-prefix      ?= /usr/local
-PREFIX      ?= $(prefix)
-exec_prefix ?= $(PREFIX)
-libdir      ?= $(exec_prefix)/lib
-LIBDIR      ?= $(libdir)
-includedir  ?= $(PREFIX)/include
-INCLUDEDIR  ?= $(includedir)
+# support both lower and uppercase (BSD), use lower in script
+PREFIX      ?= /usr/local
+prefix      ?= $(PREFIX)
+EXEC_PREFIX ?= $(prefix)
+exec_prefix ?= $(EXEC_PREFIX)
+LIBDIR      ?= $(exec_prefix)/lib
+libdir      ?= $(LIBDIR)
+INCLUDEDIR  ?= $(prefix)/include
+includedir  ?= $(INCLUDEDIR)
 
 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
-PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
+PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig
 else
-PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
+PKGCONFIGDIR ?= $(libdir)/pkgconfig
 endif
+pkgconfigdir ?= $(PKGCONFIGDIR)
 
 ifneq (,$(filter $(shell uname),SunOS))
 INSTALL ?= ginstall
@@ -158,41 +160,41 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 
 liblz4.pc: liblz4.pc.in Makefile
 	@echo creating pkgconfig
-	$(Q)sed -e 's|@PREFIX@|$(PREFIX)|' \
-         -e 's|@LIBDIR@|$(LIBDIR)|' \
-         -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
+	$(Q)sed -e 's|@PREFIX@|$(prefix)|' \
+         -e 's|@LIBDIR@|$(libdir)|' \
+         -e 's|@INCLUDEDIR@|$(includedir)|' \
          -e 's|@VERSION@|$(LIBVER)|' \
           $< >$@
 
 install: lib liblz4.pc
-	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
-	$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/
+	$(Q)$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/
 	@echo Installing libraries
 ifeq ($(BUILD_STATIC),yes)
-	$(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
-	$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a
+	$(Q)$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h
 endif
 ifeq ($(BUILD_SHARED),yes)
-	$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
+	$(Q)$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
 endif
-	@echo Installing headers in $(INCLUDEDIR)
-	$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	$(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
+	@echo Installing headers in $(includedir)
+	$(Q)$(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h
+	$(Q)$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h
+	$(Q)$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h
 	@echo lz4 libraries installed
 
 uninstall:
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER)
-	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
-	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
+	$(Q)$(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT)
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR)
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER)
+	$(Q)$(RM) $(DESTDIR)$(libdir)/liblz4.a
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4.h
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4hc.h
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame.h
+	$(Q)$(RM) $(DESTDIR)$(includedir)/lz4frame_static.h
 	@echo lz4 libraries successfully uninstalled
 
 endif
diff --git a/programs/Makefile b/programs/Makefile
index a51bd4b0f..72bdcaac1 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -120,21 +120,19 @@ lz4cat: lz4
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
-# support both lower and uppercase (BSD), use uppercase in script
-prefix      ?= /usr/local
-PREFIX      ?= $(prefix)
-exec_prefix ?= $(PREFIX)
-bindir      ?= $(exec_prefix)/bin
-BINDIR      ?= $(bindir)
-datarootdir ?= $(PREFIX)/share
-mandir      ?= $(datarootdir)/man
-man1dir     ?= $(mandir)/man1
-
-ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
-MANDIR  ?= $(PREFIX)/man/man1
-else
-MANDIR  ?= $(man1dir)
-endif
+# support both lower and uppercase (BSD), use lowercase in script
+PREFIX      ?= /usr/local
+prefix      ?= $(PREFIX)
+EXEC_PREFIX ?= $(prefix)
+exec_prefix ?= $(EXEC_PREFIX)
+BINDIR      ?= $(exec_prefix)/bin
+bindir      ?= $(BINDIR)
+DATAROOTDIR ?= $(prefix)/share
+datarootdir ?= $(DATAROOTDIR)
+MANDIR      ?= $(datarootdir)/man
+mandir      ?= $(MANDIR)
+MAN1DIR     ?= $(mandir)/man1
+man1dir     ?= $(MAN1DIR)
 
 ifneq (,$(filter $(shell uname),SunOS))
 INSTALL ?= ginstall
@@ -148,27 +146,27 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 
 install: lz4
 	@echo Installing binaries
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
-	@$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4
-	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4c
-	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
-	@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(bindir)/ $(DESTDIR)$(man1dir)/
+	@$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(bindir)/lz4
+	@ln -sf lz4 $(DESTDIR)$(bindir)/lz4c
+	@ln -sf lz4 $(DESTDIR)$(bindir)/lz4cat
+	@ln -sf lz4 $(DESTDIR)$(bindir)/unlz4
 	@echo Installing man pages
-	@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
-	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
-	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
-	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
+	@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(man1dir)/lz4.1
+	@ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4c.1
+	@ln -sf lz4.1 $(DESTDIR)$(man1dir)/lz4cat.1
+	@ln -sf lz4.1 $(DESTDIR)$(man1dir)/unlz4.1
 	@echo lz4 installation completed
 
 uninstall:
-	@$(RM) $(DESTDIR)$(BINDIR)/lz4cat
-	@$(RM) $(DESTDIR)$(BINDIR)/unlz4
-	@$(RM) $(DESTDIR)$(BINDIR)/lz4
-	@$(RM) $(DESTDIR)$(BINDIR)/lz4c
-	@$(RM) $(DESTDIR)$(MANDIR)/lz4.1
-	@$(RM) $(DESTDIR)$(MANDIR)/lz4c.1
-	@$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1
-	@$(RM) $(DESTDIR)$(MANDIR)/unlz4.1
+	@$(RM) $(DESTDIR)$(bindir)/lz4cat
+	@$(RM) $(DESTDIR)$(bindir)/unlz4
+	@$(RM) $(DESTDIR)$(bindir)/lz4
+	@$(RM) $(DESTDIR)$(bindir)/lz4c
+	@$(RM) $(DESTDIR)$(man1dir)/lz4.1
+	@$(RM) $(DESTDIR)$(man1dir)/lz4c.1
+	@$(RM) $(DESTDIR)$(man1dir)/lz4cat.1
+	@$(RM) $(DESTDIR)$(man1dir)/unlz4.1
 	@echo lz4 programs successfully uninstalled
 
 endif
diff --git a/tests/Makefile b/tests/Makefile
index d4847b1e6..196426e1c 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -141,11 +141,14 @@ endif
 DD:=dd
 
 
-test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer
+test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install
 
 test32: CFLAGS+=-m32
 test32: test
 
+test-install:
+	lz4_root=.. ./test_install.sh
+
 test-lz4-sparse: lz4 datagen
 	@echo "\n ---- test sparse file support ----"
 	./datagen -g5M  -P100 > tmplsdg5M
diff --git a/tests/test_install.sh b/tests/test_install.sh
new file mode 100755
index 000000000..2e1a5a54a
--- /dev/null
+++ b/tests/test_install.sh
@@ -0,0 +1,21 @@
+#/usr/bin/env sh
+set -e
+
+make="make -C $lz4_root"
+$make CFLAGS=-O0 > /dev/null
+for cmd in install uninstall; do
+  for upper in DUMMY PREFIX EXEC_PREFIX LIBDIR INCLUDEDIR PKGCONFIGDIR BINDIR MANDIR MAN1DIR ; do
+    lower=$(echo $upper | tr '[:upper:]' '[:lower:]')
+    tmp_lower="$(pwd)/tmp-lower-$lower/"
+    tmp_upper="$(pwd)/tmp-upper-$lower/"
+    echo $make $cmd DESTDIR="$tmp_upper" $upper="test"
+    $make $cmd DESTDIR="$tmp_upper" $upper="test" >/dev/null
+    echo $make $cmd DESTDIR="$tmp_lower" $lower="test"
+    $make $cmd DESTDIR="$tmp_lower" $lower="test" >/dev/null
+    command diff -r "$tmp_lower" "$tmp_upper" && echo "SAME!" || false
+    if [ "x$cmd" = "xuninstall" ]; then
+      test -z "$(find "$tmp_lower" -type f)" && echo "EMPTY!" || false
+      rm -rf "$tmp_upper" "$tmp_lower"
+    fi
+  done
+done

From 3e7fa1b14d9a5fc7f10c36231ef5cd2ba3ac7350 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 4 May 2018 14:33:59 -0700
Subject: [PATCH 247/257] Attempt to fix travis

---
 tests/Makefile        | 5 ++++-
 tests/test_install.sh | 1 -
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 196426e1c..d2385614a 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -71,6 +71,9 @@ all32: all
 lz4:
 	$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
 
+lib liblz4.pc:
+	$(MAKE) -C $(LZ4DIR) $@ CFLAGS="$(CFLAGS)"
+
 lz4c unlz4 lz4cat: lz4
 	ln -sf $(LZ4) $(PRGDIR)/$@
 
@@ -146,7 +149,7 @@ test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install
 test32: CFLAGS+=-m32
 test32: test
 
-test-install:
+test-install: lz4 lib liblz4.pc
 	lz4_root=.. ./test_install.sh
 
 test-lz4-sparse: lz4 datagen
diff --git a/tests/test_install.sh b/tests/test_install.sh
index 2e1a5a54a..f9de402b7 100755
--- a/tests/test_install.sh
+++ b/tests/test_install.sh
@@ -2,7 +2,6 @@
 set -e
 
 make="make -C $lz4_root"
-$make CFLAGS=-O0 > /dev/null
 for cmd in install uninstall; do
   for upper in DUMMY PREFIX EXEC_PREFIX LIBDIR INCLUDEDIR PKGCONFIGDIR BINDIR MANDIR MAN1DIR ; do
     lower=$(echo $upper | tr '[:upper:]' '[:lower:]')

From 9699ba5ddfa8485bb4ad36cc9ec78cbed542f28b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Fri, 4 May 2018 19:13:33 -0700
Subject: [PATCH 248/257] integrated chain swapper into HC match finder

slower than expected
Pattern analyzer and Chain Swapper
work slower when both activated.
Reasons unclear.
---
 lib/lz4hc.c | 121 +++++++++++++++++++++++++++++++++-------------------
 1 file changed, 76 insertions(+), 45 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 6c50db5d2..c58cd9de4 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -227,6 +227,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const dictBase = hc4->dictBase;
     int const lookBackLength = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
+    int matchChainPos = 0;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
     U32 dictMatchIndex;
@@ -241,31 +242,30 @@ LZ4HC_InsertAndGetWiderMatch (
                 matchIndex, lowestMatchIndex);
 
     while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
+        int mlt=0;
         nbAttempts--;
         assert(matchIndex < ipIndex);
         if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
             /* do nothing */
-        } else if (matchIndex >= dictLimit) {
+        } else if (matchIndex >= dictLimit) {   /* within current Prefix */
             const BYTE* const matchPtr = base + matchIndex;
             assert(matchPtr >= lowPrefixPtr);
             assert(matchPtr < ip);
             assert(longest >= 1);
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
-                    int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
                     int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
+                    mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
                     mlt -= back;
                     if (mlt > longest) {
                         longest = mlt;
-                        *matchpos = matchPtr+back;
-                        *startpos = ip+back;
+                        *matchpos = matchPtr + back;
+                        *startpos = ip + back;
             }   }   }
-        } else {   /* matchIndex < dictLimit */
+        } else {   /* lowestMatchIndex <= matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == pattern) {
                 const BYTE* const dictStart = dictBase + hc4->lowLimit;
-                int mlt;
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
@@ -280,9 +280,10 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
-            matchIndex -= nextOffset;
-            if (patternAnalysis && nextOffset==1) {
+        {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
+            if (1 && patternAnalysis && distNextMatch==1 && matchChainPos==0) {
+                U32 const matchCandidateIdx = matchIndex-1;
+                //static U64 g_nbPatternAnalysis = 0; g_nbPatternAnalysis++; if (g_nbPatternAnalysis % (((g_nbPatternAnalysis/1000000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis=%llu ", g_nbPatternAnalysis);
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
                     if ( ((pattern & 0xFFFF) == (pattern >> 16))
@@ -293,21 +294,48 @@ LZ4HC_InsertAndGetWiderMatch (
                         repeat = rep_not;
                 }   }
                 if ( (repeat == rep_confirmed)
-                  && (matchIndex >= dictLimit) ) {   /* same segment only */
-                    const BYTE* const matchPtr = base + matchIndex;
+                  && (matchCandidateIdx >= dictLimit) ) {   /* same segment only */
+                    const BYTE* const matchPtr = base + matchCandidateIdx;
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
                         size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
                         const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
                         size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
+                        //static U64 g_nbPatternAnalysis_confirmed = 0; g_nbPatternAnalysis_confirmed++; if (g_nbPatternAnalysis_confirmed%(((g_nbPatternAnalysis_confirmed/100000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis_confirmed=%llu ", g_nbPatternAnalysis_confirmed);
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
-                            matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
+                            matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
                         } else {
-                            matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
+                            matchIndex = matchCandidateIdx - (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
-        }   }   }   }
+                        matchChainPos = 0;
+                        continue;
+                }   }
+        }   }
+
+        if (mlt == longest) {  /* better match => select a better chain */
+            assert(longest >= MINMATCH);
+            if (1 && matchIndex + longest <= ipIndex) {
+                U32 distanceToNextMatch = 1;
+                int pos;
+                //static U64 g_nbChainSwapper = 0; g_nbChainSwapper++; if (g_nbChainSwapper%(((g_nbChainSwapper/100000)+1)*100)==0) DEBUGLOG(2, "g_nbChainSwapper=%llu ", g_nbChainSwapper);
+                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
+                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                    if (candidateDist > distanceToNextMatch) {
+                        distanceToNextMatch = candidateDist;
+                        matchChainPos = pos;
+                    }
+                }
+                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                matchIndex -= distanceToNextMatch;
+                continue;
+            }
+        }
+
+        /* follow current chain */
+        matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos);
+
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
     if (dict == usingDictCtx && nbAttempts && ipIndex - lowestMatchIndex < MAX_DISTANCE) {
@@ -339,6 +367,7 @@ LZ4HC_InsertAndGetWiderMatch (
             }
         }
     }
+
     return longest;
 }
 
@@ -1099,7 +1128,7 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
     int nbAttempts = maxNbAttempts;
     U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
-    int fromMStart = 0;
+    int matchChainPos = 0;
 
     DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
     /* First Match */
@@ -1109,13 +1138,11 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
                 matchIndex, lowestMatchIndex);
 
     /* find first match */
-    while ( (longest <= MINMATCH)
+    while ( 1 && (longest < MINMATCH)
       && (matchIndex>=lowestMatchIndex)
       && (nbAttempts) ) {
         const BYTE* const matchPtr = base + matchIndex;
         assert(matchIndex < ipIndex);
-        assert(matchIndex >= dictLimit);
-        assert(matchPtr >= lowPrefixPtr);
         assert(matchPtr < ip);
         assert(longest >= 1);
         nbAttempts--;
@@ -1124,7 +1151,6 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
             if (mlt > longest) {
                 longest = mlt;
                 *matchpos = matchPtr;
-                break;
         }   }
 
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
@@ -1132,39 +1158,44 @@ LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
         }
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
 
-    assert(longest > MINMATCH);
-    while ((matchIndex-fromMStart>=lowestMatchIndex) && (nbAttempts)) {
-        const BYTE* const matchPtr = base + matchIndex - fromMStart;
+    /* search longer matches */
+    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
+        const BYTE* const matchPtr = base + matchIndex;
+        int mlt = 0;
         assert(matchIndex < ipIndex);
-        assert(matchIndex >= dictLimit);
-        assert(matchPtr >= lowPrefixPtr);
         assert(matchPtr < ip);
-        assert(longest >= 1);
+        assert(longest >= MINMATCH);
         nbAttempts--;
         if (LZ4_read16(ip + longest - 1) == LZ4_read16(matchPtr + longest - 1)) {
             if (LZ4_read32(matchPtr) == pattern) {
-                int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+                mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
                 if (mlt > longest) {
                     longest = mlt;
                     *matchpos = matchPtr;
-                    matchIndex -= fromMStart;   /* beginning of match */
-                    if (1 && matchIndex + longest <= ipIndex) {
-                        U32 distanceToNextMatch = 1;
-                        int pos;
-                        for (pos = 0; pos <= longest - MINMATCH; pos++) {
-                            U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
-                            if (candidateDist > distanceToNextMatch) {
-                                distanceToNextMatch = candidateDist;
-                                fromMStart = pos;
-                            }
-                        }
-                        if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
-                        matchIndex -= distanceToNextMatch - fromMStart;
-                        continue;
+                }
+            }
+        }
+
+        /* search more promising chain */
+        if (mlt == longest) {
+            if (1 && matchIndex + longest <= ipIndex) {
+                U32 distanceToNextMatch = 1;
+                int pos;
+                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
+                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                    if (candidateDist > distanceToNextMatch) {
+                        distanceToNextMatch = candidateDist;
+                        matchChainPos = pos;
                     }
-        }   }   }
+                }
+                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                matchIndex -= distanceToNextMatch;
+                continue;
+            }
+        }
 
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
+        /* continue current chain */
+        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex+matchChainPos);
             matchIndex -= nextOffset;
         }
     }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
@@ -1190,8 +1221,8 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    //int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
-    int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    //int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */

From fa89a9e18b93741c65fa84c293d15298169d1ecb Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 13:31:03 -0700
Subject: [PATCH 249/257] lz4hc: fixed performance issue

when combining both PA and CS optimizations
---
 lib/lz4hc.c | 134 ++++++++--------------------------------------------
 1 file changed, 20 insertions(+), 114 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c58cd9de4..7e2fcb39d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -280,10 +280,29 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
+        if (mlt == longest) {  /* better match => select a better chain */
+            assert(longest >= MINMATCH);
+            if (1 && matchIndex + longest <= ipIndex) {
+                U32 distanceToNextMatch = 1;
+                int pos;
+                for (pos = 0; pos <= longest - MINMATCH; pos++) {
+                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
+                    if (candidateDist > distanceToNextMatch) {
+                        distanceToNextMatch = candidateDist;
+                        matchChainPos = pos;
+                    }
+                }
+                if (distanceToNextMatch > 1) {
+                    if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
+                    matchIndex -= distanceToNextMatch;
+                    continue;
+                }
+            }
+        }
+
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
             if (1 && patternAnalysis && distNextMatch==1 && matchChainPos==0) {
                 U32 const matchCandidateIdx = matchIndex-1;
-                //static U64 g_nbPatternAnalysis = 0; g_nbPatternAnalysis++; if (g_nbPatternAnalysis % (((g_nbPatternAnalysis/1000000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis=%llu ", g_nbPatternAnalysis);
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
                     if ( ((pattern & 0xFFFF) == (pattern >> 16))
@@ -301,7 +320,6 @@ LZ4HC_InsertAndGetWiderMatch (
                         const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
                         size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
-                        //static U64 g_nbPatternAnalysis_confirmed = 0; g_nbPatternAnalysis_confirmed++; if (g_nbPatternAnalysis_confirmed%(((g_nbPatternAnalysis_confirmed/100000)+1)*100)==0) DEBUGLOG(2, "g_nbPatternAnalysis_confirmed=%llu ", g_nbPatternAnalysis_confirmed);
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
@@ -314,25 +332,6 @@ LZ4HC_InsertAndGetWiderMatch (
                 }   }
         }   }
 
-        if (mlt == longest) {  /* better match => select a better chain */
-            assert(longest >= MINMATCH);
-            if (1 && matchIndex + longest <= ipIndex) {
-                U32 distanceToNextMatch = 1;
-                int pos;
-                //static U64 g_nbChainSwapper = 0; g_nbChainSwapper++; if (g_nbChainSwapper%(((g_nbChainSwapper/100000)+1)*100)==0) DEBUGLOG(2, "g_nbChainSwapper=%llu ", g_nbChainSwapper);
-                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
-                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
-                    if (candidateDist > distanceToNextMatch) {
-                        distanceToNextMatch = candidateDist;
-                        matchChainPos = pos;
-                    }
-                }
-                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
-                matchIndex -= distanceToNextMatch;
-                continue;
-            }
-        }
-
         /* follow current chain */
         matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos);
 
@@ -1112,98 +1111,6 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 }
 
 
-int
-LZ4HC_FindLongestMatch (LZ4HC_CCtx_internal* hc4,
-                        const BYTE* const ip,
-                        const BYTE* const iHighLimit,
-                        int longest,
-                        const BYTE** matchpos,
-                        const int maxNbAttempts)
-{
-    U16* const chainTable = hc4->chainTable;
-    U32* const HashTable = hc4->hashTable;
-    const BYTE* const base = hc4->base;
-    const U32 ipIndex = (U32)(ip - base);
-    const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE;
-    int nbAttempts = maxNbAttempts;
-    U32 const pattern = LZ4_read32(ip);
-    U32 matchIndex;
-    int matchChainPos = 0;
-
-    DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
-    /* First Match */
-    LZ4HC_Insert(hc4, ip);
-    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
-    DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",
-                matchIndex, lowestMatchIndex);
-
-    /* find first match */
-    while ( 1 && (longest < MINMATCH)
-      && (matchIndex>=lowestMatchIndex)
-      && (nbAttempts) ) {
-        const BYTE* const matchPtr = base + matchIndex;
-        assert(matchIndex < ipIndex);
-        assert(matchPtr < ip);
-        assert(longest >= 1);
-        nbAttempts--;
-        if (LZ4_read32(matchPtr) == pattern) {
-            int const mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-            if (mlt > longest) {
-                longest = mlt;
-                *matchpos = matchPtr;
-        }   }
-
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
-            matchIndex -= nextOffset;
-        }
-    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
-
-    /* search longer matches */
-    while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        const BYTE* const matchPtr = base + matchIndex;
-        int mlt = 0;
-        assert(matchIndex < ipIndex);
-        assert(matchPtr < ip);
-        assert(longest >= MINMATCH);
-        nbAttempts--;
-        if (LZ4_read16(ip + longest - 1) == LZ4_read16(matchPtr + longest - 1)) {
-            if (LZ4_read32(matchPtr) == pattern) {
-                mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-                if (mlt > longest) {
-                    longest = mlt;
-                    *matchpos = matchPtr;
-                }
-            }
-        }
-
-        /* search more promising chain */
-        if (mlt == longest) {
-            if (1 && matchIndex + longest <= ipIndex) {
-                U32 distanceToNextMatch = 1;
-                int pos;
-                for (pos = matchChainPos; pos <= longest - MINMATCH; pos++) {
-                    U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + pos);
-                    if (candidateDist > distanceToNextMatch) {
-                        distanceToNextMatch = candidateDist;
-                        matchChainPos = pos;
-                    }
-                }
-                if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
-                matchIndex -= distanceToNextMatch;
-                continue;
-            }
-        }
-
-        /* continue current chain */
-        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex+matchChainPos);
-            matchIndex -= nextOffset;
-        }
-    }  /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
-
-    return longest;
-}
-
-
 typedef struct {
     int off;
     int len;
@@ -1222,7 +1129,6 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
     int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
-    //int matchLength = LZ4HC_FindLongestMatch(ctx, ip, iHighLimit, minLen, &matchPtr, nbSearches); (void)dict;
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */

From d097bf93f853ddefdd05321d65cf7b90e3171c14 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 13:46:45 -0700
Subject: [PATCH 250/257] fixed SC.opt integration with regular HC parser

Only enabled when searching forward.

note : it slighly improves compression ratio,
but measurably decreases speed.
Trade-off to analyse.
---
 lib/lz4hc.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 7e2fcb39d..0f37f4235 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -280,9 +280,9 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
-        if (mlt == longest) {  /* better match => select a better chain */
-            assert(longest >= MINMATCH);
-            if (1 && matchIndex + longest <= ipIndex) {
+        if ( lookBackLength==0  /* search forward only */
+          && mlt==longest) {    /* better match => select a better chain */
+            if (matchIndex + longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;
                 int pos;
                 for (pos = 0; pos <= longest - MINMATCH; pos++) {
@@ -301,7 +301,7 @@ LZ4HC_InsertAndGetWiderMatch (
         }
 
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
-            if (1 && patternAnalysis && distNextMatch==1 && matchChainPos==0) {
+            if (patternAnalysis && distNextMatch==1 && matchChainPos==0) {
                 U32 const matchCandidateIdx = matchIndex-1;
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {

From a4e918d7a6e85f825f4cb1ac1c08a2cf2fc22194 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 14:10:30 -0700
Subject: [PATCH 251/257] lz4hc: SC only enabled for opt parser

the trade off is not good for regular HC parser :
compression is a little bit better, but speed cost is too large in comparison.
---
 lib/lz4hc.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0f37f4235..d29a989d7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -213,6 +213,7 @@ LZ4HC_InsertAndGetWiderMatch (
     const BYTE** startpos,
     const int maxNbAttempts,
     const int patternAnalysis,
+    const int chainSwap,
     const dictCtx_directive dict,
     const HCfavor_e favorDecSpeed)
 {
@@ -280,8 +281,8 @@ LZ4HC_InsertAndGetWiderMatch (
                     *startpos = ip + back;
         }   }   }
 
-        if ( lookBackLength==0  /* search forward only */
-          && mlt==longest) {    /* better match => select a better chain */
+        if (chainSwap && mlt==longest) {    /* better match => select a better chain */
+            assert(lookBackLength==0);   /* search forward only */
             if (matchIndex + longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;
                 int pos;
@@ -327,7 +328,6 @@ LZ4HC_InsertAndGetWiderMatch (
                         } else {
                             matchIndex = matchCandidateIdx - (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                         }
-                        matchChainPos = 0;
                         continue;
                 }   }
         }   }
@@ -382,7 +382,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, 0 /* chainSwap */, patternAnalysis, dict, favorCompressionRatio);
 }
 
 
@@ -515,7 +515,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (ip+ml <= mflimit) {
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
         } else {
             ml2 = ml;
         }
@@ -560,7 +560,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (start2 + ml2 <= mflimit) {
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
         } else {
             ml3 = ml2;
         }
@@ -1128,7 +1128,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /* patternAnalysis */, dict, favorDecSpeed);
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*chainSwap*/, 1 /*patternAnalysis*/, dict, favorDecSpeed);
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */

From cdb0275b7f7513e13a05c27adea658614c0174d9 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 14:32:57 -0700
Subject: [PATCH 252/257] lz4hc: fixed PA / SC parameter order

also :
reserved PA for levels 9+ (instead of 8+).
In most cases, speed is lower, and compression benefit is not worth.
---
 lib/lz4hc.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d29a989d7..1b3596f35 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -382,7 +382,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, 0 /* chainSwap */, patternAnalysis, dict, favorCompressionRatio);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio);
 }
 
 
@@ -477,7 +477,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
     )
 {
     const int inputSize = *srcSizePtr;
-    const int patternAnalysis = (maxNbAttempts > 64);   /* levels 8+ */
+    const int patternAnalysis = (maxNbAttempts > 128);   /* levels 9+ */
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
@@ -515,7 +515,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (ip+ml <= mflimit) {
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2,
-                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio);
         } else {
             ml2 = ml;
         }
@@ -560,7 +560,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
         if (start2 + ml2 <= mflimit) {
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
                             start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3,
-                            maxNbAttempts, 0, patternAnalysis, dict, favorCompressionRatio);
+                            maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio);
         } else {
             ml3 = ml2;
         }
@@ -1128,7 +1128,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*chainSwap*/, 1 /*patternAnalysis*/, dict, favorDecSpeed);
+    int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed);
     if (matchLength <= minLen) return match;
     if (favorDecSpeed) {
         if ((matchLength>18) & (matchLength<=36)) matchLength=18;   /* favor shortcut */

From af127334670a5e7b710bbd6adb71aa7c3ef0cd72 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 18:24:11 -0700
Subject: [PATCH 253/257] fixed frametest error

The error can be reproduced using following command :
./frametest -v -i100000000 -s1659 -t31096808

It's actually a bug in the stream LZ4 API,
when starting a new stream
and providing a first chunk to complete with size < MINMATCH.
In which case, the chunk becomes a dictionary.
No hash was generated and stored,
but the chunk is accessible as default position 0 points to dictStart,
and position 0 is still within MAX_DISTANCE.
Then, next attempt to read 32-bits from position 0 fails.

The issue would have been mitigated by starting from index 64 KB,
effectively eliminating position 0 as too far away.

The proper fix is to eliminate such "dictionary" as too small.
Which is what this patch does.
---
 lib/lz4.c         | 13 ++++++++++++-
 lib/lz4frame.c    |  1 +
 tests/frametest.c |  8 ++++++--
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 3860c5173..9a922b54d 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -767,6 +767,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
                 } else if (dictDirective==usingExtDict) {
                     if (matchIndex < startIndex) {
                         DEBUGLOG(7, "extDict candidate: matchIndex=%5u  <  startIndex=%5u", matchIndex, startIndex);
+                        assert(startIndex - matchIndex >= MINMATCH);
                         match = dictBase + matchIndex;
                         lowLimit = dictionary;
                     } else {
@@ -989,6 +990,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     if (outputLimited == fillOutput) {
         *inputConsumed = (int) (((const char*)ip)-source);
     }
+    DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, (int)(((char*)op) - dest));
     return (int)(((char*)op) - dest);
 }
 
@@ -1255,12 +1257,19 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
 {
     const tableType_t tableType = byU32;
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+    const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
     LZ4_renormDictT(streamPtr, inputSize);   /* avoid index overflow */
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
 
+    /* invalidate tiny dictionaries */
+    if (streamPtr->dictSize < 4) {
+        streamPtr->dictSize = 0;
+        streamPtr->dictionary = (const BYTE*)source;
+        dictEnd = (const BYTE*)source;
+    }
+
     /* Check overlapping input/dictionary space */
     {   const BYTE* sourceEnd = (const BYTE*) source + inputSize;
         if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) {
@@ -1402,6 +1411,8 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
     const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
 
+    DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i)", srcSize);
+
     /* Special cases */
     if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                      /* targetOutputSize too high => just decode everything */
     if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1;  /* Empty output buffer */
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index aa7889bf6..4e408161b 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -808,6 +808,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
+    DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
diff --git a/tests/frametest.c b/tests/frametest.c
index 7d69ff79a..21d883d91 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -740,8 +740,9 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un
     size_t p=0;
     const BYTE* b1=(const BYTE*)buff1;
     const BYTE* b2=(const BYTE*)buff2;
+    DISPLAY("locateBuffDiff: looking for error position \n");
     if (nonContiguous) {
-        DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
+        DISPLAY("mode %u: non-contiguous output (%zu bytes), cannot search \n", nonContiguous, size);
         return;
     }
     while (p < size && b1[p]==b2[p]) p++;
@@ -838,6 +839,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 size_t const iSize = MIN(sampleMax, (size_t)(iend-ip));
                 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
                 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
+                DISPLAYLEVEL(6, "Sending %zi bytes to compress (stableSrc:%u) \n",
+                                iSize, cOptions.stableSrc);
 
                 result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
                 CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result));
@@ -882,7 +885,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 dOptions.stableDst = FUZ_rand(&randState) & 1;
                 if (nonContiguousDst==2) dOptions.stableDst = 0;   /* overwrite mode */
                 result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
-                if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
+                if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid)
+                    locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
                 CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName(result));
                 XXH64_update(&xxh64, op, (U32)oSize);
                 totalOut += oSize;

From d7b6c726edb23fc19757dc84d4546863557bfe20 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sat, 5 May 2018 19:59:00 -0700
Subject: [PATCH 254/257] small extDict : fixed side-effect

don't fix dictionaries of size 0.
setting dictEnd == source triggers prefix mode,
thus removing possibility to use CDict.
---
 lib/lz4.c         | 9 ++++++---
 lib/lz4frame.c    | 1 +
 tests/frametest.c | 1 +
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 9a922b54d..e51a3e0a4 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -683,13 +683,12 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
     /* Init conditions */
     if (outputLimited == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;   /* Unsupported inputSize, too large (or negative) */
+    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;  /* Size too large (not within 64K limit) */
     if (tableType==byPtr) assert(dictDirective==noDict);      /* only supported use case with byPtr */
     assert(acceleration >= 1);
 
     lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0);
 
-    if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0;  /* Size too large (not within 64K limit) */
-
     /* Update context state */
     if (dictDirective == usingDictCtx) {
         /* Subsequent linked blocks can't use the dictionary. */
@@ -1259,12 +1258,16 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch
     LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse;
     const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
+    DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize);
+
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
     LZ4_renormDictT(streamPtr, inputSize);   /* avoid index overflow */
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
 
     /* invalidate tiny dictionaries */
-    if (streamPtr->dictSize < 4) {
+    if ( (streamPtr->dictSize-1 < 4)   /* intentional underflow */
+      && (dictEnd != (const BYTE*)source) ) {
+        DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary);
         streamPtr->dictSize = 0;
         streamPtr->dictionary = (const BYTE*)source;
         dictEnd = (const BYTE*)source;
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4e408161b..e1d0b1d02 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -472,6 +472,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
     const char* dictStart = (const char*)dictBuffer;
     LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
+    DEBUGLOG(4, "LZ4F_createCDict");
     if (!cdict) return NULL;
     if (dictSize > 64 KB) {
         dictStart += dictSize - 64 KB;
diff --git a/tests/frametest.c b/tests/frametest.c
index 21d883d91..4efeb6f0c 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -520,6 +520,7 @@ int basicTests(U32 seed, double compressibility)
         LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
         if (cdict == NULL) goto _output_error;
         CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+        
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
         CHECK_V(cSizeNoDict,
                 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,

From 24b9c485db34743f83496038d61598d41c3c497e Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 6 May 2018 16:47:31 -0700
Subject: [PATCH 255/257] small PA optimization

which measurably improves speed
on levels 9+
---
 lib/lz4hc.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 1b3596f35..059c4b579 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -291,15 +291,12 @@ LZ4HC_InsertAndGetWiderMatch (
                     if (candidateDist > distanceToNextMatch) {
                         distanceToNextMatch = candidateDist;
                         matchChainPos = pos;
-                    }
-                }
+                }   }
                 if (distanceToNextMatch > 1) {
                     if (distanceToNextMatch > matchIndex) break;   /* avoid overflow */
                     matchIndex -= distanceToNextMatch;
                     continue;
-                }
-            }
-        }
+        }   }   }
 
         {   U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex);
             if (patternAnalysis && distNextMatch==1 && matchChainPos==0) {
@@ -309,7 +306,7 @@ LZ4HC_InsertAndGetWiderMatch (
                     if ( ((pattern & 0xFFFF) == (pattern >> 16))
                       &  ((pattern & 0xFF)   == (pattern >> 24)) ) {
                         repeat = rep_confirmed;
-                        srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4;
+                        srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
                     } else {
                         repeat = rep_not;
                 }   }
@@ -318,19 +315,29 @@ LZ4HC_InsertAndGetWiderMatch (
                     const BYTE* const matchPtr = base + matchCandidateIdx;
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
                         size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
-                        const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
-                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
+                        const BYTE* const lowestMatchPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
+                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
                             matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
                         } else {
-                            matchIndex = matchCandidateIdx - (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
-                        }
+                            matchIndex = matchCandidateIdx - (U32)backLength;   /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */
+                            if (lookBackLength==0) {  /* no back possible */
+                                size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
+                                if ((size_t)longest < maxML) {
+                                    longest = maxML;
+                                    *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
+                                    *startpos = ip;
+                                }
+                                {   U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex);
+                                    if (distToNextPattern > matchIndex) break;  /* avoid overflow */
+                                    matchIndex -= distToNextPattern;
+                        }   }   }
                         continue;
                 }   }
-        }   }
+        }   }   /* PA optimization */
 
         /* follow current chain */
         matchIndex -= DELTANEXTU16(chainTable, matchIndex+matchChainPos);

From 200b2960d5a5d0fe76d34bb826c236c7c2941563 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Sun, 6 May 2018 18:26:14 -0700
Subject: [PATCH 256/257] fixed minor conversion warning

---
 lib/lz4hc.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 059c4b579..b89983fe0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -327,7 +327,8 @@ LZ4HC_InsertAndGetWiderMatch (
                             if (lookBackLength==0) {  /* no back possible */
                                 size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
                                 if ((size_t)longest < maxML) {
-                                    longest = maxML;
+                                    assert(maxML < 2 GB);
+                                    longest = (int)maxML;
                                     *matchpos = base + matchIndex;   /* virtual pos, relative to ip, to retrieve offset */
                                     *startpos = ip;
                                 }

From ba1c7148a56029aa01795bfcf8cc365a2551f968 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 7 May 2018 12:14:26 -0700
Subject: [PATCH 257/257] renamed variable for clarity

---
 lib/lz4hc.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b89983fe0..8108ea011 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -243,7 +243,7 @@ LZ4HC_InsertAndGetWiderMatch (
                 matchIndex, lowestMatchIndex);
 
     while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
-        int mlt=0;
+        int matchLength=0;
         nbAttempts--;
         assert(matchIndex < ipIndex);
         if (favorDecSpeed && (ipIndex - matchIndex < 8)) {
@@ -256,10 +256,10 @@ LZ4HC_InsertAndGetWiderMatch (
             if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) {
                 if (LZ4_read32(matchPtr) == pattern) {
                     int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr) : 0;
-                    mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-                    mlt -= back;
-                    if (mlt > longest) {
-                        longest = mlt;
+                    matchLength = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+                    matchLength -= back;
+                    if (matchLength > longest) {
+                        longest = matchLength;
                         *matchpos = matchPtr + back;
                         *startpos = ip + back;
             }   }   }
@@ -270,18 +270,18 @@ LZ4HC_InsertAndGetWiderMatch (
                 int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
-                mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
-                if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
-                    mlt += LZ4_count(ip+mlt, lowPrefixPtr, iHighLimit);
+                matchLength = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+                if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
+                    matchLength += LZ4_count(ip+matchLength, lowPrefixPtr, iHighLimit);
                 back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0;
-                mlt -= back;
-                if (mlt > longest) {
-                    longest = mlt;
+                matchLength -= back;
+                if (matchLength > longest) {
+                    longest = matchLength;
                     *matchpos = base + matchIndex + back;   /* virtual pos, relative to ip, to retrieve offset */
                     *startpos = ip + back;
         }   }   }
 
-        if (chainSwap && mlt==longest) {    /* better match => select a better chain */
+        if (chainSwap && matchLength==longest) {    /* better match => select a better chain */
             assert(lookBackLength==0);   /* search forward only */
             if (matchIndex + longest <= ipIndex) {
                 U32 distanceToNextMatch = 1;