Merge pull request lz4#692 from lz4/dev

v1.9.1

.travis.yml

@@ -8,7 +8,7 @@ matrix:
       os: osx
       compiler: clang
       script:
-        - make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion'
+        - make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console
         - CFLAGS=-m32 make -C tests clean test-lz4-contentSize
 
     # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)

Makefile

@@ -35,16 +35,7 @@ PRGDIR  = programs
 TESTDIR = tests
 EXDIR   = examples
 
-
-# Define nul output
-ifneq (,$(filter Windows%,$(OS)))
-EXT  = .exe
-VOID = nul
-else
-EXT  =
-VOID = /dev/null
-endif
-
+include Makefile.inc
 
 .PHONY: default
 default: lib-release lz4-release
@@ -93,7 +84,7 @@ clean:
 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
 #-----------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW32_NT-6.1 MINGW64_NT-6.1 MINGW32_NT-10.0 MINGW64_NT-10.0))
+ifeq ($(POSIX_ENV),Yes)
 HOST_OS = POSIX
 
 .PHONY: install uninstall
@@ -181,6 +172,14 @@ gpptest gpptest32: clean
 	CC=$(CC) $(MAKE) -C $(PRGDIR)  all CFLAGS="$(CFLAGS)"
 	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
 
+cxx17build : CC = "$(CXX) -Wno-deprecated"
+cxx17build : CFLAGS = -std=c++17 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror -pedantic
+cxx17build : clean
+	$(CXX) -v
+	CC=$(CC) $(MAKE) -C $(LZ4DIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(PRGDIR)  all CFLAGS="$(CFLAGS)"
+	CC=$(CC) $(MAKE) -C $(TESTDIR) all CFLAGS="$(CFLAGS)"
+
 ctocpptest: LIBCC="$(CC)"
 ctocpptest: TESTCC="$(CXX)"
 ctocpptest: CFLAGS=""

Makefile.inc

@@ -0,0 +1,87 @@
+ifeq ($(V), 1)
+Q =
+else
+Q = @
+endif
+
+TARGET_OS ?= $(shell uname)
+ifeq ($(TARGET_OS),)
+  TARGET_OS ?= $(OS)
+endif
+
+ifneq (,$(filter Windows%,$(TARGET_OS)))
+LIBLZ4 = liblz4-$(LIBVER_MAJOR)
+LIBLZ4_EXP = liblz4.lib
+WINBASED   = yes
+else
+LIBLZ4_EXP = liblz4.dll.a
+  ifneq (,$(filter MINGW%,$(TARGET_OS)))
+LIBLZ4     = liblz4
+WINBASED   = yes
+  else
+    ifneq (,$(filter MSYS%,$(TARGET_OS)))
+LIBLZ4     = msys-lz4-$(LIBVER_MAJOR)
+WINBASED   = yes
+    else
+      ifneq (,$(filter CYGWIN%,$(TARGET_OS)))
+LIBLZ4     = cyglz4-$(LIBVER_MAJOR)
+WINBASED   = yes
+      else
+LIBLZ4     = liblz4.$(SHARED_EXT_VER)
+WINBASED   = no
+EXT        =
+      endif
+    endif
+  endif
+endif
+
+ifeq ($(WINBASED),yes)
+EXT        = .exe
+WINDRES    = windres
+endif
+
+#determine if dev/nul based on host environment
+ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
+VOID := /dev/null
+else
+  ifneq (,$(filter Windows%,$(OS)))
+VOID := nul
+  else
+VOID  := /dev/null
+  endif
+endif
+
+ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW% CYGWIN% MSYS%,$(shell uname)))
+POSIX_ENV = Yes
+else
+POSIX_ENV = No
+endif
+
+# Avoid symlinks when targetting Windows or building on a Windows host
+ifeq ($(WINBASED),yes)
+LN_S  = cp -p
+LN_SF = cp -p
+else
+  ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell uname)))
+LN_S  = cp -p
+LN_SF = cp -p
+  else
+    ifneq (,$(filter Windows%,$(OS)))
+LN_S  = cp -p
+LN_SF = cp -p
+    else
+LN_S   = ln -s
+LN_SF  = ln -sf
+    endif
+  endif
+endif
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL) -m 755
+INSTALL_DATA    ?= $(INSTALL) -m 644
+INSTALL_DIR     ?= $(INSTALL) -d -m 755

NEWS

@@ -1,3 +1,10 @@
+v1.9.1
+fix : decompression functions were reading a few bytes beyond input size (introduced in v1.9.0, reported by @ppodolsky and @danlark1)
+api : fix : lz4frame initializers compatibility with c++, reported by @degski
+cli : added command --list, based on a patch by @gabrielstedman
+build: improved Windows build, by @JPeterMugaas
+build: AIX, by Norman Green
+
 v1.9.0
 perf: large decompression speed improvement on x86/x64 (up to +20%) by @djwatson
 api : changed : _destSize() compression variants are promoted to stable API

README.md

@@ -68,7 +68,7 @@ in single-thread mode.
 | [Zstandard] 1.4.0 -1    |  2.883  |   515 MB/s  |   1380 MB/s   |
 |  LZF v3.6               |  2.073  |   415 MB/s  |    910 MB/s   |
 | [zlib] deflate 1.2.11 -1|  2.730  |   100 MB/s  |    415 MB/s   |
-|**LZ4 HC -9 (v1.8.2)**   |**2.721**|    41 MB/s  | **4900 MB/s** |
+|**LZ4 HC -9 (v1.9.0)**   |**2.721**|    41 MB/s  | **4900 MB/s** |
 | [zlib] deflate 1.2.11 -6|  3.099  |    36 MB/s  |    445 MB/s   |
 
 [zlib]: http://www.zlib.net/

doc/lz4_manual.html

@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.9.0 Manual</title>
+<title>1.9.1 Manual</title>
 </head>
 <body>
-<h1>1.9.0 Manual</h1>
+<h1>1.9.1 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -454,21 +454,18 @@ <h1>1.9.0 Manual</h1>
 
 </p></pre><BR>
 
-<pre><b>LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") LZ4LIB_API
-int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
-LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") LZ4LIB_API
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
-LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") LZ4LIB_API
-int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
-</b><p>  These functions used to be a bit faster than LZ4_decompress_safe(),
-  but situation has changed in recent versions.
-  Now, `LZ4_decompress_safe()` is as fast and sometimes even faster than `LZ4_decompress_fast()`.
-  Moreover, LZ4_decompress_safe() is protected vs malformed input, while `LZ4_decompress_fast()` is not, making it a security liability.
+<pre><b></b><p>  These functions used to be faster than LZ4_decompress_safe(),
+  but it has changed, and they are now slower than LZ4_decompress_safe().
+  This is because LZ4_decompress_fast() doesn't know the input size,
+  and therefore must progress more cautiously in the input buffer to not read beyond the end of block.
+  On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
   As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
 
-  Last LZ4_decompress_fast() specificity is that it can decompress a block without knowing its compressed size.
-  Note that even that functionality could be achieved in a more secure manner if need be,
-  though it would require new prototypes, and adaptation of the implementation to this new use case.
+  The last remaining LZ4_decompress_fast() specificity is that
+  it can decompress a block without knowing its compressed size.
+  Such functionality could be achieved in a more secure manner,
+  by also providing the maximum size of input buffer,
+  but it would require new prototypes, and adaptation of the implementation to this new use case.
 
   Parameters:
   originalSize : is the uncompressed size to regenerate.
@@ -477,9 +474,9 @@ <h1>1.9.0 Manual</h1>
            The function expects to finish at block's end exactly.
            If the source stream is detected malformed, the function stops decoding and returns a negative result.
   note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
-         However, since it doesn't know its 'src' size, it may read an unknown amount of input, and overflow input buffer.
-         Also, since match offsets are not validated, match reads from 'src' may underflow.
-         These issues never happen if input data is correct.
+         However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
+         Also, since match offsets are not validated, match reads from 'src' may underflow too.
+         These issues never happen if input (compressed) data is correct.
          But they may happen if input data is invalid (error or intentional tampering).
          As a consequence, use these functions in trusted environments with trusted data **only**.
 

doc/lz4frame_manual.html

@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>1.9.0 Manual</title>
+<title>1.9.1 Manual</title>
 </head>
 <body>
-<h1>1.9.0 Manual</h1>
+<h1>1.9.1 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>

examples/Makefile

@@ -36,16 +36,7 @@ TESTFILE  = Makefile
 LZ4DIR   := ../lib
 LZ4       = ../programs/lz4
 
-
-# Define *.exe as extension for Windows systems
-ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
-VOID = nul
-else
-EXT =
-VOID = /dev/null
-endif
-
+include ../Makefile.inc
 
 default: all