Fall back to Python bindings when lz4 binary not available

Also, the dependency on python-lz4 is no longer optional, so there is no
need to test `if lz4 is None` everywhere

Author: marcelm

src/xopen/__init__.py

@@ -34,6 +34,8 @@
 )
 from types import ModuleType
 
+import lz4.frame  # type: ignore
+
 from ._version import version as __version__
 
 # 128K buffer size also used by cat, pigz etc. It is faster than the 8K default.
@@ -71,11 +73,6 @@
 except ImportError:
     zstandard = None  # type: ignore
 
-try:
-    import lz4.frame  # type: ignore
-except ImportError:
-    lz4 = None
-
 try:
     import fcntl
 
@@ -568,11 +565,11 @@ def _open_lz4(
     if compresslevel is None:
         compresslevel = XOPEN_DEFAULT_LZ4_COMPRESSION
 
-    if lz4 is not None and (mode == "rb" or (mode in ("ab", "wb") and threads == 0)):
-        # use Python bindings
+    if mode == "rb" or threads == 0:
+        # Use Python bindings
         f = lz4.frame.LZ4FrameFile(filename, mode, compression_level=compresslevel)
         return f
-    # use CLI program
+    # Use CLI program
     try:
         return _PipedCompressionProgram(
             filename,
@@ -582,15 +579,8 @@ def _open_lz4(
             program_settings=_PROGRAM_SETTINGS["lz4"],
         )
     except OSError:
-        _program_settings = _PROGRAM_SETTINGS["lz4"]
-        _program_settings.threads_flag = None
-        return _PipedCompressionProgram(
-            filename,
-            mode,
-            compresslevel,
-            threads,
-            program_settings=_program_settings,
-        )
+        # lz4 binary not available, fall back to Python bindings
+        return _open_lz4(filename, mode, compresslevel, threads=0)
 
 
 def _open_gz(

tests/test_xopen.py

@@ -1,6 +1,7 @@
 """
 Tests for the xopen.xopen function
 """
+
 import bz2
 import functools
 import gzip
@@ -19,10 +20,8 @@
 
 from xopen import _detect_format_from_content, xopen
 
-try:
-    import lz4.frame
-except ImportError:
-    lz4 = None
+import lz4.frame  # type: ignore
+
 try:
     import zstandard
 except ImportError:
@@ -113,8 +112,6 @@ def test_binary(fname):
 def test_roundtrip(ext, tmp_path, threads, mode):
     if ext == ".zst" and threads == 0 and zstandard is None:
         return
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     path = tmp_path / f"file{ext}"
     data = b"Hello" if mode == "b" else "Hello"
     with xopen(path, "w" + mode, threads=threads) as f:
@@ -209,8 +206,6 @@ def test_next(fname):
 
 
 def test_has_iter_method(ext, tmp_path):
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     path = tmp_path / f"out{ext}"
     with xopen(path, mode="w") as f:
         # Writing anything isn’t strictly necessary, but if we don’t, then
@@ -278,8 +273,6 @@ def test_invalid_compression_level(tmp_path):
 def test_append(ext, threads, tmp_path):
     if ext == ".zst" and zstandard is None and threads == 0:
         pytest.skip("No zstandard installed")
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     text = b"AB"
     reference = text + text
     path = tmp_path / f"the-file{ext}"
@@ -296,8 +289,6 @@ def test_append(ext, threads, tmp_path):
 
 @pytest.mark.parametrize("ext", extensions)
 def test_append_text(ext, tmp_path):
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     text = "AB"
     reference = text + text
     path = tmp_path / f"the-file{ext}"
@@ -377,22 +368,17 @@ def test_read_no_threads(ext):
         ".gz": gzip.GzipFile,
         ".xz": lzma.LZMAFile,
         ".zst": io.BufferedReader,
+        ".lz4": lz4.frame.LZ4FrameFile,
         "": io.BufferedReader,
     }
     if ext == ".zst" and zstandard is None:
         return
-    if ext == ".lz4" and lz4 is None:
-        return
-    if ext == ".lz4" and lz4.frame is not None:
-        klasses[".lz4"] = lz4.frame.LZ4FrameFile
     klass = klasses[ext]
     with xopen(TEST_DIR / f"file.txt{ext}", "rb", threads=0) as f:
         assert isinstance(f, klass), f
 
 
 def test_write_threads(tmp_path, ext):
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     path = tmp_path / f"out.{ext}"
     with xopen(path, mode="w", threads=3) as f:
         f.write("hello")
@@ -413,17 +399,13 @@ def test_write_no_threads(tmp_path, ext):
         ".bz2": bz2.BZ2File,
         ".gz": gzip.GzipFile,
         ".xz": lzma.LZMAFile,
+        ".lz4": lz4.frame.LZ4FrameFile,
         "": io.BufferedWriter,
     }
     if ext == ".zst":
         # Skip zst because if python-zstandard is not installed,
         # we fall back to an external process even when threads=0
         return
-    if ext == ".lz4" and lz4 is None:
-        # Skip lz4 if lz4 is not installed
-        return
-    if ext == ".lz4" and lz4.frame is not None:
-        klasses[".lz4"] = lz4.frame.LZ4FrameFile
     klass = klasses[ext]
     with xopen(tmp_path / f"out{ext}", "wb", threads=0) as f:
         if isinstance(f, io.BufferedWriter):
@@ -473,8 +455,6 @@ def test_read_pathlib_binary(fname):
 
 
 def test_write_pathlib(ext, tmp_path):
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     path = tmp_path / f"hello.txt{ext}"
     with xopen(path, mode="wt") as f:
         f.write("hello")
@@ -483,8 +463,6 @@ def test_write_pathlib(ext, tmp_path):
 
 
 def test_write_pathlib_binary(ext, tmp_path):
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     path = tmp_path / f"hello.txt{ext}"
     with xopen(path, mode="wb") as f:
         f.write(b"hello")
@@ -522,8 +500,6 @@ def test_falls_back_to_lzma_open(lacking_xz_permissions):
 
 
 def test_open_many_writers(tmp_path, ext):
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     files = []
     # Because lzma.open allocates a lot of memory,
     # open fewer files to avoid MemoryError on 32-bit architectures
@@ -569,8 +545,6 @@ def test_override_output_format_wrong_format(tmp_path):
 @pytest.mark.parametrize("opener", OPENERS)
 @pytest.mark.parametrize("extension", extensions)
 def test_text_encoding_newline_passthrough(opener, extension, tmp_path):
-    if extension == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     if extension == ".zst" and zstandard is None:
         return
     # "Eén ree\nTwee reeën\n" latin-1 encoded with \r for as line separator.
@@ -586,8 +560,6 @@ def test_text_encoding_newline_passthrough(opener, extension, tmp_path):
 @pytest.mark.parametrize("opener", OPENERS)
 @pytest.mark.parametrize("extension", extensions)
 def test_text_encoding_errors(opener, extension, tmp_path):
-    if extension == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     if extension == ".zst" and zstandard is None:
         return
     # "Eén ree\nTwee reeën\n" latin-1 encoded. This is not valid ascii.
@@ -673,11 +645,6 @@ def test_pass_bytesio_for_reading_and_writing(ext, threads):
         format = None
     if ext == ".zst" and zstandard is None:
         return
-    if ext == ".lz4" and lz4 is None and threads == 0:
-        pytest.skip("lz4 not working for BytesIO in piped write mode")
-    if ext == ".lz4" and threads != 0:
-        # _PipedCompressionProgram not working on write mode
-        pytest.skip("lz4 not working for BytesIO in piped write mode")
     first_line = CONTENT_LINES[0].encode("utf-8")
     writer = xopen(filelike, "wb", format=format, threads=threads)
     writer.write(first_line)
@@ -727,8 +694,6 @@ def test_xopen_read_from_pipe(ext, threads):
 
 @pytest.mark.parametrize("threads", (0, 1))
 def test_xopen_write_to_pipe(threads, ext):
-    if ext == ".lz4" and shutil.which("lz4") is None:
-        pytest.skip("lz4 not installed")
     if ext == ".zst" and zstandard is None:
         return
     format = ext.lstrip(".")