diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 780990ec2..b238308d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,7 @@ repos: -- repo: https://github.com/pycqa/isort - rev: '5.12.0' - hooks: - - id: isort - exclude: blosc2/c-blosc2 -- repo: https://github.com/ambv/black - rev: 23.1.0 - hooks: - - id: black - language_version: python3 - exclude: blosc2/c-blosc2 -- repo: https://github.com/charliermarsh/ruff-pre-commit - # Ruff version. - rev: 'v0.0.252' +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.1.7' hooks: - id: ruff + - id: ruff-format exclude: blosc2/c-blosc2 diff --git a/bench/compress_numpy.py b/bench/compress_numpy.py index 3811f98e0..219b32c10 100644 --- a/bench/compress_numpy.py +++ b/bench/compress_numpy.py @@ -39,9 +39,7 @@ np.copyto(out_, in_) tcpy = (time.time() - t0) / NREP print( - " *** np.copyto() *** Time for memcpy():\t{:.3f} s\t({:.2f} GB/s)".format( - tcpy, (N * 8 / tcpy) / 2**30 - ) + " *** np.copyto() *** Time for memcpy():\t{:.3f} s\t({:.2f} GB/s)".format(tcpy, (N * 8 / tcpy) / 2**30) ) print("\nTimes for compressing/decompressing:") diff --git a/bench/pack_compress.py b/bench/pack_compress.py index cbec1f699..2ca361ee3 100644 --- a/bench/pack_compress.py +++ b/bench/pack_compress.py @@ -39,8 +39,9 @@ toc = time.time() tcpy = (toc - tic) / NREP print( - " Time for copying array with np.copy: {:.3f} s ({:.2f} GB/s))" - .format(tcpy, ((N * 8 / tcpy) / 2**30)) + " Time for copying array with np.copy: {:.3f} s ({:.2f} GB/s))".format( + tcpy, ((N * 8 / tcpy) / 2**30) + ) ) if comprehensive_copy_timing: @@ -51,8 +52,9 @@ toc = time.time() tcpy = (toc - tic) / NREP print( - " Time for copying array with np.copyto and empty_like: {:.3f} s ({:.2f} GB/s))" - .format(tcpy, ((N * 8 / tcpy) / 2**30)) + " Time for copying array with np.copyto and empty_like: {:.3f} s ({:.2f} GB/s))".format( + tcpy, ((N * 8 / tcpy) / 2**30) + ) ) # Unlike numpy.zeros, numpy.zeros_like doesn't use calloc, but instead uses @@ -66,8 +68,9 @@ toc = time.time() tcpy = (toc - tic) / NREP print( - " Time for copying array with np.copyto and zeros: {:.3f} s ({:.2f} GB/s))" - .format(tcpy, ((N * 8 / tcpy) / 2**30)) + " Time for copying array with np.copyto and zeros: {:.3f} s ({:.2f} GB/s))".format( + tcpy, ((N * 8 / tcpy) / 2**30) + ) ) # Cause a page fault before the benchmark @@ -78,8 +81,9 @@ toc = time.time() tcpy = (toc - tic) / NREP print( - " Time for copying array with np.copyto and full_like: {:.3f} s ({:.2f} GB/s))" - .format(tcpy, ((N * 8 / tcpy) / 2**30)) + " Time for copying array with np.copyto and full_like: {:.3f} s ({:.2f} GB/s))".format( + tcpy, ((N * 8 / tcpy) / 2**30) + ) ) tic = time.time() @@ -89,8 +93,9 @@ toc = time.time() tcpy = (toc - tic) / NREP print( - " Time for copying array with numpy assignment: {:.3f} s ({:.2f} GB/s))" - .format(tcpy, ((N * 8 / tcpy) / 2**30)) + " Time for copying array with numpy assignment: {:.3f} s ({:.2f} GB/s))".format( + tcpy, ((N * 8 / tcpy) / 2**30) + ) ) print() @@ -122,8 +127,9 @@ tc = (ctoc - ctic) / NREP td = (dtoc - dtic) / NREP print( - " Time for compress/decompress: {:.3f}/{:.3f} s ({:.2f}/{:.2f} GB/s)) " - .format(tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30)), + " Time for compress/decompress: {:.3f}/{:.3f} s ({:.2f}/{:.2f} GB/s)) ".format( + tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30) + ), end="", ) print("\tcr: {:5.1f}x".format(in_.size * in_.dtype.itemsize * 1.0 / len(c))) @@ -141,8 +147,9 @@ tc = (ctoc - ctic) / NREP td = (dtoc - dtic) / NREP print( - " Time for pack_array2/unpack_array2: {:.3f}/{:.3f} s ({:.2f}/{:.2f} GB/s)) " - .format(tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30)), + " Time for pack_array2/unpack_array2: {:.3f}/{:.3f} s ({:.2f}/{:.2f} GB/s)) ".format( + tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30) + ), end="", ) print("\tcr: {:5.1f}x".format(in_.size * in_.dtype.itemsize * 1.0 / len(c))) @@ -160,8 +167,9 @@ tc = (ctoc - ctic) / NREP td = (dtoc - dtic) / NREP print( - " Time for pack_tensor/unpack_tensor: {:.3f}/{:.3f} s ({:.2f}/{:.2f} GB/s)) " - .format(tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30)), + " Time for pack_tensor/unpack_tensor: {:.3f}/{:.3f} s ({:.2f}/{:.2f} GB/s)) ".format( + tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30) + ), end="", ) print("\tcr: {:5.1f}x".format(in_.size * in_.dtype.itemsize * 1.0 / len(c))) diff --git a/bench/pack_tensor.py b/bench/pack_tensor.py index 1d83c8b5b..257551674 100644 --- a/bench/pack_tensor.py +++ b/bench/pack_tensor.py @@ -48,7 +48,9 @@ ctoc = time.time() tc = (ctoc - ctic) / NREP print( - " Time for tensorflow (tf.io.serialize):\t[:.3f} s ({:.2f} GB/s)) ".format(tc, ((N * 8 / tc) / 2**30)), + " Time for tensorflow (tf.io.serialize):\t{:.3f} s ({:.2f} GB/s)) ".format( + tc, ((N * 8 / tc) / 2**30) + ), end="", ) print("\tcr: {:5.1f}x".format(in_.size * in_.dtype.itemsize * 1.0 / len(c))) @@ -65,7 +67,8 @@ ctoc = time.time() tc = (ctoc - ctic) / NREP print( - " Time for torch (torch.save):\t\t\t{:.3f} s ({:.2f} GB/s)) ".format(tc, ((N * 8 / tc) / 2**30)), end="" + " Time for torch (torch.save):\t\t\t{:.3f} s ({:.2f} GB/s)) ".format(tc, ((N * 8 / tc) / 2**30)), + end="", ) buff.seek(0) c = buff.read() @@ -85,7 +88,9 @@ ctoc = time.time() tc = (ctoc - ctic) / NREP print( - " Time for tensorflow (blosc2.pack_tensor):\t{:.3f} s ({:.2f} GB/s)) ".format(tc, ((N * 8 / tc) / 2**30)), + " Time for tensorflow (blosc2.pack_tensor):\t{:.3f} s ({:.2f} GB/s)) ".format( + tc, ((N * 8 / tc) / 2**30) + ), end="", ) print("\tcr: {:5.1f}x".format(in_.size * in_.dtype.itemsize * 1.0 / len(c))) @@ -101,7 +106,9 @@ ctoc = time.time() tc = (ctoc - ctic) / NREP print( - " Time for torch (blosc2.pack_tensor):\t\t{:.3f} s ({:.2f} GB/s)) ".format(tc, ((N * 8 / tc) / 2**30)), + " Time for torch (blosc2.pack_tensor):\t\t{:.3f} s ({:.2f} GB/s)) ".format( + tc, ((N * 8 / tc) / 2**30) + ), end="", ) print("\tcr: {:5.1f}x".format(in_.size * in_.dtype.itemsize * 1.0 / len(c))) @@ -120,7 +127,9 @@ dtoc = time.time() td = (dtoc - dtic) / NREP print( - " Time for tensorflow (tf.io.parse_tensor):\t{:.3f} s ({:.2f} GB/s)) ".format(td, ((N * 8 / td) / 2**30)), + " Time for tensorflow (tf.io.parse_tensor):\t{:.3f} s ({:.2f} GB/s)) ".format( + td, ((N * 8 / td) / 2**30) + ), ) with open("serialize_torch.bin", "rb") as f: @@ -147,8 +156,9 @@ dtoc = time.time() td = (dtoc - dtic) / NREP print( - " Time for tensorflow (blosc2.unpack_tensor):\t{:.3f} s ({:.2f} GB/s)) " - .format(td, ((N * 8 / td) / 2**30)), + " Time for tensorflow (blosc2.unpack_tensor):\t{:.3f} s ({:.2f} GB/s)) ".format( + td, ((N * 8 / td) / 2**30) + ), ) assert np.array_equal(in_, out) @@ -164,6 +174,8 @@ td = (dtoc - dtic) / NREP print( - " Time for torch (blosc2.unpack_tensor):\t{:.3f} s ({:.2f} GB/s)) ".format(td, ((N * 8 / td) / 2**30)), + " Time for torch (blosc2.unpack_tensor):\t{:.3f} s ({:.2f} GB/s)) ".format( + td, ((N * 8 / td) / 2**30) + ), ) assert np.array_equal(in_, out) diff --git a/blosc2/core.py b/blosc2/core.py index e53f0b22a..96b47933f 100644 --- a/blosc2/core.py +++ b/blosc2/core.py @@ -1115,13 +1115,14 @@ def get_chunksize(blocksize, l3_minimum=2**21, l3_maximum=2**25): # value only when it is an actual int. # Also, sometimes cpuinfo does not return a correct L3 size; # so in general, enforcing L3 > L2 is a good sanity check. - l2_cache_size = cpu_info.get("l2_cache_size", "Not found") - if type(l3_cache_size) is int and type(l2_cache_size) is int and l3_cache_size > l2_cache_size: - chunksize = l3_cache_size + if isinstance(l3_cache_size, int): + l2_cache_size = cpu_info.get("l2_cache_size", "Not found") + if isinstance(l2_cache_size, int) and l3_cache_size > l2_cache_size: + chunksize = l3_cache_size else: # Chunksize should be at least the size of L2 l2_cache_size = cpu_info.get("l2_cache_size", "Not found") - if type(l2_cache_size) is int and l2_cache_size > chunksize: + if isinstance(l2_cache_size, int) and l2_cache_size > chunksize: chunksize = l2_cache_size # Ensure a minimum size diff --git a/blosc2/info.py b/blosc2/info.py index 6f50ca95e..6c211ae7c 100644 --- a/blosc2/info.py +++ b/blosc2/info.py @@ -22,7 +22,7 @@ def info_text_report(items: list) -> str: max_key_len = max(len(k) for k in keys) report = "" for k, v in items: - if type(v) is dict: + if isinstance(v, dict): # rich way, this is disabled because it doesn't work well in the notebooks # with io.StringIO() as buf: # v_sorted = {k: val for k, val in sorted(v.items())} diff --git a/blosc2/ndarray.py b/blosc2/ndarray.py index eb035a260..3789d0b6b 100644 --- a/blosc2/ndarray.py +++ b/blosc2/ndarray.py @@ -368,9 +368,9 @@ def squeeze(self): def _check_shape(shape): - if type(shape) is int: + if isinstance(shape, int): shape = (shape,) - if type(shape) not in (tuple, list): + elif not isinstance(shape, (tuple, list)): raise ValueError("shape should be a tuple or a list!") return shape diff --git a/examples/btune.py b/examples/btune.py index e5a826427..cb4d486a8 100644 --- a/examples/btune.py +++ b/examples/btune.py @@ -11,10 +11,10 @@ # https://github.com/Blosc/blosc2_btune/blob/main/README.md ####################################################################### +import blosc2_btune import numpy as np import blosc2 -import blosc2_btune nchunks = 10 # Set the compression and decompression parameters, use BTUNE tuner @@ -28,7 +28,7 @@ # Set the Btune configuration to use btune_conf = {"tradeoff": 0.3, "perf_mode": blosc2_btune.PerformanceMode.DECOMP} -blosc2_btune.set_params_defaults(**kwargs) +blosc2_btune.set_params_defaults() # Create the SChunk data = np.arange(200 * 1000 * nchunks) diff --git a/pyproject.toml b/pyproject.toml index 800688100..b41e01bbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,12 +47,6 @@ documentation = "https://www.blosc.org/python-blosc2/python-blosc2.html" [tool.cibuildwheel] skip = "cp36-* cp37-* pp37-* cp38-* pp38-* *-manylinux_i686 *_ppc64le *_s390x" -[tool.black] -line-length = 109 - [tool.ruff] line-length = 109 - -[tool.isort] -profile = "black" -multi_line_output = 3 +extend-select = ["I"] diff --git a/tests/test_compress2.py b/tests/test_compress2.py index 1ae1d6711..1e35f8ca5 100644 --- a/tests/test_compress2.py +++ b/tests/test_compress2.py @@ -109,7 +109,7 @@ def test_raise_error(object, cparams, dparams, gil): blosc2.decompress2(c, dst=dest) dest3 = blosc2.decompress2(c) - if type(object) is bytes: + if isinstance(object, bytes): assert dest3 == object else: assert dest3 == object.tobytes() diff --git a/tests/test_decompress.py b/tests/test_decompress.py index d34f758c7..3ce9fd88f 100644 --- a/tests/test_decompress.py +++ b/tests/test_decompress.py @@ -83,13 +83,13 @@ def test_raise_error(object, codec): blosc2.decompress(c, dst=dest) dest3 = blosc2.decompress(c) - if type(object) is bytes: + if isinstance(object, bytes): assert dest3 == object else: assert dest3 == object.tobytes() dest4 = blosc2.decompress(c, as_bytearray=True) - if type(object) is bytes: + if isinstance(object, bytes): assert dest4 == object else: assert dest4 == object.tobytes() diff --git a/tests/test_pathlib.py b/tests/test_pathlib.py index d109710e1..7814d27d5 100644 --- a/tests/test_pathlib.py +++ b/tests/test_pathlib.py @@ -6,11 +6,10 @@ # LICENSE file in the root directory of this source tree) ####################################################################### -import os +import pathlib import numpy as np import pytest -import pathlib import blosc2 diff --git a/tests/test_ucodecs.py b/tests/test_ucodecs.py index 48df45eb5..d4ff8f071 100644 --- a/tests/test_ucodecs.py +++ b/tests/test_ucodecs.py @@ -142,11 +142,10 @@ def test_dynamic_ucodecs_error(cparams, dparams): chunk_len = 100 dtype = np.dtype(np.int32) nchunks = 1 - fill_value = 341 data = np.arange(chunk_len * nchunks, dtype=dtype) with pytest.raises(RuntimeError): - schunk = blosc2.SChunk( + _ = blosc2.SChunk( chunksize=chunk_len * dtype.itemsize, data=data, cparams=cparams, diff --git a/tests/test_ufilters.py b/tests/test_ufilters.py index 2690e4178..6b129f3b5 100644 --- a/tests/test_ufilters.py +++ b/tests/test_ufilters.py @@ -91,15 +91,17 @@ def backward2(input, output, meta, schunk): blosc2.remove_urlpath(urlpath) + @pytest.mark.parametrize( "cparams, dparams", [ ({"nthreads": 4, "filters": [255, blosc2.Filter.SHUFFLE], "filters_meta": [0, 0]}, {"nthreads": 1}), - ({"nthreads": 1, "filters": [255], "filters_meta": [4]}, {"nthreads": 4}) + ({"nthreads": 1, "filters": [255], "filters_meta": [4]}, {"nthreads": 4}), ], ) def test_pyufilters_error(cparams, dparams): dtype = np.dtype(np.int32) + def forward(input, output, meta, schunk): nd_input = input.view(dtype) nd_output = output.view(dtype) @@ -111,6 +113,7 @@ def backward(input, output, meta, schunk): nd_output = output.view(dtype) nd_output[:] = nd_input - 1 + if 255 not in blosc2.ufilters_registry: blosc2.register_filter(255, forward, backward) @@ -132,7 +135,7 @@ def backward(input, output, meta, schunk): "cparams, dparams", [ ({"nthreads": 4, "filters": [163, blosc2.Filter.SHUFFLE], "filters_meta": [0, 0]}, {"nthreads": 1}), - ({"nthreads": 1, "filters": [163], "filters_meta": [4]}, {"nthreads": 4}) + ({"nthreads": 1, "filters": [163], "filters_meta": [4]}, {"nthreads": 4}), ], ) def test_dynamic_ufilters_error(cparams, dparams):