Package review PR

.github/workflows/ci.yml

@@ -0,0 +1,131 @@
+name: ci
+
+on:
+  push:
+  pull_request:
+
+env:
+  CARGO_TERM_COLOR: always
+  VALKEY_REPO_URL: https://github.com/valkey-io/valkey.git
+
+jobs:
+  build-ubuntu-latest:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        server_version: ['unstable', '8.0.0']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set the server verison for python integeration tests
+      run: echo "SERVER_VERSION=${{ matrix.server_version }}" >> $GITHUB_ENV
+    - name: Run cargo and clippy format check
+      run: |
+        cargo fmt --check
+        cargo clippy --profile release --all-targets -- -D clippy::all
+    - name: Release Build
+      run: |
+        if [ "${{ matrix.server_version }}" = "8.0.0" ]; then
+          RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
+        else
+          RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
+        fi
+    - name: Run unit tests
+      run: cargo test --features enable-system-alloc
+    - name: Make valkey-server binary
+      run: |
+        mkdir -p "tests/.build/binaries/${{ matrix.server_version }}"
+        cd tests/.build
+        git clone "${{ env.VALKEY_REPO_URL }}"
+        cd valkey
+        git checkout ${{ matrix.server_version }}
+        make -j
+        cp src/valkey-server ../binaries/${{ matrix.server_version }}/
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Update module path
+      run: echo "MODULE_PATH=$(realpath target/release/libvalkey_bloom.so)" >> $GITHUB_ENV
+    - name: Run integration tests 
+      run: python -m pytest --cache-clear -v "tests/" 
+
+  build-macos-latest:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Run cargo and clippy format check
+      run: |
+        cargo fmt --check
+        cargo clippy --profile release --all-targets -- -D clippy::all
+    - name: Release Build
+      run: RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
+    - name: Run unit tests
+      run: cargo test --features enable-system-alloc
+
+  asan-build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        server_version: ['unstable', '8.0.0']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set the server verison for python integeration tests
+      run: echo "SERVER_VERSION=${{ matrix.server_version }}" >> $GITHUB_ENV
+    - name: Run cargo and clippy format check
+      run: |
+        cargo fmt --check
+        cargo clippy --profile release --all-targets -- -D clippy::all
+    - name: Release Build
+      run: |
+        if [ "${{ matrix.server_version }}" = "8.0.0" ]; then
+          RUSTFLAGS="-D warnings" cargo build --all --all-targets --release --features valkey_8_0
+        else
+          RUSTFLAGS="-D warnings" cargo build --all --all-targets --release
+        fi
+    - name: Run unit tests
+      run: cargo test --features enable-system-alloc
+    - name: Make Valkey-server binary with asan
+      run: |
+        mkdir -p "tests/.build/binaries/${{ matrix.server_version }}"
+        cd tests/.build
+        git clone "${{ env.VALKEY_REPO_URL }}"
+        cd valkey
+        git checkout ${{ matrix.server_version }}
+        make distclean
+        make -j SANITIZER=address SERVER_CFLAGS='-Werror' BUILD_TLS=module
+        cp src/valkey-server ../binaries/${{ matrix.server_version }}/
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Update module path
+      run: echo "MODULE_PATH=$(realpath target/release/libvalkey_bloom.so)" >> $GITHUB_ENV
+    - name: Run integration tests 
+      run: |
+        python -m pytest --capture=sys --cache-clear -v "tests/" -m "not skip_for_asan" 2>&1 | tee test_output.tmp
+
+        if grep -q "LeakSanitizer: detected memory leaks" test_output.tmp; then
+          RED='\033[0;31m'
+          echo -e "${RED}Memory leaks detected in the following tests:"
+          LEAKING_TESTS=$(grep -B 2 "LeakSanitizer: detected memory leaks" test_output.tmp | \
+                          grep -v "LeakSanitizer" | \
+                          grep ".*\.py::")
+          LEAK_COUNT=$(echo "$LEAKING_TESTS" | wc -l)        
+          echo "$LEAKING_TESTS" | while read -r line; do
+              echo "::error::Test with leak: $line"
+          done
+          echo -e "\n$LEAK_COUNT python integration tests have leaks detected in them"
+          rm test_output.tmp
+          exit 1
+        fi
+        rm test_output.tmp

.gitignore

@@ -0,0 +1,6 @@
+Cargo.lock
+target
+tests/.build
+__pycache__
+test-data
+.attach_pid*

00-RELEASENOTES

@@ -0,0 +1,14 @@
+Hello! This file is just a placeholder, since this is the "unstable" branch
+of Valkey-Bloom, the place where all the development happens.
+
+There is no release notes for this branch, it gets forked into another branch
+every time there is a partial feature freeze in order to eventually create
+a new stable release.
+
+Usually "unstable" is stable enough for you to use it in development environments
+however you should never use it in production environments. You can find the
+latest stable release here:
+
+    https://github.com/valkey-io/valkey-bloom/releases
+
+Happy hacking!

Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "valkey-bloom"
+authors = ["Karthik Subbarao"]
+version = "0.1.0-dev"
+edition = "2021"
+license = "BSD-3-Clause"
+repository = "https://github.com/valkey-io/valkey-bloom"
+readme = "README.md"
+description = "A bloom filter module for Valkey"
+homepage = "https://github.com/valkey-io/valkey-bloom"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+valkey-module = { version = "0.1.5", features = ["min-valkey-compatibility-version-8-0", "min-redis-compatibility-version-7-2"]}
+valkey-module-macros = "0"
+linkme = "0"
+bloomfilter = { version = "3.0.1", features = ["serde"] }
+lazy_static = "1.4.0"
+libc = "0.2"
+serde = { version = "1.0", features = ["derive"] }
+bincode = "1.3"
+
+[dev-dependencies]
+rand = "0.8"
+rstest = "0.23.0"
+
+[lib]
+crate-type = ["cdylib"]
+name = "valkey_bloom"
+
+[profile.dev]
+opt-level = 0
+debug = 2
+debug-assertions = true
+
+[features]
+default = ["min-valkey-compatibility-version-8-0"]
+enable-system-alloc = ["valkey-module/enable-system-alloc"]
+min-valkey-compatibility-version-8-0 = []
+valkey_8_0 = []  # Valkey-bloom is intended to be loaded on server versions >= Valkey 8.1 and by default it is built this way (unless this flag is provided). It is however compatible with Valkey version 8.0 if the user explicitly provides this feature flag in their cargo build command.
+use-redismodule-api = [] # We don't support this feature flag which is why it is empty. 

LICENSE

@@ -0,0 +1,12 @@
+BSD 3-Clause License
+
+Copyright (c) 2024-present, Valkey contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1 +1,74 @@
-# valkey-bloom
+# valkey-bloom
+
+Valkey-Bloom (BSD-3-Clause) is a Rust Valkey-Module which brings a native and space efficient probabilistic Module data type to Valkey. With this, users can create filters (space-efficient probabilistic Module data type) to add elements, perform “check” operation to test whether an element exists, auto scale their filters, perform RDB Save and load operations, etc.
+
+Valkey-Bloom is built using `bloomfilter::Bloom` (https://crates.io/crates/bloomfilter which has a BSD-2-Clause license).
+
+It is compatible with the BloomFilter (BF.*) command APIs in Redis offerings.
+
+## Supported commands
+```
+BF.EXISTS
+BF.ADD
+BF.MEXISTS
+BF.MADD
+BF.CARD
+BF.RESERVE
+BF.INFO
+BF.INSERT
+BF.LOAD
+```
+
+## Build instructions
+```
+curl https://sh.rustup.rs -sSf | sh
+sudo yum install clang
+git clone https://github.com/valkey-io/valkey-bloom.git
+cd valkey-bloom
+cargo build --all --all-targets  --release
+valkey-server --loadmodule ./target/release/libvalkey_bloom.so
+```
+
+#### Local development script to build, run format checks, run unit / integration tests, and for cargo release:
+```
+# Builds the valkey-server (unstable) for integration testing.
+SERVER_VERSION=unstable
+./build.sh
+# Same as above, but uses valkey-server (8.0.0) for integration testing.
+SERVER_VERSION=8.0.0
+./build.sh
+# Build with asan, you may need to remove the old valkey binary if you have used ./build.sh before. You can do this by deleting the `.build` folder in the `tests` folder 
+ASAN_BUILD=true
+./build.sh
+```
+
+## Load the Module
+To test the module with a Valkey, you can load the module in the following ways:
+
+#### Using valkey.conf:
+```
+1. Add the following to valkey.conf:
+    loadmodule /path/to/libvalkey_bloom.so
+2. Start valkey-server:
+    valkey-server /path/to/valkey.conf
+```
+
+#### Starting Valkey with the `--loadmodule` option:
+```text
+valkey-server --loadmodule /path/to/libvalkey_bloom.so
+```
+
+#### Using the Valkey command `MODULE LOAD`:
+```
+1. Connect to a running Valkey instance using valkey-cli
+2. Execute Valkey command:
+    MODULE LOAD /path/to/libvalkey_bloom.so
+```
+## Feature Flags
+
+* valkey_8_0: valkey-bloom is intended to be loaded on server versions >= Valkey 8.1 and by default it is built this way (unless this flag is provided). It is however compatible with Valkey version 8.0 if the user explicitly provides this feature flag in their cargo build command.
+```
+cargo build --release --features valkey_8_0
+```
+
+This can also be done by specifiyng SERVER_VERSION=8.0.0 and then running `./build.sh`