diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index 3be8c549f..88b628e1e 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -57,7 +57,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run Benchmarks - run: just bench-ci main release ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} + run: just bench-ci main ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index c266c25cc..b93042c50 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -186,5 +186,5 @@ jobs: - name: Run benchmarks run: | - just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} + just bench-ci main ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} if: ${{ matrix.config == 'release' }} diff --git a/Justfile b/Justfile index 9ae49e3d2..c364937e7 100644 --- a/Justfile +++ b/Justfile @@ -326,11 +326,13 @@ bench-download os hypervisor cpu tag="": tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1 # Warning: compares to and then OVERWRITES the given baseline -bench-ci baseline target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} +bench-ci baseline features="": + @# Benchmarks are always run with release builds for meaningful results + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} -bench target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose +bench features="": + @# Benchmarks are always run with release builds for meaningful results + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose ############### ### FUZZING ### diff --git a/docs/benchmarking-hyperlight.md b/docs/benchmarking-hyperlight.md index 3ae56d55c..5ac2662f4 100644 --- a/docs/benchmarking-hyperlight.md +++ b/docs/benchmarking-hyperlight.md @@ -72,4 +72,6 @@ Found 1 outliers among 100 measurements (1.00%) ## Running benchmarks locally -Use `just bench [debug/release]` parameter to run benchmarks. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. +Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. + +**Important**: The `just bench` command uses release builds by default to ensure meaningful performance measurements. For profiling purposes, you can compile benchmarks with debug symbols by running `cargo bench` directly.