feat: add storage and OCI lifecycle latency benchmark

.github/workflows/build.yml

@@ -0,0 +1,39 @@
+name: Release Benchmark Harness
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    strategy:
+      matrix:
+        goarch: [ amd64, arm64 ]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: "1.26.4"
+
+      - name: Download Go modules
+        run: go mod tidy
+
+      - name: Build Runtime Benchmark
+        run: |
+          GOOS=linux GOARCH=${{ matrix.goarch }} go build  -ldflags "-X 'github.com/urunc-dev/evaluation_suite/internal/cli.Version=${{ github.event.release.tag_name }}'" -o build/runtime-benchmark-${{ github.event.release.tag_name }}-linux-${{ matrix.goarch }} cmd/harness/main.go
+
+      - name: Upload Release Asset
+        uses: softprops/action-gh-release@v3
+        with:
+          files: |
+            experiment.yml
+            build/runtime-benchmark-${{ github.event.release.tag_name }}-linux-${{ matrix.goarch }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,16 @@
+# go buildd
+build/
+
+# experiment results
+results/
+
+# C builds
+**/bin
+
+harness
+
+.venv
+
+# linux image
+
+images/fio-urunc/bzImage

.python-version

@@ -0,0 +1 @@
+3.14

Makefile

@@ -0,0 +1,16 @@
+CC = gcc
+CFLAGS = -Wall -Wextra -O2
+
+MINIMAL_C_SRC = workloads/minimal-c/main.c
+MINIMAL_C_TARGET = images/minimal-c/bin/minimal
+
+all: $(MINIMAL_C_TARGET)
+
+$(MINIMAL_C_TARGET): $(MINIMAL_C_SRC)
+	mkdir -p images/minimal-c/bin
+	$(CC) $(CFLAGS) -o $(MINIMAL_C_TARGET) $(MINIMAL_C_SRC)
+
+clean:
+	rm -rf images/minimal-c/bin
+	rm -rf build
+

README.md

@@ -0,0 +1,17 @@
+# Reproducible benchmarking suite for urunc
+
+This reposiitory contains an evaluation suite for container runtimes. It currently supports
+
+- urunc
+- Kata containers
+- runc
+- gVisor
+
+## Dependency tools
+
+To run this program, you need the following tools installed
+
+- `nerdctl`
+- `containerd`
+- `skopeo`
+- `umoci`

cmd/harness/main.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2026-present, urunc-dev
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/urunc-dev/evaluation_suite/internal/cli"
+)
+
+func main() {
+	cli.Execute()
+}

experiment.yml

@@ -0,0 +1,37 @@
+
+runtimes:
+  - name: runc
+    handler: io.containerd.runc.v2
+
+  - name: kata
+    handler: io.containerd.kata.v2
+
+  - name: runsc
+    handler: io.containerd.runsc.v1
+
+  - name: urunc
+    handler: io.containerd.urunc.v2
+
+
+experiments:
+
+  lifecycle:
+    workloads:
+      default:
+        image: docker.io/library/nginx:alpine
+
+      other:
+      - image: docker.io/library/minimalc:test
+        runtime: urunc
+
+  storage:
+    workloads:
+      default:
+        image: docker.io/jimjuniorb/fio:0.1
+        snapshotter: overlayfs
+      other:
+
+      - image: docker.io/jimjuniorb/fio-urunc:0.4
+        runtime: urunc
+        snapshotter: devmapper
+

go.mod

@@ -0,0 +1,70 @@
+module github.com/urunc-dev/evaluation_suite
+
+go 1.26.4
+
+require (
+	github.com/containerd/containerd v1.7.34
+	github.com/containerd/containerd/api v1.11.1
+	github.com/containerd/typeurl/v2 v2.2.3
+	github.com/pelletier/go-toml/v2 v2.4.2
+	github.com/spf13/cobra v1.10.2
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	cyphar.com/go-pathrs v0.2.1 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect
+	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
+	github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
+	github.com/Microsoft/hcsshim v0.12.5 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/cgroups/v3 v3.0.3 // indirect
+	github.com/containerd/continuity v0.5.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v1.0.0-rc.4 // indirect
+	github.com/containerd/ttrpc v1.2.8 // indirect
+	github.com/cyphar/filepath-securejoin v0.6.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0 // indirect
+	github.com/opencontainers/selinux v1.13.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/procfs v0.19.2 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
+	golang.org/x/sync v0.21.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/text v0.38.0 // indirect
+	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect
+	google.golang.org/grpc v1.80.0 // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+)