Enable MicroVM Benchmarking

benchmarking/automation/Dockerfile

@@ -30,6 +30,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         python3-pip \
         python3-venv \
         docker.io \
+        unzip \
+        zstd \
     && rm -rf /var/lib/apt/lists/*
 
 # Go must be new enough to honor the `tool` directive in

benchmarking/automation/README.md

@@ -15,16 +15,30 @@ Each run uploads results to GCS via `benchmarking/locust/runner.py`.
 4. `docker build && docker push` builds the locust image tagged with the commit
    hash and pushes it to `${KO_DOCKER_REPO}/locust-test:<commit>`.
 5. `hack/install-ate.sh --deploy-ate-system` + `benchmarking/workloads/deploy.sh
-   --deploy` (these build & push substrate / workload images via `ko` as part
-   of their deploy steps — there's no separate `make build-images` step).
+   --deploy --sandbox-class <class>` (these build & push substrate / workload
+   images via `ko` as part of their deploy steps — there's no separate
+   `make build-images` step). For a `microvm` test the orchestrator also
+   runs `hack/install-microvm-deps.sh --install` between the two, which
+   stages kata + cloud-hypervisor + virtiofsd assets to the cluster's object
+   store bucket and applies the cluster-wide `microvm` SandboxConfig.
 6. For each test in `tests.yaml`:
    - Submits a Job using the just-built locust image; the Job runs
      `runner.py -f <file> -t <duration> -u <users> --tag <commit> --name <name>
      --dest <dest>`.
    - Polls until complete/failed/timeout; tails logs; deletes the Job.
-   - Tears down substrate + workloads.
+   - Tears down workloads + micro-VM deps (if any) + substrate.
    - If not the last test, redeploys them so the next run starts clean.
 
+## Choosing a sandbox class
+
+Each entry in `tests.yaml` may set `sandboxClass: gvisor | microvm` (default
+`gvisor`). This controls both `spec.sandboxClass` on the benchmark WorkerPool
+and its `ateomImage` (`ateom-gvisor` vs `ateom-microvm`).
+
+For `microvm` tests the target cluster must have KVM-capable nodes and the
+object store bucket named in its `.ate-dev-env.sh` must be writable by the
+orchestrator's Workload Identity principal.
+
 ## Setup
 
 ```bash

benchmarking/automation/orchestrator.py

@@ -272,6 +272,9 @@ def wait_for_job(name: str, timeout_seconds: int) -> str:
     return "timeout"
 
 
+SANDBOX_CLASSES = ("gvisor", "microvm")
+
+
 def deploy_substrate() -> None:
     run(["hack/install-ate.sh", "--deploy-ate-system"])
 
@@ -280,13 +283,29 @@ def teardown_substrate() -> None:
     run_no_check(["hack/install-ate.sh", "--delete-ate-system"])
 
 
-def deploy_workloads(worker_count: int = 1) -> None:
+def install_microvm_deps() -> None:
+    """Stage kata/cloud-hypervisor assets and apply the cluster-wide
+    microvm SandboxConfig. Required before a microvm WorkerPool can
+    schedule; must run after deploy_substrate() (which installs the CRDs)."""
+    run(["hack/install-microvm-deps.sh", "--install"])
+
+
+def teardown_microvm_deps() -> None:
+    """Remove the microvm SandboxConfig. Must run before
+    teardown_substrate(), which deletes the SandboxConfig CRD (and would
+    prevent this from succeeding via kubectl)."""
+    run_no_check(["hack/install-microvm-deps.sh", "--delete"])
+
+
+def deploy_workloads(worker_count: int = 1, sandbox_class: str = "gvisor") -> None:
     run(
         [
             "benchmarking/workloads/deploy.sh",
             "--deploy",
             "--worker-count",
             str(worker_count),
+            "--sandbox-class",
+            sandbox_class,
         ]
     )
     # Block until ActorTemplates are Ready
@@ -365,6 +384,12 @@ def main() -> None:
             sys.exit(
                 f"test {t.get('name')!r} missing required 'targetCluster' field"
             )
+        sandbox_class = t.get("sandboxClass", "gvisor")
+        if sandbox_class not in SANDBOX_CLASSES:
+            sys.exit(
+                f"test {t.get('name')!r} has invalid sandboxClass "
+                f"{sandbox_class!r} (want one of {list(SANDBOX_CLASSES)})"
+            )
 
     # Per-target-cluster caches: re-running setup for the same target
     # cluster is wasted work, so we track what was last set up and only
@@ -400,15 +425,23 @@ def main() -> None:
             # Idempotent sweep before anything else: a previous CronJob
             # fire that crashed mid-test (or any other process that left
             # state behind) would otherwise leak its substrate + workloads
-            # into this run. Both teardowns use --ignore-not-found, so
-            # this is cheap on a clean cluster.
+            # into this run. All teardowns use --ignore-not-found, so
+            # this is cheap on a clean cluster. Order matters:
+            # microvm-deps deletes a SandboxConfig CR, which requires the
+            # SandboxConfig CRD that teardown_substrate removes.
             teardown_workloads()
+            teardown_microvm_deps()
             teardown_substrate()
 
+            sandbox_class = test.get("sandboxClass", "gvisor")
             status = "error"
             try:
                 deploy_substrate()
-                deploy_workloads(test.get("workerCount", 1))
+                # install-microvm-deps needs the CRDs from deploy_substrate;
+                # deploy_workloads needs the microvm SandboxConfig.
+                if sandbox_class == "microvm":
+                    install_microvm_deps()
+                deploy_workloads(test.get("workerCount", 1), sandbox_class)
                 try:
                     status = run_test(test, locust_image, args.dest, commit)
                 except Exception as e:
@@ -418,7 +451,10 @@ def main() -> None:
             finally:
                 # Always tear down, even if deploy or run failed, so the
                 # next test (and the next CronJob fire) starts clean.
+                # microvm-deps must go before substrate for the same reason
+                # as above.
                 teardown_workloads()
+                teardown_microvm_deps()
                 teardown_substrate()
             results.append((test["name"], status))
         finally:

benchmarking/automation/tests.yaml

@@ -24,6 +24,9 @@
 # the deploy that runs before this test. The orchestrator redeploys
 # workloads between tests, so each test can pick its own scale.
 #
+# Optional `sandboxClass` (default "gvisor") picks the sandbox runtime for
+# the benchmark WorkerPool. Allowed values: "gvisor" | "microvm".
+#
 # Required `targetCluster` names a config file under
 # /etc/orchestrator/target-clusters/ (e.g. `targetCluster: dev` ->
 # /etc/orchestrator/target-clusters/dev.sh). The orchestrator copies that

benchmarking/workloads/deploy.sh

@@ -37,20 +37,34 @@ if [[ ! -f "${MANIFEST_TEMPLATE}" ]]; then
 fi
 
 WORKER_COUNT=1
+SANDBOX_CLASS="gvisor"
 
 usage() {
   echo "Usage: $0 [options]"
   echo ""
   echo "Options:"
-  echo "  --deploy             Substitute env vars and deploy workloads to the cluster using ko apply"
-  echo "  --delete             Substitute env vars and delete workloads from the cluster"
-  echo "  --worker-count N     Number of WorkerPool replicas (default: 1)"
-  echo "  -h, --help           Show this help message"
+  echo "  --deploy                    Substitute env vars and deploy workloads to the cluster using ko apply"
+  echo "  --delete                    Substitute env vars and delete workloads from the cluster"
+  echo "  --worker-count N            Number of WorkerPool replicas (default: 1)"
+  echo "  --sandbox-class CLASS       Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)."
+  echo "                              microvm requires hack/install-microvm-deps.sh --install to have run."
+  echo "  -h, --help                  Show this help message"
 }
 
 substitute() {
+  # SandboxConfig names are pinned per class (rather than defaulted) so a stale
+  # config from a dirty teardown fails loudly instead of silently binding this
+  # pool. gvisor-default is applied by hack/install-ate.sh; microvm is applied
+  # by hack/install-microvm-deps.sh.
+  local sandbox_config_name
+  case "${SANDBOX_CLASS}" in
+    gvisor)  sandbox_config_name="gvisor-default" ;;
+    microvm) sandbox_config_name="microvm"        ;;
+  esac
   sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \
       -e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \
+      -e "s|\${SANDBOX_CLASS}|${SANDBOX_CLASS}|g" \
+      -e "s|\${SANDBOX_CONFIG_NAME}|${sandbox_config_name}|g" \
       "${MANIFEST_TEMPLATE}"
 }
 
@@ -87,6 +101,13 @@ while [[ "$#" -gt 0 ]]; do
     --worker-count=*)
       WORKER_COUNT="${1#*=}"
       ;;
+    --sandbox-class)
+      shift
+      SANDBOX_CLASS="$1"
+      ;;
+    --sandbox-class=*)
+      SANDBOX_CLASS="${1#*=}"
+      ;;
     -h|--help)
       usage
       exit 0
@@ -100,6 +121,14 @@ while [[ "$#" -gt 0 ]]; do
   shift
 done
 
+case "${SANDBOX_CLASS}" in
+  gvisor|microvm) ;;
+  *)
+    echo "Error: --sandbox-class must be gvisor or microvm, got '${SANDBOX_CLASS}'" >&2
+    exit 1
+    ;;
+esac
+
 if [[ "${action}" == "deploy" ]]; then
   deploy
 elif [[ "${action}" == "delete" ]]; then

benchmarking/workloads/manifests/workloads.yaml.tmpl

@@ -28,7 +28,9 @@ metadata:
     workload: benchmark-ateom
 spec:
   replicas: ${WORKER_COUNT}
-  ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+  sandboxClass: ${SANDBOX_CLASS}
+  sandboxConfigName: ${SANDBOX_CONFIG_NAME}
+  ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-${SANDBOX_CLASS}
 
 ---
 
@@ -38,6 +40,8 @@ metadata:
   name: sleep
   namespace: benchmark-workloads
 spec:
+  # Must match the WorkerPool's sandboxClass so snapshots stay within a class.
+  sandboxClass: ${SANDBOX_CLASS}
   pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
   containers:
   - name: sleep
@@ -57,6 +61,7 @@ metadata:
   name: glutton
   namespace: benchmark-workloads
 spec:
+  sandboxClass: ${SANDBOX_CLASS}
   pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da"
   containers:
   - name: glutton

demos/counter/counter-microvm.yaml.tmpl

@@ -16,18 +16,8 @@
 # in-RAM (atomic uint64), so a successful suspend/resume across pods shows the
 # count continuing — proving the guest memory snapshot round-tripped.
 #
-# The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs,
-# base configuration.toml) live on a cluster-scoped SandboxConfig, FETCHED at runtime
-# from the cluster object store bucket ${BUCKET_NAME} under kata-assets/ (rustfs on
-# kind, GCS on GKE) — NOT baked into the worker image. ateom boots cloud-hypervisor
-# itself and gives the actor an overlay rootfs (virtio-fs RO lower + guest-tmpfs
-# upper), so it fetches virtiofsd (v1.14.0 — the first release with the vhost-0.16
-# snapshot/restore fix; the kata-bundled v1.13.3 hangs restore). The kata containerd
-# shim is NOT fetched (ateom drives the kata-agent directly). kata assets are 4.0.0.
-# The per-arch sha256 values below are the asset sets produced by
-# hack/microvm-assets/assemble.sh; atelet selects the block matching the node's
-# architecture, and each cluster's bucket holds that arch's binaries at these paths
-# (staged by run-microvm-demo[-kind].sh).
+# The cluster-wide `microvm` SandboxConfig referenced below by name is
+# installed by hack/install-microvm-deps.sh
 
 apiVersion: v1
 kind: Namespace
@@ -36,58 +26,6 @@ metadata:
 
 ---
 
-apiVersion: ate.dev/v1alpha1
-kind: SandboxConfig
-metadata:
-  name: counter-microvm
-spec:
-  sandboxClass: microvm
-  assets:
-    arm64:
-      cloud-hypervisor:
-        url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor"
-        sha256: "bf004ddc1a148f47caa87ac49a783b8dbd6bf9bc27abe522ed197df7b982d3b1"
-      # virtiofsd serves the overlay RO lower (virtio-fs); v1.14.0 carries the
-      # vhost-0.16 snapshot/restore fix the kata-bundled v1.13.3 lacks.
-      virtiofsd:
-        url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd"
-        # Upstream publishes no arm64 prebuilt, so assemble.sh builds the v1.14.0 tag
-        # from source; the binary is not byte-reproducible across toolchains and can't
-        # carry a fixed pin. run-microvm-demo.sh computes this from the freshly-staged
-        # binary at deploy.
-        sha256: "${VIRTIOFSD_SHA256}"
-      kata-kernel:
-        url: "gs://${BUCKET_NAME}/kata-assets/vmlinux"
-        sha256: "4a8998a2e7ac12d6ad1f15b5e7d00571e4518ea9f33db1a1a568310373ca428d"
-      kata-image:
-        url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img"
-        sha256: "24d77700749846355f3be25b87974c1fdea9fd2a9534256b4e4bbdde5df23588"
-      kata-config:
-        url: "gs://${BUCKET_NAME}/kata-assets/configuration-clh.toml"
-        sha256: "20f5de4f705423ddd1ee93318c784c6aede30c07c78dc6b311a771cc2f6859c7"
-    # amd64 assets are kata 4.0.0 + virtiofsd v1.14.0 (assemble.sh ARCH=amd64).
-    amd64:
-      cloud-hypervisor:
-        url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor"
-        sha256: "829af01ff075bb96c4f183905134c453a88d68cbabdc6b87df21098842581ee9"
-      virtiofsd:
-        url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd"
-        # The x86_64-musl binary attached to the upstream v1.14.0 release (downloaded
-        # by assemble.sh, reproducible bytes), so it carries a fixed pin like the
-        # other downloaded assets.
-        sha256: "15b2e72a78cc08a9bd8a6943e89fb69c88cb3cbeb63069efceade835342ac7d4"
-      kata-kernel:
-        url: "gs://${BUCKET_NAME}/kata-assets/vmlinux"
-        sha256: "6abc48fa83c58e3db314037105d04ec00c1bc80d85eb2dfb2e2854f414573bca"
-      kata-image:
-        url: "gs://${BUCKET_NAME}/kata-assets/rootfs.img"
-        sha256: "fbbd40de605ab2f99dc82c777dc5b41f9a00dad3a0f699f2247c306ac9c1204d"
-      kata-config:
-        url: "gs://${BUCKET_NAME}/kata-assets/configuration-clh.toml"
-        sha256: "1e67eabdfb9d900bf95a00edb52cacc1efe64f23ea095f8b49ee81c7434bce90"
-
----
-
 apiVersion: ate.dev/v1alpha1
 kind: WorkerPool
 metadata:
@@ -98,7 +36,7 @@ metadata:
 spec:
   replicas: 2
   sandboxClass: microvm
-  sandboxConfigName: counter-microvm
+  sandboxConfigName: microvm
   ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm
 
 ---