ci(e2e): reuse prebuilt VM driver artifact

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

.github/actions/setup-e2e-vm-driver/action.yml

@@ -0,0 +1,38 @@
+name: Setup E2E VM Driver
+description: Download a prebuilt OpenShell VM driver for E2E tests
+
+inputs:
+  artifact-name:
+    description: Artifact name to download, such as driver-vm-linux-amd64
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Download prebuilt VM driver
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: .e2e/prebuilt-vm-driver
+
+    - name: Configure prebuilt VM driver
+      shell: bash
+      run: |
+        set -euo pipefail
+        extract_dir="$GITHUB_WORKSPACE/.e2e/prebuilt-vm-driver/bin"
+        mkdir -p "$extract_dir"
+        archive="$(find "$GITHUB_WORKSPACE/.e2e/prebuilt-vm-driver" -maxdepth 1 -type f -name 'openshell-driver-vm-*.tar.gz' -print -quit)"
+        if [[ -z "$archive" ]]; then
+          echo "downloaded artifact is missing openshell-driver-vm-*.tar.gz" >&2
+          find "$GITHUB_WORKSPACE/.e2e/prebuilt-vm-driver" -maxdepth 2 -type f -print >&2 || true
+          exit 1
+        fi
+        tar -xzf "$archive" -C "$extract_dir"
+        driver="$extract_dir/openshell-driver-vm"
+        if [[ ! -f "$driver" ]]; then
+          echo "downloaded artifact is missing $driver" >&2
+          exit 1
+        fi
+        chmod +x "$driver"
+        "$driver" --version
+        echo "OPENSHELL_VM_DRIVER_BIN=$driver" >> "$GITHUB_ENV"

.github/workflows/branch-e2e.yml

@@ -104,8 +104,22 @@ jobs:
       component: cli
       platform: linux/amd64,linux/arm64
 
+  build-driver-vm-linux:
+    needs: [pr_metadata]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
+    permissions:
+      contents: read
+      packages: read
+    uses: ./.github/workflows/driver-vm-linux.yml
+    with:
+      image-tag: ${{ github.sha }}
+      checkout-ref: ${{ github.sha }}
+      runtime-platforms: linux-x86_64
+      driver-targets: >-
+        [{"arch":"amd64","runner":"linux-amd64-cpu8","target":"x86_64-unknown-linux-gnu","zig_target":"x86_64-unknown-linux-gnu.2.28","platform":"linux-x86_64","guest_arch":"x86_64"}]
+
   e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -117,6 +131,7 @@ jobs:
       runner: linux-arm64-cpu8
       cli-artifact-prefix: rust-binary-cli
       gateway-artifact-prefix: rust-binary-gateway
+      vm-driver-artifact-name: driver-vm-linux-amd64
 
   gpu-e2e:
     needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
@@ -179,7 +194,7 @@ jobs:
 
   core-e2e-result:
     name: Core E2E result
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, e2e, kubernetes-e2e]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e]
     if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     runs-on: ubuntu-latest
     steps:
@@ -188,6 +203,7 @@ jobs:
           BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
           BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
           BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
+          BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }}
           E2E_RESULT: ${{ needs.e2e.result }}
           KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }}
         run: |
@@ -197,6 +213,7 @@ jobs:
             "build-gateway:$BUILD_GATEWAY_RESULT" \
             "build-supervisor:$BUILD_SUPERVISOR_RESULT" \
             "build-cli:$BUILD_CLI_RESULT" \
+            "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \
             "e2e:$E2E_RESULT" \
             "kubernetes-e2e:$KUBERNETES_E2E_RESULT"; do
             name="${item%%:*}"

.github/workflows/driver-vm-linux.yml

@@ -4,14 +4,26 @@ on:
   workflow_call:
     inputs:
       cargo-version:
-        required: true
+        required: false
         type: string
+        default: ""
       image-tag:
         required: true
         type: string
       checkout-ref:
         required: true
         type: string
+      runtime-platforms:
+        description: "Comma-separated VM runtime platforms to download: linux-aarch64,linux-x86_64"
+        required: false
+        type: string
+        default: "linux-aarch64,linux-x86_64"
+      driver-targets:
+        description: "JSON matrix entries for Linux VM driver targets"
+        required: false
+        type: string
+        default: >-
+          [{"arch":"arm64","runner":"linux-arm64-cpu8","target":"aarch64-unknown-linux-gnu","zig_target":"aarch64-unknown-linux-gnu.2.28","platform":"linux-aarch64","guest_arch":"aarch64"},{"arch":"amd64","runner":"linux-amd64-cpu8","target":"x86_64-unknown-linux-gnu","zig_target":"x86_64-unknown-linux-gnu.2.28","platform":"linux-x86_64","guest_arch":"x86_64"}]
 
 permissions:
   contents: read
@@ -39,11 +51,29 @@ jobs:
       - name: Download Linux runtime tarballs
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUNTIME_PLATFORMS: ${{ inputs['runtime-platforms'] }}
         run: |
           set -euo pipefail
           mkdir -p runtime-artifacts
 
-          for platform in linux-aarch64 linux-x86_64; do
+          runtime_platforms="${RUNTIME_PLATFORMS//[[:space:]]/}"
+          IFS=',' read -r -a platforms <<< "$runtime_platforms"
+          if [ "${#platforms[@]}" -eq 0 ]; then
+            echo "::error::No VM runtime platforms requested"
+            exit 1
+          fi
+          for platform in "${platforms[@]}"; do
+            case "$platform" in
+              linux-aarch64 | linux-x86_64) ;;
+              *)
+                echo "::error::Unsupported VM runtime platform '$platform' in runtime-platforms"
+                exit 1
+                ;;
+            esac
+          done
+          printf '%s\n' "${platforms[@]}" > runtime-artifacts/platforms.txt
+
+          for platform in "${platforms[@]}"; do
             asset="vm-runtime-${platform}.tar.zst"
             echo "Downloading ${asset}..."
             if ! gh release download vm-runtime \
@@ -61,9 +91,9 @@ jobs:
       - name: Verify downloads
         run: |
           set -euo pipefail
-          for platform in linux-aarch64 linux-x86_64; do
+          while IFS= read -r platform; do
             test -f "runtime-artifacts/vm-runtime-${platform}.tar.zst"
-          done
+          done < runtime-artifacts/platforms.txt
 
       - name: Upload runtime artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -77,19 +107,7 @@ jobs:
     needs: [download-kernel-runtime]
     strategy:
       matrix:
-        include:
-          - arch: arm64
-            runner: linux-arm64-cpu8
-            target: aarch64-unknown-linux-gnu
-            zig_target: aarch64-unknown-linux-gnu.2.28
-            platform: linux-aarch64
-            guest_arch: aarch64
-          - arch: amd64
-            runner: linux-amd64-cpu8
-            target: x86_64-unknown-linux-gnu
-            zig_target: x86_64-unknown-linux-gnu.2.28
-            platform: linux-x86_64
-            guest_arch: x86_64
+        include: ${{ fromJSON(inputs['driver-targets']) }}
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     container:

.github/workflows/e2e-test.yml

@@ -27,6 +27,11 @@ on:
         required: false
         type: string
         default: ""
+      vm-driver-artifact-name:
+        description: "Optional prebuilt VM driver artifact name"
+        required: false
+        type: string
+        default: ""
 
 permissions:
   actions: read
@@ -257,6 +262,24 @@ jobs:
           ref: ${{ inputs['checkout-ref'] || github.sha }}
           persist-credentials: false
 
+      - name: Use prebuilt OpenShell CLI
+        if: inputs.cli-artifact-prefix != ''
+        uses: ./.github/actions/setup-e2e-cli
+        with:
+          artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+
+      - name: Use prebuilt OpenShell gateway
+        if: inputs.gateway-artifact-prefix != ''
+        uses: ./.github/actions/setup-e2e-gateway
+        with:
+          artifact-prefix: ${{ inputs.gateway-artifact-prefix }}
+
+      - name: Use prebuilt OpenShell VM driver
+        if: inputs.vm-driver-artifact-name != ''
+        uses: ./.github/actions/setup-e2e-vm-driver
+        with:
+          artifact-name: ${{ inputs.vm-driver-artifact-name }}
+
       - name: Enable KVM access
         run: |
           set -euo pipefail

CI.md

@@ -20,7 +20,7 @@ Three opt-in labels enable the long-running E2E suites:
 - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay
   (`replicaCount: 2` and bundled PostgreSQL) in `Branch E2E Checks`
 
-When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, and MCP E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI.
+When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact.
 The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while HA behavior is under active iteration: failures are visible in the workflow run but do not publish a required CI gate status.
 
 The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly.

e2e/rust/e2e-vm.sh

@@ -23,9 +23,12 @@
 #   mise run e2e:vm
 #
 # What the script does:
-#   1. Ensures the VM runtime (libkrun + gvproxy) and bundled supervisor are staged.
+#   1. When no prebuilt VM driver is supplied, ensures the VM runtime
+#      (libkrun + gvproxy) and bundled supervisor are staged.
 #   2. Builds `openshell-gateway`, `openshell-driver-vm`, and the
-#      `openshell` CLI with the embedded runtime.
+#      `openshell` CLI with the embedded runtime as needed. When CI supplies
+#      OPENSHELL_GATEWAY_BIN, OPENSHELL_VM_DRIVER_BIN, or OPENSHELL_BIN, the
+#      matching prebuilt binary is reused instead of rebuilt.
 #   3. On macOS, codesigns the VM driver (libkrun needs the
 #      `com.apple.security.hypervisor` entitlement).
 #   4. Writes a per-run gateway config with `[openshell.drivers.vm]`
@@ -35,7 +38,7 @@
 #   5. Tears the gateway down and (on failure) preserves the gateway
 #      log and every VM serial console log for post-mortem.
 #
-# Prerequisites (handled automatically by this script if missing):
+# Prerequisites (handled automatically by this script for local VM-driver builds):
 #   - `mise run vm:setup`      — downloads / builds the libkrun runtime.
 #   - `mise run vm:supervisor` — builds the bundled sandbox supervisor.
 
@@ -45,8 +48,9 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 source "${ROOT}/e2e/support/gateway-common.sh"
 
 COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed"
-GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway"
-DRIVER_BIN="${ROOT}/target/debug/openshell-driver-vm"
+GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}"
+DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm}"
+CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}"
 E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}"
 E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}"
 
@@ -73,25 +77,59 @@ if [ -n "${RUSTC_WRAPPER:-}" ] && [ "${OPENSHELL_E2E_VM_ALLOW_RUSTC_WRAPPER:-0}"
   unset RUSTC_WRAPPER
 fi
 
-mkdir -p "${COMPRESSED_DIR}"
+if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then
+  mkdir -p "${COMPRESSED_DIR}"
 
-if ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q .; then
-  echo "==> Preparing embedded VM runtime (mise run vm:setup)"
-  mise run vm:setup
+  if ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q .; then
+    echo "==> Preparing embedded VM runtime (mise run vm:setup)"
+    mise run vm:setup
+  fi
+
+  if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then
+    echo "==> Building bundled VM supervisor (mise run vm:supervisor)"
+    mise run vm:supervisor
+  fi
+
+  export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}"
+else
+  echo "==> Skipping embedded VM runtime prep; prebuilt VM driver was supplied"
 fi
 
-if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then
-  echo "==> Building bundled VM supervisor (mise run vm:supervisor)"
-  mise run vm:supervisor
+build_packages=()
+if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then
+  build_packages+=(-p openshell-server)
+else
+  echo "==> Using prebuilt openshell-gateway at ${GATEWAY_BIN}"
+fi
+if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then
+  build_packages+=(-p openshell-driver-vm)
+else
+  echo "==> Using prebuilt openshell-driver-vm at ${DRIVER_BIN}"
+fi
+if [ -z "${OPENSHELL_BIN:-}" ]; then
+  build_packages+=(-p openshell-cli)
+else
+  echo "==> Using prebuilt openshell CLI at ${CLI_BIN}"
 fi
 
-export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}"
+if [ "${#build_packages[@]}" -gt 0 ]; then
+  echo "==> Building ${build_packages[*]}"
+  cargo build "${build_packages[@]}"
+fi
 
-echo "==> Building openshell-gateway, openshell-driver-vm, openshell (CLI)"
-cargo build \
-  -p openshell-server \
-  -p openshell-driver-vm \
-  -p openshell-cli
+for pair in \
+  "openshell-gateway:${GATEWAY_BIN}" \
+  "openshell-driver-vm:${DRIVER_BIN}" \
+  "openshell CLI:${CLI_BIN}"; do
+  label="${pair%%:*}"
+  path="${pair#*:}"
+  if [ ! -x "${path}" ]; then
+    echo "ERROR: expected ${label} binary at ${path}" >&2
+    exit 1
+  fi
+done
+export OPENSHELL_BIN="${CLI_BIN}"
+DRIVER_DIR="$(dirname "${DRIVER_BIN}")"
 
 if [ "$(uname -s)" = "Darwin" ]; then
   echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)"
@@ -224,7 +262,7 @@ ttl_secs = 0
 
 [openshell.drivers.vm]
 grpc_endpoint = "https://host.openshell.internal:${HOST_PORT}"
-driver_dir = "${ROOT}/target/debug"
+driver_dir = "${DRIVER_DIR}"
 state_dir = "${RUN_STATE_DIR}"
 guest_tls_ca = "${PKI_DIR}/ca.crt"
 guest_tls_cert = "${PKI_DIR}/client/tls.crt"