ci(deploy): isolate non-release runner runtime caches

.github/workflows/e2e-dev.yml

@@ -0,0 +1,163 @@
+# E2E suite against the dev cloud stack — SDK → dev API → dev Runner → VM.
+#
+# Builds Python SDK, Node SDK, and the CLI from source using
+# BOXLITE_DEPS_STUB=1 (skips vendor submodule builds — those only
+# matter inside the runner, not the SDK client), then runs the REST
+# e2e pytest suite against the dev environment.
+#
+# C and Go SDK tests are excluded — those SDKs wrap libboxlite.a over
+# REST purely to test FFI bindings, not a real user path. Python,
+# Node, and CLI cover the actual user-facing REST surface.
+#
+# Does NOT build or deploy the API or runner — the dev stack is
+# assumed to be already running the version you want to test against.
+#
+# Authentication: boxlite API key stored as GitHub repo secret
+# `BOXLITE_DEV_API_KEY`. No AWS credentials needed.
+
+name: E2E dev
+
+on:
+  push:
+    branches:
+      - chore/e2e-required-merge-gate
+  workflow_dispatch:
+    inputs:
+      api_url:
+        description: 'Dev API base URL'
+        type: string
+        required: false
+        default: 'https://api.dev.boxlite.ai/api'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: e2e-dev
+  cancel-in-progress: true
+
+env:
+  BOXLITE_DEPS_STUB: "1"
+  API_URL: ${{ inputs.api_url || 'https://api.dev.boxlite.ai/api' }}
+
+jobs:
+  e2e:
+    name: E2E suite (dev)
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v5
+
+      # ── Health check ──────────────────────────────────────────────
+      - name: Probe API health
+        run: |
+          curl -fsS --retry 6 --retry-delay 5 --retry-all-errors --max-time 10 \
+               -o /dev/null "${API_URL}/health"
+          echo "::notice::${API_URL}/health returned 2xx"
+
+      # ── System deps ──────────────────────────────────────────────
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends protobuf-compiler
+
+      # ── Rust toolchain ───────────────────────────────────────────
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      # ── CLI (cargo build, warms the shared boxlite core) ─────────
+      - name: Build CLI
+        run: |
+          cargo build --release -p boxlite-cli
+          sudo install -m 0755 target/release/boxlite /usr/local/bin/boxlite
+          boxlite --version
+
+      # ── Node SDK (napi-rs) ───────────────────────────────────────
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Build Node SDK (napi + tsc)
+        run: |
+          cd sdks/node
+          npm install --ignore-scripts
+          npm run build:native
+          npm run build
+          ls -lh native/ dist/
+
+      # ── Python SDK (maturin + PyO3) ──────────────────────────────
+      - name: Build & install Python SDK
+        run: |
+          cd sdks/python
+          pip install --break-system-packages --quiet maturin
+          maturin build --release
+          pip install --break-system-packages --force-reinstall ../../target/wheels/boxlite-*.whl
+          pip install --break-system-packages --quiet pytest pytest-asyncio pytest-timeout
+
+      # ── Configure pytest credentials ─────────────────────────────
+      - name: Configure pytest profile p1
+        env:
+          BOXLITE_DEV_API_KEY: ${{ secrets.BOXLITE_DEV_API_KEY }}
+        run: |
+          set -euo pipefail
+
+          if [ -z "$BOXLITE_DEV_API_KEY" ]; then
+            echo "::error::Secret BOXLITE_DEV_API_KEY is not set. Add it in repo Settings → Secrets."
+            exit 1
+          fi
+          echo "::add-mask::$BOXLITE_DEV_API_KEY"
+
+          PATH_PREFIX=$(curl -fsS -H "Authorization: Bearer $BOXLITE_DEV_API_KEY" \
+            "${API_URL}/v1/me" | python3 -c 'import json,sys; print(json.load(sys.stdin)["path_prefix"])')
+          [ -n "$PATH_PREFIX" ] || { echo "::error::Empty path_prefix from /v1/me"; exit 1; }
+          echo "Resolved path_prefix=${PATH_PREFIX}"
+
+          mkdir -p ~/.boxlite
+          chmod 700 ~/.boxlite
+          {
+              printf '[profiles.p1]\n'
+              printf 'url = "%s"\n'         "$API_URL"
+              printf 'api_key = "%s"\n'     "$BOXLITE_DEV_API_KEY"
+              printf 'path_prefix = "%s"\n' "$PATH_PREFIX"
+          } > ~/.boxlite/credentials.toml
+          chmod 600 ~/.boxlite/credentials.toml
+
+      # ── Run pytest ───────────────────────────────────────────────
+      # --ignore rationale:
+      #  - C/Go entry+coverage: FFI-binding tests whose SDKs aren't built here.
+      #  - node-entry: needs runner journal access (unreachable from GHA
+      #    against remote dev); node-coverage already covers the Node REST
+      #    surface without journal checks.
+      #  - exec-timeout: the timeout fires and kills the process on time, but
+      #    the guest kills only the direct child — an orphaned grandchild
+      #    (e.g. `sh`'s `sleep`) keeps the stdout pipe open, so the attach
+      #    stream never EOFs and ex.wait() hangs to the 45s bound. Guest-side
+      #    root cause + fix tracked in #910. Un-ignore once that lands. The
+      #    kill itself is covered by sdks/python/tests/test_exec_timeout_sigalrm.py.
+      # test_path_verification is NOT ignored — despite the name it never
+      # touches the journal; it proves the API→Runner chain via the
+      # X-BoxLite-Api-Version response header + an exec marker.
+      - name: Run E2E suite
+        env:
+          BOXLITE_E2E_SKIP_PATH_VERIFY: '1'
+          BOXLITE_E2E_PROFILE: p1
+        run: |
+          timeout 35m python3 -m pytest scripts/test/e2e/cases/ \
+              --ignore=scripts/test/e2e/cases/test_c_entry.py \
+              --ignore=scripts/test/e2e/cases/test_c_coverage.py \
+              --ignore=scripts/test/e2e/cases/test_go_entry.py \
+              --ignore=scripts/test/e2e/cases/test_go_coverage.py \
+              --ignore=scripts/test/e2e/cases/test_node_entry.py \
+              --ignore=scripts/test/e2e/cases/test_exec_timeout.py \
+              -v -s --tb=short --no-header -p no:cacheprovider \
+              --timeout=180 --timeout-method=thread \
+              --junit-xml=pytest-junit.xml
+
+      - name: Upload pytest junit XML
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pytest-junit
+          path: pytest-junit.xml
+          if-no-files-found: ignore

scripts/deploy/build-runner-binary.sh

@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+# Build the Linux amd64 boxlite-runner tarball without GitHub Actions.
+#
+# Run this from the checkout you want to deploy. To deploy latest main to dev:
+#   git checkout main
+#   git pull --ff-only origin main
+#   scripts/deploy/build-runner-binary.sh --output-dir dist
+# The generated tarball is intentionally named
+# `v{VERSION}-dev-{BUILD_SEQUENCE}-{GUEST_HASH}`. This script is for
+# non-published builds, so the embedded runtime cache must not share the
+# official release directory `v{VERSION}`.
+#
+# The runner is a cgo binary that links libboxlite.a. Build the embedded runtime
+# inputs first (boxlite-shim + boxlite-guest), force boxlite's build.rs to rerun
+# so those binaries are embedded, then build/stage the C SDK archive for Go.
+#
+# Usage:
+#   scripts/deploy/build-runner-binary.sh
+#   scripts/deploy/build-runner-binary.sh --output-dir /tmp
+#   BOXLITE_RUNNER_BUILD_NUMBER=123 scripts/deploy/build-runner-binary.sh
+#   scripts/deploy/build-runner-binary.sh --skip-setup
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+OUTPUT_DIR="$ROOT_DIR/dist"
+RUN_SETUP=1
+BUILD_SEQUENCE="${BOXLITE_RUNNER_BUILD_NUMBER:-${GITHUB_RUN_NUMBER:-}}"
+
+usage() {
+  sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'
+  cat <<'EOF'
+Options:
+  --output-dir DIR   Directory for the runner tarball. Default: ./dist
+  --build-number ID  Ordered dev build sequence. Default: BOXLITE_RUNNER_BUILD_NUMBER,
+                     GITHUB_RUN_NUMBER, or current UTC timestamp.
+  --skip-setup       Skip `make setup:build`.
+  -h, --help         Show this help.
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --output-dir)
+      [[ $# -ge 2 ]] || { echo "error: --output-dir requires a value" >&2; exit 1; }
+      OUTPUT_DIR="$2"
+      shift 2
+      ;;
+    --build-number)
+      [[ $# -ge 2 ]] || { echo "error: --build-number requires a value" >&2; exit 1; }
+      BUILD_SEQUENCE="$2"
+      shift 2
+      ;;
+    --skip-setup)
+      RUN_SETUP=0
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "error: unknown argument $1" >&2
+      usage >&2
+      exit 1
+      ;;
+  esac
+done
+
+cd "$ROOT_DIR"
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || { echo "error: required command not found: $1" >&2; exit 1; }
+}
+
+require_cmd cargo
+require_cmd go
+require_cmd make
+require_cmd sha256sum
+require_cmd tar
+
+find_embedded_runtime_dir() {
+  local guest_sha256="$1"
+  local runtime_dir
+  local runtime_guest_sha256
+
+  while IFS= read -r runtime_dir; do
+    [ -f "$runtime_dir/boxlite-guest" ] || continue
+    runtime_guest_sha256="$(sha256sum "$runtime_dir/boxlite-guest" | awk '{print $1}')"
+    if [[ "$runtime_guest_sha256" == "$guest_sha256" ]]; then
+      printf '%s\n' "$runtime_dir"
+      return 0
+    fi
+  done < <(find "$ROOT_DIR/target/release/build" -maxdepth 3 -path '*/boxlite-*/out/runtime' -type d 2>/dev/null | sort)
+
+  echo "error: embedded runtime payload not found for guest hash ${guest_sha256:0:12}" >&2
+  exit 1
+}
+
+if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then
+  echo "error: runner tarball build currently requires a Linux x86_64 builder" >&2
+  exit 1
+fi
+
+VERSION="$(grep -m 1 '^version' "$ROOT_DIR/Cargo.toml" | sed -E 's/^version *= *"([^"]+)".*/\1/')"
+if [[ -z "$VERSION" ]]; then
+  echo "error: could not read version from Cargo.toml" >&2
+  exit 1
+fi
+
+if [[ -z "$BUILD_SEQUENCE" ]]; then
+  BUILD_SEQUENCE="$(date -u +%Y%m%d%H%M%S)"
+fi
+BUILD_SEQUENCE="$(printf '%s' "$BUILD_SEQUENCE" | tr -c 'A-Za-z0-9._-' '-')"
+BUILD_SEQUENCE="${BUILD_SEQUENCE#v}"
+[[ -n "$BUILD_SEQUENCE" ]] || { echo "error: build sequence cannot be empty" >&2; exit 1; }
+
+TMP_DIR="$(mktemp -d)"
+GO_WORK_BACKUP="$TMP_DIR/go.work.backup"
+GO_WORK_SUM_BACKUP="$TMP_DIR/go.work.sum.backup"
+HAD_GO_WORK=0
+HAD_GO_WORK_SUM=0
+
+restore_go_work() {
+  if [[ "$HAD_GO_WORK" -eq 1 ]]; then
+    cp "$GO_WORK_BACKUP" "$ROOT_DIR/apps/go.work"
+  else
+    rm -f "$ROOT_DIR/apps/go.work"
+  fi
+
+  if [[ "$HAD_GO_WORK_SUM" -eq 1 ]]; then
+    cp "$GO_WORK_SUM_BACKUP" "$ROOT_DIR/apps/go.work.sum"
+  else
+    rm -f "$ROOT_DIR/apps/go.work.sum"
+  fi
+
+  rm -rf "$TMP_DIR"
+}
+trap restore_go_work EXIT
+
+if [[ -f "$ROOT_DIR/apps/go.work" ]]; then
+  HAD_GO_WORK=1
+  cp "$ROOT_DIR/apps/go.work" "$GO_WORK_BACKUP"
+fi
+if [[ -f "$ROOT_DIR/apps/go.work.sum" ]]; then
+  HAD_GO_WORK_SUM=1
+  cp "$ROOT_DIR/apps/go.work.sum" "$GO_WORK_SUM_BACKUP"
+fi
+
+GO_VERSION="$(awk '/^go / { print $2; exit }' "$ROOT_DIR/apps/runner/go.mod")"
+if [[ -z "$GO_VERSION" ]]; then
+  echo "error: could not read Go version from apps/runner/go.mod" >&2
+  exit 1
+fi
+
+cat > "$ROOT_DIR/apps/go.work" <<EOF
+go $GO_VERSION
+
+use (
+	./runner
+	./common-go
+	./api-client-go
+	../sdks/go
+)
+EOF
+
+echo "==> Building boxlite-runner from package v$VERSION at $(git rev-parse --short HEAD)"
+echo "==> Non-release artifact version prefix: v${VERSION}-dev-${BUILD_SEQUENCE}"
+
+echo "==> Cleaning runner build artifacts"
+cargo clean -p boxlite -p boxlite-c -p boxlite-shim -p boxlite-guest
+rm -f \
+  "$ROOT_DIR/sdks/go/libboxlite.a" \
+  "$ROOT_DIR/target/release/libboxlite.a" \
+  "$ROOT_DIR/target/release/libboxlite.so" \
+  "$ROOT_DIR/target/release/boxlite-shim"
+rm -f "$ROOT_DIR/target/x86_64-unknown-linux-gnu/release/boxlite-shim"
+rm -f "$ROOT_DIR/target/x86_64-unknown-linux-musl/release/boxlite-guest"
+rm -rf "$ROOT_DIR/sdks/c/dist"
+
+if [[ "$RUN_SETUP" -eq 1 ]]; then
+  make setup:build
+fi
+
+scripts/build/build-shim.sh --profile release
+scripts/build/build-guest.sh --profile release
+GUEST_BIN="$ROOT_DIR/target/x86_64-unknown-linux-musl/release/boxlite-guest"
+[[ -x "$GUEST_BIN" ]] || { echo "error: guest binary not found after build: $GUEST_BIN" >&2; exit 1; }
+GUEST_SHA256="$(sha256sum "$GUEST_BIN" | awk '{print $1}')"
+RUNTIME_SUFFIX="${GUEST_SHA256:0:12}"
+RUNTIME_CACHE_SUFFIX="dev-${BUILD_SEQUENCE}-${RUNTIME_SUFFIX}"
+RUNNER_VERSION="${VERSION}-${RUNTIME_CACHE_SUFFIX}"
+
+echo "==> Building libboxlite with runtime cache key v${RUNNER_VERSION}"
+make dist:c
+echo "==> Fixing libboxlite Go runtime symbols"
+bash "$ROOT_DIR/scripts/build/fix-go-symbols.sh" "$ROOT_DIR/target/release/libboxlite.a"
+RUNTIME_DIR="$(find_embedded_runtime_dir "$GUEST_SHA256")"
+tar czf "$TMP_DIR/boxlite-runtime.tar.gz" -C "$RUNTIME_DIR" .
+echo "==> Wrote embedded runtime payload from $RUNTIME_DIR"
+cp "$ROOT_DIR/target/release/libboxlite.a" "$ROOT_DIR/sdks/go/libboxlite.a"
+
+go -C apps/runner mod download
+go -C apps/common-go mod download
+go -C apps/api-client-go mod download
+
+RUNNER_BIN="$TMP_DIR/boxlite-runner"
+CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -C apps \
+  -ldflags "-X github.com/boxlite-ai/runner/internal.Version=${RUNNER_VERSION}" \
+  -o "$RUNNER_BIN" ./runner/cmd/runner
+printf '%s  boxlite-guest\n' "$GUEST_SHA256" > "$TMP_DIR/boxlite-runner.guest.sha256"
+echo "==> Wrote guest hash sidecar ${GUEST_SHA256:0:12}"
+printf '%s\n' "$RUNTIME_CACHE_SUFFIX" > "$TMP_DIR/boxlite-runner.runtime-suffix"
+echo "==> Runtime cache directory: v${RUNNER_VERSION}"
+
+mkdir -p "$OUTPUT_DIR"
+TARBALL="$OUTPUT_DIR/boxlite-runner-v${RUNNER_VERSION}-linux-amd64.tar.gz"
+tar czf "$TARBALL" -C "$TMP_DIR" \
+  boxlite-runner \
+  boxlite-runner.guest.sha256 \
+  boxlite-runner.runtime-suffix \
+  boxlite-runtime.tar.gz
+
+if command -v sha256sum >/dev/null 2>&1; then
+  sha256sum "$TARBALL" > "$TARBALL.sha256"
+else
+  shasum -a 256 "$TARBALL" > "$TARBALL.sha256"
+fi
+
+echo "==> Wrote $TARBALL"
+echo "==> Wrote $TARBALL.sha256"