chore(ci): switch sccache from local disk to memcached backend (#68)

Author: drew

.github/workflows/checks.yml

@@ -9,15 +9,20 @@ env:
   CARGO_TERM_COLOR: always
   CARGO_INCREMENTAL: "0"
   MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
 
 permissions:
   contents: read
   packages: read
 
 jobs:
   rust:
-    name: Rust
-    runs-on: build-amd64
+    name: Rust (${{ matrix.runner }})
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [build-amd64, build-arm64]
+    runs-on: ${{ matrix.runner }}
     container:
       image: ghcr.io/nvidia/nv-agent-env/ci:latest
       credentials:
@@ -26,13 +31,17 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Cache sccache
-        uses: actions/cache@v4
+      - name: Install tools
+        run: mise install
+
+      - name: Cache Rust target and registry
+        uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
         with:
-          path: .cache/sccache
-          key: sccache-rust-${{ runner.arch }}-${{ hashFiles('Cargo.lock') }}
-          restore-keys: |
-            sccache-rust-${{ runner.arch }}-
+          # Separate caches for clippy (check-like) vs test (full build)
+          # so they don't thrash each other's artifacts
+          shared-key: rust-checks-${{ matrix.runner }}
+          # Cache the sccache directory too
+          cache-directories: .cache/sccache
 
       - name: Format
         run: mise run rust:format:check
@@ -45,11 +54,15 @@ jobs:
 
       - name: sccache stats
         if: always()
-        run: sccache --show-stats
+        run: mise x -- sccache --show-stats
 
   python:
-    name: Python
-    runs-on: build-amd64
+    name: Python (${{ matrix.runner }})
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [build-amd64, build-arm64]
+    runs-on: ${{ matrix.runner }}
     container:
       image: ghcr.io/nvidia/nv-agent-env/ci:latest
       credentials:
@@ -58,6 +71,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install tools
+        run: mise install
+
       - name: Install dependencies
         run: uv sync --frozen
 

.github/workflows/docker-build.yml

@@ -20,6 +20,7 @@ on:
 
 env:
   MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
 
 permissions:
   contents: read

.github/workflows/publish.yml

@@ -50,6 +50,7 @@ jobs:
         - /var/run/docker.sock:/var/run/docker.sock
     env:
       MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_DEFAULT_REGION: us-west-2
@@ -93,6 +94,7 @@ jobs:
         - /var/run/docker.sock:/var/run/docker.sock
     env:
       MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
       NAV_PYPI_S3_BUCKET: navigator-pypi-artifacts
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

architecture/build-containers.md

@@ -121,7 +121,7 @@ A k3s image with bundled Helm charts and Kubernetes manifests for single-contain
 
 Two Dockerfiles produce Python wheels for the CLI package distribution. These are not deployed as running containers.
 
-- **`Dockerfile.python-wheels`** -- Builds Linux amd64/arm64 wheels using Maturin with a two-pass Rust build (dependency prebuild + final wheel build), BuildKit cache mounts for cargo registry/git/target and sccache, and `cross-build.sh` for conditional cross-toolchain installation. The final build step patches workspace version inside the container layer from `NAVIGATOR_CARGO_VERSION` (computed before Docker build), preserving cacheable dependency layers and avoiding dirty working-tree edits. Output stage is `scratch` with only the `.whl` files.
+- **`Dockerfile.python-wheels`** -- Builds Linux amd64/arm64 wheels using Maturin with a two-pass Rust build (dependency prebuild + final wheel build), BuildKit cache mounts for cargo registry/git/target, sccache (backed by memcached when `SCCACHE_MEMCACHED_ENDPOINT` build arg is provided), and `cross-build.sh` for conditional cross-toolchain installation. The final build step patches workspace version inside the container layer from `NAVIGATOR_CARGO_VERSION` (computed before Docker build), preserving cacheable dependency layers and avoiding dirty working-tree edits. Output stage is `scratch` with only the `.whl` files.
 - **`Dockerfile.python-wheels-macos`** -- Builds macOS arm64 wheels using osxcross (cross-compiling from Linux) with the same two-pass dependency caching pattern and cargo cache mounts. Version injection uses the same in-container workspace-version patch from `NAVIGATOR_CARGO_VERSION`, avoiding host-side file edits that break Docker layer caching. Uses `crazymax/osxcross:latest` as the cross-toolchain source. The `OSXCROSS_IMAGE` build arg allows using a mirrored registry image instead of Docker Hub.
 
 ### CI Runner Image (`navigator-ci`)
@@ -386,7 +386,7 @@ Container builds use Docker BuildKit with local cache directories:
 - `build/scripts/docker-build-component.sh` stores per-component caches in `.cache/buildkit/<component>`.
 - `build/scripts/docker-build-cluster.sh` stores the cluster image cache in `.cache/buildkit/cluster`.
 - `mise run python:build:multiarch` stores per-platform wheel caches in `.cache/buildkit/python-wheels/<platform>` for local builds when using a `docker-container` buildx driver.
-- Rust-heavy Dockerfiles use BuildKit cache mounts for cargo registry and target directories, keyed by image name and `TARGETARCH`, with `sharing=locked` to prevent concurrent cache corruption in parallel CI builds.
+- Rust-heavy Dockerfiles use BuildKit cache mounts for cargo registry, cargo target, and sccache local disk directories, keyed by image name and `TARGETARCH`, with `sharing=locked` to prevent concurrent cache corruption in parallel CI builds. The cargo target mount gives cargo a persistent `target/` directory for true incremental rebuilds on source-only changes. sccache uses memcached in CI (`SCCACHE_MEMCACHED_ENDPOINT`) and falls back to the local disk cache mount for local dev builds, providing a second layer of caching at the compilation unit level.
 - When the active buildx driver is `docker` (not `docker-container`), local cache import/export flags are skipped automatically because the docker driver cannot export local caches. In CI, cache export is also skipped.
 - For local single-arch builds, the scripts auto-select a builder with the native `docker` driver (matching the active Docker context) so images land directly in the Docker image store without slow tarball export.
 
@@ -395,7 +395,7 @@ Container builds use Docker BuildKit with local cache directories:
 In CI pipelines:
 
 - Remote BuildKit daemons (`buildkit-amd64` and `buildkit-arm64`) are used as persistent builders via `driver: remote`. Their built-in layer cache persists across builds, so no external cache (registry-backed or otherwise) is needed in CI.
-- Rust lint/test jobs cache `.cache/sccache/` and `target/` with keys derived from `Cargo.lock` and Rust task config files, scoped per runner architecture.
+- Rust lint/test jobs cache `target/` with keys derived from `Cargo.lock` and Rust task config files, scoped per runner architecture. sccache uses a shared memcached backend (`SCCACHE_MEMCACHED_ENDPOINT`) instead of local disk.
 - CI sets `CARGO_INCREMENTAL=0` to favor deterministic clean builds over incremental metadata churn.
 - Publish jobs mirror `crazymax/osxcross:latest` into `$CI_REGISTRY_IMAGE/third_party/osxcross:latest` (when missing) and set `OSXCROSS_IMAGE` so macOS wheel Docker builds consume the mirrored image instead of pulling from Docker Hub on each run.
 - The sandbox e2e test job tags and pushes component images to the GitLab project registry (`$CI_REGISTRY_IMAGE`) and configures cluster bootstrap to pull from that remote registry with CI credentials.

build/scripts/docker-build-component.sh

@@ -93,10 +93,16 @@ if [[ "${DOCKER_PUSH:-}" == "1" ]]; then
   OUTPUT_FLAG="--push"
 fi
 
+SCCACHE_ARGS=()
+if [[ -n "${SCCACHE_MEMCACHED_ENDPOINT:-}" ]]; then
+  SCCACHE_ARGS=(--build-arg "SCCACHE_MEMCACHED_ENDPOINT=${SCCACHE_MEMCACHED_ENDPOINT}")
+fi
+
 docker buildx build \
   ${BUILDER_ARGS[@]+"${BUILDER_ARGS[@]}"} \
   ${DOCKER_PLATFORM:+--platform ${DOCKER_PLATFORM}} \
   ${CACHE_ARGS[@]+"${CACHE_ARGS[@]}"} \
+  ${SCCACHE_ARGS[@]+"${SCCACHE_ARGS[@]}"} \
   -f "${DOCKERFILE}" \
   -t "${IMAGE_NAME}:${IMAGE_TAG}" \
   --provenance=false \

build/scripts/docker-publish-multiarch.sh

@@ -121,6 +121,9 @@ for component in sandbox server; do
   if [ "$component" = "sandbox" ]; then
     BUILD_ARGS="--build-arg RUST_BUILD_PROFILE=${RUST_BUILD_PROFILE:-release}"
   fi
+  if [ -n "${SCCACHE_MEMCACHED_ENDPOINT:-}" ]; then
+    BUILD_ARGS="${BUILD_ARGS} --build-arg SCCACHE_MEMCACHED_ENDPOINT=${SCCACHE_MEMCACHED_ENDPOINT}"
+  fi
   DOCKERFILE=$(resolve_dockerfile "${component}")
   FULL_IMAGE="${REGISTRY}/${IMAGE_PREFIX}${component}"
   docker buildx build \

deploy/docker/Dockerfile.python-wheels

@@ -26,7 +26,7 @@ ARG TARGETARCH
 ARG BUILDARCH
 ARG NAVIGATOR_CARGO_VERSION
 
-ENV SCCACHE_DIR=/tmp/sccache
+ARG SCCACHE_MEMCACHED_ENDPOINT
 
 WORKDIR /build
 
@@ -55,6 +55,8 @@ RUN mkdir -p crates/navigator-cli/src crates/navigator-core/src crates/navigator
     touch crates/navigator-bootstrap/src/lib.rs
 
 # Build dependencies only (cached unless Cargo.toml/lock changes).
+# sccache uses memcached in CI or the local disk cache mount for local dev.
+# The cargo-target mount gives cargo a persistent target/ dir for incremental rebuilds.
 RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \
     --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \
     --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH},sharing=locked,target=/build/target \
@@ -68,8 +70,6 @@ COPY dev-sandbox-policy.yaml ./
 COPY python/ python/
 
 # Touch source files to ensure they're rebuilt (not the cached dummy).
-# Touch build.rs and proto files to force proto code regeneration when the
-# cargo target cache mount retains stale OUT_DIR artifacts from prior builds.
 RUN touch crates/navigator-cli/src/main.rs \
     crates/navigator-cli/src/lib.rs \
     crates/navigator-bootstrap/src/lib.rs \

deploy/docker/Dockerfile.server

@@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY deploy/docker/cross-build.sh /usr/local/bin/
 RUN . cross-build.sh && install_cross_toolchain && install_sccache && add_rust_target
 
-ENV SCCACHE_DIR=/tmp/sccache
+ARG SCCACHE_MEMCACHED_ENDPOINT
 
 WORKDIR /build
 
@@ -42,7 +42,10 @@ RUN mkdir -p crates/navigator-cli/src crates/navigator-core/src crates/navigator
 # Copy proto files needed for build
 COPY proto/ proto/
 
-# Build dependencies only (cached unless Cargo.toml/lock changes)
+# Build dependencies only (cached unless Cargo.toml/lock changes).
+# sccache uses memcached in CI (SCCACHE_MEMCACHED_ENDPOINT) or the local
+# disk cache mount for local dev builds. The cargo-target mount gives cargo
+# a persistent target/ dir for true incremental rebuilds on source changes.
 RUN --mount=type=cache,id=cargo-registry-server-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
     --mount=type=cache,id=cargo-target-server-${TARGETARCH},sharing=locked,target=/build/target \
     --mount=type=cache,id=sccache-server-${TARGETARCH},sharing=locked,target=/tmp/sccache \

deploy/docker/cross-build.sh

@@ -81,12 +81,28 @@ export_cross_env() {
 # Automatically wraps with sccache when available.
 cargo_cross_build() {
   export_cross_env
+  # Unset empty SCCACHE_MEMCACHED_ENDPOINT so sccache falls back to the
+  # local disk cache instead of erroring on an empty endpoint string.
+  if [ -z "${SCCACHE_MEMCACHED_ENDPOINT:-}" ]; then
+    unset SCCACHE_MEMCACHED_ENDPOINT 2>/dev/null || true
+  fi
+  # Default sccache local disk cache to /tmp/sccache (matches BuildKit
+  # cache mount target in Dockerfiles) when no dir is explicitly set.
+  export SCCACHE_DIR="${SCCACHE_DIR:-/tmp/sccache}"
   if command -v sccache >/dev/null 2>&1; then
     export RUSTC_WRAPPER=sccache
   fi
   local target_flag=""
   if is_cross; then target_flag="--target $(rust_target)"; fi
-  cargo build $target_flag "$@"
+  # Retry once after cleaning if the build fails. BuildKit cargo-target cache
+  # mounts can retain stale .rmeta files from prior builds with different
+  # dependency versions; cargo clean purges them so the retry succeeds.
+  # sccache still has the compiled objects, so the clean rebuild is fast.
+  if ! cargo build $target_flag "$@"; then
+    echo "cargo build failed; cleaning stale target cache and retrying..." >&2
+    cargo clean 2>/dev/null || true
+    cargo build $target_flag "$@"
+  fi
 }
 
 # Print the directory containing the compiled binary.

deploy/docker/sandbox/Dockerfile.base

@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY deploy/docker/cross-build.sh /usr/local/bin/
 RUN . cross-build.sh && install_cross_toolchain && install_sccache && add_rust_target
 
-ENV SCCACHE_DIR=/tmp/sccache
+ARG SCCACHE_MEMCACHED_ENDPOINT
 
 WORKDIR /build
 
@@ -44,7 +44,9 @@ RUN mkdir -p crates/navigator-cli/src crates/navigator-core/src crates/navigator
 COPY proto/ proto/
 COPY dev-sandbox-policy.rego ./
 
-# Build dependencies only (cached unless Cargo.toml/lock changes)
+# Build dependencies only (cached unless Cargo.toml/lock changes).
+# sccache uses memcached in CI or the local disk cache mount for local dev.
+# The cargo-target mount gives cargo a persistent target/ dir for incremental rebuilds.
 RUN --mount=type=cache,id=cargo-registry-sandbox-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
     --mount=type=cache,id=cargo-target-sandbox-${TARGETARCH},sharing=locked,target=/build/target \
     --mount=type=cache,id=sccache-sandbox-${TARGETARCH},sharing=locked,target=/tmp/sccache \